diff --git a/go.mod b/go.mod index 60d01b7ac58..67181512886 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/SigNoz/signoz-otel-collector v0.144.2 github.com/antlr4-go/antlr/v4 v4.13.1 github.com/antonmedv/expr v1.15.3 - github.com/bytedance/sonic v1.14.1 github.com/cespare/xxhash/v2 v2.3.0 github.com/coreos/go-oidc/v3 v3.17.0 github.com/dgraph-io/ristretto/v2 v2.3.0 @@ -106,6 +105,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect github.com/aws/smithy-go v1.24.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.14.1 // indirect github.com/bytedance/sonic/loader v0.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/gabriel-vasile/mimetype v1.4.8 // indirect diff --git a/pkg/errors/http.go b/pkg/errors/http.go index 25a77fc40f3..ab08e2a8540 100644 --- a/pkg/errors/http.go +++ b/pkg/errors/http.go @@ -18,13 +18,17 @@ type responseerroradditional struct { func AsJSON(cause error) *JSON { // See if this is an instance of the base error or not - _, c, m, _, u, a := Unwrapb(cause) + _, c, m, err, u, a := Unwrapb(cause) rea := make([]responseerroradditional, len(a)) for k, v := range a { rea[k] = responseerroradditional{v} } + if err != nil { + rea = append(rea, responseerroradditional{err.Error()}) + } + return &JSON{ Code: c.String(), Message: m, diff --git a/pkg/modules/promote/implpromote/module.go b/pkg/modules/promote/implpromote/module.go index a5ff24d0f5d..c05e6eb875d 100644 --- a/pkg/modules/promote/implpromote/module.go +++ b/pkg/modules/promote/implpromote/module.go @@ -78,7 +78,7 @@ func (m *module) ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetype // add the paths that are not promoted but have indexes for path, indexes := range aggr { - path := strings.TrimPrefix(path, telemetrylogs.BodyJSONColumnPrefix) + path := strings.TrimPrefix(path, telemetrylogs.BodyV2ColumnPrefix) path = telemetrytypes.BodyJSONStringSearchPrefix + path response = append(response, promotetypes.PromotePath{ Path: path, @@ -163,7 +163,7 @@ func (m *module) PromoteAndIndexPaths( } } if len(it.Indexes) > 0 { - parentColumn := telemetrylogs.LogsV2BodyJSONColumn + parentColumn := telemetrylogs.LogsV2BodyV2Column // if the path is already promoted or is being promoted, add it to the promoted column if _, promoted := existingPromotedPaths[it.Path]; promoted || it.Promote { parentColumn = telemetrylogs.LogsV2BodyPromotedColumn diff --git a/pkg/querier/builder_query.go b/pkg/querier/builder_query.go index 8ec2fab7bb4..d3a382a343a 100644 --- a/pkg/querier/builder_query.go +++ b/pkg/querier/builder_query.go @@ -10,13 +10,11 @@ import ( "github.com/ClickHouse/clickhouse-go/v2" "github.com/SigNoz/signoz/pkg/errors" - "github.com/SigNoz/signoz/pkg/telemetrylogs" "github.com/SigNoz/signoz/pkg/telemetrystore" "github.com/SigNoz/signoz/pkg/types/ctxtypes" "github.com/SigNoz/signoz/pkg/types/instrumentationtypes" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" - "github.com/bytedance/sonic" ) type builderQuery[T any] struct { @@ -262,40 +260,6 @@ func (q *builderQuery[T]) executeWithContext(ctx context.Context, query string, return nil, err } - // merge body_json and promoted into body - if q.spec.Signal == telemetrytypes.SignalLogs { - switch typedPayload := payload.(type) { - case *qbtypes.RawData: - for _, rr := range typedPayload.Rows { - seeder := func() error { - body, ok := rr.Data[telemetrylogs.LogsV2BodyJSONColumn].(map[string]any) - if !ok { - return nil - } - promoted, ok := rr.Data[telemetrylogs.LogsV2BodyPromotedColumn].(map[string]any) - if !ok { - return nil - } - seed(promoted, body) - str, err := sonic.MarshalString(body) - if err != nil { - return errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "failed to marshal body") - } - rr.Data["body"] = str - return nil - } - err := seeder() - if err != nil { - return nil, err - } - - delete(rr.Data, telemetrylogs.LogsV2BodyJSONColumn) - delete(rr.Data, telemetrylogs.LogsV2BodyPromotedColumn) - } - payload = typedPayload - } - } - return &qbtypes.Result{ Type: q.kind, Value: payload, @@ -423,18 +387,3 @@ func decodeCursor(cur string) (int64, error) { } return strconv.ParseInt(string(b), 10, 64) } - -func seed(promoted map[string]any, body map[string]any) { - for key, fromValue := range promoted { - if toValue, ok := body[key]; !ok { - body[key] = fromValue - } else { - if fromValue, ok := fromValue.(map[string]any); ok { - if toValue, ok := toValue.(map[string]any); ok { - seed(fromValue, toValue) - body[key] = toValue - } - } - } - } -} diff --git a/pkg/querier/consume.go b/pkg/querier/consume.go index 58c3df871cb..72ce0856576 100644 --- a/pkg/querier/consume.go +++ b/pkg/querier/consume.go @@ -14,7 +14,6 @@ import ( "github.com/ClickHouse/clickhouse-go/v2/lib/driver" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" - "github.com/bytedance/sonic" ) var ( @@ -394,17 +393,11 @@ func readAsRaw(rows driver.Rows, queryName string) (*qbtypes.RawData, error) { // de-reference the typed pointer to any val := reflect.ValueOf(cellPtr).Elem().Interface() - - // Post-process JSON columns: normalize into structured values + // Post-process JSON columns: normalize into String value if strings.HasPrefix(strings.ToUpper(colTypes[i].DatabaseTypeName()), "JSON") { switch x := val.(type) { case []byte: - if len(x) > 0 { - var v any - if err := sonic.Unmarshal(x, &v); err == nil { - val = v - } - } + val = string(x) default: // already a structured type (map[string]any, []any, etc.) } diff --git a/pkg/query-service/rules/base_rule_test.go b/pkg/query-service/rules/base_rule_test.go index 32472c68ccb..b2cf19bb0e0 100644 --- a/pkg/query-service/rules/base_rule_test.go +++ b/pkg/query-service/rules/base_rule_test.go @@ -658,7 +658,7 @@ func TestBaseRule_FilterNewSeries(t *testing.T) { telemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &queryMatcherAny{}) // Setup mock metadata store - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) // Create query parser queryParser := queryparser.New(settings) diff --git a/pkg/querybuilder/fallback_expr.go b/pkg/querybuilder/fallback_expr.go index dbc2bba995d..d4d4f417ee9 100644 --- a/pkg/querybuilder/fallback_expr.go +++ b/pkg/querybuilder/fallback_expr.go @@ -204,7 +204,10 @@ func DataTypeCollisionHandledFieldName(key *telemetrytypes.TelemetryFieldKey, va // While we expect user not to send the mixed data types, it inevitably happens // So we handle the data type collisions here switch key.FieldDataType { - case telemetrytypes.FieldDataTypeString, telemetrytypes.FieldDataTypeArrayString: + case telemetrytypes.FieldDataTypeString, telemetrytypes.FieldDataTypeArrayString, telemetrytypes.FieldDataTypeJSON: + if key.FieldDataType == telemetrytypes.FieldDataTypeJSON { + tblFieldName = fmt.Sprintf("toString(%s)", tblFieldName) + } switch v := value.(type) { case float64: // try to convert the string value to to number @@ -219,7 +222,6 @@ func DataTypeCollisionHandledFieldName(key *telemetrytypes.TelemetryFieldKey, va // we don't have a toBoolOrNull in ClickHouse, so we need to convert the bool to a string value = fmt.Sprintf("%t", v) } - case telemetrytypes.FieldDataTypeInt64, telemetrytypes.FieldDataTypeArrayInt64, telemetrytypes.FieldDataTypeNumber, diff --git a/pkg/querybuilder/where_clause_visitor.go b/pkg/querybuilder/where_clause_visitor.go index 35a1bb1afb9..855f3b37dfc 100644 --- a/pkg/querybuilder/where_clause_visitor.go +++ b/pkg/querybuilder/where_clause_visitor.go @@ -313,37 +313,30 @@ func (v *filterExpressionVisitor) VisitPrimary(ctx *grammar.PrimaryContext) any return "" } child := ctx.GetChild(0) + var searchText string if keyCtx, ok := child.(*grammar.KeyContext); ok { // create a full text search condition on the body field - - keyText := keyCtx.GetText() - cond, err := v.conditionBuilder.ConditionFor(context.Background(), v.fullTextColumn, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(keyText), v.builder, v.startNs, v.endNs) - if err != nil { - v.errors = append(v.errors, fmt.Sprintf("failed to build full text search condition: %s", err.Error())) - return "" - } - return cond + searchText = keyCtx.GetText() } else if valCtx, ok := child.(*grammar.ValueContext); ok { - var text string if valCtx.QUOTED_TEXT() != nil { - text = trimQuotes(valCtx.QUOTED_TEXT().GetText()) + searchText = trimQuotes(valCtx.QUOTED_TEXT().GetText()) } else if valCtx.NUMBER() != nil { - text = valCtx.NUMBER().GetText() + searchText = valCtx.NUMBER().GetText() } else if valCtx.BOOL() != nil { - text = valCtx.BOOL().GetText() + searchText = valCtx.BOOL().GetText() } else if valCtx.KEY() != nil { - text = valCtx.KEY().GetText() + searchText = valCtx.KEY().GetText() } else { v.errors = append(v.errors, fmt.Sprintf("unsupported value type: %s", valCtx.GetText())) return "" } - cond, err := v.conditionBuilder.ConditionFor(context.Background(), v.fullTextColumn, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(text), v.builder, v.startNs, v.endNs) - if err != nil { - v.errors = append(v.errors, fmt.Sprintf("failed to build full text search condition: %s", err.Error())) - return "" - } - return cond } + cond, err := v.conditionBuilder.ConditionFor(context.Background(), v.fullTextColumn, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(searchText), v.builder, v.startNs, v.endNs) + if err != nil { + v.errors = append(v.errors, fmt.Sprintf("failed to build full text search condition: %s", err.Error())) + return "" + } + return cond } return "" // Should not happen with valid input @@ -383,6 +376,7 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext for _, key := range keys { condition, err := v.conditionBuilder.ConditionFor(context.Background(), key, op, nil, v.builder, v.startNs, v.endNs) if err != nil { + v.errors = append(v.errors, fmt.Sprintf("failed to build condition: %s", err.Error())) return "" } conds = append(conds, condition) @@ -648,7 +642,6 @@ func (v *filterExpressionVisitor) VisitValueList(ctx *grammar.ValueListContext) // VisitFullText handles standalone quoted strings for full-text search func (v *filterExpressionVisitor) VisitFullText(ctx *grammar.FullTextContext) any { - if v.skipFullTextFilter { return "" } @@ -670,6 +663,7 @@ func (v *filterExpressionVisitor) VisitFullText(ctx *grammar.FullTextContext) an v.errors = append(v.errors, fmt.Sprintf("failed to build full text search condition: %s", err.Error())) return "" } + return cond } diff --git a/pkg/telemetrylogs/condition_builder.go b/pkg/telemetrylogs/condition_builder.go index bc1de017977..254465f7cda 100644 --- a/pkg/telemetrylogs/condition_builder.go +++ b/pkg/telemetrylogs/condition_builder.go @@ -3,14 +3,12 @@ package telemetrylogs import ( "context" "fmt" - "slices" schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator" "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/querybuilder" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" - "golang.org/x/exp/maps" "github.com/huandu/go-sqlbuilder" ) @@ -35,7 +33,7 @@ func (c *conditionBuilder) conditionFor( return "", err } - if column.Type.GetType() == schema.ColumnTypeEnumJSON && querybuilder.BodyJSONQueryEnabled { + if column.Type.GetType() == schema.ColumnTypeEnumJSON && querybuilder.BodyJSONQueryEnabled && !key.Materialized { valueType, value := InferDataType(value, operator, key) cond, err := NewJSONConditionBuilder(key, valueType).buildJSONCondition(operator, value, sb) if err != nil { @@ -54,7 +52,7 @@ func (c *conditionBuilder) conditionFor( } // Check if this is a body JSON search - either by FieldContext - if key.FieldContext == telemetrytypes.FieldContextBody { + if key.FieldContext == telemetrytypes.FieldContextBody && !querybuilder.BodyJSONQueryEnabled { tblFieldName, value = GetBodyJSONKey(ctx, key, operator, value) } @@ -108,7 +106,6 @@ func (c *conditionBuilder) conditionFor( return sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil case qbtypes.FilterOperatorNotContains: return sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil - case qbtypes.FilterOperatorRegexp: // Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs // Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder) @@ -176,9 +173,16 @@ func (c *conditionBuilder) conditionFor( var value any switch column.Type.GetType() { case schema.ColumnTypeEnumJSON: - if operator == qbtypes.FilterOperatorExists { - return sb.IsNotNull(tblFieldName), nil - } else { + switch key.FieldDataType { + case telemetrytypes.FieldDataTypeJSON: + if operator == qbtypes.FilterOperatorExists { + return sb.EQ(fmt.Sprintf("empty(%s)", tblFieldName), false), nil + } + return sb.EQ(fmt.Sprintf("empty(%s)", tblFieldName), true), nil + default: + if operator == qbtypes.FilterOperatorExists { + return sb.IsNotNull(tblFieldName), nil + } return sb.IsNull(tblFieldName), nil } case schema.ColumnTypeEnumLowCardinality: @@ -247,19 +251,32 @@ func (c *conditionBuilder) ConditionFor( return "", err } - if !(key.FieldContext == telemetrytypes.FieldContextBody && querybuilder.BodyJSONQueryEnabled) && operator.AddDefaultExistsFilter() { - // skip adding exists filter for intrinsic fields - // with an exception for body json search - field, _ := c.fm.FieldFor(ctx, key) - if slices.Contains(maps.Keys(IntrinsicFields), field) && key.FieldContext != telemetrytypes.FieldContextBody { + // Skip adding exists filter for intrinsic fields i.e. Table level log context fields + buildExistCondition := operator.AddDefaultExistsFilter() + switch key.FieldContext { + case telemetrytypes.FieldContextLog, telemetrytypes.FieldContextScope: + // pass; No need to build exist condition for top level columns + // immidiately return + return condition, nil + case telemetrytypes.FieldContextResource, telemetrytypes.FieldContextAttribute: + buildExistCondition = buildExistCondition && true + case telemetrytypes.FieldContextBody: + // Querying JSON fields already account for Nullability of fields + // so additional exists checks are not needed + if querybuilder.BodyJSONQueryEnabled { return condition, nil } + buildExistCondition = buildExistCondition && true + } + + if buildExistCondition { existsCondition, err := c.conditionFor(ctx, key, qbtypes.FilterOperatorExists, nil, sb) if err != nil { return "", err } return sb.And(condition, existsCondition), nil } + return condition, nil } diff --git a/pkg/telemetrylogs/condition_builder_test.go b/pkg/telemetrylogs/condition_builder_test.go index 739b21fb316..269c86c0174 100644 --- a/pkg/telemetrylogs/condition_builder_test.go +++ b/pkg/telemetrylogs/condition_builder_test.go @@ -127,7 +127,8 @@ func TestConditionFor(t *testing.T) { { name: "Contains operator - body", key: telemetrytypes.TelemetryFieldKey{ - Name: "body", + Name: "body", + FieldContext: telemetrytypes.FieldContextLog, }, operator: qbtypes.FilterOperatorContains, value: 521509198310, diff --git a/pkg/telemetrylogs/const.go b/pkg/telemetrylogs/const.go index e05cd548f46..db5ca59af9b 100644 --- a/pkg/telemetrylogs/const.go +++ b/pkg/telemetrylogs/const.go @@ -1,7 +1,11 @@ package telemetrylogs import ( + "fmt" + + schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator" "github.com/SigNoz/signoz-otel-collector/constants" + "github.com/SigNoz/signoz/pkg/querybuilder" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" ) @@ -17,7 +21,7 @@ const ( LogsV2TimestampColumn = "timestamp" LogsV2ObservedTimestampColumn = "observed_timestamp" LogsV2BodyColumn = "body" - LogsV2BodyJSONColumn = constants.BodyV2Column + LogsV2BodyV2Column = constants.BodyV2Column LogsV2BodyPromotedColumn = constants.BodyPromotedColumn LogsV2TraceIDColumn = "trace_id" LogsV2SpanIDColumn = "span_id" @@ -34,11 +38,25 @@ const ( LogsV2ResourcesStringColumn = "resources_string" LogsV2ScopeStringColumn = "scope_string" - BodyJSONColumnPrefix = constants.BodyV2ColumnPrefix + BodyV2ColumnPrefix = constants.BodyV2ColumnPrefix BodyPromotedColumnPrefix = constants.BodyPromotedColumnPrefix + MessageBodyField = "message" + MessageSubColumn = "body_v2.message" + bodySearchDefaultWarning = "body searches default to `body.message:string`. Use `body.` to search a different field inside body" ) var ( + // BodyLogicalFieldJSONMapping is the canonical key for the "message" type hint. + // It lives under body (FieldContextBody) but Materialized=true signals the field + // mapper to access it as a direct sub-column (body_v2.message) rather than via + // dynamicElement(). Its name is the bare field name ("message"), not the column path. + BodyLogicalFieldJSONMapping = &telemetrytypes.TelemetryFieldKey{ + Name: MessageBodyField, + Signal: telemetrytypes.SignalLogs, + FieldContext: telemetrytypes.FieldContextBody, + FieldDataType: telemetrytypes.FieldDataTypeString, + Materialized: true, + } DefaultFullTextColumn = &telemetrytypes.TelemetryFieldKey{ Name: "body", Signal: telemetrytypes.SignalLogs, @@ -118,3 +136,31 @@ var ( }, } ) + +func bodyAliasExpression() string { + if !querybuilder.BodyJSONQueryEnabled { + return LogsV2BodyColumn + } + + return fmt.Sprintf("%s as body", LogsV2BodyV2Column) +} + +func enrichMapsForJSONBodyEnabled() { + if querybuilder.BodyJSONQueryEnabled { + DefaultFullTextColumn = BodyLogicalFieldJSONMapping + IntrinsicFields["body"] = *BodyLogicalFieldJSONMapping + + // Register all key names that should resolve to the message type-hint column so + // QB can look them up directly: bare "message" and qualified "body_v2.message". + IntrinsicFields[MessageSubColumn] = *BodyLogicalFieldJSONMapping + IntrinsicFields[MessageBodyField] = *BodyLogicalFieldJSONMapping + logsV2Columns[MessageSubColumn] = &schema.Column{ + Name: MessageSubColumn, + Type: schema.ColumnTypeString, + } + } +} + +func init() { + enrichMapsForJSONBodyEnabled() +} diff --git a/pkg/telemetrylogs/field_mapper.go b/pkg/telemetrylogs/field_mapper.go index dbea2e5bfa2..4600838c26c 100644 --- a/pkg/telemetrylogs/field_mapper.go +++ b/pkg/telemetrylogs/field_mapper.go @@ -30,7 +30,7 @@ var ( "severity_text": {Name: "severity_text", Type: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}}, "severity_number": {Name: "severity_number", Type: schema.ColumnTypeUInt8}, "body": {Name: "body", Type: schema.ColumnTypeString}, - LogsV2BodyJSONColumn: {Name: LogsV2BodyJSONColumn, Type: schema.JSONColumnType{ + LogsV2BodyV2Column: {Name: LogsV2BodyV2Column, Type: schema.JSONColumnType{ MaxDynamicTypes: utils.ToPointer(uint(32)), MaxDynamicPaths: utils.ToPointer(uint(0)), }}, @@ -88,10 +88,17 @@ func (m *fieldMapper) getColumn(_ context.Context, key *telemetrytypes.Telemetry return logsV2Columns["attributes_bool"], nil } case telemetrytypes.FieldContextBody: + // Type hints (Materialized=true) have a direct physical sub-column in body_v2. + // Return a synthetic String column so the condition builder uses the direct path + // instead of the JSON condition builder (which expects a JSONPlan). + if key.Materialized { + // Type hints have a direct physical sub-column in body_v2 (e.g. body_v2.message). + return logsV2Columns[fmt.Sprintf("%s.%s", LogsV2BodyV2Column, key.Name)], nil + } // Body context is for JSON body fields - // Use body_json if feature flag is enabled + // Use body_v2 if feature flag is enabled if querybuilder.BodyJSONQueryEnabled { - return logsV2Columns[LogsV2BodyJSONColumn], nil + return logsV2Columns[LogsV2BodyV2Column], nil } // Fall back to legacy body column return logsV2Columns["body"], nil @@ -100,9 +107,9 @@ func (m *fieldMapper) getColumn(_ context.Context, key *telemetrytypes.Telemetry if !ok { // check if the key has body JSON search if strings.HasPrefix(key.Name, telemetrytypes.BodyJSONStringSearchPrefix) { - // Use body_json if feature flag is enabled and we have a body condition builder + // Use body_v2 if feature flag is enabled and we have a body condition builder if querybuilder.BodyJSONQueryEnabled { - return logsV2Columns[LogsV2BodyJSONColumn], nil + return logsV2Columns[LogsV2BodyV2Column], nil } // Fall back to legacy body column return logsV2Columns["body"], nil @@ -147,6 +154,8 @@ func (m *fieldMapper) FieldFor(ctx context.Context, key *telemetrytypes.Telemetr } return m.buildFieldForJSON(key) + case telemetrytypes.FieldContextLog: + return column.Name, nil default: return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource/body context fields are supported for json columns, got %s", key.FieldContext.String) } @@ -246,34 +255,37 @@ func (m *fieldMapper) buildFieldForJSON(key *telemetrytypes.TelemetryFieldKey) ( node := plan[0] expr := fmt.Sprintf("dynamicElement(%s, '%s')", node.FieldPath(), node.TerminalConfig.ElemType.StringValue()) - if key.Materialized { - if len(plan) < 2 { - return "", errors.Newf(errors.TypeUnexpected, CodePromotedPlanMissing, - "plan length is less than 2 for promoted path: %s", key.Name) - } - - node := plan[1] - promotedExpr := fmt.Sprintf( - "dynamicElement(%s, '%s')", - node.FieldPath(), - node.TerminalConfig.ElemType.StringValue(), - ) - - // dynamicElement returns NULL for scalar types or an empty array for array types. - if node.TerminalConfig.ElemType.IsArray { - expr = fmt.Sprintf( - "if(length(%s) > 0, %s, %s)", - promotedExpr, - promotedExpr, - expr, - ) - } else { - // promoted column first then body_json column - // TODO(Piyush): Change this in future for better performance - expr = fmt.Sprintf("coalesce(%s, %s)", promotedExpr, expr) - } - - } + // TODO(Piyush): Promoted path logic commented out. Materialized now means type hint + // promotion will be extracted from key field evolution + // (direct sub-column access), not a promoted body_promoted.* column. + // if key.Materialized { + // if len(plan) < 2 { + // return "", errors.Newf(errors.TypeUnexpected, CodePromotedPlanMissing, + // "plan length is less than 2 for promoted path: %s", key.Name) + // } + + // node := plan[1] + // promotedExpr := fmt.Sprintf( + // "dynamicElement(%s, '%s')", + // node.FieldPath(), + // node.TerminalConfig.ElemType.StringValue(), + // ) + + // // dynamicElement returns NULL for scalar types or an empty array for array types. + // if node.TerminalConfig.ElemType.IsArray { + // expr = fmt.Sprintf( + // "if(length(%s) > 0, %s, %s)", + // promotedExpr, + // promotedExpr, + // expr, + // ) + // } else { + // // promoted column first then body_json column + // // TODO(Piyush): Change this in future for better performance + // expr = fmt.Sprintf("coalesce(%s, %s)", promotedExpr, expr) + // } + + // } return expr, nil } diff --git a/pkg/telemetrylogs/json_condition_builder.go b/pkg/telemetrylogs/json_condition_builder.go index c0e1b0e6df1..0665317493c 100644 --- a/pkg/telemetrylogs/json_condition_builder.go +++ b/pkg/telemetrylogs/json_condition_builder.go @@ -30,7 +30,7 @@ func NewJSONConditionBuilder(key *telemetrytypes.TelemetryFieldKey, valueType te return &jsonConditionBuilder{key: key, valueType: telemetrytypes.MappingFieldDataTypeToJSONDataType[valueType]} } -// BuildCondition builds the full WHERE condition for body_json JSON paths +// BuildCondition builds the full WHERE condition for body_v2 JSON paths func (c *jsonConditionBuilder) buildJSONCondition(operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder) (string, error) { conditions := []string{} for _, node := range c.key.JSONPlan { @@ -40,6 +40,7 @@ func (c *jsonConditionBuilder) buildJSONCondition(operator qbtypes.FilterOperato } conditions = append(conditions, condition) } + return sb.Or(conditions...), nil } @@ -288,9 +289,9 @@ func (c *jsonConditionBuilder) applyOperator(sb *sqlbuilder.SelectBuilder, field } return sb.NotIn(fieldExpr, values...), nil case qbtypes.FilterOperatorExists: - return fmt.Sprintf("%s IS NOT NULL", fieldExpr), nil + return sb.IsNotNull(fieldExpr), nil case qbtypes.FilterOperatorNotExists: - return fmt.Sprintf("%s IS NULL", fieldExpr), nil + return sb.IsNull(fieldExpr), nil // between and not between case qbtypes.FilterOperatorBetween, qbtypes.FilterOperatorNotBetween: values, ok := value.([]any) diff --git a/pkg/telemetrylogs/json_stmt_builder_test.go b/pkg/telemetrylogs/json_stmt_builder_test.go index 1590bf8e0b3..b46dcfb53a1 100644 --- a/pkg/telemetrylogs/json_stmt_builder_test.go +++ b/pkg/telemetrylogs/json_stmt_builder_test.go @@ -17,10 +17,9 @@ import ( ) func TestStmtBuilderTimeSeriesBodyGroupByJSON(t *testing.T) { - enableBodyJSONQuery(t) - defer func() { - disableBodyJSONQuery(t) - }() + enable, disable := jsonQueryTestUtil(t) + enable() + defer disable() statementBuilder := buildJSONTestStatementBuilder(t) cases := []struct { @@ -97,11 +96,13 @@ func TestStmtBuilderTimeSeriesBodyGroupByJSON(t *testing.T) { } } +/* Promoted path tests commented out — Materialized now means type hint (direct sub-column), + not a body_promoted.* column. These tests assumed the old coalesce(body_promoted.x, body_v2.x) path. + func TestStmtBuilderTimeSeriesBodyGroupByPromoted(t *testing.T) { - enableBodyJSONQuery(t) - defer func() { - disableBodyJSONQuery(t) - }() + enable, disable := jsonQueryTestUtil(t) + enable() + defer disable() statementBuilder := buildJSONTestStatementBuilder(t, "user.age", "user.name") cases := []struct { @@ -158,12 +159,12 @@ func TestStmtBuilderTimeSeriesBodyGroupByPromoted(t *testing.T) { }) } } +*/ func TestStatementBuilderListQueryBodyHas(t *testing.T) { - enableBodyJSONQuery(t) - defer func() { - disableBodyJSONQuery(t) - }() + enable, disable := jsonQueryTestUtil(t) + enable() + defer disable() statementBuilder := buildJSONTestStatementBuilder(t) cases := []struct { @@ -182,7 +183,7 @@ func TestStatementBuilderListQueryBodyHas(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (has(arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))), ?) OR has(arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))), ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (has(arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))), ?) OR has(arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))), ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), 1.65, 1.65, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, Warnings: []string{ "Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64,jsondatatype=Array(Nullable(Float64)) name=education[].parameters,context=body,datatype=[]dynamic,jsondatatype=Array(Dynamic)].", @@ -199,7 +200,7 @@ func TestStatementBuilderListQueryBodyHas(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND hasAll(dynamicElement(body_v2.`user.permissions`, 'Array(Nullable(String))'), ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND hasAll(dynamicElement(body_v2.`user.permissions`, 'Array(Nullable(String))'), ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), []any{[]any{"read", "write"}}, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -213,7 +214,7 @@ func TestStatementBuilderListQueryBodyHas(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND hasAny(arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))), ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND hasAny(arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))), ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), []any{[]any{"Piyush", "Tushar"}}, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -238,10 +239,9 @@ func TestStatementBuilderListQueryBodyHas(t *testing.T) { } func TestStatementBuilderListQueryBody(t *testing.T) { - enableBodyJSONQuery(t) - defer func() { - disableBodyJSONQuery(t) - }() + enable, disable := jsonQueryTestUtil(t) + enable() + defer disable() statementBuilder := buildJSONTestStatementBuilder(t) cases := []struct { @@ -260,7 +260,7 @@ func TestStatementBuilderListQueryBody(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (dynamicElement(body_v2.`user.name`, 'String') = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (dynamicElement(body_v2.`user.name`, 'String') = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "x", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -274,7 +274,7 @@ func TestStatementBuilderListQueryBody(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> dynamicElement(`body_v2.education`.`name`, 'String') IS NOT NULL, dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> dynamicElement(`body_v2.education`.`name`, 'String') IS NOT NULL, dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -288,7 +288,7 @@ func TestStatementBuilderListQueryBody(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') IS NOT NULL, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') IS NOT NULL, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') IS NOT NULL, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') IS NOT NULL, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -302,7 +302,7 @@ func TestStatementBuilderListQueryBody(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') = ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') = ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "Iron Award", "Iron Award", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -316,7 +316,7 @@ func TestStatementBuilderListQueryBody(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) OR (arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayMap(x->dynamicElement(x, 'Float64'), arrayFilter(x->(dynamicType(x) = 'Float64'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)')))) OR arrayExists(x -> toFloat64(x) = ?, arrayMap(x->dynamicElement(x, 'Float64'), arrayFilter(x->(dynamicType(x) = 'Float64'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) OR (arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayMap(x->dynamicElement(x, 'Float64'), arrayFilter(x->(dynamicType(x) = 'Float64'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)')))) OR arrayExists(x -> toFloat64(x) = ?, arrayMap(x->dynamicElement(x, 'Float64'), arrayFilter(x->(dynamicType(x) = 'Float64'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "%1.65%", 1.65, "%1.65%", 1.65, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64,jsondatatype=Array(Nullable(Float64)) name=education[].parameters,context=body,datatype=[]dynamic,jsondatatype=Array(Dynamic)]."}, }, @@ -331,7 +331,7 @@ func TestStatementBuilderListQueryBody(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> LOWER(dynamicElement(`body_v2.education`.`name`, 'String')) LIKE LOWER(?), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> LOWER(dynamicElement(`body_v2.education`.`name`, 'String')) LIKE LOWER(?), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "%IIT%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -345,7 +345,7 @@ func TestStatementBuilderListQueryBody(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> x = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) OR (arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayMap(x->dynamicElement(x, 'Bool'), arrayFilter(x->(dynamicType(x) = 'Bool'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)')))) OR arrayExists(x -> x = ?, arrayMap(x->dynamicElement(x, 'Bool'), arrayFilter(x->(dynamicType(x) = 'Bool'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> x = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) OR (arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayMap(x->dynamicElement(x, 'Bool'), arrayFilter(x->(dynamicType(x) = 'Bool'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)')))) OR arrayExists(x -> x = ?, arrayMap(x->dynamicElement(x, 'Bool'), arrayFilter(x->(dynamicType(x) = 'Bool'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "%true%", true, "%true%", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64,jsondatatype=Array(Nullable(Float64)) name=education[].parameters,context=body,datatype=[]dynamic,jsondatatype=Array(Dynamic)]."}, }, @@ -360,7 +360,7 @@ func TestStatementBuilderListQueryBody(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> toString(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) OR (arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(x) LIKE LOWER(?), arrayMap(x->dynamicElement(x, 'String'), arrayFilter(x->(dynamicType(x) = 'String'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)')))) OR arrayExists(x -> x = ?, arrayMap(x->dynamicElement(x, 'String'), arrayFilter(x->(dynamicType(x) = 'String'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> toString(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) OR (arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(x) LIKE LOWER(?), arrayMap(x->dynamicElement(x, 'String'), arrayFilter(x->(dynamicType(x) = 'String'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)')))) OR arrayExists(x -> x = ?, arrayMap(x->dynamicElement(x, 'String'), arrayFilter(x->(dynamicType(x) = 'String'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "%passed%", "passed", "%passed%", "passed", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64,jsondatatype=Array(Nullable(Float64)) name=education[].parameters,context=body,datatype=[]dynamic,jsondatatype=Array(Dynamic)]."}, }, @@ -375,7 +375,7 @@ func TestStatementBuilderListQueryBody(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((arrayExists(`body_v2.education`-> (arrayExists(x -> toFloat64(x) IN (?, ?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) OR (arrayExists(`body_v2.education`-> (arrayExists(x -> x IN (?, ?), arrayMap(x->dynamicElement(x, 'Array(Nullable(Float64))'), arrayFilter(x->(dynamicType(x) = 'Array(Nullable(Float64))'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((arrayExists(`body_v2.education`-> (arrayExists(x -> toFloat64(x) IN (?, ?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) OR (arrayExists(`body_v2.education`-> (arrayExists(x -> x IN (?, ?), arrayMap(x->dynamicElement(x, 'Array(Nullable(Float64))'), arrayFilter(x->(dynamicType(x) = 'Array(Nullable(Float64))'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), 1.65, 1.99, 1.65, 1.99, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64,jsondatatype=Array(Nullable(Float64)) name=education[].parameters,context=body,datatype=[]dynamic,jsondatatype=Array(Dynamic)]."}, }, @@ -390,8 +390,8 @@ func TestStatementBuilderListQueryBody(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) BETWEEN ? AND ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) BETWEEN ? AND ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) BETWEEN ? AND ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) BETWEEN ? AND ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", - Args: []any{float64(2), float64(4), float64(2), float64(4), uint64(1747945619), uint64(1747983448), float64(2), float64(4), float64(2), float64(4), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) BETWEEN ? AND ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) BETWEEN ? AND ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{uint64(1747945619), uint64(1747983448), float64(2), float64(4), float64(2), float64(4), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, }, @@ -404,8 +404,8 @@ func TestStatementBuilderListQueryBody(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) IN (?, ?), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) IN (?, ?), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) IN (?, ?), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) IN (?, ?), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", - Args: []any{float64(2), float64(4), float64(2), float64(4), uint64(1747945619), uint64(1747983448), float64(2), float64(4), float64(2), float64(4), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) IN (?, ?), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) IN (?, ?), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{uint64(1747945619), uint64(1747983448), float64(2), float64(4), float64(2), float64(4), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, }, @@ -418,7 +418,7 @@ func TestStatementBuilderListQueryBody(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`type`, 'String') = ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`type`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`type`, 'String') = ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`type`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "sports", "sports", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -432,7 +432,7 @@ func TestStatementBuilderListQueryBody(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(`body_v2.interests[].entities[].reviews`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(Int64))')) OR arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(Int64))'))), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata`.`positions`, 'Array(JSON(max_dynamic_types=0, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews[].entries`.`metadata`, 'Array(JSON(max_dynamic_types=1, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews`.`entries`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities`.`reviews`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) OR (arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(`body_v2.interests[].entities[].reviews`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`-> (arrayExists(x -> LOWER(x) LIKE LOWER(?), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(String))')) OR arrayExists(x -> toFloat64OrNull(x) = ?, dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(String))'))), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata`.`positions`, 'Array(JSON(max_dynamic_types=0, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews[].entries`.`metadata`, 'Array(JSON(max_dynamic_types=1, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews`.`entries`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities`.`reviews`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(`body_v2.interests[].entities[].reviews`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(Int64))')) OR arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(Int64))'))), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata`.`positions`, 'Array(JSON(max_dynamic_types=0, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews[].entries`.`metadata`, 'Array(JSON(max_dynamic_types=1, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews`.`entries`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities`.`reviews`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) OR (arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(`body_v2.interests[].entities[].reviews`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`-> (arrayExists(x -> LOWER(x) LIKE LOWER(?), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(String))')) OR arrayExists(x -> toFloat64OrNull(x) = ?, dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(String))'))), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata`.`positions`, 'Array(JSON(max_dynamic_types=0, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews[].entries`.`metadata`, 'Array(JSON(max_dynamic_types=1, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews`.`entries`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities`.`reviews`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "%4%", float64(4), "%4%", float64(4), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, Warnings: []string{"Key `interests[].entities[].reviews[].entries[].metadata[].positions[].ratings` is ambiguous, found 2 different combinations of field context / data type: [name=interests[].entities[].reviews[].entries[].metadata[].positions[].ratings,context=body,datatype=[]int64,jsondatatype=Array(Nullable(Int64)) name=interests[].entities[].reviews[].entries[].metadata[].positions[].ratings,context=body,datatype=[]string,jsondatatype=Array(Nullable(String))]."}, }, @@ -447,7 +447,7 @@ func TestStatementBuilderListQueryBody(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(`body_v2.interests[].entities[].reviews`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(Int64))')) OR arrayExists(x -> toString(x) = ?, dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(Int64))'))), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata`.`positions`, 'Array(JSON(max_dynamic_types=0, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews[].entries`.`metadata`, 'Array(JSON(max_dynamic_types=1, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews`.`entries`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities`.`reviews`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) OR (arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(`body_v2.interests[].entities[].reviews`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`-> (arrayExists(x -> LOWER(x) LIKE LOWER(?), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(String))')) OR arrayExists(x -> x = ?, dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(String))'))), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata`.`positions`, 'Array(JSON(max_dynamic_types=0, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews[].entries`.`metadata`, 'Array(JSON(max_dynamic_types=1, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews`.`entries`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities`.`reviews`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(`body_v2.interests[].entities[].reviews`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(Int64))')) OR arrayExists(x -> toString(x) = ?, dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(Int64))'))), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata`.`positions`, 'Array(JSON(max_dynamic_types=0, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews[].entries`.`metadata`, 'Array(JSON(max_dynamic_types=1, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews`.`entries`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities`.`reviews`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) OR (arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(`body_v2.interests[].entities[].reviews`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`-> (arrayExists(x -> LOWER(x) LIKE LOWER(?), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(String))')) OR arrayExists(x -> x = ?, dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(String))'))), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata`.`positions`, 'Array(JSON(max_dynamic_types=0, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews[].entries`.`metadata`, 'Array(JSON(max_dynamic_types=1, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews`.`entries`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities`.`reviews`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "%Good%", "Good", "%Good%", "Good", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, Warnings: []string{"Key `interests[].entities[].reviews[].entries[].metadata[].positions[].ratings` is ambiguous, found 2 different combinations of field context / data type: [name=interests[].entities[].reviews[].entries[].metadata[].positions[].ratings,context=body,datatype=[]int64,jsondatatype=Array(Nullable(Int64)) name=interests[].entities[].reviews[].entries[].metadata[].positions[].ratings,context=body,datatype=[]string,jsondatatype=Array(Nullable(String))]."}, }, @@ -462,7 +462,7 @@ func TestStatementBuilderListQueryBody(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=0))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=0))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "%Civil%", "%Civil%", "%Civil%", "%Civil%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -476,7 +476,7 @@ func TestStatementBuilderListQueryBody(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((LOWER(toString(dynamicElement(body_v2.`user.age`, 'Int64'))) LIKE LOWER(?)) OR (LOWER(dynamicElement(body_v2.`user.age`, 'String')) LIKE LOWER(?))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((LOWER(toString(dynamicElement(body_v2.`user.age`, 'Int64'))) LIKE LOWER(?)) OR (LOWER(dynamicElement(body_v2.`user.age`, 'String')) LIKE LOWER(?))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "%25%", "%25%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, Warnings: []string{"Key `user.age` is ambiguous, found 2 different combinations of field context / data type: [name=user.age,context=body,datatype=int64,jsondatatype=Int64 name=user.age,context=body,datatype=string,jsondatatype=String]."}, }, @@ -491,7 +491,7 @@ func TestStatementBuilderListQueryBody(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (LOWER(toString(dynamicElement(body_v2.`user.height`, 'Float64'))) LIKE LOWER(?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (LOWER(toString(dynamicElement(body_v2.`user.height`, 'Float64'))) LIKE LOWER(?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "%5.8%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -505,7 +505,7 @@ func TestStatementBuilderListQueryBody(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> LOWER(toString(dynamicElement(`body_v2.education`.`year`, 'Int64'))) LIKE LOWER(?), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> LOWER(toString(dynamicElement(`body_v2.education`.`year`, 'Int64'))) LIKE LOWER(?), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "%2020%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -529,11 +529,13 @@ func TestStatementBuilderListQueryBody(t *testing.T) { } } +/* Promoted path list-query tests commented out — Materialized now means type hint + (direct sub-column access), not a body_promoted.* column. + func TestStatementBuilderListQueryBodyPromoted(t *testing.T) { - enableBodyJSONQuery(t) - defer func() { - disableBodyJSONQuery(t) - }() + enable, disable := jsonQueryTestUtil(t) + enable() + defer disable() statementBuilder := buildJSONTestStatementBuilder(t, "education", "tags") cases := []struct { @@ -552,7 +554,7 @@ func TestStatementBuilderListQueryBodyPromoted(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND has(if(length(dynamicElement(body_promoted.`tags`, 'Array(Nullable(String))')) > 0, dynamicElement(body_promoted.`tags`, 'Array(Nullable(String))'), dynamicElement(body_v2.`tags`, 'Array(Nullable(String))')), ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND has(if(length(dynamicElement(body_promoted.`tags`, 'Array(Nullable(String))')) > 0, dynamicElement(body_promoted.`tags`, 'Array(Nullable(String))'), dynamicElement(body_v2.`tags`, 'Array(Nullable(String))')), ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "production", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -566,7 +568,7 @@ func TestStatementBuilderListQueryBodyPromoted(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> dynamicElement(`body_v2.education`.`name`, 'String') IS NOT NULL, dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> dynamicElement(`body_promoted.education`.`name`, 'String') IS NOT NULL, dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> dynamicElement(`body_v2.education`.`name`, 'String') IS NOT NULL, dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> dynamicElement(`body_promoted.education`.`name`, 'String') IS NOT NULL, dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -580,7 +582,7 @@ func TestStatementBuilderListQueryBodyPromoted(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') IS NOT NULL, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') IS NOT NULL, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(`body_promoted.education[].awards`-> dynamicElement(`body_promoted.education[].awards`.`name`, 'String') IS NOT NULL, dynamicElement(`body_promoted.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')) OR arrayExists(`body_promoted.education[].awards`-> dynamicElement(`body_promoted.education[].awards`.`name`, 'String') IS NOT NULL, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_promoted.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') IS NOT NULL, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') IS NOT NULL, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(`body_promoted.education[].awards`-> dynamicElement(`body_promoted.education[].awards`.`name`, 'String') IS NOT NULL, dynamicElement(`body_promoted.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')) OR arrayExists(`body_promoted.education[].awards`-> dynamicElement(`body_promoted.education[].awards`.`name`, 'String') IS NOT NULL, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_promoted.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -594,7 +596,7 @@ func TestStatementBuilderListQueryBodyPromoted(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') = ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(`body_promoted.education[].awards`-> dynamicElement(`body_promoted.education[].awards`.`name`, 'String') = ?, dynamicElement(`body_promoted.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')) OR arrayExists(`body_promoted.education[].awards`-> dynamicElement(`body_promoted.education[].awards`.`name`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_promoted.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') = ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(`body_promoted.education[].awards`-> dynamicElement(`body_promoted.education[].awards`.`name`, 'String') = ?, dynamicElement(`body_promoted.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')) OR arrayExists(`body_promoted.education[].awards`-> dynamicElement(`body_promoted.education[].awards`.`name`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_promoted.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "Iron Award", "Iron Award", "Iron Award", "Iron Award", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -608,7 +610,7 @@ func TestStatementBuilderListQueryBodyPromoted(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_promoted.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_promoted.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'))) OR (arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayMap(x->dynamicElement(x, 'Float64'), arrayFilter(x->(dynamicType(x) = 'Float64'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)')))) OR arrayExists(x -> toFloat64(x) = ?, arrayMap(x->dynamicElement(x, 'Float64'), arrayFilter(x->(dynamicType(x) = 'Float64'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayMap(x->dynamicElement(x, 'Float64'), arrayFilter(x->(dynamicType(x) = 'Float64'), dynamicElement(`body_promoted.education`.`parameters`, 'Array(Dynamic)')))) OR arrayExists(x -> toFloat64(x) = ?, arrayMap(x->dynamicElement(x, 'Float64'), arrayFilter(x->(dynamicType(x) = 'Float64'), dynamicElement(`body_promoted.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_promoted.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_promoted.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'))) OR (arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayMap(x->dynamicElement(x, 'Float64'), arrayFilter(x->(dynamicType(x) = 'Float64'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)')))) OR arrayExists(x -> toFloat64(x) = ?, arrayMap(x->dynamicElement(x, 'Float64'), arrayFilter(x->(dynamicType(x) = 'Float64'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayMap(x->dynamicElement(x, 'Float64'), arrayFilter(x->(dynamicType(x) = 'Float64'), dynamicElement(`body_promoted.education`.`parameters`, 'Array(Dynamic)')))) OR arrayExists(x -> toFloat64(x) = ?, arrayMap(x->dynamicElement(x, 'Float64'), arrayFilter(x->(dynamicType(x) = 'Float64'), dynamicElement(`body_promoted.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "%1.65%", 1.65, "%1.65%", 1.65, "%1.65%", 1.65, "%1.65%", 1.65, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64,materialized=true,jsondatatype=Array(Nullable(Float64)) name=education[].parameters,context=body,datatype=[]dynamic,materialized=true,jsondatatype=Array(Dynamic)]."}, }, @@ -623,7 +625,7 @@ func TestStatementBuilderListQueryBodyPromoted(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> LOWER(dynamicElement(`body_v2.education`.`name`, 'String')) LIKE LOWER(?), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> LOWER(dynamicElement(`body_promoted.education`.`name`, 'String')) LIKE LOWER(?), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> LOWER(dynamicElement(`body_v2.education`.`name`, 'String')) LIKE LOWER(?), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> LOWER(dynamicElement(`body_promoted.education`.`name`, 'String')) LIKE LOWER(?), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "%IIT%", "%IIT%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -637,7 +639,7 @@ func TestStatementBuilderListQueryBodyPromoted(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> x = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_promoted.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> x = ?, dynamicElement(`body_promoted.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'))) OR (arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayMap(x->dynamicElement(x, 'Bool'), arrayFilter(x->(dynamicType(x) = 'Bool'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)')))) OR arrayExists(x -> x = ?, arrayMap(x->dynamicElement(x, 'Bool'), arrayFilter(x->(dynamicType(x) = 'Bool'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayMap(x->dynamicElement(x, 'Bool'), arrayFilter(x->(dynamicType(x) = 'Bool'), dynamicElement(`body_promoted.education`.`parameters`, 'Array(Dynamic)')))) OR arrayExists(x -> x = ?, arrayMap(x->dynamicElement(x, 'Bool'), arrayFilter(x->(dynamicType(x) = 'Bool'), dynamicElement(`body_promoted.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> x = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_promoted.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> x = ?, dynamicElement(`body_promoted.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'))) OR (arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayMap(x->dynamicElement(x, 'Bool'), arrayFilter(x->(dynamicType(x) = 'Bool'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)')))) OR arrayExists(x -> x = ?, arrayMap(x->dynamicElement(x, 'Bool'), arrayFilter(x->(dynamicType(x) = 'Bool'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayMap(x->dynamicElement(x, 'Bool'), arrayFilter(x->(dynamicType(x) = 'Bool'), dynamicElement(`body_promoted.education`.`parameters`, 'Array(Dynamic)')))) OR arrayExists(x -> x = ?, arrayMap(x->dynamicElement(x, 'Bool'), arrayFilter(x->(dynamicType(x) = 'Bool'), dynamicElement(`body_promoted.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "%true%", true, "%true%", true, "%true%", true, "%true%", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64,materialized=true,jsondatatype=Array(Nullable(Float64)) name=education[].parameters,context=body,datatype=[]dynamic,materialized=true,jsondatatype=Array(Dynamic)]."}, }, @@ -652,7 +654,7 @@ func TestStatementBuilderListQueryBodyPromoted(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> toString(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_promoted.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> toString(x) = ?, dynamicElement(`body_promoted.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'))) OR (arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(x) LIKE LOWER(?), arrayMap(x->dynamicElement(x, 'String'), arrayFilter(x->(dynamicType(x) = 'String'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)')))) OR arrayExists(x -> x = ?, arrayMap(x->dynamicElement(x, 'String'), arrayFilter(x->(dynamicType(x) = 'String'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(x -> LOWER(x) LIKE LOWER(?), arrayMap(x->dynamicElement(x, 'String'), arrayFilter(x->(dynamicType(x) = 'String'), dynamicElement(`body_promoted.education`.`parameters`, 'Array(Dynamic)')))) OR arrayExists(x -> x = ?, arrayMap(x->dynamicElement(x, 'String'), arrayFilter(x->(dynamicType(x) = 'String'), dynamicElement(`body_promoted.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> toString(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_promoted.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> toString(x) = ?, dynamicElement(`body_promoted.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'))) OR (arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(x) LIKE LOWER(?), arrayMap(x->dynamicElement(x, 'String'), arrayFilter(x->(dynamicType(x) = 'String'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)')))) OR arrayExists(x -> x = ?, arrayMap(x->dynamicElement(x, 'String'), arrayFilter(x->(dynamicType(x) = 'String'), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(x -> LOWER(x) LIKE LOWER(?), arrayMap(x->dynamicElement(x, 'String'), arrayFilter(x->(dynamicType(x) = 'String'), dynamicElement(`body_promoted.education`.`parameters`, 'Array(Dynamic)')))) OR arrayExists(x -> x = ?, arrayMap(x->dynamicElement(x, 'String'), arrayFilter(x->(dynamicType(x) = 'String'), dynamicElement(`body_promoted.education`.`parameters`, 'Array(Dynamic)'))))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "%passed%", "passed", "%passed%", "passed", "%passed%", "passed", "%passed%", "passed", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64,materialized=true,jsondatatype=Array(Nullable(Float64)) name=education[].parameters,context=body,datatype=[]dynamic,materialized=true,jsondatatype=Array(Dynamic)]."}, }, @@ -667,7 +669,7 @@ func TestStatementBuilderListQueryBodyPromoted(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`type`, 'String') = ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`type`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(`body_promoted.education[].awards`-> dynamicElement(`body_promoted.education[].awards`.`type`, 'String') = ?, dynamicElement(`body_promoted.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')) OR arrayExists(`body_promoted.education[].awards`-> dynamicElement(`body_promoted.education[].awards`.`type`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_promoted.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`type`, 'String') = ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`type`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(`body_promoted.education[].awards`-> dynamicElement(`body_promoted.education[].awards`.`type`, 'String') = ?, dynamicElement(`body_promoted.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')) OR arrayExists(`body_promoted.education[].awards`-> dynamicElement(`body_promoted.education[].awards`.`type`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_promoted.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "sports", "sports", "sports", "sports", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -681,7 +683,7 @@ func TestStatementBuilderListQueryBodyPromoted(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=0))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(`body_promoted.education[].awards`-> (arrayExists(`body_promoted.education[].awards[].participated`-> arrayExists(`body_promoted.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_promoted.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_promoted.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=4))')), dynamicElement(`body_promoted.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=16))')) OR arrayExists(`body_promoted.education[].awards[].participated`-> arrayExists(`body_promoted.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_promoted.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_promoted.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_promoted.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_promoted.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')) OR arrayExists(`body_promoted.education[].awards`-> (arrayExists(`body_promoted.education[].awards[].participated`-> arrayExists(`body_promoted.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_promoted.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_promoted.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')), dynamicElement(`body_promoted.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')) OR arrayExists(`body_promoted.education[].awards[].participated`-> arrayExists(`body_promoted.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_promoted.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_promoted.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_promoted.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_promoted.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=0))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_promoted.education`-> (arrayExists(`body_promoted.education[].awards`-> (arrayExists(`body_promoted.education[].awards[].participated`-> arrayExists(`body_promoted.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_promoted.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_promoted.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=4))')), dynamicElement(`body_promoted.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=16))')) OR arrayExists(`body_promoted.education[].awards[].participated`-> arrayExists(`body_promoted.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_promoted.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_promoted.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_promoted.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_promoted.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')) OR arrayExists(`body_promoted.education[].awards`-> (arrayExists(`body_promoted.education[].awards[].participated`-> arrayExists(`body_promoted.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_promoted.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_promoted.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')), dynamicElement(`body_promoted.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')) OR arrayExists(`body_promoted.education[].awards[].participated`-> arrayExists(`body_promoted.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_promoted.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_promoted.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_promoted.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_promoted.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_promoted.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{uint64(1747945619), uint64(1747983448), "%Civil%", "%Civil%", "%Civil%", "%Civil%", "%Civil%", "%Civil%", "%Civil%", "%Civil%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -705,27 +707,14 @@ func TestStatementBuilderListQueryBodyPromoted(t *testing.T) { }) } } +*/ func TestStatementBuilderListQueryBodyMessage(t *testing.T) { - enableBodyJSONQuery(t) - defer func() { - disableBodyJSONQuery(t) - }() + enable, disable := jsonQueryTestUtil(t) + enable() + defer disable() statementBuilder := buildJSONTestStatementBuilder(t) - indexed := []*telemetrytypes.TelemetryFieldKey{ - { - Name: "message", - Indexes: []telemetrytypes.JSONDataTypeIndex{ - { - Type: telemetrytypes.String, - ColumnExpression: "body_promoted.message", - IndexExpression: "(lower(assumeNotNull(dynamicElement(body_promoted.message, 'String'))))", - }, - }, - }, - } - testAddIndexedPaths(t, statementBuilder, indexed...) cases := []struct { name string requestType qbtypes.RequestType @@ -738,57 +727,57 @@ func TestStatementBuilderListQueryBodyMessage(t *testing.T) { requestType: qbtypes.RequestTypeRaw, query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{ Signal: telemetrytypes.SignalLogs, - Filter: &qbtypes.Filter{Expression: "body.message Exists"}, - Limit: 10, - }, - expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (dynamicElement(body_v2.`message`, 'String') IS NOT NULL OR dynamicElement(body_promoted.`message`, 'String') IS NOT NULL) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", - Args: []any{uint64(1747945619), uint64(1747983448), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, - }, - expectedErr: nil, - }, - { - name: "body.message equals to empty string", - requestType: qbtypes.RequestTypeRaw, - query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{ - Signal: telemetrytypes.SignalLogs, - Filter: &qbtypes.Filter{Expression: "body.message = ''"}, - Limit: 10, - }, - expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (dynamicElement(body_v2.`message`, 'String') = ? OR dynamicElement(body_promoted.`message`, 'String') = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", - Args: []any{uint64(1747945619), uint64(1747983448), "", "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, - }, - expectedErr: nil, - }, - { - name: "body.message equals to 'Iron Award'", - requestType: qbtypes.RequestTypeRaw, - query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{ - Signal: telemetrytypes.SignalLogs, - Filter: &qbtypes.Filter{Expression: "body.message = 'Iron Award'"}, - Limit: 10, - }, - expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (dynamicElement(body_v2.`message`, 'String') = ? OR dynamicElement(body_promoted.`message`, 'String') = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", - Args: []any{uint64(1747945619), uint64(1747983448), "Iron Award", "Iron Award", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, - }, - expectedErr: nil, - }, - { - name: "body.message contains 'Iron Award'", - requestType: qbtypes.RequestTypeRaw, - query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{ - Signal: telemetrytypes.SignalLogs, - Filter: &qbtypes.Filter{Expression: "body.message Contains 'Iron Award'"}, - Limit: 10, - }, - expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_v2, body_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (LOWER(dynamicElement(body_v2.`message`, 'String')) LIKE LOWER(?) OR LOWER(dynamicElement(body_promoted.`message`, 'String')) LIKE LOWER(?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", - Args: []any{uint64(1747945619), uint64(1747983448), "%Iron Award%", "%Iron Award%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, - }, - expectedErr: nil, - }, + Filter: &qbtypes.Filter{Expression: "message Exists"}, + Limit: 10, + }, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND body_v2.message <> ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{uint64(1747945619), uint64(1747983448), "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + }, + expectedErr: nil, + }, + // { + // name: "body.message equals to empty string", + // requestType: qbtypes.RequestTypeRaw, + // query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{ + // Signal: telemetrytypes.SignalLogs, + // Filter: &qbtypes.Filter{Expression: "body.message = ''"}, + // Limit: 10, + // }, + // expected: qbtypes.Statement{ + // Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND body_v2.message = ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + // Args: []any{uint64(1747945619), uint64(1747983448), "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + // }, + // expectedErr: nil, + // }, + // { + // name: "body.message equals to 'Iron Award'", + // requestType: qbtypes.RequestTypeRaw, + // query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{ + // Signal: telemetrytypes.SignalLogs, + // Filter: &qbtypes.Filter{Expression: "body.message = 'Iron Award'"}, + // Limit: 10, + // }, + // expected: qbtypes.Statement{ + // Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND body_v2.message = ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + // Args: []any{uint64(1747945619), uint64(1747983448), "Iron Award", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + // }, + // expectedErr: nil, + // }, + // { + // name: "message contains 'Iron Award'", + // requestType: qbtypes.RequestTypeRaw, + // query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{ + // Signal: telemetrytypes.SignalLogs, + // Filter: &qbtypes.Filter{Expression: "message Contains 'Iron Award'"}, + // Limit: 10, + // }, + // expected: qbtypes.Statement{ + // Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND LOWER(body_v2.message) LIKE LOWER(?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + // Args: []any{uint64(1747945619), uint64(1747983448), "%Iron Award%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + // }, + // expectedErr: nil, + // }, } for _, c := range cases { @@ -810,16 +799,13 @@ func TestStatementBuilderListQueryBodyMessage(t *testing.T) { } func buildTestTelemetryMetadataStore(t *testing.T, promotedPaths ...string) *telemetrytypestest.MockMetadataStore { - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() - + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(IntrinsicFields) types, _ := telemetrytypes.TestJSONTypeSet() for path, jsonTypes := range types { promoted := false split := strings.Split(path, telemetrytypes.ArraySep) - if path == "message" { - promoted = true - } else if slices.Contains(promotedPaths, split[0]) { + if slices.Contains(promotedPaths, split[0]) { promoted = true } // Create a TelemetryFieldKey for each JSONDataType for this path @@ -834,7 +820,7 @@ func buildTestTelemetryMetadataStore(t *testing.T, promotedPaths ...string) *tel Materialized: promoted, } err := key.SetJSONAccessPlan(telemetrytypes.JSONColumnMetadata{ - BaseColumn: LogsV2BodyJSONColumn, + BaseColumn: LogsV2BodyV2Column, PromotedColumn: LogsV2BodyPromotedColumn, }, types) require.NoError(t, err) @@ -850,11 +836,14 @@ func buildJSONTestStatementBuilder(t *testing.T, promotedPaths ...string) *logQu fm := NewFieldMapper() cb := NewConditionBuilder(fm) + resourceFilterFM := resourcefilter.NewFieldMapper() + resourceFilterCB := resourcefilter.NewConditionBuilder(resourceFilterFM) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil) resourceFilterStmtBuilder := resourcefilter.NewLogResourceFilterStatementBuilder( instrumentationtest.New().ToProviderSettings(), - fm, - cb, + resourceFilterFM, + resourceFilterCB, mockMetadataStore, DefaultFullTextColumn, GetBodyJSONKey, @@ -874,23 +863,27 @@ func buildJSONTestStatementBuilder(t *testing.T, promotedPaths ...string) *logQu return statementBuilder } -func testAddIndexedPaths(t *testing.T, statementBuilder *logQueryStatementBuilder, telemetryFieldKeys ...*telemetrytypes.TelemetryFieldKey) { - mockMetadataStore := statementBuilder.metadataStore.(*telemetrytypestest.MockMetadataStore) - for _, key := range telemetryFieldKeys { - if strings.Contains(key.Name, telemetrytypes.ArraySep) || strings.Contains(key.Name, telemetrytypes.ArrayAnyIndex) { - t.Fatalf("array paths are not supported: %s", key.Name) - } - - for _, storedKey := range mockMetadataStore.KeysMap[key.Name] { - storedKey.Indexes = append(storedKey.Indexes, key.Indexes...) - } +func jsonQueryTestUtil(_ *testing.T) (func(), func()) { + querybuilder.BodyJSONQueryEnabled = true + base := telemetrytypes.TelemetryFieldKey{ + Name: "body", + Signal: telemetrytypes.SignalLogs, + FieldContext: telemetrytypes.FieldContextLog, + FieldDataType: telemetrytypes.FieldDataTypeString, } -} -func enableBodyJSONQuery(_ *testing.T) { - querybuilder.BodyJSONQueryEnabled = true -} + enable := func() { + querybuilder.BodyJSONQueryEnabled = true + enrichMapsForJSONBodyEnabled() + } + disable := func() { + querybuilder.BodyJSONQueryEnabled = false + DefaultFullTextColumn = &base + IntrinsicFields["body"] = base + delete(IntrinsicFields, MessageSubColumn) + delete(IntrinsicFields, MessageBodyField) + delete(logsV2Columns, MessageSubColumn) + } -func disableBodyJSONQuery(_ *testing.T) { - querybuilder.BodyJSONQueryEnabled = false + return enable, disable } diff --git a/pkg/telemetrylogs/statement_builder.go b/pkg/telemetrylogs/statement_builder.go index 40751914721..a0179632cef 100644 --- a/pkg/telemetrylogs/statement_builder.go +++ b/pkg/telemetrylogs/statement_builder.go @@ -65,7 +65,7 @@ func (b *logQueryStatementBuilder) Build( start = querybuilder.ToNanoSecs(start) end = querybuilder.ToNanoSecs(end) - keySelectors := getKeySelectors(query) + keySelectors, warnings := getKeySelectors(query) keys, _, err := b.metadataStore.GetKeysMulti(ctx, keySelectors) if err != nil { return nil, err @@ -76,20 +76,32 @@ func (b *logQueryStatementBuilder) Build( // Create SQL builder q := sqlbuilder.NewSelectBuilder() + var stmt *qbtypes.Statement switch requestType { case qbtypes.RequestTypeRaw, qbtypes.RequestTypeRawStream: - return b.buildListQuery(ctx, q, query, start, end, keys, variables) + stmt, err = b.buildListQuery(ctx, q, query, start, end, keys, variables) case qbtypes.RequestTypeTimeSeries: - return b.buildTimeSeriesQuery(ctx, q, query, start, end, keys, variables) + stmt, err = b.buildTimeSeriesQuery(ctx, q, query, start, end, keys, variables) case qbtypes.RequestTypeScalar: - return b.buildScalarQuery(ctx, q, query, start, end, keys, false, variables) + stmt, err = b.buildScalarQuery(ctx, q, query, start, end, keys, false, variables) + default: + return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported request type: %s", requestType) } - return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported request type: %s", requestType) + if err != nil { + return nil, err + } + + if stmt != nil && len(warnings) > 0 { + stmt.Warnings = append(stmt.Warnings, warnings...) + } + + return stmt, nil } -func getKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]) []*telemetrytypes.FieldKeySelector { +func getKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]) ([]*telemetrytypes.FieldKeySelector, []string) { var keySelectors []*telemetrytypes.FieldKeySelector + var warnings []string for idx := range query.Aggregations { aggExpr := query.Aggregations[idx] @@ -136,7 +148,19 @@ func getKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]) [] keySelectors[idx].SelectorMatchType = telemetrytypes.FieldSelectorMatchTypeExact } - return keySelectors + // When the new JSON body experience is enabled, warn the user if they use the bare + // "body" key in the filter — queries on plain "body" default to body.message:string. + // TODO(Piyush): Setup better for coming FTS support. + if querybuilder.BodyJSONQueryEnabled { + for _, sel := range keySelectors { + if sel.Name == LogsV2BodyColumn { + warnings = append(warnings, bodySearchDefaultWarning) + break + } + } + } + + return keySelectors, warnings } func (b *logQueryStatementBuilder) adjustKeys(ctx context.Context, keys map[string][]*telemetrytypes.TelemetryFieldKey, query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation], requestType qbtypes.RequestType) qbtypes.QueryBuilderQuery[qbtypes.LogAggregation] { @@ -203,7 +227,6 @@ func (b *logQueryStatementBuilder) adjustKeys(ctx context.Context, keys map[stri } func (b *logQueryStatementBuilder) adjustKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) []string { - // First check if it matches with any intrinsic fields var intrinsicOrCalculatedField telemetrytypes.TelemetryFieldKey if _, ok := IntrinsicFields[key.Name]; ok { @@ -212,7 +235,6 @@ func (b *logQueryStatementBuilder) adjustKey(key *telemetrytypes.TelemetryFieldK } return querybuilder.AdjustKey(key, keys, nil) - } // buildListQuery builds a query for list panel type @@ -249,11 +271,7 @@ func (b *logQueryStatementBuilder) buildListQuery( sb.SelectMore(LogsV2SeverityNumberColumn) sb.SelectMore(LogsV2ScopeNameColumn) sb.SelectMore(LogsV2ScopeVersionColumn) - sb.SelectMore(LogsV2BodyColumn) - if querybuilder.BodyJSONQueryEnabled { - sb.SelectMore(LogsV2BodyJSONColumn) - sb.SelectMore(LogsV2BodyPromotedColumn) - } + sb.SelectMore(bodyAliasExpression()) sb.SelectMore(LogsV2AttributesStringColumn) sb.SelectMore(LogsV2AttributesNumberColumn) sb.SelectMore(LogsV2AttributesBoolColumn) diff --git a/pkg/telemetrylogs/stmt_builder_test.go b/pkg/telemetrylogs/stmt_builder_test.go index 15810c966aa..51b3611918e 100644 --- a/pkg/telemetrylogs/stmt_builder_test.go +++ b/pkg/telemetrylogs/stmt_builder_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest" "github.com/SigNoz/signoz/pkg/querybuilder" "github.com/SigNoz/signoz/pkg/querybuilder/resourcefilter" @@ -17,7 +18,7 @@ import ( func resourceFilterStmtBuilder() qbtypes.StatementBuilder[qbtypes.LogAggregation] { fm := resourcefilter.NewFieldMapper() cb := resourcefilter.NewConditionBuilder(fm) - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) keysMap := buildCompleteFieldKeyMap() for _, keys := range keysMap { for _, key := range keys { @@ -195,7 +196,7 @@ func TestStatementBuilderTimeSeries(t *testing.T) { }, } - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() fm := NewFieldMapper() cb := NewConditionBuilder(fm) @@ -315,7 +316,7 @@ func TestStatementBuilderListQuery(t *testing.T) { }, } - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() fm := NewFieldMapper() cb := NewConditionBuilder(fm) @@ -455,7 +456,7 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) { }, } - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() fm := NewFieldMapper() cb := NewConditionBuilder(fm) @@ -531,7 +532,7 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) { }, } - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() fm := NewFieldMapper() cb := NewConditionBuilder(fm) @@ -626,7 +627,7 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) { }, } - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) mockMetadataStore.KeysMap = buildCompleteFieldKeyMapCollision() fm := NewFieldMapper() cb := NewConditionBuilder(fm) @@ -849,7 +850,7 @@ func TestAdjustKey(t *testing.T) { } fm := NewFieldMapper() - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) mockMetadataStore.KeysMap = buildCompleteFieldKeyMapCollision() cb := NewConditionBuilder(fm) @@ -886,3 +887,152 @@ func TestAdjustKey(t *testing.T) { }) } } + +func TestStmtBuilderBodyField(t *testing.T) { + cases := []struct { + name string + requestType qbtypes.RequestType + query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation] + enableBodyJSONQuery bool + expected qbtypes.Statement + expectedErr error + }{ + { + name: "body_exists", + requestType: qbtypes.RequestTypeRaw, + query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{ + Signal: telemetrytypes.SignalLogs, + Filter: &qbtypes.Filter{Expression: "body Exists"}, + Limit: 10, + }, + enableBodyJSONQuery: true, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND body_v2.message <> ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{uint64(1747945619), uint64(1747983448), "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + Warnings: []string{bodySearchDefaultWarning}, + }, + expectedErr: nil, + }, + { + name: "body_exists_disabled", + requestType: qbtypes.RequestTypeRaw, + query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{ + Signal: telemetrytypes.SignalLogs, + Filter: &qbtypes.Filter{Expression: "body Exists"}, + Limit: 10, + }, + enableBodyJSONQuery: false, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND body <> ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{uint64(1747945619), uint64(1747983448), "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + }, + expectedErr: nil, + }, + { + name: "body_empty", + requestType: qbtypes.RequestTypeRaw, + query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{ + Signal: telemetrytypes.SignalLogs, + Filter: &qbtypes.Filter{Expression: "body == ''"}, + Limit: 10, + }, + enableBodyJSONQuery: true, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND body_v2.message = ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{uint64(1747945619), uint64(1747983448), "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + Warnings: []string{bodySearchDefaultWarning}, + }, + expectedErr: nil, + }, + { + name: "body_empty_disabled", + requestType: qbtypes.RequestTypeRaw, + query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{ + Signal: telemetrytypes.SignalLogs, + Filter: &qbtypes.Filter{Expression: "body == ''"}, + Limit: 10, + }, + enableBodyJSONQuery: false, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND body = ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{uint64(1747945619), uint64(1747983448), "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + }, + expectedErr: nil, + }, + { + name: "body_contains", + requestType: qbtypes.RequestTypeRaw, + query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{ + Signal: telemetrytypes.SignalLogs, + Filter: &qbtypes.Filter{Expression: "body CONTAINS 'error'"}, + Limit: 10, + }, + enableBodyJSONQuery: true, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND LOWER(body_v2.message) LIKE LOWER(?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{uint64(1747945619), uint64(1747983448), "%error%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + Warnings: []string{bodySearchDefaultWarning}, + }, + expectedErr: nil, + }, + { + name: "body_contains_disabled", + requestType: qbtypes.RequestTypeRaw, + query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{ + Signal: telemetrytypes.SignalLogs, + Filter: &qbtypes.Filter{Expression: "body CONTAINS 'error'"}, + Limit: 10, + }, + enableBodyJSONQuery: false, + expected: qbtypes.Statement{ + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND LOWER(body) LIKE LOWER(?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{uint64(1747945619), uint64(1747983448), "%error%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + }, + expectedErr: nil, + }, + } + + fm := NewFieldMapper() + cb := NewConditionBuilder(fm) + + enable, disable := jsonQueryTestUtil(t) + defer disable() + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if c.enableBodyJSONQuery { + enable() + } else { + disable() + } + // build the key map after enabling/disabling body JSON query + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(IntrinsicFields) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil) + resourceFilterStmtBuilder := resourceFilterStmtBuilder() + statementBuilder := NewLogQueryStatementBuilder( + instrumentationtest.New().ToProviderSettings(), + mockMetadataStore, + fm, + cb, + resourceFilterStmtBuilder, + aggExprRewriter, + DefaultFullTextColumn, + GetBodyJSONKey, + ) + + q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil) + if c.expectedErr != nil { + require.Error(t, err) + require.Contains(t, err.Error(), c.expectedErr.Error()) + } else { + if err != nil { + _, _, _, _, _, add := errors.Unwrapb(err) + t.Logf("error additionals: %v", add) + } + require.NoError(t, err) + require.Equal(t, c.expected.Query, q.Query) + require.Equal(t, c.expected.Args, q.Args) + require.Equal(t, c.expected.Warnings, q.Warnings) + } + }) + } +} diff --git a/pkg/telemetrylogs/test_data.go b/pkg/telemetrylogs/test_data.go index 4b208e5b636..ae52227101f 100644 --- a/pkg/telemetrylogs/test_data.go +++ b/pkg/telemetrylogs/test_data.go @@ -27,13 +27,6 @@ func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey { FieldDataType: telemetrytypes.FieldDataTypeString, }, }, - "body": { - { - Name: "body", - FieldContext: telemetrytypes.FieldContextLog, - FieldDataType: telemetrytypes.FieldDataTypeString, - }, - }, "http.status_code": { { Name: "http.status_code", @@ -938,6 +931,13 @@ func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey { Materialized: true, }, }, + "body": { + { + Name: "body", + FieldContext: telemetrytypes.FieldContextLog, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + }, } for _, keys := range keysMap { @@ -945,6 +945,7 @@ func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey { key.Signal = telemetrytypes.SignalLogs } } + return keysMap } diff --git a/pkg/telemetrymetadata/body_json_metadata.go b/pkg/telemetrymetadata/body_json_metadata.go index 3a6e456af28..accc6535a15 100644 --- a/pkg/telemetrymetadata/body_json_metadata.go +++ b/pkg/telemetrymetadata/body_json_metadata.go @@ -54,6 +54,7 @@ func (t *telemetryMetaStore) fetchBodyJSONPaths(ctx context.Context, instrumentationtypes.CodeNamespace: "metadata", instrumentationtypes.CodeFunctionName: "fetchBodyJSONPaths", }) + query, args, limit := buildGetBodyJSONPathsQuery(fieldKeySelectors) rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...) if err != nil { @@ -184,6 +185,9 @@ func buildGetBodyJSONPathsQuery(fieldKeySelectors []*telemetrytypes.FieldKeySele limit += fieldKeySelector.Limit } sb.Where(sb.Or(orClauses...)) + // mesasge field is skipped; since it is a type hint and is handled by the field mapper + // TODO(Piyush): If typehints increases in future, use aftership parser to skip type hints here + sb.Where(sb.NotEqual("path", telemetrylogs.MessageBodyField)) // Group by path to get unique paths with aggregated types sb.GroupBy("path") @@ -319,7 +323,7 @@ func (t *telemetryMetaStore) ListJSONValues(ctx context.Context, path string, li if promoted { path = telemetrylogs.BodyPromotedColumnPrefix + path } else { - path = telemetrylogs.BodyJSONColumnPrefix + path + path = telemetrylogs.BodyV2ColumnPrefix + path } from := fmt.Sprintf("%s.%s", telemetrylogs.DBName, telemetrylogs.LogsV2TableName) @@ -522,7 +526,7 @@ func (t *telemetryMetaStore) GetPromotedPaths(ctx context.Context, paths ...stri // TODO(Piyush): Remove this function func CleanPathPrefixes(path string) string { path = strings.TrimPrefix(path, telemetrytypes.BodyJSONStringSearchPrefix) - path = strings.TrimPrefix(path, telemetrylogs.BodyJSONColumnPrefix) + path = strings.TrimPrefix(path, telemetrylogs.BodyV2ColumnPrefix) path = strings.TrimPrefix(path, telemetrylogs.BodyPromotedColumnPrefix) return path } diff --git a/pkg/telemetrymetadata/metadata.go b/pkg/telemetrymetadata/metadata.go index 5e4f9bb0f6b..bb6500f4ed4 100644 --- a/pkg/telemetrymetadata/metadata.go +++ b/pkg/telemetrymetadata/metadata.go @@ -102,7 +102,7 @@ func NewTelemetryMetaStore( jsonColumnMetadata: map[telemetrytypes.Signal]map[telemetrytypes.FieldContext]telemetrytypes.JSONColumnMetadata{ telemetrytypes.SignalLogs: { telemetrytypes.FieldContextBody: telemetrytypes.JSONColumnMetadata{ - BaseColumn: telemetrylogs.LogsV2BodyJSONColumn, + BaseColumn: telemetrylogs.LogsV2BodyV2Column, PromotedColumn: telemetrylogs.LogsV2BodyPromotedColumn, }, }, @@ -351,7 +351,7 @@ func (t *telemetryMetaStore) logsTblStatementToFieldKeys(ctx context.Context) ([ } // getLogsKeys returns the keys from the spans that match the field selection criteria -func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors []*telemetrytypes.FieldKeySelector) ([]*telemetrytypes.TelemetryFieldKey, bool, error) { +func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors []*telemetrytypes.FieldKeySelector) (map[string][]*telemetrytypes.TelemetryFieldKey, bool, error) { ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{ instrumentationtypes.TelemetrySignal: telemetrytypes.SignalLogs.StringValue(), instrumentationtypes.CodeNamespace: "metadata", @@ -367,9 +367,10 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors if err != nil { return nil, false, err } - mapOfKeys := make(map[string]*telemetrytypes.TelemetryFieldKey) + // setOfKeys to reuse the same key object for qualified names + setOfKeys := make(map[string]*telemetrytypes.TelemetryFieldKey) for _, key := range matKeys { - mapOfKeys[key.Name+";"+key.FieldContext.StringValue()+";"+key.FieldDataType.StringValue()] = key + setOfKeys[key.Text()] = key } // queries for both attribute and resource keys tables @@ -470,7 +471,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors if len(queries) == 0 { // No matching contexts, return empty result - return []*telemetrytypes.TelemetryFieldKey{}, true, nil + return nil, true, nil } // Combine queries with UNION ALL @@ -498,7 +499,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors } defer rows.Close() - keys := []*telemetrytypes.TelemetryFieldKey{} + mapOfKeys := make(map[string][]*telemetrytypes.TelemetryFieldKey) rowCount := 0 searchTexts := []string{} dataTypes := []telemetrytypes.FieldDataType{} @@ -526,7 +527,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors if err != nil { return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error()) } - key, ok := mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] + key, ok := setOfKeys[fieldContext.StringValue()+"."+name+":"+fieldDataType.StringValue()] // if there is no materialised column, create a key with the field context and data type if !ok { @@ -538,8 +539,8 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors } } - keys = append(keys, key) - mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] = key + mapOfKeys[key.Name] = append(mapOfKeys[key.Name], key) + setOfKeys[key.Text()] = key } if rows.Err() != nil { @@ -565,17 +566,16 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors if found { if field, exists := telemetrylogs.IntrinsicFields[key]; exists { - if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added { - keys = append(keys, &field) + // Register by physical name only once and only when it differs from the + // logical key — if they are the same, the always-register below covers it. + if _, added := setOfKeys[field.Text()]; !added { + mapOfKeys[field.Text()] = append(mapOfKeys[field.Text()], &field) } - continue + // Always register the logical key so that every alias in IntrinsicFields + // (e.g. "message", "body_v2.message") independently resolves to the same + // physical field in the keys map. + mapOfKeys[key] = append(mapOfKeys[key], &field) } - - keys = append(keys, &telemetrytypes.TelemetryFieldKey{ - Name: key, - FieldContext: telemetrytypes.FieldContextLog, - Signal: telemetrytypes.SignalLogs, - }) } } @@ -584,10 +584,13 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors if err != nil { t.logger.ErrorContext(ctx, "failed to extract body JSON paths", "error", err) } - keys = append(keys, bodyJSONPaths...) + for _, key := range bodyJSONPaths { + mapOfKeys[key.Name] = append(mapOfKeys[key.Name], key) + } complete = complete && finished } - return keys, complete, nil + + return mapOfKeys, complete, nil } func getPriorityForContext(ctx telemetrytypes.FieldContext) int { @@ -882,12 +885,20 @@ func (t *telemetryMetaStore) GetKeys(ctx context.Context, fieldKeySelector *tele if fieldKeySelector != nil { selectors = []*telemetrytypes.FieldKeySelector{fieldKeySelector} } + mapOfKeys := make(map[string][]*telemetrytypes.TelemetryFieldKey) switch fieldKeySelector.Signal { case telemetrytypes.SignalTraces: keys, complete, err = t.getTracesKeys(ctx, selectors) case telemetrytypes.SignalLogs: - keys, complete, err = t.getLogsKeys(ctx, selectors) + mapOfLogKeys, logsComplete, err := t.getLogsKeys(ctx, selectors) + if err != nil { + return nil, false, err + } + for keyName, keys := range mapOfLogKeys { + mapOfKeys[keyName] = append(mapOfKeys[keyName], keys...) + } + complete = complete && logsComplete case telemetrytypes.SignalMetrics: if fieldKeySelector.Source == telemetrytypes.SourceMeter { keys, complete, err = t.getMeterSourceMetricKeys(ctx, selectors) @@ -903,12 +914,13 @@ func (t *telemetryMetaStore) GetKeys(ctx context.Context, fieldKeySelector *tele keys = append(keys, tracesKeys...) // get logs keys - logsKeys, logsComplete, err := t.getLogsKeys(ctx, selectors) + mapOfLogKeys, logsComplete, err := t.getLogsKeys(ctx, selectors) if err != nil { return nil, false, err } - keys = append(keys, logsKeys...) - + for keyName, keys := range mapOfLogKeys { + mapOfKeys[keyName] = append(mapOfKeys[keyName], keys...) + } // get metrics keys metricsKeys, metricsComplete, err := t.getMetricsKeys(ctx, selectors) if err != nil { @@ -922,7 +934,6 @@ func (t *telemetryMetaStore) GetKeys(ctx context.Context, fieldKeySelector *tele return nil, false, err } - mapOfKeys := make(map[string][]*telemetrytypes.TelemetryFieldKey) for _, key := range keys { mapOfKeys[key.Name] = append(mapOfKeys[key.Name], key) } @@ -959,7 +970,7 @@ func (t *telemetryMetaStore) GetKeysMulti(ctx context.Context, fieldKeySelectors } } - logsKeys, logsComplete, err := t.getLogsKeys(ctx, logsSelectors) + mapOfLogKeys, logsComplete, err := t.getLogsKeys(ctx, logsSelectors) if err != nil { return nil, false, err } @@ -980,8 +991,8 @@ func (t *telemetryMetaStore) GetKeysMulti(ctx context.Context, fieldKeySelectors complete := logsComplete && tracesComplete && metricsComplete mapOfKeys := make(map[string][]*telemetrytypes.TelemetryFieldKey) - for _, key := range logsKeys { - mapOfKeys[key.Name] = append(mapOfKeys[key.Name], key) + for keyName, keys := range mapOfLogKeys { + mapOfKeys[keyName] = append(mapOfKeys[keyName], keys...) } for _, key := range tracesKeys { mapOfKeys[key.Name] = append(mapOfKeys[key.Name], key) diff --git a/pkg/telemetrymeter/stmt_builder_test.go b/pkg/telemetrymeter/stmt_builder_test.go index 2b2c39bd7da..8b198b81bf6 100644 --- a/pkg/telemetrymeter/stmt_builder_test.go +++ b/pkg/telemetrymeter/stmt_builder_test.go @@ -159,7 +159,7 @@ func TestStatementBuilder(t *testing.T) { fm := telemetrymetrics.NewFieldMapper() cb := telemetrymetrics.NewConditionBuilder(fm) - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) keys, err := telemetrytypestest.LoadFieldKeysFromJSON("testdata/keys_map.json") if err != nil { t.Fatalf("failed to load field keys: %v", err) diff --git a/pkg/telemetrymetrics/stmt_builder_test.go b/pkg/telemetrymetrics/stmt_builder_test.go index 8ecbd4c1a78..1a5fcd75796 100644 --- a/pkg/telemetrymetrics/stmt_builder_test.go +++ b/pkg/telemetrymetrics/stmt_builder_test.go @@ -221,7 +221,7 @@ func TestStatementBuilder(t *testing.T) { fm := NewFieldMapper() cb := NewConditionBuilder(fm) - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) keys, err := telemetrytypestest.LoadFieldKeysFromJSON("testdata/keys_map.json") if err != nil { t.Fatalf("failed to load field keys: %v", err) diff --git a/pkg/telemetrytraces/stmt_builder_test.go b/pkg/telemetrytraces/stmt_builder_test.go index 026e83062b7..e527ec1dda2 100644 --- a/pkg/telemetrytraces/stmt_builder_test.go +++ b/pkg/telemetrytraces/stmt_builder_test.go @@ -18,7 +18,7 @@ import ( func resourceFilterStmtBuilder() qbtypes.StatementBuilder[qbtypes.TraceAggregation] { fm := resourcefilter.NewFieldMapper() cb := resourcefilter.NewConditionBuilder(fm) - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() return resourcefilter.NewTraceResourceFilterStatementBuilder( @@ -368,7 +368,7 @@ func TestStatementBuilder(t *testing.T) { fm := NewFieldMapper() cb := NewConditionBuilder(fm) - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil) @@ -664,7 +664,7 @@ func TestStatementBuilderListQuery(t *testing.T) { fm := NewFieldMapper() cb := NewConditionBuilder(fm) - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil) @@ -771,7 +771,7 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) { t.Run(c.name, func(t *testing.T) { fm := NewFieldMapper() cb := NewConditionBuilder(fm) - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) mockMetadataStore.KeysMap = c.keysMap if mockMetadataStore.KeysMap == nil { mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() @@ -927,7 +927,7 @@ func TestStatementBuilderTraceQuery(t *testing.T) { fm := NewFieldMapper() cb := NewConditionBuilder(fm) - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil) @@ -1145,7 +1145,7 @@ func TestAdjustKey(t *testing.T) { fm := NewFieldMapper() cb := NewConditionBuilder(fm) - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil) resourceFilterStmtBuilder := resourceFilterStmtBuilder() @@ -1420,7 +1420,7 @@ func TestAdjustKeys(t *testing.T) { fm := NewFieldMapper() cb := NewConditionBuilder(fm) - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil) resourceFilterStmtBuilder := resourceFilterStmtBuilder() diff --git a/pkg/telemetrytraces/trace_operator_cte_builder_test.go b/pkg/telemetrytraces/trace_operator_cte_builder_test.go index f7cb8bde74a..7ed7750ebc5 100644 --- a/pkg/telemetrytraces/trace_operator_cte_builder_test.go +++ b/pkg/telemetrytraces/trace_operator_cte_builder_test.go @@ -388,7 +388,7 @@ func TestTraceOperatorStatementBuilder(t *testing.T) { fm := NewFieldMapper() cb := NewConditionBuilder(fm) - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil) @@ -504,7 +504,7 @@ func TestTraceOperatorStatementBuilderErrors(t *testing.T) { fm := NewFieldMapper() cb := NewConditionBuilder(fm) - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil) diff --git a/pkg/telemetrytraces/trace_time_range_test.go b/pkg/telemetrytraces/trace_time_range_test.go index 08fe4175367..0446f94f88d 100644 --- a/pkg/telemetrytraces/trace_time_range_test.go +++ b/pkg/telemetrytraces/trace_time_range_test.go @@ -19,7 +19,7 @@ func TestTraceTimeRangeOptimization(t *testing.T) { fm := NewFieldMapper() cb := NewConditionBuilder(fm) - mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore := telemetrytypestest.NewMockMetadataStore(nil) mockMetadataStore.KeysMap = buildCompleteFieldKeyMap() mockMetadataStore.KeysMap["trace_id"] = []*telemetrytypes.TelemetryFieldKey{{ diff --git a/pkg/types/telemetrytypes/field.go b/pkg/types/telemetrytypes/field.go index 24bad0ee49e..4829101ddec 100644 --- a/pkg/types/telemetrytypes/field.go +++ b/pkg/types/telemetrytypes/field.go @@ -40,7 +40,7 @@ type TelemetryFieldKey struct { JSONDataType *JSONDataType `json:"-"` JSONPlan JSONAccessPlan `json:"-"` Indexes []JSONDataTypeIndex `json:"-"` - Materialized bool `json:"-"` // refers to promoted in case of body.... fields + Materialized bool `json:"-"` // refers to type hint in case of JSON column fields } func (f *TelemetryFieldKey) KeyNameContainsArray() bool { diff --git a/pkg/types/telemetrytypes/field_datatype.go b/pkg/types/telemetrytypes/field_datatype.go index e9e0735765b..83f7b97d0da 100644 --- a/pkg/types/telemetrytypes/field_datatype.go +++ b/pkg/types/telemetrytypes/field_datatype.go @@ -21,6 +21,7 @@ var ( // int64 and number are synonyms for float64 FieldDataTypeInt64 = FieldDataType{valuer.NewString("int64")} FieldDataTypeNumber = FieldDataType{valuer.NewString("number")} + FieldDataTypeJSON = FieldDataType{valuer.NewString("json")} FieldDataTypeUnspecified = FieldDataType{valuer.NewString("")} FieldDataTypeArrayString = FieldDataType{valuer.NewString("[]string")} diff --git a/pkg/types/telemetrytypes/json_access_plan.go b/pkg/types/telemetrytypes/json_access_plan.go index 03a5454b063..6102152fd21 100644 --- a/pkg/types/telemetrytypes/json_access_plan.go +++ b/pkg/types/telemetrytypes/json_access_plan.go @@ -40,7 +40,7 @@ type JSONAccessNode struct { // Node information Name string IsTerminal bool - isRoot bool // marked true for only body_json and body_json_promoted + isRoot bool // marked true for only body_v2 and body_promoted // Precomputed type information (single source of truth) AvailableTypes []JSONDataType diff --git a/pkg/types/telemetrytypes/json_access_plan_test.go b/pkg/types/telemetrytypes/json_access_plan_test.go index fbf30492183..156f4cf28ab 100644 --- a/pkg/types/telemetrytypes/json_access_plan_test.go +++ b/pkg/types/telemetrytypes/json_access_plan_test.go @@ -1,12 +1,19 @@ package telemetrytypes import ( + "fmt" "testing" + otelconstants "github.com/SigNoz/signoz-otel-collector/constants" "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" ) +const ( + bodyV2Column = otelconstants.BodyV2Column + bodyPromotedColumn = otelconstants.BodyPromotedColumn +) + // ============================================================================ // Helper Functions for Test Data Creation // ============================================================================ @@ -109,8 +116,8 @@ func TestNode_Alias(t *testing.T) { }{ { name: "Root node returns name as-is", - node: NewRootJSONAccessNode("body_json", 32, 0), - expected: "body_json", + node: NewRootJSONAccessNode(bodyV2Column, 32, 0), + expected: bodyV2Column, }, { name: "Node without parent returns backticked name", @@ -124,9 +131,9 @@ func TestNode_Alias(t *testing.T) { name: "Node with root parent uses dot separator", node: &JSONAccessNode{ Name: "age", - Parent: NewRootJSONAccessNode("body_json", 32, 0), + Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0), }, - expected: "`" + "body_json" + ".age`", + expected: "`" + bodyV2Column + ".age`", }, { name: "Node with non-root parent uses array separator", @@ -134,10 +141,10 @@ func TestNode_Alias(t *testing.T) { Name: "name", Parent: &JSONAccessNode{ Name: "education", - Parent: NewRootJSONAccessNode("body_json", 32, 0), + Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0), }, }, - expected: "`" + "body_json" + ".education[].name`", + expected: "`" + bodyV2Column + ".education[].name`", }, { name: "Nested array path with multiple levels", @@ -147,11 +154,11 @@ func TestNode_Alias(t *testing.T) { Name: "awards", Parent: &JSONAccessNode{ Name: "education", - Parent: NewRootJSONAccessNode("body_json", 32, 0), + Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0), }, }, }, - expected: "`" + "body_json" + ".education[].awards[].type`", + expected: "`" + bodyV2Column + ".education[].awards[].type`", }, } @@ -173,18 +180,18 @@ func TestNode_FieldPath(t *testing.T) { name: "Simple field path from root", node: &JSONAccessNode{ Name: "user", - Parent: NewRootJSONAccessNode("body_json", 32, 0), + Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0), }, // FieldPath() always wraps the field name in backticks - expected: "body_json" + ".`user`", + expected: bodyV2Column + ".`user`", }, { name: "Field path with backtick-required key", node: &JSONAccessNode{ Name: "user-name", // requires backtick - Parent: NewRootJSONAccessNode("body_json", 32, 0), + Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0), }, - expected: "body_json" + ".`user-name`", + expected: bodyV2Column + ".`user-name`", }, { name: "Nested field path", @@ -192,11 +199,11 @@ func TestNode_FieldPath(t *testing.T) { Name: "age", Parent: &JSONAccessNode{ Name: "user", - Parent: NewRootJSONAccessNode("body_json", 32, 0), + Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0), }, }, // FieldPath() always wraps the field name in backticks - expected: "`" + "body_json" + ".user`.`age`", + expected: "`" + bodyV2Column + ".user`.`age`", }, { name: "Array element field path", @@ -204,11 +211,11 @@ func TestNode_FieldPath(t *testing.T) { Name: "name", Parent: &JSONAccessNode{ Name: "education", - Parent: NewRootJSONAccessNode("body_json", 32, 0), + Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0), }, }, // FieldPath() always wraps the field name in backticks - expected: "`" + "body_json" + ".education`.`name`", + expected: "`" + bodyV2Column + ".education`.`name`", }, } @@ -236,36 +243,36 @@ func TestPlanJSON_BasicStructure(t *testing.T) { { name: "Simple path not promoted", key: makeKey("user.name", String, false), - expectedYAML: ` + expectedYAML: fmt.Sprintf(` - name: user.name - column: body_json + column: %s availableTypes: - String maxDynamicTypes: 16 isTerminal: true elemType: String -`, +`, bodyV2Column), }, { name: "Simple path promoted", key: makeKey("user.name", String, true), - expectedYAML: ` + expectedYAML: fmt.Sprintf(` - name: user.name - column: body_json + column: %s availableTypes: - String maxDynamicTypes: 16 isTerminal: true elemType: String - name: user.name - column: body_json_promoted + column: %s availableTypes: - String maxDynamicTypes: 16 maxDynamicPaths: 256 isTerminal: true elemType: String -`, +`, bodyV2Column, bodyPromotedColumn), }, { name: "Empty path returns error", @@ -278,8 +285,8 @@ func TestPlanJSON_BasicStructure(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { err := tt.key.SetJSONAccessPlan(JSONColumnMetadata{ - BaseColumn: "body_json", - PromotedColumn: "body_json_promoted", + BaseColumn: bodyV2Column, + PromotedColumn: bodyPromotedColumn, }, types) if tt.expectErr { require.Error(t, err) @@ -304,9 +311,9 @@ func TestPlanJSON_ArrayPaths(t *testing.T) { { name: "Single array level - JSON branch only", path: "education[].name", - expectedYAML: ` + expectedYAML: fmt.Sprintf(` - name: education - column: body_json + column: %s availableTypes: - Array(JSON) maxDynamicTypes: 16 @@ -318,14 +325,14 @@ func TestPlanJSON_ArrayPaths(t *testing.T) { maxDynamicTypes: 8 isTerminal: true elemType: String -`, +`, bodyV2Column), }, { name: "Single array level - both JSON and Dynamic branches", path: "education[].awards[].type", - expectedYAML: ` + expectedYAML: fmt.Sprintf(` - name: education - column: body_json + column: %s availableTypes: - Array(JSON) maxDynamicTypes: 16 @@ -352,14 +359,14 @@ func TestPlanJSON_ArrayPaths(t *testing.T) { maxDynamicPaths: 256 isTerminal: true elemType: String -`, +`, bodyV2Column), }, { name: "Deeply nested array path", path: "interests[].entities[].reviews[].entries[].metadata[].positions[].name", - expectedYAML: ` + expectedYAML: fmt.Sprintf(` - name: interests - column: body_json + column: %s availableTypes: - Array(JSON) maxDynamicTypes: 16 @@ -399,14 +406,14 @@ func TestPlanJSON_ArrayPaths(t *testing.T) { - String isTerminal: true elemType: String -`, +`, bodyV2Column), }, { name: "ArrayAnyIndex replacement [*] to []", path: "education[*].name", - expectedYAML: ` + expectedYAML: fmt.Sprintf(` - name: education - column: body_json + column: %s availableTypes: - Array(JSON) maxDynamicTypes: 16 @@ -418,7 +425,7 @@ func TestPlanJSON_ArrayPaths(t *testing.T) { maxDynamicTypes: 8 isTerminal: true elemType: String -`, +`, bodyV2Column), }, } @@ -426,8 +433,8 @@ func TestPlanJSON_ArrayPaths(t *testing.T) { t.Run(tt.name, func(t *testing.T) { key := makeKey(tt.path, String, false) err := key.SetJSONAccessPlan(JSONColumnMetadata{ - BaseColumn: "body_json", - PromotedColumn: "body_json_promoted", + BaseColumn: bodyV2Column, + PromotedColumn: bodyPromotedColumn, }, types) require.NoError(t, err) require.NotNil(t, key.JSONPlan) @@ -445,15 +452,15 @@ func TestPlanJSON_PromotedVsNonPromoted(t *testing.T) { t.Run("Non-promoted plan", func(t *testing.T) { key := makeKey(path, String, false) err := key.SetJSONAccessPlan(JSONColumnMetadata{ - BaseColumn: "body_json", - PromotedColumn: "body_json_promoted", + BaseColumn: bodyV2Column, + PromotedColumn: bodyPromotedColumn, }, types) require.NoError(t, err) require.Len(t, key.JSONPlan, 1) - expectedYAML := ` + expectedYAML := fmt.Sprintf(` - name: education - column: body_json + column: %s availableTypes: - Array(JSON) maxDynamicTypes: 16 @@ -480,7 +487,7 @@ func TestPlanJSON_PromotedVsNonPromoted(t *testing.T) { maxDynamicPaths: 256 isTerminal: true elemType: String -` +`, bodyV2Column) got := plansToYAML(t, key.JSONPlan) require.YAMLEq(t, expectedYAML, got) }) @@ -488,15 +495,15 @@ func TestPlanJSON_PromotedVsNonPromoted(t *testing.T) { t.Run("Promoted plan", func(t *testing.T) { key := makeKey(path, String, true) err := key.SetJSONAccessPlan(JSONColumnMetadata{ - BaseColumn: "body_json", - PromotedColumn: "body_json_promoted", + BaseColumn: bodyV2Column, + PromotedColumn: bodyPromotedColumn, }, types) require.NoError(t, err) require.Len(t, key.JSONPlan, 2) - expectedYAML := ` + expectedYAML := fmt.Sprintf(` - name: education - column: body_json + column: %s availableTypes: - Array(JSON) maxDynamicTypes: 16 @@ -524,7 +531,7 @@ func TestPlanJSON_PromotedVsNonPromoted(t *testing.T) { isTerminal: true elemType: String - name: education - column: body_json_promoted + column: %s availableTypes: - Array(JSON) maxDynamicTypes: 16 @@ -554,7 +561,7 @@ func TestPlanJSON_PromotedVsNonPromoted(t *testing.T) { maxDynamicPaths: 256 isTerminal: true elemType: String -` +`, bodyV2Column, bodyPromotedColumn) got := plansToYAML(t, key.JSONPlan) require.YAMLEq(t, expectedYAML, got) }) @@ -575,11 +582,11 @@ func TestPlanJSON_EdgeCases(t *testing.T) { expectErr: true, }, { - name: "Very deep nesting - validates progression doesn't go negative", - path: "interests[].entities[].reviews[].entries[].metadata[].positions[].name", - expectedYAML: ` + name: "Very deep nesting - validates progression doesn't go negative", + path: "interests[].entities[].reviews[].entries[].metadata[].positions[].name", + expectedYAML: fmt.Sprintf(` - name: interests - column: body_json + column: %s availableTypes: - Array(JSON) maxDynamicTypes: 16 @@ -619,14 +626,14 @@ func TestPlanJSON_EdgeCases(t *testing.T) { - String isTerminal: true elemType: String -`, +`, bodyV2Column), }, { - name: "Path with mixed scalar and array types", - path: "education[].type", - expectedYAML: ` + name: "Path with mixed scalar and array types", + path: "education[].type", + expectedYAML: fmt.Sprintf(` - name: education - column: body_json + column: %s availableTypes: - Array(JSON) maxDynamicTypes: 16 @@ -639,20 +646,20 @@ func TestPlanJSON_EdgeCases(t *testing.T) { maxDynamicTypes: 8 isTerminal: true elemType: String -`, +`, bodyV2Column), }, { - name: "Exists with only array types available", - path: "education", - expectedYAML: ` + name: "Exists with only array types available", + path: "education", + expectedYAML: fmt.Sprintf(` - name: education - column: body_json + column: %s availableTypes: - Array(JSON) maxDynamicTypes: 16 isTerminal: true elemType: Array(JSON) -`, +`, bodyV2Column), }, } @@ -668,8 +675,8 @@ func TestPlanJSON_EdgeCases(t *testing.T) { } key := makeKey(tt.path, keyType, false) err := key.SetJSONAccessPlan(JSONColumnMetadata{ - BaseColumn: "body_json", - PromotedColumn: "body_json_promoted", + BaseColumn: bodyV2Column, + PromotedColumn: bodyPromotedColumn, }, types) if tt.expectErr { require.Error(t, err) @@ -687,15 +694,15 @@ func TestPlanJSON_TreeStructure(t *testing.T) { path := "education[].awards[].participated[].team[].branch" key := makeKey(path, String, false) err := key.SetJSONAccessPlan(JSONColumnMetadata{ - BaseColumn: "body_json", - PromotedColumn: "body_json_promoted", + BaseColumn: bodyV2Column, + PromotedColumn: bodyPromotedColumn, }, types) require.NoError(t, err) require.Len(t, key.JSONPlan, 1) - expectedYAML := ` + expectedYAML := fmt.Sprintf(` - name: education - column: body_json + column: %s availableTypes: - Array(JSON) maxDynamicTypes: 16 @@ -780,7 +787,7 @@ func TestPlanJSON_TreeStructure(t *testing.T) { maxDynamicPaths: 64 isTerminal: true elemType: String -` +`, bodyV2Column) got := plansToYAML(t, key.JSONPlan) require.YAMLEq(t, expectedYAML, got) diff --git a/pkg/types/telemetrytypes/telemetrytypestest/metadata_store.go b/pkg/types/telemetrytypes/telemetrytypestest/metadata_store.go index fed12b08327..d8a9309f5b7 100644 --- a/pkg/types/telemetrytypes/telemetrytypestest/metadata_store.go +++ b/pkg/types/telemetrytypes/telemetrytypestest/metadata_store.go @@ -2,6 +2,7 @@ package telemetrytypestest import ( "context" + "slices" "strings" schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator" @@ -20,10 +21,20 @@ type MockMetadataStore struct { PromotedPathsMap map[string]bool LogsJSONIndexesMap map[string][]schemamigrator.Index LookupKeysMap map[telemetrytypes.MetricMetadataLookupKey]int64 + // StaticFields holds signal-specific intrinsic field definitions (e.g. telemetrylogs.IntrinsicFields). + // It is injected into GetKeys / GetKey results, mirroring what the real metadata store does when + // it reads telemetrylogs.IntrinsicFields directly. Callers pass their package's IntrinsicFields + // map at construction time to avoid a circular import. + StaticFields map[string]telemetrytypes.TelemetryFieldKey } -// NewMockMetadataStore creates a new instance of MockMetadataStore with initialized maps -func NewMockMetadataStore() *MockMetadataStore { +// NewMockMetadataStore creates a new instance of MockMetadataStore with initialized maps. +// Pass the signal-specific intrinsic fields (e.g. telemetrylogs.IntrinsicFields) so the mock +// mirrors what the real metadata store does when injecting those definitions into key results. +func NewMockMetadataStore(intrinsicFields map[string]telemetrytypes.TelemetryFieldKey) *MockMetadataStore { + if intrinsicFields == nil { + intrinsicFields = make(map[string]telemetrytypes.TelemetryFieldKey) + } return &MockMetadataStore{ KeysMap: make(map[string][]*telemetrytypes.TelemetryFieldKey), RelatedValuesMap: make(map[string][]string), @@ -33,12 +44,13 @@ func NewMockMetadataStore() *MockMetadataStore { PromotedPathsMap: make(map[string]bool), LogsJSONIndexesMap: make(map[string][]schemamigrator.Index), LookupKeysMap: make(map[telemetrytypes.MetricMetadataLookupKey]int64), + StaticFields: intrinsicFields, } } // GetKeys returns a map of field keys types.TelemetryFieldKey by name func (m *MockMetadataStore) GetKeys(ctx context.Context, fieldKeySelector *telemetrytypes.FieldKeySelector) (map[string][]*telemetrytypes.TelemetryFieldKey, bool, error) { - + setOfKeys := make(map[string]*telemetrytypes.TelemetryFieldKey) result := make(map[string][]*telemetrytypes.TelemetryFieldKey) // If selector is nil, return all keys @@ -46,22 +58,42 @@ func (m *MockMetadataStore) GetKeys(ctx context.Context, fieldKeySelector *telem return m.KeysMap, true, nil } - // Apply selector logic + // Apply selector logic from KeysMap for name, keys := range m.KeysMap { - // Check if name matches if matchesName(fieldKeySelector, name) { - filteredKeys := []*telemetrytypes.TelemetryFieldKey{} for _, key := range keys { if matchesKey(fieldKeySelector, key) { - filteredKeys = append(filteredKeys, key) + if _, exists := setOfKeys[key.Text()]; !exists { + result[name] = append(result[name], key) + setOfKeys[key.Text()] = key + } } } - if len(filteredKeys) > 0 { - result[name] = filteredKeys - } } } + // Inject StaticFields (e.g. IntrinsicFields), mirroring the real metadata store. + // StaticFields take precedence over KeysMap entries for the same logical key. + // Each logical key always gets its own entry (so "message" and "body.message" both + // resolve to the IntrinsicField independently). The physical name is registered + // only once to avoid duplicate entries in that slot. + for key, field := range m.StaticFields { + if !matchesName(fieldKeySelector, key) { + continue + } + + // Register by physical name only once and only when it differs from the + // logical key — if they are the same, the always-register below covers it. + if _, exists := setOfKeys[field.Text()]; !exists { + result[field.Text()] = append(result[field.Text()], &field) + setOfKeys[field.Text()] = &field + } + // Always register the logical key so that every alias in IntrinsicFields + // (e.g. "message", "body_v2.message") independently resolves to the same + // physical field in the keys map. + result[key] = append(result[key], &field) + } + return result, true, nil } @@ -108,7 +140,7 @@ func (m *MockMetadataStore) GetKey(ctx context.Context, fieldKeySelector *teleme result := []*telemetrytypes.TelemetryFieldKey{} - // Find keys matching the selector + // Find keys matching the selector from KeysMap for name, keys := range m.KeysMap { if matchesName(fieldKeySelector, name) { for _, key := range keys { @@ -119,6 +151,14 @@ func (m *MockMetadataStore) GetKey(ctx context.Context, fieldKeySelector *teleme } } + // Add matching StaticFields (e.g. IntrinsicFields), same as the real metadata store does + for key, field := range m.StaticFields { + if fieldKeySelector.Name == "" || strings.Contains(key, fieldKeySelector.Name) { + fieldCopy := field + result = append(result, &fieldCopy) + } + } + return result, nil } @@ -178,8 +218,9 @@ func matchesKey(selector *telemetrytypes.FieldKeySelector, key *telemetrytypes.T return true } + matchNameExceptions := []string{"body"} // Check name (already checked in matchesName, but double-check here) - if selector.Name != "" && !matchesName(selector, key.Name) { + if selector.Name != "" && !matchesName(selector, key.Name) && slices.Contains(matchNameExceptions, key.Name) { return false } @@ -289,7 +330,7 @@ func (m *MockMetadataStore) FetchTemporalityMulti(ctx context.Context, queryTime return result, nil } -// FetchTemporalityMulti fetches the temporality for multiple metrics +// FetchTemporalityAndTypeMulti fetches the temporality and type for multiple metrics func (m *MockMetadataStore) FetchTemporalityAndTypeMulti(ctx context.Context, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricNames ...string) (map[string]metrictypes.Temporality, map[string]metrictypes.Type, error) { temporalities := make(map[string]metrictypes.Temporality) types := make(map[string]metrictypes.Type) diff --git a/pkg/types/telemetrytypes/test_data.go b/pkg/types/telemetrytypes/test_data.go index c79821adb21..50e17f7ae5c 100644 --- a/pkg/types/telemetrytypes/test_data.go +++ b/pkg/types/telemetrytypes/test_data.go @@ -64,8 +64,7 @@ func TestJSONTypeSet() (map[string][]JSONDataType, MetadataStore) { "interests[].entities[].reviews[].entries[].metadata[].positions[].duration": {Int64, Float64}, "interests[].entities[].reviews[].entries[].metadata[].positions[].unit": {String}, "interests[].entities[].reviews[].entries[].metadata[].positions[].ratings": {ArrayInt64, ArrayString}, - "message": {String}, - "tags": {ArrayString}, + "tags": {ArrayString}, } return types, nil diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index bc840e10227..f1602ac50c6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,5 +1,19 @@ +import json +import os +import tempfile + import pytest +if not os.environ.get("DOCKER_CONFIG"): + os.environ["DOCKER_CONFIG"] = tempfile.mkdtemp(prefix="docker-config-") +os.environ.setdefault("DOCKER_CREDENTIAL_STORE", "") +os.environ.setdefault("DOCKER_CREDENTIAL_HELPER", "") + +docker_config_path = os.path.join(os.environ["DOCKER_CONFIG"], "config.json") +if not os.path.exists(docker_config_path): + with open(docker_config_path, "w", encoding="utf-8") as config_file: + json.dump({"auths": {}, "credsStore": ""}, config_file) + pytest_plugins = [ "fixtures.auth", "fixtures.clickhouse", @@ -22,6 +36,7 @@ "fixtures.notification_channel", "fixtures.alerts", "fixtures.cloudintegrations", + "fixtures.jsontypeexporter", ] @@ -59,7 +74,7 @@ def pytest_addoption(parser: pytest.Parser): parser.addoption( "--clickhouse-version", action="store", - default="25.5.6", + default="25.8.6", help="clickhouse version", ) parser.addoption( @@ -74,3 +89,5 @@ def pytest_addoption(parser: pytest.Parser): default="v0.129.7", help="schema migrator version", ) + + diff --git a/tests/integration/fixtures/jsontypeexporter.py b/tests/integration/fixtures/jsontypeexporter.py new file mode 100644 index 00000000000..066fe3c9762 --- /dev/null +++ b/tests/integration/fixtures/jsontypeexporter.py @@ -0,0 +1,421 @@ +""" +Simpler version of jsontypeexporter for test fixtures. +This exports JSON type metadata to the path_types table by parsing JSON bodies +and extracting all paths with their types, similar to how the real jsontypeexporter works. +""" +import datetime +import json +from abc import ABC +from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, List, Optional, Set, Union + +import numpy as np +import pytest + +from fixtures import types + +if TYPE_CHECKING: + from fixtures.logs import Logs + + +class JSONPathType(ABC): + """Represents a JSON path with its type information""" + path: str + type: str + last_seen: np.uint64 + + def __init__( + self, + path: str, + type: str, # pylint: disable=redefined-builtin + last_seen: Optional[datetime.datetime] = None, + ) -> None: + self.path = path + self.type = type + if last_seen is None: + last_seen = datetime.datetime.now() + self.last_seen = np.uint64(int(last_seen.timestamp() * 1e9)) + + def np_arr(self) -> np.array: + """Return path type data as numpy array for database insertion""" + return np.array([self.path, self.type, self.last_seen]) + + +# Constants matching jsontypeexporter +ARRAY_SEPARATOR = "[]." # Used in paths like "education[].name" +ARRAY_SUFFIX = "[]" # Used when traversing into array element objects + + +def _infer_array_type_from_type_strings(types: List[str]) -> Optional[str]: + """ + Infer array type from a list of pre-classified type strings. + Matches jsontypeexporter's inferArrayMask logic (v0.144.2+). + + Type strings are: "JSON", "String", "Bool", "Float64", "Int64" + + SuperTyping rules (matching Go inferArrayMask): + - JSON alone → Array(JSON) + - JSON + any primitive → Array(Dynamic) + - String alone → Array(Nullable(String)); String + other → Array(Dynamic) + - Float64 wins over Int64 and Bool + - Int64 wins over Bool + - Bool alone → Array(Nullable(Bool)) + """ + if len(types) == 0: + return None + + unique = set(types) + + has_json = "JSON" in unique + # hasPrimitive mirrors Go: (hasJSON && len(unique) > 1) || (!hasJSON && len(unique) > 0) + has_primitive = (has_json and len(unique) > 1) or (not has_json and len(unique) > 0) + + if has_json: + if not has_primitive: + return "Array(JSON)" + return "Array(Dynamic)" + + # ---- Primitive Type Resolution (Float > Int > Bool) ---- + if "String" in unique: + if len(unique) > 1: + return "Array(Dynamic)" + return "Array(Nullable(String))" + + if "Float64" in unique: + return "Array(Nullable(Float64))" + if "Int64" in unique: + return "Array(Nullable(Int64))" + if "Bool" in unique: + return "Array(Nullable(Bool))" + + return "Array(Dynamic)" + + +def _infer_array_type(elements: List[Any]) -> Optional[str]: + """ + Infer array type from raw Python list elements. + Classifies each element then delegates to _infer_array_type_from_type_strings. + """ + if len(elements) == 0: + return None + + types = [] + for elem in elements: + if elem is None: + continue + if isinstance(elem, dict): + types.append("JSON") + elif isinstance(elem, str): + types.append("String") + elif isinstance(elem, bool): # must be before int (bool is subclass of int) + types.append("Bool") + elif isinstance(elem, float): + types.append("Float64") + elif isinstance(elem, int): + types.append("Int64") + + return _infer_array_type_from_type_strings(types) + + +def _python_type_to_clickhouse_type(value: Any) -> str: + """ + Convert Python type to ClickHouse JSON type string. + Maps Python types to ClickHouse JSON data types. + """ + if value is None: + return "String" # Default for null values + + if isinstance(value, bool): + return "Bool" + elif isinstance(value, int): + return "Int64" + elif isinstance(value, float): + return "Float64" + elif isinstance(value, str): + return "String" + elif isinstance(value, list): + # Use the sophisticated array type inference + array_type = _infer_array_type(value) + return array_type if array_type else "Array(Dynamic)" + elif isinstance(value, dict): + return "JSON" + else: + return "String" # Default fallback + + +def _extract_json_paths( + obj: Any, + current_path: str = "", + path_types: Optional[Dict[str, Set[str]]] = None, + level: int = 0, +) -> Dict[str, Set[str]]: + """ + Recursively extract all paths and their types from a JSON object. + Matches jsontypeexporter's analyzePValue logic. + + Args: + obj: The JSON object to traverse + current_path: Current path being built (e.g., "user.name") + path_types: Dictionary mapping paths to sets of types found + level: Current nesting level (for depth limiting) + + Returns: + Dictionary mapping paths to sets of type strings + """ + if path_types is None: + path_types = {} + + if obj is None: + if current_path: + if current_path not in path_types: + path_types[current_path] = set() + path_types[current_path].add("String") # Null defaults to String + return path_types + + if isinstance(obj, dict): + # For objects, add the object itself and recurse into keys + if current_path: + if current_path not in path_types: + path_types[current_path] = set() + path_types[current_path].add("JSON") + + for key, value in obj.items(): + # Build the path for this key + if current_path: + new_path = f"{current_path}.{key}" + else: + new_path = key + + # Recurse into the value + _extract_json_paths(value, new_path, path_types, level + 1) + + elif isinstance(obj, list): + # Skip empty arrays + if len(obj) == 0: + return path_types + + # Collect types from array elements (matching Go: types := make([]pcommon.ValueType, 0, s.Len())) + types = [] + + for item in obj: + if isinstance(item, dict): + # When traversing into array element objects, use ArraySuffix ([]) + # This matches: prefix+ArraySuffix in the Go code + # Example: if current_path is "education", we use "education[]" to traverse into objects + array_prefix = current_path + ARRAY_SUFFIX if current_path else "" + for key, value in item.items(): + if array_prefix: + # Use array separator: education[].name + array_path = f"{array_prefix}.{key}" + else: + array_path = key + # Recurse without increasing level (matching Go behavior) + _extract_json_paths(value, array_path, path_types, level) + types.append("JSON") + elif isinstance(item, list): + # Arrays inside arrays are not supported - skip the whole path + # Matching Go: e.logger.Error("arrays inside arrays are not supported!", ...); return nil + return path_types + elif isinstance(item, str): + types.append("String") + elif isinstance(item, bool): + types.append("Bool") + elif isinstance(item, float): + types.append("Float64") + elif isinstance(item, int): + types.append("Int64") + + # Infer array type from collected types (matching Go: if mask := inferArrayMask(types); mask != 0) + if len(types) > 0: + array_type = _infer_array_type_from_type_strings(types) + if array_type and current_path: + if current_path not in path_types: + path_types[current_path] = set() + path_types[current_path].add(array_type) + + else: + # Primitive value (string, number, bool) + if current_path: + if current_path not in path_types: + path_types[current_path] = set() + obj_type = _python_type_to_clickhouse_type(obj) + path_types[current_path].add(obj_type) + + return path_types + + +def _parse_json_bodies_and_extract_paths( + json_bodies: List[str], + timestamp: Optional[datetime.datetime] = None, +) -> List[JSONPathType]: + """ + Parse JSON bodies and extract all paths with their types. + This mimics the behavior of jsontypeexporter. + + Args: + json_bodies: List of JSON body strings to parse + timestamp: Timestamp to use for last_seen (defaults to now) + + Returns: + List of JSONPathType objects with all discovered paths and types + """ + if timestamp is None: + timestamp = datetime.datetime.now() + + # Aggregate all paths and their types across all JSON bodies + all_path_types: Dict[str, Set[str]] = {} + + for json_body in json_bodies: + try: + parsed = json.loads(json_body) + _extract_json_paths(parsed, "", all_path_types, level=0) + except (json.JSONDecodeError, TypeError): + # Skip invalid JSON + continue + + # Convert to list of JSONPathType objects + # Each path can have multiple types, so we create one JSONPathType per type + path_type_objects: List[JSONPathType] = [] + for path, types_set in all_path_types.items(): + for type_str in types_set: + path_type_objects.append( + JSONPathType(path=path, type=type_str, last_seen=timestamp) + ) + + return path_type_objects + + +@pytest.fixture(name="export_json_types", scope="function") +def export_json_types( + clickhouse: types.TestContainerClickhouse, + request: pytest.FixtureRequest, # To access migrator fixture +) -> Generator[Callable[[Union[List[JSONPathType], List[str], List[Any]]], None], Any, None]: + """ + Fixture for exporting JSON type metadata to the path_types table. + This is a simpler version of jsontypeexporter for test fixtures. + + The function can accept: + 1. List of JSONPathType objects (manual specification) + 2. List of JSON body strings (auto-extract paths) + 3. List of Logs objects (extract from body_json field) + + Usage examples: + # Manual specification + export_json_types([ + JSONPathType(path="user.name", type="String"), + JSONPathType(path="user.age", type="Int64"), + ]) + + # Auto-extract from JSON strings + export_json_types([ + '{"user": {"name": "alice", "age": 25}}', + '{"user": {"name": "bob", "age": 30}}', + ]) + + # Auto-extract from Logs objects + export_json_types(logs_list) + """ + # Ensure migrator has run to create the table + try: + request.getfixturevalue("migrator") + except Exception: + # If migrator fixture is not available, that's okay - table might already exist + pass + + def _export_json_types( + data: Union[List[JSONPathType], List[str], List[Any]], # List[Logs] but avoiding circular import + ) -> None: + """ + Export JSON type metadata to signoz_metadata.distributed_json_path_types table. + This table stores path and type information for body JSON fields. + """ + path_types: List[JSONPathType] = [] + + if len(data) == 0: + return + + # Determine input type and convert to JSONPathType list + first_item = data[0] + + if isinstance(first_item, JSONPathType): + # Already JSONPathType objects + path_types = data # type: ignore + elif isinstance(first_item, str): + # List of JSON strings - parse and extract paths + path_types = _parse_json_bodies_and_extract_paths(data) # type: ignore + else: + # Assume it's a list of Logs objects - extract body_v2 + json_bodies: List[str] = [] + for log in data: # type: ignore + # Try to get body_v2 attribute + if hasattr(log, "body_v2") and log.body_v2: + json_bodies.append(log.body_v2) + elif hasattr(log, "body") and log.body: + # Fallback to body if body_v2 not available + try: + # Try to parse as JSON + json.loads(log.body) + json_bodies.append(log.body) + except (json.JSONDecodeError, TypeError): + pass + + if json_bodies: + path_types = _parse_json_bodies_and_extract_paths(json_bodies) + + if len(path_types) == 0: + return + + clickhouse.conn.insert( + database="signoz_metadata", + table="distributed_json_path_types", + data=[path_type.np_arr() for path_type in path_types], + column_names=[ + "path", + "type", + "last_seen", + ], + ) + + yield _export_json_types + + # Cleanup - truncate the local table after tests (following pattern from logs fixture) + clickhouse.conn.query( + f"TRUNCATE TABLE signoz_metadata.json_path_types ON CLUSTER '{clickhouse.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}' SYNC" + ) + + +@pytest.fixture(name="export_promoted_paths", scope="function") +def export_promoted_paths( + clickhouse: types.TestContainerClickhouse, + request: pytest.FixtureRequest, # To access migrator fixture +) -> Generator[Callable[[List[str]], None], Any, None]: + """ + Fixture for exporting promoted JSON paths to the promoted paths table. + """ + # Ensure migrator has run to create the table + try: + request.getfixturevalue("migrator") + except Exception: + # If migrator fixture is not available, that's okay - table might already exist + pass + + def _export_promoted_paths(paths: List[str]) -> None: + if len(paths) == 0: + return + + now_ms = int(datetime.datetime.now().timestamp() * 1000) + rows = [(path, now_ms) for path in paths] + clickhouse.conn.insert( + database="signoz_metadata", + table="distributed_json_promoted_paths", + data=rows, + column_names=[ + "path", + "created_at", + ], + ) + + yield _export_promoted_paths + + clickhouse.conn.query( + f"TRUNCATE TABLE signoz_metadata.json_promoted_paths ON CLUSTER '{clickhouse.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}' SYNC" + ) diff --git a/tests/integration/fixtures/logs.py b/tests/integration/fixtures/logs.py index 916ea50de72..7ed8843f428 100644 --- a/tests/integration/fixtures/logs.py +++ b/tests/integration/fixtures/logs.py @@ -100,6 +100,8 @@ class Logs(ABC): severity_text: str severity_number: np.uint8 body: str + body_v2: str + body_promoted: str attributes_string: dict[str, str] attributes_number: dict[str, np.float64] attributes_bool: dict[str, bool] @@ -119,6 +121,8 @@ def __init__( resources: dict[str, Any] = {}, attributes: dict[str, Any] = {}, body: str = "default body", + body_v2: Optional[str] = None, + body_promoted: Optional[str] = None, severity_text: str = "INFO", trace_id: str = "", span_id: str = "", @@ -163,6 +167,33 @@ def __init__( # Set body self.body = body + # Set body_v2 - if body is JSON, parse and stringify it, otherwise use empty string + # ClickHouse accepts String input for JSON column + if body_v2 is not None: + self.body_v2 = body_v2 + else: + # Try to parse body as JSON; if successful use it directly, + # otherwise wrap as {"message": body} matching the normalize operator behavior. + try: + json.loads(body) + self.body_v2 = body + except (json.JSONDecodeError, TypeError): + self.body_v2 = json.dumps({"message": body}) + + # Set body_promoted - must be valid JSON + # Tests will explicitly pass promoted column's content, but we validate it + if body_promoted is not None: + # Validate that it's valid JSON + try: + json.loads(body_promoted) + self.body_promoted = body_promoted + except (json.JSONDecodeError, TypeError): + # If invalid, default to empty JSON object + self.body_promoted = "{}" + else: + # Default to empty JSON object (valid JSON) + self.body_promoted = "{}" + # Process resources and attributes self.resources_string = {k: str(v) for k, v in resources.items()} for k, v in self.resources_string.items(): @@ -319,6 +350,8 @@ def np_arr(self) -> np.array: self.severity_text, self.severity_number, self.body, + self.body_v2, + self.body_promoted, self.attributes_string, self.attributes_number, self.attributes_bool, @@ -463,6 +496,8 @@ def _insert_logs(logs: List[Logs]) -> None: "severity_text", "severity_number", "body", + "body_v2", + "body_promoted", "attributes_string", "attributes_number", "attributes_bool", diff --git a/tests/integration/fixtures/migrator.py b/tests/integration/fixtures/migrator.py index 42ab9ab0998..c1508a1dd5f 100644 --- a/tests/integration/fixtures/migrator.py +++ b/tests/integration/fixtures/migrator.py @@ -20,12 +20,14 @@ def migrator( """ def create() -> None: - version = request.config.getoption("--schema-migrator-version") + # Hardcode version for new QB tests + version = "v0.144.3-rc.1" client = docker.from_env() container = client.containers.run( image=f"signoz/signoz-schema-migrator:{version}", command=f"sync --replication=true --cluster-name=cluster --up= --dsn={clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN"]}", + environment={"ENABLE_LOGS_MIGRATIONS_V2": "1"}, detach=True, auto_remove=False, network=network.id, @@ -44,6 +46,7 @@ def create() -> None: container = client.containers.run( image=f"signoz/signoz-schema-migrator:{version}", command=f"async --replication=true --cluster-name=cluster --up= --dsn={clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN"]}", + environment={"ENABLE_LOGS_MIGRATIONS_V2": "1"}, detach=True, auto_remove=False, network=network.id, diff --git a/tests/integration/fixtures/signoz.py b/tests/integration/fixtures/signoz.py index bec11170936..16a93608770 100644 --- a/tests/integration/fixtures/signoz.py +++ b/tests/integration/fixtures/signoz.py @@ -73,6 +73,7 @@ def create() -> types.SigNoz: "SIGNOZ_ALERTMANAGER_SIGNOZ_POLL__INTERVAL": "5s", "SIGNOZ_ALERTMANAGER_SIGNOZ_ROUTE_GROUP__WAIT": "1s", "SIGNOZ_ALERTMANAGER_SIGNOZ_ROUTE_GROUP__INTERVAL": "5s", + "BODY_JSON_QUERY_ENABLED": "true", } | sqlstore.env | clickhouse.env diff --git a/tests/integration/src/querier/02_logs_json_body_new_qb.py b/tests/integration/src/querier/02_logs_json_body_new_qb.py new file mode 100644 index 00000000000..904daa7aa77 --- /dev/null +++ b/tests/integration/src/querier/02_logs_json_body_new_qb.py @@ -0,0 +1,1628 @@ +import json +from datetime import datetime, timedelta, timezone +from http import HTTPStatus +from typing import Any, Callable, Dict, List + +import pytest +import requests + +from fixtures import types +from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD +from fixtures.jsontypeexporter import export_json_types, export_promoted_paths +from fixtures.logs import Logs + + +def _assert_ok(response: requests.Response) -> None: + if response.status_code != HTTPStatus.OK: + raise AssertionError(f"HTTP {response.status_code}: {response.text}") + + +def _post_query_range( + signoz: types.SigNoz, token: str, payload: Dict[str, Any] +) -> requests.Response: + return requests.post( + signoz.self.host_configs["8080"].get("/api/v5/query_range"), + timeout=2, + headers={"authorization": f"Bearer {token}"}, + json=payload, + ) + + +def _build_query_payload( + now: datetime, case: Dict[str, Any], start_offset_seconds: int = 10 +) -> Dict[str, Any]: + start_ms = case.get( + "startMs", int((now - timedelta(seconds=start_offset_seconds)).timestamp() * 1000) + ) + end_ms = case.get("endMs", int(now.timestamp() * 1000)) + expression = case.get("expression") + group_by = case.get("groupBy") + aggregation = case.get("aggregation") + order = case.get("order") + step_interval = case.get("stepInterval") + + if aggregation and not isinstance(aggregation, list): + aggregation = [{"expression": aggregation}] + if order is None and case["requestType"] == "raw": + order = [{"key": {"name": "timestamp"}, "direction": "desc"}] + + payload: Dict[str, Any] = { + "schemaVersion": "v1", + "start": start_ms, + "end": end_ms, + "requestType": case["requestType"], + "compositeQuery": { + "queries": [ + { + "type": "builder_query", + "spec": { + "name": case["name"], + "signal": "logs", + "disabled": False, + "limit": case.get("limit", 100), + "offset": 0, + "filter": {"expression": expression} if expression else None, + "groupBy": group_by, + "aggregations": aggregation, + "order": order, + "stepInterval": step_interval, + }, + } + ] + }, + "formatOptions": {"formatTableResultForUI": False, "fillGaps": False}, + } + return payload + + +def _get_results(response: requests.Response) -> List[Dict[str, Any]]: + assert response.json()["status"] == "success" + results = response.json()["data"]["data"]["results"] + assert len(results) == 1 + return results + + +def _get_rows(response: requests.Response) -> List[Dict[str, Any]]: + results = _get_results(response) + return results[0]["rows"] + + +def _run_query_case( + signoz: types.SigNoz, token: str, now: datetime, case: Dict[str, Any] +) -> None: + payload = _build_query_payload(now, case) + response = _post_query_range(signoz, token, payload) + _assert_ok(response) + assert case["validate"](response) + + +def _labels_to_map(labels: Any) -> Dict[str, Any]: + if isinstance(labels, list): + mapped: Dict[str, Any] = {} + for entry in labels: + key = entry.get("key", {}) if isinstance(entry, dict) else {} + name = key.get("name") + if name: + mapped[name] = entry.get("value") + return mapped + if isinstance(labels, dict): + return labels + return {} + + +# ============================================================================ +# NEW QB TESTS - Comprehensive integration tests for new JSON Query Builder +# ============================================================================ +# These tests use body_v2 and body_promoted columns and require +# BODY_JSON_QUERY_ENABLED=true environment variable +# Breadcrumbs for promoted-path tests: +# - body is empty for JSON logs; the API returns a merged view from JSON columns +# - body_v2 contains only non-promoted fields for post-promotion logs +# (pre-promotion logs still carry the full JSON in body_v2) +# - body_promoted contains only promoted paths (no full JSON duplication) +# - export_json_types is fed full JSON payloads so metadata paths/types exist +# even if a log only stores promoted fields in body_promoted +# - export_promoted_paths seeds signoz_metadata.distributed_json_promoted_paths +# ============================================================================ + + +def test_logs_json_body_new_qb_simple_searches( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[List[Logs]], None], + export_json_types: Callable[[List[Logs]], None], +) -> None: + """ + Setup: + Insert logs with JSON bodies using new QB columns (body_v2, body_promoted) + Export JSON type metadata + + Tests: + 1. Search by body.message Contains "value" + 2. Search by body.status = 200 (numeric) + 3. Search by body.active = true (boolean) + 4. Search by body.level = "error" with CONTAINS + 5. Search by body.code > 100 (comparison) + 6. Search with body_promoted column + """ + now = datetime.now(tz=timezone.utc) + + # Log with simple JSON body + log1_body = json.dumps( + { + "message": "User logged in successfully", + "status": 200, + "active": True, + "level": "info", + "code": 100, + } + ) + + log2_body = json.dumps( + { + "message": "User authentication failed", + "status": 401, + "active": False, + "level": "error", + "code": 401, + } + ) + + log3_body = json.dumps( + { + "message": "Database connection established", + "status": 200, + "active": True, + "level": "info", + "code": 200, + } + ) + + logs_list = [ + Logs( + timestamp=now - timedelta(seconds=3), + resources={"service.name": "auth-service"}, + attributes={}, + body_v2=log1_body, + body_promoted="", + severity_text="INFO", + ), + Logs( + timestamp=now - timedelta(seconds=2), + resources={"service.name": "auth-service"}, + attributes={}, + body_v2=log2_body, + body_promoted="", + severity_text="ERROR", + ), + Logs( + timestamp=now - timedelta(seconds=1), + resources={"service.name": "db-service"}, + attributes={}, + body_v2=log3_body, + body_promoted="", + severity_text="INFO", + ), + ] + + # Export JSON type metadata - auto-extract from logs (jsontypeexporter behavior) + export_json_types(logs_list) + + insert_logs(logs_list) + + token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD) + + cases = [ + { + "name": "simple_search.message_contains", + "requestType": "raw", + "expression": 'body.message CONTAINS "logged in"', + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 1 + and "logged in" + in json.loads(_get_rows(response)[0]["data"]["body"])["message"] + ), + }, + { + "name": "simple_search.status_200", + "requestType": "raw", + "expression": "body.status = 200", + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 2 + and all( + json.loads(row["data"]["body"])["status"] == 200 + for row in _get_rows(response) + ) + ), + }, + { + "name": "simple_search.active_true", + "requestType": "raw", + "expression": "body.active = true", + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 2 + and all( + json.loads(row["data"]["body"])["active"] is True + for row in _get_rows(response) + ) + ), + }, + { + "name": "simple_search.level_error", + "requestType": "raw", + "expression": 'body.level CONTAINS "error"', + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 1 + and json.loads(_get_rows(response)[0]["data"]["body"])["level"] == "error" + ), + }, + { + "name": "simple_search.code_gt_100", + "requestType": "raw", + "expression": "body.code > 100", + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 2 + and all( + json.loads(row["data"]["body"])["code"] > 100 + for row in _get_rows(response) + ) + ), + }, + ] + + for case in cases: + _run_query_case(signoz, token, now, case) + + # NOTE: Tests using body_promoted are temporarily disabled until backend behavior is stable. + # The following promoted-only search is intentionally commented out: + # + # # Test 6: Search with promoted column (body_promoted) + # # Insert a log where message is only in promoted JSON + # log4_body = json.dumps({}) + # log4_promoted = json.dumps({"message": "Promoted message"}) + # + # insert_logs( + # [ + # Logs( + # timestamp=now, + # resources={"service.name": "promoted-service"}, + # attributes={}, + # body_v2=log4_body, + # body_promoted=log4_promoted, + # severity_text="INFO", + # ), + # ] + # ) + # + # promoted_case = { + # "name": "simple_search.promoted_message", + # "requestType": "raw", + # "expression": 'body.message = "Promoted message"', + # "groupBy": None, + # "limit": 100, + # "aggregation": "count()", + # "stepInterval": None, + # "endMs": int((now + timedelta(seconds=5)).timestamp() * 1000), + # "validate": lambda response: ( + # len(_get_rows(response)) >= 1 + # and any( + # "Promoted message" in json.loads(row["data"]["body"])["message"] + # for row in _get_rows(response) + # ) + # ), + # } + # + # _run_query_case(signoz, token, now, promoted_case) + + +def test_logs_json_body_new_qb_nested_keys( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[List[Logs]], None], + export_json_types: Callable[[List[Logs]], None], +) -> None: + """ + Setup: + Insert logs with nested JSON bodies using new QB + + Tests: + 1. Search by body.user.name = "value" + 2. Search by body.request.secure = true (boolean) + 3. Search by body.response.latency = 123.45 (floating point) + 4. Search by body.response.status.code = 200 (deeply nested) + 5. Search with EXISTS operator + """ + now = datetime.now(tz=timezone.utc) + + log1_body = json.dumps( + { + "user": { + "name": "john_doe", + "id": 12345, + "email": "john@example.com", + }, + "request": { + "method": "GET", + "secure": True, + "headers": { + "content_type": "application/json", + }, + }, + "response": { + "status": { + "code": 200, + "message": "OK", + }, + "latency": 123.45, + }, + } + ) + + log2_body = json.dumps( + { + "user": { + "name": "jane_smith", + "id": 67890, + }, + "request": { + "method": "POST", + "secure": False, + }, + "response": { + "status": { + "code": 201, + }, + "latency": 456.78, + }, + } + ) + + logs_list = [ + Logs( + timestamp=now - timedelta(seconds=2), + resources={"service.name": "api-service"}, + attributes={}, + body_v2=log1_body, + body_promoted="", + severity_text="INFO", + ), + Logs( + timestamp=now - timedelta(seconds=1), + resources={"service.name": "api-service"}, + attributes={}, + body_v2=log2_body, + body_promoted="", + severity_text="INFO", + ), + ] + + export_json_types(logs_list) + insert_logs(logs_list) + + token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD) + + cases = [ + { + "name": "nested_keys.user_name", + "requestType": "raw", + "expression": 'body.user.name = "john_doe"', + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 1 + and json.loads(_get_rows(response)[0]["data"]["body"])["user"]["name"] + == "john_doe" + ), + }, + { + "name": "nested_keys.request_secure", + "requestType": "raw", + "expression": "body.request.secure = true", + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 1 + and json.loads(_get_rows(response)[0]["data"]["body"])["request"]["secure"] + is True + ), + }, + { + "name": "nested_keys.response_latency", + "requestType": "raw", + "expression": "body.response.latency = 123.45", + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 1 + and json.loads(_get_rows(response)[0]["data"]["body"])["response"][ + "latency" + ] + == 123.45 + ), + }, + { + "name": "nested_keys.response_status_code", + "requestType": "raw", + "expression": "body.response.status.code = 200", + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 1 + and json.loads(_get_rows(response)[0]["data"]["body"])["response"][ + "status" + ]["code"] + == 200 + ), + }, + { + "name": "nested_keys.user_email_exists", + "requestType": "raw", + "expression": "body.user.email EXISTS", + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 1 + and "email" + in json.loads(_get_rows(response)[0]["data"]["body"])["user"] + ), + }, + ] + + for case in cases: + _run_query_case(signoz, token, now, case) + + +def test_logs_json_body_new_qb_array_paths( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[List[Logs]], None], + export_json_types: Callable[[List[Logs]], None], +) -> None: + """ + Setup: + Insert logs with JSON bodies containing arrays + + Tests: + 1. Search by body.education[].name EXISTS (array path exists) + 2. Search by body.education[].name = "IIT" (array path equals) + 3. Search by body.education[].awards[].name = "Iron Award" (nested array) + 4. Search by body.education[].parameters CONTAINS 1.65 (array contains) + """ + now = datetime.now(tz=timezone.utc) + + log1_body = json.dumps( + { + "education": [ + { + "name": "IIT", + "year": 2020, + "awards": [ + {"name": "Iron Award", "type": "sports"}, + {"name": "Gold Award", "type": "academic"}, + ], + "parameters": [1.65, 2.5, 3.0], + }, + { + "name": "MIT", + "year": 2022, + "awards": [ + {"name": "Silver Award", "type": "research"}, + ], + "parameters": [4.0, 5.0], + }, + ] + } + ) + + log2_body = json.dumps( + { + "education": [ + { + "name": "Stanford", + "year": 2021, + "awards": [ + {"name": "Bronze Award", "type": "sports"}, + ], + "parameters": [1.65, 6.0], + } + ] + } + ) + + logs_list = [ + Logs( + timestamp=now - timedelta(seconds=2), + resources={"service.name": "app-service"}, + attributes={}, + body_v2=log1_body, + body_promoted="", + severity_text="INFO", + ), + Logs( + timestamp=now - timedelta(seconds=1), + resources={"service.name": "app-service"}, + attributes={}, + body_v2=log2_body, + body_promoted="", + severity_text="INFO", + ), + ] + + export_json_types(logs_list) + insert_logs(logs_list) + + token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD) + cases = [ + { + "name": "array_paths.exists", + "requestType": "raw", + "expression": "body.education[].name EXISTS", + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: len(_get_rows(response)) == 2, + }, + { + "name": "array_paths.equals_iit", + "requestType": "raw", + "expression": 'body.education[].name = "IIT"', + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 1 + and any( + "IIT" in edu["name"] + for edu in json.loads(_get_rows(response)[0]["data"]["body"])[ + "education" + ] + ) + ), + }, + { + "name": "array_paths.nested_awards", + "requestType": "raw", + "expression": 'body.education[].awards[].name = "Iron Award"', + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 1 + and any( + any(award["name"] == "Iron Award" for award in edu.get("awards", [])) + for edu in json.loads(_get_rows(response)[0]["data"]["body"])[ + "education" + ] + ) + ), + }, + { + "name": "array_paths.parameters_contains", + "requestType": "raw", + "expression": "body.education[].parameters CONTAINS 1.65", + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 2 + and all( + any( + 1.65 in edu.get("parameters", []) + for edu in json.loads(row["data"]["body"])["education"] + ) + for row in _get_rows(response) + ) + ), + }, + ] + + for case in cases: + _run_query_case(signoz, token, now, case) + + +@pytest.mark.skip(reason="Promotion is temporarily not supported; uncomment when promotion is back") +def test_logs_json_body_new_qb_promoted_time_windows( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[List[Logs]], None], + export_json_types: Callable[[List[Logs]], None], + export_promoted_paths: Callable[[List[str]], None], +) -> None: + """ + Setup: + Insert logs before/after a promotion timestamp so some fields are read + from body_v2 (before) and body_promoted (after). + + Tests: + 1. startMs/endMs before promotion timestamp + 2. startMs before and endMs after promotion timestamp + 3. startMs/endMs after promotion timestamp + """ + now = datetime.now(tz=timezone.utc) + promotion_ts = now - timedelta(minutes=20) + + log1_full = { + "id": "pre_15m", + "message": "before alpha", + "user": {"name": "alice", "age": 25}, + "tags": ["prod", "alpha"], + "status": 200, + "education": [ + { + "name": "IIT", + "year": 2020, + "awards": [{"name": "Iron Award", "type": "sports"}], + "parameters": [1.65, 2.5], + } + ], + } + log2_full = { + "id": "pre_5m", + "message": "before beta", + "user": {"name": "bob", "age": 30}, + "tags": ["stage", "beta"], + "status": 201, + "education": [ + { + "name": "MIT", + "year": 2022, + "awards": [{"name": "Silver Award", "type": "research"}], + "parameters": [2.75, 3.5], + } + ], + } + log3_full = { + "id": "post_2m", + "message": "after gamma", + "user": {"name": "carol", "age": 35}, + "tags": ["prod", "gamma"], + "status": 202, + "education": [ + { + "name": "Stanford", + "year": 2023, + "awards": [{"name": "Gold Award", "type": "research"}], + "parameters": [3.5], + } + ], + } + log4_full = { + "id": "post_10m", + "message": "after delta", + "user": {"name": "dan", "age": 40}, + "tags": ["stage", "delta"], + "status": 203, + "education": [ + { + "name": "Harvard", + "year": 2024, + "awards": [{"name": "Silver Award", "type": "research"}], + "parameters": [2.75], + } + ], + } + + promoted_paths = [ + "message", + "user.name", + "user.age", + "tags", + "education", + ] + + def _promoted_part(payload: Dict[str, Any]) -> Dict[str, Any]: + return { + "message": payload["message"], + "user": payload["user"], + "tags": payload["tags"], + "education": payload["education"], + } + + logs_list = [ + Logs( + timestamp=promotion_ts - timedelta(minutes=15), + resources={"service.name": "app-service"}, + attributes={}, + body_v2=json.dumps(log1_full), + body_promoted="", + severity_text="INFO", + ), + Logs( + timestamp=promotion_ts - timedelta(minutes=5), + resources={"service.name": "app-service"}, + attributes={}, + body_v2=json.dumps(log2_full), + body_promoted="", + severity_text="INFO", + ), + Logs( + timestamp=promotion_ts + timedelta(minutes=2), + resources={"service.name": "app-service"}, + attributes={}, + body_v2=json.dumps(log3_full), + body_promoted=json.dumps(_promoted_part(log3_full)), + severity_text="INFO", + ), + Logs( + timestamp=promotion_ts + timedelta(minutes=10), + resources={"service.name": "app-service"}, + attributes={}, + body_v2=json.dumps(log4_full), + body_promoted=json.dumps(_promoted_part(log4_full)), + severity_text="INFO", + ), + ] + + export_json_types( + [ + json.dumps(log1_full), + json.dumps(log2_full), + json.dumps(log3_full), + json.dumps(log4_full), + ] + ) + export_promoted_paths(promoted_paths) + insert_logs(logs_list) + + token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD) + + def _row_ids( + response: requests.Response, context: str, expected_ids: set[str] + ) -> set[str]: + results = _get_results(response) + rows = results[0].get("rows") + if rows is None: + if expected_ids: + raise AssertionError( + f"{context}: rows is null, response={response.json()}" + ) + return set() + return {json.loads(row["data"]["body"])["id"] for row in rows} + + def _assert_expected_ids( + response: requests.Response, context: str, expected_ids: set[str] + ) -> None: + actual_ids = _row_ids(response, context, expected_ids) + if actual_ids != expected_ids: + raise AssertionError( + f"{context}: expected_ids={expected_ids}, " + f"actual_ids={actual_ids}, response={response.json()}" + ) + + windows = [ + { + "name": "before_promotion", + "startMs": int( + (promotion_ts - timedelta(minutes=20)).timestamp() * 1000 + ), + "endMs": int((promotion_ts - timedelta(minutes=1)).timestamp() * 1000), + }, + { + "name": "spanning_promotion", + "startMs": int( + (promotion_ts - timedelta(minutes=20)).timestamp() * 1000 + ), + "endMs": int((promotion_ts + timedelta(minutes=20)).timestamp() * 1000), + }, + { + "name": "after_promotion", + "startMs": int((promotion_ts + timedelta(minutes=1)).timestamp() * 1000), + "endMs": int((promotion_ts + timedelta(minutes=20)).timestamp() * 1000), + }, + ] + + cases = [ + { + "name": "promoted_time_window.message_contains_before", + "expression": 'body.message CONTAINS "before"', + "expected_ids": { + "before_promotion": {"pre_15m", "pre_5m"}, + "spanning_promotion": {"pre_15m", "pre_5m"}, + "after_promotion": set(), + }, + }, + { + "name": "promoted_time_window.user_name_equals_bob", + "expression": 'body.user.name = "bob"', + "expected_ids": { + "before_promotion": {"pre_5m"}, + "spanning_promotion": {"pre_5m"}, + "after_promotion": set(), + }, + }, + { + "name": "promoted_time_window.awards_equals_silver", + "expression": 'body.education[].awards[].name = "Silver Award"', + "expected_ids": { + "before_promotion": {"pre_5m"}, + "spanning_promotion": {"pre_5m", "post_10m"}, + "after_promotion": {"post_10m"}, + }, + }, + { + "name": "promoted_time_window.parameters_contains", + "expression": "body.education[].parameters CONTAINS 2.75", + "expected_ids": { + "before_promotion": {"pre_5m"}, + "spanning_promotion": {"pre_5m", "post_10m"}, + "after_promotion": {"post_10m"}, + }, + }, + { + "name": "promoted_time_window.tags_has_prod", + "expression": 'has(body.tags, "prod")', + "expected_ids": { + "before_promotion": {"pre_15m"}, + "spanning_promotion": {"pre_15m", "post_2m"}, + "after_promotion": {"post_2m"}, + }, + }, + { + "name": "promoted_time_window.user_age_gt_30", + "expression": "body.user.age > 30", + "expected_ids": { + "before_promotion": set(), + "spanning_promotion": {"post_2m", "post_10m"}, + "after_promotion": {"post_2m", "post_10m"}, + }, + }, + ] + + for case in cases: + for window in windows: + expected = case["expected_ids"][window["name"]] + case_name = f"{case['name']}.{window['name']}" + run_case = { + "name": case_name, + "requestType": "raw", + "expression": case["expression"], + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "startMs": window["startMs"], + "endMs": window["endMs"], + "validate": lambda response, expected_ids=expected, name=case_name: ( + _assert_expected_ids(response, name, expected_ids) is None + ), + } + _run_query_case(signoz, token, now, run_case) + + +def test_logs_json_body_new_qb_groupby_timeseries( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[List[Logs]], None], + export_json_types: Callable[[List[Logs]], None], +) -> None: + """ + Setup: + Insert logs with JSON bodies for GroupBy testing + + Tests: + 1. Time series query with GroupBy on body.user.age + 2. Time series query with GroupBy on body.user.name + 3. Time series query with multiple GroupBy fields + """ + now = datetime.now(tz=timezone.utc) + + logs_data = [ + {"user": {"name": "alice", "age": 25}, "status": 200}, + {"user": {"name": "bob", "age": 30}, "status": 200}, + {"user": {"name": "alice", "age": 25}, "status": 201}, + {"user": {"name": "charlie", "age": 35}, "status": 200}, + {"user": {"name": "alice", "age": 25}, "status": 200}, + ] + + logs_list = [ + Logs( + timestamp=now - timedelta(seconds=5 - i), + resources={"service.name": "api-service"}, + attributes={}, + body_v2=json.dumps(log_data), + body_promoted="", + severity_text="INFO", + ) + for i, log_data in enumerate(logs_data) + ] + + export_json_types(logs_list) + insert_logs(logs_list) + + token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD) + cases = [ + { + "name": "groupby_timeseries.age", + "requestType": "time_series", + "expression": None, + "groupBy": [ + { + "name": "body.user.age", + "fieldDataType": "int64", + } + ], + "limit": 100, + "aggregation": "count()", + "stepInterval": 60, + "validate": lambda response: ( + { + _labels_to_map(s.get("labels")).get("user.age") + for s in _get_results(response)[0]["aggregations"][0]["series"] + if _labels_to_map(s.get("labels")).get("user.age") is not None + } + in ({"25", "30", "35"}, {25, 30, 35}) + ), + }, + { + "name": "groupby_timeseries.name", + "requestType": "time_series", + "expression": None, + "groupBy": [ + { + "name": "body.user.name", + "fieldDataType": "string", + } + ], + "limit": 100, + "aggregation": "count()", + "stepInterval": 60, + "validate": lambda response: {"alice", "bob", "charlie"}.issubset( + { + _labels_to_map(s.get("labels")).get("user.name") + for s in _get_results(response)[0]["aggregations"][0]["series"] + if _labels_to_map(s.get("labels")).get("user.name") is not None + } + ), + }, + { + "name": "groupby_timeseries.multi", + "requestType": "time_series", + "expression": None, + "groupBy": [ + { + "name": "body.user.name", + "fieldDataType": "string", + }, + { + "name": "body.user.age", + "fieldDataType": "int64", + }, + ], + "limit": 100, + "aggregation": "count()", + "stepInterval": 60, + "validate": lambda response: ( + { + ( + _labels_to_map(s.get("labels")).get("user.name"), + _labels_to_map(s.get("labels")).get("user.age"), + ) + for s in _get_results(response)[0]["aggregations"][0]["series"] + if _labels_to_map(s.get("labels")).get("user.name") is not None + and _labels_to_map(s.get("labels")).get("user.age") is not None + }.issuperset( + {("alice", "25"), ("bob", "30"), ("charlie", "35")} + ) + or { + ( + _labels_to_map(s.get("labels")).get("user.name"), + _labels_to_map(s.get("labels")).get("user.age"), + ) + for s in _get_results(response)[0]["aggregations"][0]["series"] + if _labels_to_map(s.get("labels")).get("user.name") is not None + and _labels_to_map(s.get("labels")).get("user.age") is not None + }.issuperset({("alice", 25), ("bob", 30), ("charlie", 35)}) + ), + }, + ] + + for case in cases: + _run_query_case(signoz, token, now, case) + + + +@pytest.mark.skip(reason="Promotion is temporarily not supported; uncomment when promotion is back") +def test_logs_json_body_new_qb_groupby_timeseries_promoted( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[List[Logs]], None], + export_json_types: Callable[[List[Logs]], None], + export_promoted_paths: Callable[[List[str]], None], +) -> None: + """ + Setup: + Insert logs before/after a promotion timestamp so group-by values are + present in body_v2 (before) and body_promoted (after). + + Tests: + 1. startMs/endMs before promotion timestamp + 2. startMs before and endMs after promotion timestamp + 3. startMs/endMs after promotion timestamp + """ + now = datetime.now(tz=timezone.utc) + promotion_ts = now - timedelta(minutes=20) + + log1_full = {"id": "pre_15m", "user": {"name": "alice", "age": 25}, "status": 200} + log2_full = {"id": "pre_5m", "user": {"name": "bob", "age": 30}, "status": 201} + log3_full = {"id": "post_2m", "user": {"name": "carol", "age": 35}, "status": 202} + log4_full = {"id": "post_10m", "user": {"name": "dan", "age": 40}, "status": 203} + + promoted_paths = [ + "user.name", + "user.age", + ] + + def _promoted_part(payload: Dict[str, Any]) -> Dict[str, Any]: + return {"user": payload["user"]} + + logs_list = [ + Logs( + timestamp=promotion_ts - timedelta(minutes=15), + resources={"service.name": "api-service"}, + attributes={}, + body_v2=json.dumps(log1_full), + body_promoted="", + severity_text="INFO", + ), + Logs( + timestamp=promotion_ts - timedelta(minutes=5), + resources={"service.name": "api-service"}, + attributes={}, + body_v2=json.dumps(log2_full), + body_promoted="", + severity_text="INFO", + ), + Logs( + timestamp=promotion_ts + timedelta(minutes=2), + resources={"service.name": "api-service"}, + attributes={}, + body_v2=json.dumps(log3_full), + body_promoted=json.dumps(_promoted_part(log3_full)), + severity_text="INFO", + ), + Logs( + timestamp=promotion_ts + timedelta(minutes=10), + resources={"service.name": "api-service"}, + attributes={}, + body_v2=json.dumps(log4_full), + body_promoted=json.dumps(_promoted_part(log4_full)), + severity_text="INFO", + ), + ] + + export_json_types( + [ + json.dumps(log1_full), + json.dumps(log2_full), + json.dumps(log3_full), + json.dumps(log4_full), + ] + ) + export_promoted_paths(promoted_paths) + insert_logs(logs_list) + + token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD) + + def _label_values(response: requests.Response, key: str) -> set[Any]: + return { + _labels_to_map(s.get("labels")).get(key) + for s in _get_results(response)[0]["aggregations"][0]["series"] + if _labels_to_map(s.get("labels")).get(key) is not None + } + + def _label_pairs(response: requests.Response) -> set[tuple[Any, Any]]: + return { + ( + _labels_to_map(s.get("labels")).get("user.name"), + _labels_to_map(s.get("labels")).get("user.age"), + ) + for s in _get_results(response)[0]["aggregations"][0]["series"] + if _labels_to_map(s.get("labels")).get("user.name") is not None + and _labels_to_map(s.get("labels")).get("user.age") is not None + } + + windows = [ + { + "name": "before_promotion", + "startMs": int( + (promotion_ts - timedelta(minutes=20)).timestamp() * 1000 + ), + "endMs": int((promotion_ts - timedelta(minutes=1)).timestamp() * 1000), + }, + { + "name": "spanning_promotion", + "startMs": int( + (promotion_ts - timedelta(minutes=20)).timestamp() * 1000 + ), + "endMs": int((promotion_ts + timedelta(minutes=20)).timestamp() * 1000), + }, + { + "name": "after_promotion", + "startMs": int((promotion_ts + timedelta(minutes=1)).timestamp() * 1000), + "endMs": int((promotion_ts + timedelta(minutes=20)).timestamp() * 1000), + }, + ] + + cases = [ + { + "name": "groupby_timeseries.promoted_age", + "groupBy": [{"name": "body.user.age", "fieldDataType": "int64"}], + "expected": { + "before_promotion": {25, 30}, + "spanning_promotion": {25, 30, 35, 40}, + "after_promotion": {35, 40}, + }, + }, + { + "name": "groupby_timeseries.promoted_name", + "groupBy": [{"name": "body.user.name", "fieldDataType": "string"}], + "expected": { + "before_promotion": {"alice", "bob"}, + "spanning_promotion": {"alice", "bob", "carol", "dan"}, + "after_promotion": {"carol", "dan"}, + }, + }, + { + "name": "groupby_timeseries.promoted_multi", + "groupBy": [ + { + "name": "body.user.name", + "fieldDataType": "string", + }, + { + "name": "body.user.age", + "fieldDataType": "int64", + }, + ], + "expected": { + "before_promotion": {("alice", 25), ("bob", 30)}, + "spanning_promotion": { + ("alice", 25), + ("bob", 30), + ("carol", 35), + ("dan", 40), + }, + "after_promotion": {("carol", 35), ("dan", 40)}, + }, + }, + ] + + for case in cases: + for window in windows: + expected = case["expected"][window["name"]] + run_case = { + "name": f"{case['name']}.{window['name']}", + "requestType": "time_series", + "expression": None, + "groupBy": case["groupBy"], + "limit": 100, + "aggregation": "count()", + "stepInterval": 60, + "startMs": window["startMs"], + "endMs": window["endMs"], + "validate": None, + } + if case["name"].endswith("promoted_age"): + expected_str = {str(v) for v in expected} + run_case["validate"] = lambda response, exp=expected, exp_str=expected_str: ( + _label_values(response, "user.age") == exp + or _label_values(response, "user.age") == exp_str + ) + elif case["name"].endswith("promoted_name"): + run_case["validate"] = lambda response, exp=expected: ( + _label_values(response, "user.name") == exp + ) + else: + expected_str = {(name, str(age)) for name, age in expected} + run_case["validate"] = lambda response, exp=expected, exp_str=expected_str: ( + _label_pairs(response) == exp or _label_pairs(response) == exp_str + ) + _run_query_case(signoz, token, now, run_case) + + +def test_logs_json_body_array_membership( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[List[Logs]], None], + export_json_types: Callable[[List[Logs]], None], +) -> None: + """ + Setup: + Insert logs with JSON bodies containing arrays + + Tests: + 1. Search by has(body.tags, "value") - string array + 2. Search by has(body.ids, 123) - numeric array + 3. Search by has(body.flags, true) - boolean array + """ + now = datetime.now(tz=timezone.utc) + + log1_body = json.dumps( + { + "tags": ["production", "api", "critical"], + "ids": [100, 200, 300], + "flags": [True, False, True], + "users": [ + {"name": "alice", "role": "admin"}, + {"name": "bob", "role": "user"}, + ], + } + ) + + log2_body = json.dumps( + { + "tags": ["staging", "api", "test"], + "ids": [200, 400, 500], + "flags": [False, False, True], + "users": [ + {"name": "charlie", "role": "user"}, + {"name": "david", "role": "admin"}, + ], + } + ) + + log3_body = json.dumps( + { + "tags": ["production", "web", "important"], + "ids": [100, 600, 700], + "flags": [True, True, False], + "users": [ + {"name": "alice", "role": "admin"}, + {"name": "eve", "role": "user"}, + ], + } + ) + + logs_list = [ + Logs( + timestamp=now - timedelta(seconds=3), + resources={"service.name": "app-service"}, + attributes={}, + body_v2=log1_body, + body_promoted="", + severity_text="INFO", + ), + Logs( + timestamp=now - timedelta(seconds=2), + resources={"service.name": "app-service"}, + attributes={}, + body_v2=log2_body, + body_promoted="", + severity_text="INFO", + ), + Logs( + timestamp=now - timedelta(seconds=1), + resources={"service.name": "app-service"}, + attributes={}, + body_v2=log3_body, + body_promoted="", + severity_text="INFO", + ), + ] + + export_json_types(logs_list) + insert_logs(logs_list) + + token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD) + + cases = [ + { + "name": "array_membership.tags_contains_production", + "requestType": "raw", + "expression": 'has(body.tags, "production")', + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 2 + and all( + "production" in json.loads(row["data"]["body"])["tags"] + for row in _get_rows(response) + ) + ), + }, + { + "name": "array_membership.ids_contains_200", + "requestType": "raw", + "expression": "has(body.ids, 200)", + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 2 + and all( + 200 in json.loads(row["data"]["body"])["ids"] + for row in _get_rows(response) + ) + ), + }, + { + "name": "array_membership.flags_contains_true", + "requestType": "raw", + "expression": "has(body.flags, true)", + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 3 + and all( + True in json.loads(row["data"]["body"])["flags"] + for row in _get_rows(response) + ) + ), + }, + ] + + for case in cases: + _run_query_case(signoz, token, now, case) + + +def test_logs_json_body_new_qb_message_searches( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[List[Logs]], None], + export_json_types: Callable[[List[Logs]], None], +) -> None: + """ + Verifies that full text search (body), body.message, and message key all + resolve to body_v2.message when BodyJSONQueryEnabled=true. + + When a log has a plain-text body the normalize operator wraps it as + {"message": }, so all four search variants must be able to find it. + + Setup: + - log1: plain-text body "Payment processed successfully" + -> body_v2 = {"message": "Payment processed successfully"} + - log2: JSON body {"message": "Payment failed with error", "code": 500} + - log3: control JSON body {"message": "Database connection established"} + (does NOT contain "Payment") + + Tests: + 1. body CONTAINS "Payment" - full text search / body logical key + 2. body.message CONTAINS "Payment" - explicit body.message path + 3. message CONTAINS "Payment" - bare message key + 4. body = "Payment processed ..." - FTS exact match on text-body log + 5. body.message = "Payment proc..." - body.message exact match + 6. message = "Payment proc..." - message key exact match + 7. body NOT CONTAINS "Payment" - negative: only control log returned + 8. body.message NOT CONTAINS "Pay" - same negative via body.message + 9. message NOT CONTAINS "Payment" - same negative via message key + """ + now = datetime.now(tz=timezone.utc) + + # Plain-text body: Logs.__init__ sets body_v2 = {"message": } + # because "Payment processed successfully" is not valid JSON. + text_log = Logs( + timestamp=now - timedelta(seconds=3), + resources={"service.name": "payment-service"}, + body="Payment processed successfully", + severity_text="INFO", + ) + + # JSON body with an explicit message field + json_log = Logs( + timestamp=now - timedelta(seconds=2), + resources={"service.name": "payment-service"}, + body_v2=json.dumps({"message": "Payment failed with error", "code": 500}), + body_promoted="", + severity_text="ERROR", + ) + + # Control log - message does NOT contain "Payment" + control_log = Logs( + timestamp=now - timedelta(seconds=1), + resources={"service.name": "db-service"}, + body_v2=json.dumps({"message": "Database connection established", "code": 200}), + body_promoted="", + severity_text="INFO", + ) + + logs_list = [text_log, json_log, control_log] + export_json_types(logs_list) + insert_logs(logs_list) + + token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD) + + def _body_messages(response: requests.Response) -> List[str]: + """Return the 'message' values from every row's body JSON.""" + return [ + json.loads(row["data"]["body"]).get("message", "") + for row in _get_rows(response) + ] + + payment_messages = { + "Payment processed successfully", + "Payment failed with error", + } + + cases = [ + # 1. Full text search / body logical key: body resolves to body_v2.message + # when BodyJSONQueryEnabled=true. + { + "name": "msg_search.fts_body_contains", + "requestType": "raw", + "expression": 'body CONTAINS "Payment"', + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 2 + and set(_body_messages(response)) == payment_messages + ), + }, + # 2. body.message CONTAINS + { + "name": "msg_search.body_message_contains", + "requestType": "raw", + "expression": 'body.message CONTAINS "Payment"', + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 2 + and set(_body_messages(response)) == payment_messages + ), + }, + # 3. message CONTAINS (bare key - also maps to body_v2.message) + { + "name": "msg_search.message_key_contains", + "requestType": "raw", + "expression": 'message CONTAINS "Payment"', + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 2 + and set(_body_messages(response)) == payment_messages + ), + }, + # 4. FTS exact match - text-body log (normalized to {"message": }) + { + "name": "msg_search.fts_body_exact", + "requestType": "raw", + "expression": '"Payment"', + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 2 + and all("Payment" in msg for msg in _body_messages(response)) + ), + }, + # 5. FTS exact match - text-body log (normalized to {"message": }) + { + "name": "msg_search.fts_body_exact", + "requestType": "raw", + "expression": 'Payment', + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 2 + and all("Payment" in msg for msg in _body_messages(response)) + ), + }, + # 5. body.message exact match + { + "name": "msg_search.body_message_exact", + "requestType": "raw", + "expression": 'body.message = "Payment processed successfully"', + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 1 + and _body_messages(response)[0] == "Payment processed successfully" + ), + }, + # 6. message key exact match + { + "name": "msg_search.message_key_exact", + "requestType": "raw", + "expression": 'message = "Payment processed successfully"', + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 1 + and _body_messages(response)[0] == "Payment processed successfully" + ), + }, + # 7. Negative: body NOT CONTAINS -> only control log + { + "name": "msg_search.fts_body_not_contains", + "requestType": "raw", + "expression": 'body NOT CONTAINS "Payment"', + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 1 + and _body_messages(response)[0] == "Database connection established" + ), + }, + # 8. Negative: body.message NOT CONTAINS + { + "name": "msg_search.body_message_not_contains", + "requestType": "raw", + "expression": 'body.message NOT CONTAINS "Payment"', + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 1 + and _body_messages(response)[0] == "Database connection established" + ), + }, + # 9. Negative: message NOT CONTAINS + { + "name": "msg_search.message_key_not_contains", + "requestType": "raw", + "expression": 'message NOT CONTAINS "Payment"', + "groupBy": None, + "limit": 100, + "aggregation": "count()", + "stepInterval": None, + "validate": lambda response: ( + len(_get_rows(response)) == 1 + and _body_messages(response)[0] == "Database connection established" + ), + }, + ] + + for case in cases: + _run_query_case(signoz, token, now, case)