dbqctl v0.5.0 - updated core, new checks format, checks output improvements

sholokhov · sholokhov · commit ca377513cdb5 · 2025-08-25T16:02:08.000+02:00
diff --git a/checks.yaml b/checks.yaml
@@ -1,72 +1,155 @@
 version: "1"
-validations:
+rules:
   # https://clickhouse.com/docs/getting-started/example-datasets/nyc-taxi
   - dataset: ch@[nyc_taxi.trips_small]
-    # common pre-filter for every check, e.g. to run daily check only for yesterday
     where: "pickup_datetime > '2014-01-01'"
     checks:
-      - id: row_count > 0
-        description: "data should be present" # optional
-        on_error: alert # optional (ignore, alert), default "alert"
-
-      - id: row_count between 100 and 30000
-        description: "expected rows count"
-        on_error: ignore
-
-      - id: null_count(pickup_ntaname) == 0
-        description: "no nulls are allowed in column: pickup_ntaname"
-
-      - id: min(pickup_datetime) < now() - interval 3 day
-        description: "min(pickup_datetime) should not be earlier than 3 days"
-
-      - id: stddevPop(trip_distance) < 100_000
-        description: "check stddev value"
-
-      - id: sum(fare_amount) <= 10_000_000
-        description: "sum of value"
-
-      - id: countIf(trip_id == 1) == 1
-        description: "check trip id"
-
-      - id: raw_query
-        description: "raw query quality test"
-        query: |
-          select countIf(trip_distance == 0) > 0 from {{table}} where 1=1
+      # schema-level checks
+      - schema_check:
+          expect_columns_ordered:
+            columns_order: [trip_id, pickup_datetime, dropoff_datetime, pickup_longitude, pickup_latitude,
+                            dropoff_longitude, dropoff_latitude, passenger_count, trip_distance, fare_amount, extra,
+                            tip_amount, tolls_amount, total_amount, payment_type, pickup_ntaname, dropoff_ntaname]
+        desc: "Ensure table columns are in the expected order"
+        on_fail: error
+
+      - schema_check:
+          expect_columns:
+            columns: [trip_id, fare_amount]
+        desc: "Ensure required columns exist"
+        on_fail: error
+
+      - schema_check:
+          columns_not_present:
+            columns: [credit_card_number, credit_card_cvv]
+            pattern: "pii_*"
+        desc: "Ensure PII and credit card info is not present in the table"
+        on_fail: error
+
+      # table-level checks
+      - row_count between 10000 and 3500000:
+          desc: "Dataset should contain a reasonable number of trips"
+          on_fail: error
+
+      # column existence and nullability
+      - not_null(trip_id):
+          desc: "Trip ID is mandatory"
+      - not_null(pickup_datetime)
+      - not_null(dropoff_datetime)
+
+      # data freshness
+      - freshness(pickup_datetime) < 7d:
+          desc: "Data should be no older than 7 days"
+          on_fail: warn
+
+      # uniqueness constraints
+      - uniqueness(trip_id):
+          desc: "Trip IDs must be unique"
+          on_fail: error
+
+      # numeric validations
+      - min(trip_distance) >= 0:
+          desc: "Trip distance cannot be negative"
+      - max(trip_distance) < 1000:
+          desc: "Maximum trip distance seems unrealistic"
+          on_fail: warn
+      - avg(trip_distance) between 1.0 and 20.0:
+          desc: "Average trip distance should be reasonable"
+      - stddev(trip_distance) < 100:
+          desc: "Trip distance variation should be within normal range"
+
+      # fare validations
+      - min(fare_amount) > 0:
+          desc: "Fare amount should be positive"
+      - max(fare_amount) < 1000:
+          desc: "Maximum fare seems too high"
+      - sum(fare_amount) between 10000 and 10000000:
+          desc: "Total fare amount should be within expected range"
+
+      # custom validation with raw query
+      - raw_query:
+          desc: "Check for trips with zero distance but positive fare"
+          query: "select count() from {{dataset}} where trip_distance = 0 and fare_amount > 0"
+          on_fail: warn
 
   # https://wiki.postgresql.org/wiki/Sample_Databases
   - dataset: pg@[public.land_registry_price_paid_uk]
-    # exclude January for example
-    where: "transfer_date >= '2025-02-01 00:00:00.000000'"
+    where: "transfer_date >= '2025-01-01'"
     checks:
-      - id: row_count > 0
-        description: "data should be present"
-        on_error: alert
-
-      - id: row_count between 200000 and 300000
-        description: "expected rows count"
-        on_error: ignore
-
-      - id: min(price) > 0
-        description: "min(price) should be greater than zero"
-
-      - id: max(price) < 100000000
-        description: "max(price) should be less than 100_000_000"
-
-      - id: stddev_pop(price) < 500000
-        description: "price stddev"
-
-  # https://github.com/datacharmer/test_db
+      # schema validation
+      - schema_check:
+          expect_columns_ordered:
+            columns_order: [transaction, price, transfer_date, postcode, property_type, newly_built, duration, paon, saon,
+                      street, locality, city, district, county, ppd_category_type, record_status]
+        desc: "Validate expected column order for data consistency"
+        on_fail: warn
+
+      - schema_check:
+          expect_columns:
+            columns: [transaction, price, property_type]
+        desc: "Ensure critical columns exist"
+        on_fail: error
+
+      - row_count() between 100 and 250000:
+          desc: "Recent property transactions should be within expected volume"
+
+      # price checks
+      - not_null(price):
+          desc: "Property price is mandatory"
+      - min(price) >= 100:
+          desc: "Minimum price should be realistic"
+      - max(price) < 50000000:
+          desc: "Maximum price should be within UK market range"
+      - avg(price) between 200000 and 800000:
+          desc: "Average property price should align with market data"
+      - stddev(price) < 500000:
+          desc: "Price standard deviation should indicate reasonable market variation"
+
+      # property type validations
+      - not_null(property_type)
+      - uniqueness(transaction):
+          desc: "Each transaction must have a unique identifier"
+          on_fail: error
+
+      # date validations
+      - freshness(transfer_date) < 1d:
+          desc: "Transfer date should be very recent"
+          on_fail: warn
+
+
+#  # https://github.com/datacharmer/test_db
   - dataset: mysql@[employees.salaries]
     checks:
-      - id: row_count > 0
-        description: "data should be present"
-        on_error: alert
-
-      - id: min(salary) > 0
-        description: "min(salary) should be greater than zero"
-
-      - id: max(salary) < 150000
-        description: "max(salary) should be less than 150000"
-
-      - id: stddev_pop(salary) < 50000
-        description: "price stddev"
+      # schema validation
+      - schema_check:
+          expect_columns_ordered:
+            columns_order: [order_id, customer_id, order_status, total_amount, item_count, created_at, shipped_date]
+        desc: "Ensure order table maintains expected column structure"
+        on_fail: error
+
+      - schema_check:
+          expect_columns:
+            columns: [order_id, customer_id, order_status, total_amount]
+        desc: "Ensure essential order columns exist"
+        on_fail: error
+
+      # order volume validation
+      - row_count between 100 and 10000:
+          desc: "Monthly order volume should be within business expectations"
+          on_fail: warn
+
+      # customer data integrity
+      - not_null(customer_id):
+          desc: "Every order must have a customer ID"
+          on_fail: error
+      - not_null(order_status)
+
+      # order value validations
+      - min(total_amount) > 0:
+          desc: "Order total must be positive"
+          on_fail: error
+      - max(total_amount) < 10000:
+          desc: "Unusually high order amount detected"
+          on_fail: warn
+      - avg(total_amount) between 25.0 and 200.0:
+          desc: "Average order value should align with business metrics"
diff --git a/cmd/check.go b/cmd/check.go
@@ -27,9 +27,10 @@ import (
 )
 
 type FailedCheckDetails struct {
-	ID      string
-	Dataset string
-	Err     error
+	Expression string
+	Dataset    string
+	ActualVal  string
+	Err        string
 }
 
 func NewCheckCommand(app internal.DbqCliApp) *cobra.Command {
@@ -56,7 +57,7 @@ By automating these checks, you can proactively identify and address data qualit
 			passedCount := 0
 			var failedChecks []FailedCheckDetails
 
-			for _, rule := range checksCfg.Validations {
+			for _, rule := range checksCfg.Rules {
 				dataSourceId, datasets, err := parseDatasetString(rule.Dataset)
 				if err != nil {
 					return fmt.Errorf("error while parsing dataset property: %w", err)
@@ -70,20 +71,26 @@ By automating these checks, you can proactively identify and address data qualit
 				for _, dataset := range datasets {
 					fmt.Printf("running %d quality checks for '%s'\n", len(rule.Checks), dataset)
 					for _, check := range rule.Checks {
-						pass, _, err := app.RunCheck(&check, dataSource, dataset, rule.Where)
+						validationResult := app.RunCheck(&check, dataSource, dataset, rule.Where)
 
-						fmt.Printf("  check %s ... %s\n", check.Description, getCheckResultLabel(pass))
-						if pass {
+						checkLabel := check.Expression
+						if check.Description != "" {
+							checkLabel = check.Description
+						}
+
+						fmt.Printf("  %s: %s \n", getCheckResultLabel(validationResult.Pass), checkLabel)
+						if validationResult.Pass {
 							passedCount += 1
 						} else {
 							failedChecks = append(failedChecks, FailedCheckDetails{
-								ID:      check.ID,
-								Dataset: dataset,
-								Err:     err,
+								Expression: check.Expression,
+								Dataset:    dataset,
+								ActualVal:  validationResult.QueryResultValue,
+								Err:        validationResult.Error,
 							})
 						}
 
-						if !pass && strGetOrDefault(string(check.OnError), string(dbqcore.OnErrorActionAlert)) == string(dbqcore.OnErrorActionAlert) {
+						if !validationResult.Pass && strGetOrDefault(string(check.OnFail), string(dbqcore.OnFailActionError)) == string(dbqcore.OnFailActionError) {
 							exitCode = 1
 						}
 					}
@@ -93,8 +100,18 @@ By automating these checks, you can proactively identify and address data qualit
 			if len(failedChecks) != 0 {
 				for _, result := range failedChecks {
 					fmt.Println()
-					fmt.Printf("--- %s ---\n", result.ID)
-					fmt.Printf("error: %s\n", result.Err)
+					fmt.Printf("--- %s : %s ---\n", result.Dataset, result.Expression)
+					if result.ActualVal != "" {
+						units := ""
+						if strings.HasPrefix(result.Expression, "freshness") {
+							units = " (diff in seconds)"
+						}
+
+						fmt.Printf("actual value: %s%s\n", result.ActualVal, units)
+					}
+					if result.Err != "" {
+						fmt.Printf("error: %s\n", result.Err)
+					}
 				}
 			}
 
diff --git a/cmd/version.go b/cmd/version.go
@@ -22,7 +22,7 @@ import (
 )
 
 const (
-	DbqCtlVersion = "v0.4.0"
+	DbqCtlVersion = "v0.5.0"
 )
 
 func NewVersionCommand() *cobra.Command {
diff --git a/go.mod b/go.mod
@@ -3,7 +3,7 @@ module github.com/DataBridgeTech/dbqctl
 go 1.24.5
 
 require (
-	github.com/DataBridgeTech/dbqcore v0.4.0
+	github.com/DataBridgeTech/dbqcore v0.5.2
 	github.com/spf13/cobra v1.9.1
 	github.com/spf13/pflag v1.0.7
 	github.com/spf13/viper v1.20.1
diff --git a/go.sum b/go.sum
@@ -7,6 +7,10 @@ github.com/ClickHouse/clickhouse-go/v2 v2.39.0/go.mod h1:m13KylpdcPzpIjznlfXp53I
 github.com/DataBridgeTech/dbqcore v0.3.0/go.mod h1:frSX2soCK0jiZMu1rcPPbvHar2KMtKL0/Hh2fHiQJ6s=
 github.com/DataBridgeTech/dbqcore v0.4.0 h1:GEoH9o6ByltB6VUil02Wu6/vA2qarVcFd7j18h5Eeiw=
 github.com/DataBridgeTech/dbqcore v0.4.0/go.mod h1:frSX2soCK0jiZMu1rcPPbvHar2KMtKL0/Hh2fHiQJ6s=
+github.com/DataBridgeTech/dbqcore v0.5.1 h1:s/30Mqp3uhBUB5moLYLQ1gc2TRDgnHqihH1u2P8gm0A=
+github.com/DataBridgeTech/dbqcore v0.5.1/go.mod h1:/uaxo2GiU3UjeIQ1KC2XcIHXZkDbE5wLoab+eR1PIgg=
+github.com/DataBridgeTech/dbqcore v0.5.2 h1:3vQLhskFYfLrGngFHvc7LlJRs1+ZaswMaFZ51r5yeZw=
+github.com/DataBridgeTech/dbqcore v0.5.2/go.mod h1:/uaxo2GiU3UjeIQ1KC2XcIHXZkDbE5wLoab+eR1PIgg=
 github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
 github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
diff --git a/internal/app.go b/internal/app.go
diff --git a/readme.md b/readme.md

Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,7 @@ import (`
`22`	`22`	`)`
`23`	`23`
`24`	`24`	`const (`
`25`		`- DbqCtlVersion = "v0.4.0"`
	`25`	`+ DbqCtlVersion = "v0.5.0"`
`26`	`26`	`)`
`27`	`27`
`28`	`28`	`func NewVersionCommand() *cobra.Command {`