Transform JSON null checks

Author: matt-aitken

apps/webapp/app/v3/querySchemas.ts

@@ -315,13 +315,15 @@ export const runsSchema: TableSchema = {
     },
 
     // Output & error (JSON columns)
+    // For JSON columns, NULL checks are transformed to check for empty object '{}'
+    // So `error IS NULL` becomes `error = '{}'` and `error IS NOT NULL` becomes `error != '{}'`
     output: {
       name: "output",
       ...column("JSON", {
         description: "The data you returned from the task.",
         example: '{"result": "success"}',
       }),
-      expression: "if(output = '{}', NULL, output)",
+      nullValue: "'{}'", // Transform NULL checks to compare against empty object
     },
     error: {
       name: "error",
@@ -330,7 +332,7 @@ export const runsSchema: TableSchema = {
           "If a run completely failed (after all attempts) then this error will be populated.",
         example: '{"message": "Task failed"}',
       }),
-      expression: "if(error = '{}', NULL, error)",
+      nullValue: "'{}'", // Transform NULL checks to compare against empty object
     },
 
     // Tags & versions

internal-packages/tsql/src/query/printer.test.ts

@@ -441,6 +441,112 @@ describe("ClickHousePrinter", () => {
     });
   });
 
+  describe("nullValue transformation for JSON columns", () => {
+    // Create a schema with JSON columns that have nullValue set
+    const jsonSchema: TableSchema = {
+      name: "runs",
+      clickhouseName: "trigger_dev.task_runs_v2",
+      columns: {
+        id: { name: "id", ...column("String") },
+        error: {
+          name: "error",
+          ...column("JSON"),
+          nullValue: "'{}'", // Empty object represents NULL
+        },
+        output: {
+          name: "output",
+          ...column("JSON"),
+          nullValue: "'{}'", // Empty object represents NULL
+        },
+        status: { name: "status", ...column("String") },
+        organization_id: { name: "organization_id", ...column("String") },
+        project_id: { name: "project_id", ...column("String") },
+        environment_id: { name: "environment_id", ...column("String") },
+      },
+      tenantColumns: {
+        organizationId: "organization_id",
+        projectId: "project_id",
+        environmentId: "environment_id",
+      },
+    };
+
+    function createJsonContext() {
+      const schema = createSchemaRegistry([jsonSchema]);
+      return createPrinterContext({
+        organizationId: "org_test",
+        projectId: "proj_test",
+        environmentId: "env_test",
+        schema,
+      });
+    }
+
+    it("should transform IS NULL to equals empty object for JSON columns with nullValue", () => {
+      const ctx = createJsonContext();
+      const { sql } = printQuery("SELECT * FROM runs WHERE error IS NULL", ctx);
+
+      // Should use equals with '{}' instead of isNull
+      expect(sql).toContain("equals(error, '{}')");
+      expect(sql).not.toContain("isNull(error)");
+    });
+
+    it("should transform IS NOT NULL to notEquals empty object for JSON columns with nullValue", () => {
+      const ctx = createJsonContext();
+      const { sql } = printQuery("SELECT * FROM runs WHERE error IS NOT NULL", ctx);
+
+      // Should use notEquals with '{}' instead of isNotNull
+      expect(sql).toContain("notEquals(error, '{}')");
+      expect(sql).not.toContain("isNotNull(error)");
+    });
+
+    it("should transform = NULL to equals empty object for JSON columns with nullValue", () => {
+      const ctx = createJsonContext();
+      const { sql } = printQuery("SELECT * FROM runs WHERE error = NULL", ctx);
+
+      expect(sql).toContain("equals(error, '{}')");
+      expect(sql).not.toContain("isNull(error)");
+    });
+
+    it("should transform != NULL to notEquals empty object for JSON columns with nullValue", () => {
+      const ctx = createJsonContext();
+      const { sql } = printQuery("SELECT * FROM runs WHERE error != NULL", ctx);
+
+      expect(sql).toContain("notEquals(error, '{}')");
+      expect(sql).not.toContain("isNotNull(error)");
+    });
+
+    it("should not affect regular columns without nullValue", () => {
+      const ctx = createJsonContext();
+      const { sql } = printQuery("SELECT * FROM runs WHERE status IS NULL", ctx);
+
+      // Regular column should still use isNull
+      expect(sql).toContain("isNull(status)");
+    });
+
+    it("should work with multiple JSON column NULL checks", () => {
+      const ctx = createJsonContext();
+      const { sql } = printQuery(
+        "SELECT * FROM runs WHERE error IS NOT NULL AND output IS NULL",
+        ctx
+      );
+
+      expect(sql).toContain("notEquals(error, '{}')");
+      expect(sql).toContain("equals(output, '{}')");
+    });
+
+    it("should allow GROUP BY on JSON columns without error", () => {
+      const ctx = createJsonContext();
+      const { sql } = printQuery(
+        "SELECT error, count() AS error_count FROM runs WHERE error IS NOT NULL GROUP BY error",
+        ctx
+      );
+
+      // Should filter with notEquals
+      expect(sql).toContain("notEquals(error, '{}')");
+      // Should group by the raw column
+      expect(sql).toContain("GROUP BY error");
+    });
+  });
+
   describe("ORDER BY clauses", () => {
     it("should print ORDER BY ASC", () => {
       const { sql } = printQuery("SELECT * FROM task_runs ORDER BY created_at ASC");
@@ -1348,9 +1454,7 @@ describe("Virtual columns", () => {
       // GROUP BY should use the alias, not the expression (ClickHouse allows this)
       expect(sql).toContain("GROUP BY is_long_running");
       // Should NOT have the expression in GROUP BY
-      expect(sql).not.toContain(
-        "GROUP BY (if(completed_at IS NOT NULL AND started_at IS NOT NULL"
-      );
+      expect(sql).not.toContain("GROUP BY (if(completed_at IS NOT NULL AND started_at IS NOT NULL");
     });
 
     it("should use alias for virtual column in GROUP BY with WHERE and aggregation", () => {
@@ -1369,9 +1473,7 @@ describe("Virtual columns", () => {
       // Should NOT have the expression in GROUP BY
       expect(sql).not.toContain("GROUP BY (dateDiff('millisecond'");
       // WHERE should still use the expression
-      expect(sql).toContain(
-        "isNotNull((dateDiff('millisecond', started_at, completed_at)))"
-      );
+      expect(sql).toContain("isNotNull((dateDiff('millisecond', started_at, completed_at)))");
     });
   });
 

internal-packages/tsql/src/query/printer.ts

@@ -1510,6 +1510,7 @@ export class ClickHousePrinter {
   private visitCompareOperation(node: CompareOperation): string {
     // Check if we need to transform values using valueMap
     const columnSchema = this.extractColumnSchemaFromExpression(node.left);
+    const rightColumnSchema = this.extractColumnSchemaFromExpression(node.right);
 
     // Transform the right side if it contains user-friendly values
     const transformedRight = this.transformValueMapExpression(node.right, columnSchema);
@@ -1524,12 +1525,20 @@ export class ClickHousePrinter {
           (transformedRight as Constant).expression_type === "constant" &&
           (transformedRight as Constant).value === null
         ) {
+          // Check if the column has a custom nullValue (e.g., '{}' for JSON columns)
+          if (columnSchema?.nullValue) {
+            return `equals(${left}, ${columnSchema.nullValue})`;
+          }
           return `isNull(${left})`;
         }
         if (
           (node.left as Constant).expression_type === "constant" &&
           (node.left as Constant).value === null
         ) {
+          // Check if the column has a custom nullValue (e.g., '{}' for JSON columns)
+          if (rightColumnSchema?.nullValue) {
+            return `equals(${right}, ${rightColumnSchema.nullValue})`;
+          }
           return `isNull(${right})`;
         }
         return `equals(${left}, ${right})`;
@@ -1540,12 +1549,20 @@ export class ClickHousePrinter {
           (transformedRight as Constant).expression_type === "constant" &&
           (transformedRight as Constant).value === null
         ) {
+          // Check if the column has a custom nullValue (e.g., '{}' for JSON columns)
+          if (columnSchema?.nullValue) {
+            return `notEquals(${left}, ${columnSchema.nullValue})`;
+          }
           return `isNotNull(${left})`;
         }
         if (
           (node.left as Constant).expression_type === "constant" &&
           (node.left as Constant).value === null
         ) {
+          // Check if the column has a custom nullValue (e.g., '{}' for JSON columns)
+          if (rightColumnSchema?.nullValue) {
+            return `notEquals(${right}, ${rightColumnSchema.nullValue})`;
+          }
           return `isNotNull(${right})`;
         }
         return `notEquals(${left}, ${right})`;

internal-packages/tsql/src/query/schema.ts

@@ -180,6 +180,23 @@ export interface ColumnSchema {
    * ```
    */
   whereTransform?: (value: string) => string;
+  /**
+   * Value to use when comparing to NULL for this column.
+   *
+   * When set, NULL comparisons (IS NULL, IS NOT NULL, = NULL, != NULL) are
+   * transformed to compare against this value instead. This is useful for
+   * JSON/Object columns where "empty" is represented as '{}' rather than NULL.
+   *
+   * @example
+   * ```typescript
+   * {
+   *   name: "error",
+   *   type: "JSON",
+   *   nullValue: "'{}'",  // error IS NULL → error = '{}'
+   * }
+   * ```
+   */
+  nullValue?: string;
 }
 
 /**