refactor: support agg_ops.ShiftOp and DiffOp for the sqlglot compiler (#2156)

Author: chelsea-lin

bigframes/core/compile/sqlglot/aggregations/unary_compiler.py

@@ -151,6 +151,23 @@ def _(
     )
 
 
+@UNARY_OP_REGISTRATION.register(agg_ops.DiffOp)
+def _(
+    op: agg_ops.DiffOp,
+    column: typed_expr.TypedExpr,
+    window: typing.Optional[window_spec.WindowSpec] = None,
+) -> sge.Expression:
+    shift_op_impl = UNARY_OP_REGISTRATION[agg_ops.ShiftOp(0)]
+    shifted = shift_op_impl(agg_ops.ShiftOp(op.periods), column, window)
+    if column.dtype in (dtypes.BOOL_DTYPE, dtypes.INT_DTYPE, dtypes.FLOAT_DTYPE):
+        if column.dtype == dtypes.BOOL_DTYPE:
+            return sge.NEQ(this=column.expr, expression=shifted)
+        else:
+            return sge.Sub(this=column.expr, expression=shifted)
+    else:
+        raise TypeError(f"Cannot perform diff on type {column.dtype}")
+
+
 @UNARY_OP_REGISTRATION.register(agg_ops.MaxOp)
 def _(
     op: agg_ops.MaxOp,
@@ -240,6 +257,27 @@ def _(
     return apply_window_if_present(sge.func("COUNT", sge.convert(1)), window)
 
 
+@UNARY_OP_REGISTRATION.register(agg_ops.ShiftOp)
+def _(
+    op: agg_ops.ShiftOp,
+    column: typed_expr.TypedExpr,
+    window: typing.Optional[window_spec.WindowSpec] = None,
+) -> sge.Expression:
+    if op.periods == 0:  # No-op
+        return column.expr
+    if op.periods > 0:
+        return apply_window_if_present(
+            sge.func("LAG", column.expr, sge.convert(op.periods)),
+            window,
+            include_framing_clauses=False,
+        )
+    return apply_window_if_present(
+        sge.func("LEAD", column.expr, sge.convert(-op.periods)),
+        window,
+        include_framing_clauses=False,
+    )
+
+
 @UNARY_OP_REGISTRATION.register(agg_ops.SumOp)
 def _(
     op: agg_ops.SumOp,

tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff/diff_bool.sql

@@ -0,0 +1,13 @@
+WITH `bfcte_0` AS (
+  SELECT
+    `bool_col` AS `bfcol_0`
+  FROM `bigframes-dev`.`sqlglot_test`.`scalar_types`
+), `bfcte_1` AS (
+  SELECT
+    *,
+    `bfcol_0` <> LAG(`bfcol_0`, 1) OVER (ORDER BY `bfcol_0` IS NULL ASC NULLS LAST, `bfcol_0` ASC NULLS LAST) AS `bfcol_1`
+  FROM `bfcte_0`
+)
+SELECT
+  `bfcol_1` AS `diff_bool`
+FROM `bfcte_1`

tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff/diff_int.sql

@@ -0,0 +1,13 @@
+WITH `bfcte_0` AS (
+  SELECT
+    `int64_col` AS `bfcol_0`
+  FROM `bigframes-dev`.`sqlglot_test`.`scalar_types`
+), `bfcte_1` AS (
+  SELECT
+    *,
+    `bfcol_0` - LAG(`bfcol_0`, 1) OVER (ORDER BY `bfcol_0` IS NULL ASC NULLS LAST, `bfcol_0` ASC NULLS LAST) AS `bfcol_1`
+  FROM `bfcte_0`
+)
+SELECT
+  `bfcol_1` AS `diff_int`
+FROM `bfcte_1`

tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_shift/lag.sql

@@ -0,0 +1,13 @@
+WITH `bfcte_0` AS (
+  SELECT
+    `int64_col` AS `bfcol_0`
+  FROM `bigframes-dev`.`sqlglot_test`.`scalar_types`
+), `bfcte_1` AS (
+  SELECT
+    *,
+    LAG(`bfcol_0`, 1) OVER (ORDER BY `bfcol_0` IS NULL ASC NULLS LAST, `bfcol_0` ASC NULLS LAST) AS `bfcol_1`
+  FROM `bfcte_0`
+)
+SELECT
+  `bfcol_1` AS `lag`
+FROM `bfcte_1`

tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_shift/lead.sql

@@ -0,0 +1,13 @@
+WITH `bfcte_0` AS (
+  SELECT
+    `int64_col` AS `bfcol_0`
+  FROM `bigframes-dev`.`sqlglot_test`.`scalar_types`
+), `bfcte_1` AS (
+  SELECT
+    *,
+    LEAD(`bfcol_0`, 1) OVER (ORDER BY `bfcol_0` IS NULL ASC NULLS LAST, `bfcol_0` ASC NULLS LAST) AS `bfcol_1`
+  FROM `bfcte_0`
+)
+SELECT
+  `bfcol_1` AS `lead`
+FROM `bfcte_1`

tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_shift/noop.sql

@@ -0,0 +1,13 @@
+WITH `bfcte_0` AS (
+  SELECT
+    `int64_col` AS `bfcol_0`
+  FROM `bigframes-dev`.`sqlglot_test`.`scalar_types`
+), `bfcte_1` AS (
+  SELECT
+    *,
+    `bfcol_0` AS `bfcol_1`
+  FROM `bfcte_0`
+)
+SELECT
+  `bfcol_1` AS `noop`
+FROM `bfcte_1`

tests/unit/core/compile/sqlglot/aggregations/test_unary_compiler.py

@@ -127,6 +127,28 @@ def test_dense_rank(scalar_types_df: bpd.DataFrame, snapshot):
     snapshot.assert_match(sql, "out.sql")
 
 
+def test_diff(scalar_types_df: bpd.DataFrame, snapshot):
+    # Test integer
+    int_col = "int64_col"
+    bf_df_int = scalar_types_df[[int_col]]
+    window = window_spec.WindowSpec(ordering=(ordering.ascending_over(int_col),))
+    int_op = agg_exprs.UnaryAggregation(
+        agg_ops.DiffOp(periods=1), expression.deref(int_col)
+    )
+    int_sql = _apply_unary_window_op(bf_df_int, int_op, window, "diff_int")
+    snapshot.assert_match(int_sql, "diff_int.sql")
+
+    # Test boolean
+    bool_col = "bool_col"
+    bf_df_bool = scalar_types_df[[bool_col]]
+    window = window_spec.WindowSpec(ordering=(ordering.ascending_over(bool_col),))
+    bool_op = agg_exprs.UnaryAggregation(
+        agg_ops.DiffOp(periods=1), expression.deref(bool_col)
+    )
+    bool_sql = _apply_unary_window_op(bf_df_bool, bool_op, window, "diff_bool")
+    snapshot.assert_match(bool_sql, "diff_bool.sql")
+
+
 def test_first(scalar_types_df: bpd.DataFrame, snapshot):
     if sys.version_info < (3, 12):
         pytest.skip(
@@ -271,6 +293,33 @@ def test_rank(scalar_types_df: bpd.DataFrame, snapshot):
     snapshot.assert_match(sql, "out.sql")
 
 
+def test_shift(scalar_types_df: bpd.DataFrame, snapshot):
+    col_name = "int64_col"
+    bf_df = scalar_types_df[[col_name]]
+    window = window_spec.WindowSpec(ordering=(ordering.ascending_over(col_name),))
+
+    # Test lag
+    lag_op = agg_exprs.UnaryAggregation(
+        agg_ops.ShiftOp(periods=1), expression.deref(col_name)
+    )
+    lag_sql = _apply_unary_window_op(bf_df, lag_op, window, "lag")
+    snapshot.assert_match(lag_sql, "lag.sql")
+
+    # Test lead
+    lead_op = agg_exprs.UnaryAggregation(
+        agg_ops.ShiftOp(periods=-1), expression.deref(col_name)
+    )
+    lead_sql = _apply_unary_window_op(bf_df, lead_op, window, "lead")
+    snapshot.assert_match(lead_sql, "lead.sql")
+
+    # Test no-op
+    noop_op = agg_exprs.UnaryAggregation(
+        agg_ops.ShiftOp(periods=0), expression.deref(col_name)
+    )
+    noop_sql = _apply_unary_window_op(bf_df, noop_op, window, "noop")
+    snapshot.assert_match(noop_sql, "noop.sql")
+
+
 def test_sum(scalar_types_df: bpd.DataFrame, snapshot):
     bf_df = scalar_types_df[["int64_col", "bool_col"]]
     agg_ops_map = {