feat: Add concat pushdown for hybrid engine (#1891)

Author: TrevorBergeron

bigframes/core/compile/polars/compiler.py

@@ -547,6 +547,11 @@ def compile_concat(self, node: nodes.ConcatNode):
         child_frames = [
             frame.rename(
                 {col: id.sql for col, id in zip(frame.columns, node.output_ids)}
+            ).cast(
+                {
+                    field.id.sql: _bigframes_dtype_to_polars_dtype(field.dtype)
+                    for field in node.fields
+                }
             )
             for frame in child_frames
         ]

bigframes/core/nodes.py

@@ -424,7 +424,7 @@ def remap_refs(
 
 @dataclasses.dataclass(frozen=True, eq=False)
 class ConcatNode(BigFrameNode):
-    # TODO: Explcitly map column ids from each child
+    # TODO: Explcitly map column ids from each child?
     children: Tuple[BigFrameNode, ...]
     output_ids: Tuple[identifiers.ColumnId, ...]
 

bigframes/session/polars_executor.py

@@ -36,6 +36,7 @@
     nodes.SliceNode,
     nodes.AggregateNode,
     nodes.FilterNode,
+    nodes.ConcatNode,
 )
 
 _COMPATIBLE_SCALAR_OPS = (

tests/system/small/engines/test_concat.py

@@ -0,0 +1,51 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from bigframes.core import array_value, ordering
+from bigframes.session import polars_executor
+from bigframes.testing.engine_utils import assert_equivalence_execution
+
+pytest.importorskip("polars")
+
+# Polars used as reference as its fast and local. Generally though, prefer gbq engine where they disagree.
+REFERENCE_ENGINE = polars_executor.PolarsExecutor()
+
+
+@pytest.mark.parametrize("engine", ["polars", "bq"], indirect=True)
+def test_engines_concat_self(
+    scalars_array_value: array_value.ArrayValue,
+    engine,
+):
+    result = scalars_array_value.concat([scalars_array_value, scalars_array_value])
+
+    assert_equivalence_execution(result.node, REFERENCE_ENGINE, engine)
+
+
+@pytest.mark.parametrize("engine", ["polars", "bq"], indirect=True)
+def test_engines_concat_filtered_sorted(
+    scalars_array_value: array_value.ArrayValue,
+    engine,
+):
+    input_1 = scalars_array_value.select_columns(["float64_col", "int64_col"]).order_by(
+        [ordering.ascending_over("int64_col")]
+    )
+    input_2 = scalars_array_value.filter_by_id("bool_col").select_columns(
+        ["float64_col", "int64_too"]
+    )
+
+    result = input_1.concat([input_2, input_1, input_2])
+
+    assert_equivalence_execution(result.node, REFERENCE_ENGINE, engine)