fix(sql): pagination remove extra conversions (#1702)

* fix(sql): pagination remove extra conversions

* update comment

Author: ArslanSaleem

pandasai/query_builders/paginator.py

@@ -107,6 +107,9 @@ def apply_pagination(
         if not pagination:
             return query, params
 
+        # Convert query from target dialect to postgres to generate standardized pagination query
+        query = sqlglot.transpile(query, read=target_dialect, write="postgres")[0]
+
         filtering_query = f"SELECT * FROM ({query}) AS filtered_data"
         conditions = []
 
@@ -118,26 +121,26 @@ def apply_pagination(
                 column_type = column["type"]
 
                 if column_type == "string":
-                    search_conditions.append(f"{column_name} ILIKE %s")
+                    search_conditions.append(f'"{column_name}" ILIKE %s')
                     params.append(f"%{pagination.search}%")
 
                 elif column_type == "float" and DatasetPaginator.is_float(
                     pagination.search
                 ):
-                    search_conditions.append(f"{column_name} = %s")
+                    search_conditions.append(f'"{column_name}" = %s')
                     params.append(pagination.search)
 
                 elif (
                     column_type in ["number", "integer"]
                     and pagination.search.isnumeric()
                 ):
-                    search_conditions.append(f"{column_name} = %s")
+                    search_conditions.append(f'"{column_name}" = %s')
                     params.append(pagination.search)
 
                 elif column_type == "datetime" and DatasetPaginator.is_valid_datetime(
                     pagination.search
                 ):
-                    search_conditions.append(f"{column_name} = %s")
+                    search_conditions.append(f'"{column_name}" = %s')
                     params.append(
                         datetime.datetime.strptime(
                             pagination.search, "%Y-%m-%d %H:%M:%S"
@@ -147,13 +150,13 @@ def apply_pagination(
                 elif column_type == "boolean" and DatasetPaginator.is_valid_boolean(
                     pagination.search
                 ):
-                    search_conditions.append(f"{column_name} = %s")
+                    search_conditions.append(f'"{column_name}" = %s')
                     params.append(pagination.search)
 
                 elif column_type == "uuid" and DatasetPaginator.is_valid_uuid(
                     pagination.search
                 ):
-                    search_conditions.append(f"{column_name}::TEXT = %s")
+                    search_conditions.append(f'"{column_name}"::TEXT = %s')
                     params.append(pagination.search)
 
             if search_conditions:
@@ -171,7 +174,7 @@ def apply_pagination(
                     if not isinstance(values, list):
                         values = [values]
                     placeholders = ", ".join(["%s"] * len(values))
-                    conditions.append(f"{column} IN ({placeholders})")
+                    conditions.append(f'"{column}" IN ({placeholders})')
                     params.extend(values)
             except json.JSONDecodeError as e:
                 raise ValueError(f"Invalid filters format: {e}")
@@ -188,7 +191,7 @@ def apply_pagination(
                 )
 
             filtering_query += (
-                f" ORDER BY {pagination.sort_by} {pagination.sort_order.upper()}"
+                f' ORDER BY "{pagination.sort_by}" {pagination.sort_order.upper()}'
             )
 
         # Handle page and page_size
@@ -198,12 +201,4 @@ def apply_pagination(
                 [pagination.page_size, (pagination.page - 1) * pagination.page_size]
             )
 
-        # Replace placeholders for target dialect
-        placeholder = "___PLACEHOLDER___"
-        temp_query = filtering_query.replace("%s", placeholder)
-        transpiled_query = sqlglot.transpile(
-            temp_query, read="postgres", write=target_dialect
-        )[0]
-        final_query = transpiled_query.replace(placeholder, "%s")
-
-        return final_query, params
+        return filtering_query, params

tests/unit_tests/query_builders/test_paginator.py

@@ -82,7 +82,7 @@ def test_search_string_column(self, sample_query, sample_columns):
         query, parameters = DatasetPaginator.apply_pagination(
             sample_query, sample_columns, params
         )
-        assert "name ILIKE %s" in query
+        assert '"name" ILIKE %s' in query
         assert parameters[0] == "%John%"  # First parameter is search term
         assert len(parameters) == 3  # search + LIMIT/OFFSET
 
@@ -92,8 +92,8 @@ def test_search_numeric_columns(self, sample_query, sample_columns):
         query, parameters = DatasetPaginator.apply_pagination(
             sample_query, sample_columns, params
         )
-        assert "id = %s" in query
-        assert "age = %s" in query
+        assert '"id" = %s' in query
+        assert '"age" = %s' in query
         assert parameters.count("25") >= 2  # At least id and age columns
         assert len(parameters) > 2  # search params + LIMIT/OFFSET
 
@@ -103,7 +103,7 @@ def test_search_datetime(self, sample_query, sample_columns):
         query, parameters = DatasetPaginator.apply_pagination(
             sample_query, sample_columns, params
         )
-        assert "created_at = %s" in query
+        assert '"created_at" = %s' in query
         # Convert the datetime string to expected format
         expected_dt = datetime.datetime.strptime(
             "2023-01-01 12:00:00", "%Y-%m-%d %H:%M:%S"
@@ -120,7 +120,7 @@ def test_filters(self, sample_query, sample_columns):
         query, parameters = DatasetPaginator.apply_pagination(
             sample_query, sample_columns, params
         )
-        assert "age IN (%s, %s, %s)" in query
+        assert '"age" IN (%s, %s, %s)' in query
         assert all(
             x in parameters for x in [25, 30, 35]
         )  # Filter values are in parameters
@@ -134,7 +134,7 @@ def test_sorting(self, sample_query, sample_columns):
         query, parameters = DatasetPaginator.apply_pagination(
             sample_query, sample_columns, params
         )
-        assert "ORDER BY age DESC" in query
+        assert 'ORDER BY "age" DESC' in query
 
     def test_invalid_sort_column(self, sample_query, sample_columns):
         """Test error on invalid sort column"""
@@ -183,7 +183,7 @@ def test_boolean_search(self, sample_query, sample_columns):
         query, parameters = DatasetPaginator.apply_pagination(
             sample_query, sample_columns, params
         )
-        assert "is_active = %s" in query
+        assert '"is_active" = %s' in query
         assert "true" in [str(p).lower() for p in parameters]
 
     def test_uuid_search(self, sample_query, sample_columns):
@@ -193,7 +193,7 @@ def test_uuid_search(self, sample_query, sample_columns):
         query, parameters = DatasetPaginator.apply_pagination(
             sample_query, sample_columns, params
         )
-        assert "CAST(user_id AS TEXT) = %s" in query
+        assert '"user_id"::TEXT = %s' in query
         assert uuid_value in parameters
 
     def test_filter_single_value(self, sample_query, sample_columns):
@@ -206,7 +206,7 @@ def test_filter_single_value(self, sample_query, sample_columns):
         query, parameters = DatasetPaginator.apply_pagination(
             sample_query, sample_columns, params
         )
-        assert "age IN (%s)" in query
+        assert '"age" IN (%s)' in query
         assert 25 in parameters
 
     def test_invalid_json_filter(self, sample_query, sample_columns):