Fixes pandas date method + fix ingest tests race conditions (#615)

* Import changes from other branch

* Added tests

* Is this test really fails it?

* Fixed test

* Fixed test

* skip

* Remove

* back

* test

* test

* test

* fixed warning

* fixed warning

* rename

* rename

* Increate timeout

* Increate timeout

* Same group

* Separate table per test

* Close clients

* Close clients

* Close clients

* Removed shared state

* Separate managed streaming

* Fixed test

* w

* w

* w

* Update azure-kusto-data/azure/kusto/data/helpers.py

Co-authored-by: Copilot <[email protected]>

* Update azure-kusto-ingest/tests/test_e2e_ingest.py

Co-authored-by: Copilot <[email protected]>

* Fix

* Fix

* Add retry

* Update azure-kusto-ingest/tests/test_e2e_ingest.py

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Update azure-kusto-ingest/tests/test_e2e_ingest.py

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Add sleep

* Try a different way

---------

Co-authored-by: Copilot <[email protected]>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Author: 3 people

CHANGELOG.md

@@ -5,10 +5,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [6.0.1] - 2025-12-25
 
 ### Fixed
 - Changed extra's name back to `aio`
+- Fixed handling of datetime columns in old pandas versions.
 - Fixed encoding error in `ingest_from_dataframe` when using csv data format.
 
 ## [6.0.0] - 2025-11-26

azure-kusto-data/azure/kusto/data/helpers.py

@@ -1,17 +1,17 @@
 import json
 from functools import lru_cache
 from pathlib import Path
-from typing import TYPE_CHECKING, Union, Callable, Dict, Optional
+from typing import TYPE_CHECKING, Any, Union, Callable, Optional
 
 if TYPE_CHECKING:
     import pandas as pd
     from azure.kusto.data._models import KustoResultTable, KustoStreamingResultTable
 
 # Alias for dataframe_from_result_table converter type
-Converter = Dict[str, Union[str, Callable[[str, "pd.DataFrame"], "pd.Series"]]]
+Converter = dict[str, Union[str, Callable[[str, "pd.DataFrame"], "pd.Series"]]]
 
 
-def load_bundled_json(file_name: str) -> Dict:
+def load_bundled_json(file_name: str) -> dict[Any, Any]:
     filename = Path(__file__).absolute().parent.joinpath(file_name)
     with filename.open("r", encoding="utf-8") as data:
         return json.load(data)
@@ -113,23 +113,24 @@ def parse_float(frame, col):
     import numpy as np
     import pandas as pd
 
-    frame[col] = frame[col].replace("NaN", np.nan).replace("Infinity", np.inf).replace("-Infinity", -np.inf)
+    frame[col] = frame[col].infer_objects(copy=False).replace({"NaN": np.nan, "Infinity": np.inf, "-Infinity": -np.inf})
     frame[col] = pd.to_numeric(frame[col], errors="coerce").astype(pd.Float64Dtype())  # pyright: ignore[reportCallIssue,reportArgumentType]
+
     return frame[col]
 
 
-def parse_datetime(frame, col):
+def parse_datetime(frame, col, force_version: Optional[str] = None) -> "pd.Series":
     # Pandas before version 2 doesn't support the "format" arg
     import pandas as pd
 
     args = {}
-    if pd.__version__.startswith("2."):
+    if (force_version or pd.__version__).startswith("2."):
         args = {"format": "ISO8601", "utc": True}
     else:
         # if frame contains ".", replace "Z" with ".000Z"
-        # == False is not a mistake - that's the pandas way to do it
-        contains_dot = frame[col].str.contains(".")
-        frame.loc[not contains_dot, col] = frame.loc[not contains_dot, col].str.replace("Z", ".000Z")
+        # Using bitwise NOT (~) on the boolean Series is the idiomatic pandas way to negate the mask
+        contains_dot = frame[col].str.contains("\\.")
+        frame.loc[~contains_dot, col] = frame.loc[~contains_dot, col].str.replace("Z", ".000Z")
     frame[col] = pd.to_datetime(frame[col], errors="coerce", **args)
     return frame[col]
 

azure-kusto-data/tests/test_helpers.py

@@ -7,7 +7,7 @@
 import pytest
 
 from azure.kusto.data._models import KustoResultTable
-from azure.kusto.data.helpers import dataframe_from_result_table
+from azure.kusto.data.helpers import dataframe_from_result_table, parse_datetime
 from azure.kusto.data.response import KustoResponseDataSetV2
 import pandas
 import numpy
@@ -128,3 +128,31 @@ def test_pandas_mixed_date():
 
     assert df["Date"][0] == pandas.Timestamp(year=2023, month=12, day=12, hour=1, minute=59, second=59, microsecond=352000, tzinfo=datetime.timezone.utc)
     assert df["Date"][1] == pandas.Timestamp(year=2023, month=12, day=12, hour=1, minute=54, second=44, tzinfo=datetime.timezone.utc)
+
+
+def test_datetime_parsing():
+    """Test parse_datetime function with different pandas versions and datetime formats"""
+
+    # Test with pandas v2 behavior (force version 2)
+    df_v2 = pandas.DataFrame(
+        {
+            "mixed": ["2023-12-12T01:59:59.352Z", "2023-12-12T01:54:44Z"],
+        }
+    )
+
+    # Force pandas v2 behavior
+    result_v2 = parse_datetime(df_v2, "mixed", force_version="2.0.0")
+    assert str(result_v2[0]) == "2023-12-12 01:59:59.352000+00:00"
+    assert str(result_v2[1]) == "2023-12-12 01:54:44+00:00"
+    # Test with pandas v1 behavior (force version 1)
+
+    df_v1 = pandas.DataFrame(
+        {
+            "mixed": ["2023-12-12T01:59:59.352Z", "2023-12-12T01:54:44Z"],
+        }
+    )
+
+    # Force pandas v1 behavior - it should add .000 to dates without milliseconds
+    result_v1 = parse_datetime(df_v1, "mixed", force_version="1.5.3")
+    assert str(result_v1[0]) == "2023-12-12 01:59:59.352000+00:00"
+    assert str(result_v1[1]) == "2023-12-12 01:54:44+00:00"