Fixes all tests pass

Author: bosd

src/odoo_data_flow/import_threaded.py

@@ -565,7 +565,7 @@ def _process_external_id_fields(
     return converted_vals, external_id_fields
 
 
-def _handle_create_error(
+def _handle_create_error(  # noqa: C901
     i: int,
     create_error: Exception,
     line: list[Any],
@@ -978,36 +978,10 @@ def _execute_load_batch(  # noqa: C901
             res = model.load(load_header, load_lines, context=context)
 
             if res.get("messages"):
-                error = res["messages"][0].get("message", "Batch load failed.")
+                res["messages"][0].get("message", "Batch load failed.")
                 # Don't raise immediately, log and continue to capture in fail file
-            log.debug(
-                f"Expected records: {len(load_lines)}, "
-                f"Created records: {len(created_ids)}"
-            )
-
-            # Always log detailed information about record creation
-            if len(created_ids) != len(load_lines):
-                log.warning(
-                    f"Record creation mismatch: Expected {len(load_lines)} records, "
-                    f"but only {len(created_ids)} were created"
-                )
-                if len(created_ids) == 0:
-                    log.error(
-                        f"No records were created in this batch of {len(load_lines)}. "
-                        f"This may indicate silent failures in the Odoo load operation. "
-                        f"Check Odoo server logs for validation errors."
-                    )
-                    # Log the actual data being sent for debugging
-                    if load_lines:
-                        log.debug("First few lines being sent:")
-                        for i, line in enumerate(load_lines[:3]):
-                            log.debug(f"  Line {i}: {dict(zip(load_header, line))}")
-                else:
-                    log.warning(
-                        f"Partial record creation: {len(created_ids)}/{len(load_lines)} "
-                        f"records were created. Some records may have failed validation."
-                    )
-            # Check for any Odoo server errors in the response that should halt processing
+            # Check for any Odoo server errors in the response that should halt
+            # processing
             if res.get("messages"):
                 for message in res["messages"]:
                     msg_type = message.get("type", "unknown")
@@ -1023,7 +997,8 @@ def _execute_load_batch(  # noqa: C901
 
             created_ids = res.get("ids", [])
             log.debug(
-                f"Expected records: {len(load_lines)}, Created records: {len(created_ids)}"
+                f"Expected records: {len(load_lines)}, "
+                f"Created records: {len(created_ids)}"
             )
 
             # Always log detailed information about record creation
@@ -1035,8 +1010,8 @@ def _execute_load_batch(  # noqa: C901
                 if len(created_ids) == 0:
                     log.error(
                         f"No records were created in this batch of {len(load_lines)}. "
-                        f"This may indicate silent failures in the Odoo load operation. "
-                        f"Check Odoo server logs for validation errors."
+                        f"This may indicate silent failures in the Odoo load operation."
+                        f" Check Odoo server logs for validation errors."
                     )
                     # Log the actual data being sent for debugging
                     if load_lines:
@@ -1045,14 +1020,16 @@ def _execute_load_batch(  # noqa: C901
                             log.debug(f"  Line {i}: {dict(zip(load_header, line))}")
                 else:
                     log.warning(
-                        f"Partial record creation: {len(created_ids)}/{len(load_lines)} "
-                        f"records were created. Some records may have failed validation."
+                        f"Partial record creation: {len(created_ids)}/{len(load_lines)}"
+                        f"records were created. "
+                        f"Some records may have failed validation."
                     )
 
             # Instead of raising an exception, capture failures for the fail file
             # But still create what records we can
             if res.get("messages"):
-                # Extract error information and add to failed_lines to be written to fail file
+                # Extract error information and add to failed_lines to be written
+                # to fail file
                 error_msg = res["messages"][0].get("message", "Batch load failed.")
                 log.error(f"Capturing load failure for fail file: {error_msg}")
                 # We'll add the failed lines to aggregated_failed_lines at the end
@@ -1094,13 +1071,19 @@ def _execute_load_batch(  # noqa: C901
                         failed_line = [*list(line), f"Load failed: {error_msg}"]
                         aggregated_failed_lines.append(failed_line)
 
-                        # Create a new dictionary containing only the items with integer values
+                        # Create a new dictionary containing only the items with
+                        # integer values
                         filtered_id_map = {
                             key: value
                             for key, value in id_map.items()
                             if isinstance(value, int)
                         }
-                        aggregated_id_map.update(filtered_id_map)
+            # Always update the aggregated map with successful records
+            # Create a new dictionary containing only the items with integer values
+            filtered_id_map = {
+                key: value for key, value in id_map.items() if isinstance(value, int)
+            }
+            aggregated_id_map.update(filtered_id_map)
             lines_to_process = lines_to_process[chunk_size:]
 
             # Reset serialization retry counter on successful processing

src/odoo_data_flow/lib/preflight.py

@@ -4,6 +4,7 @@
 systemic errors early (e.g., missing languages, incorrect configuration).
 """
 
+import csv
 import tempfile
 from typing import Any, Callable, Optional, Union, cast
 
@@ -396,28 +397,52 @@ def _validate_header(
     return True
 
 
-def _auto_correct_field_types(  # noqa: C901
+@register_check
+def type_correction_check(  # noqa: C901
+    preflight_mode: "PreflightMode",
+    model: str,
     filename: str,
-    header: list[str],
-    odoo_fields: dict[str, Any],
-    separator: str,
-    encoding: str,
-) -> Optional[str]:
-    """Auto-correct field type mismatches using Polars casting.
+    config: Union[str, dict[str, Any]],
+    import_plan: dict[str, Any],
+    **kwargs: Any,
+) -> bool:
+    """Auto-correct field type mismatches using Polars casting and store in import_plan.
 
     Args:
+        preflight_mode: The preflight mode (NORMAL or FAIL_MODE)
+        model: The target Odoo model name
         filename: Path to the source CSV file
-        header: List of CSV header columns
-        odoo_fields: Dictionary of Odoo model fields and their metadata
-        separator: CSV field separator
-        encoding: File encoding
+        config: Odoo connection configuration
+        import_plan: Dictionary to store import-related data including corrected file
+        **kwargs: Additional parameters like separator, encoding, etc.
 
     Returns:
-        Path to corrected CSV file, or None if no corrections needed/error occurred
+        True if the check passed (regardless of whether corrections were needed)
     """
     try:
         log.info("Running type validation and auto-correction...")
 
+        separator = kwargs.get("separator", ";")
+        encoding = kwargs.get("encoding", "utf-8")
+
+        # Read header to validate
+        with open(filename, encoding=encoding, newline="") as f:
+            reader = csv.reader(f, delimiter=separator)
+            header = next(reader)
+
+        if not header:
+            log.warning("Source file has no header, skipping type correction check")
+            return True
+
+        # Get Odoo model fields information
+        odoo_fields = _get_odoo_fields(config, model)
+        if not odoo_fields:
+            log.warning(
+                "Could not retrieve Odoo field information, "
+                "skipping type correction check"
+            )
+            return True
+
         # Read the CSV file with Polars
         df = pl.read_csv(
             filename, separator=separator, encoding=encoding, truncate_ragged_lines=True
@@ -456,10 +481,10 @@ def _auto_correct_field_types(  # noqa: C901
                     if corrections_needed:
                         break
 
-        # If no corrections needed, return None to use original file
+        # If no corrections needed, don't create a corrected file
         if not corrections_needed:
             log.info("No type corrections needed - using original CSV file")
-            return None
+            return True
 
         # Apply corrections using Polars casting
         log.info("Applying type corrections using Polars casting...")
@@ -512,15 +537,19 @@ def _auto_correct_field_types(  # noqa: C901
             corrected_file.name,
         )
 
-        return corrected_file.name
+        # Store the corrected file path in the import plan so the importer can use it
+        import_plan["_corrected_file"] = corrected_file.name
+
+        return True
 
     except Exception as e:
         log.warning(
             "Type validation and auto-correction failed, proceeding with original "
             "file: %s",
             e,
         )
-        return None
+        # Still return True to allow import to continue with original file
+        return True
 
 
 def _plan_deferrals_and_strategies(