[ENH]: Verify fields before import

Author: bosd

docs/guides/importing_data.md

@@ -20,6 +20,18 @@ odoo-data-flow import --file path/to/res_partner.csv
 * `--skip`: The number of initial lines to skip in the source file before reading the header.
 * `--sep`: The character separating columns. Defaults to a semicolon (`;`).
 
+### Verifying Fields Before Import (`--verify-fields`)
+
+To prevent common errors, you can add the `--verify-fields` flag to your import command. This is a "pre-flight check" that connects to Odoo and verifies that every column in your CSV header exists as a field on the target model before the import begins.
+
+This is highly recommended as it allows you to "fail fast" with a clear error message, rather than waiting for a large import to fail on a single typo in a column name.
+
+**Example Usage:**
+```bash
+odoo-data-flow import --file path/to/my_data.csv --model res.partner --verify-fields
+```
+If `my_data.csv` contains a column that does not exist on the `res.partner` model, the command will abort with an error message listing the invalid fields.
+
 ## The "Upsert" Strategy: How External IDs Work
 
 A core feature of `odoo-data-flow` is its ability to safely handle both creating new records and updating existing ones in a single process. This is often called an "upsert" (update or insert) operation, and it is the default behavior of the tool.

src/odoo_data_flow/__main__.py

@@ -125,15 +125,13 @@ def install_languages_cmd(config: str, languages_str: str) -> None:
 
 
 # --- Workflow Command Group ---
-# This defines 'workflow' as a subcommand of 'cli'.
 @cli.group(name="workflow")
 def workflow_group() -> None:
     """Run legacy or complex post-import processing workflows."""
     pass
 
 
 # --- Invoice v9 Workflow Sub-command ---
-# This command is now correctly nested under the 'workflow' group.
 @workflow_group.command(name="invoice-v9")
 @click.option(
     "-c",
@@ -185,7 +183,6 @@ def invoice_v9_cmd(**kwargs: Any) -> None:
 
 
 # --- Import Command ---
-# This command is attached directly to the main 'cli' group.
 @cli.command(name="import")
 @click.option(
     "-c",
@@ -200,6 +197,13 @@ def invoice_v9_cmd(**kwargs: Any) -> None:
     default=None,
     help="Odoo model to import into. If not provided, it's inferred from the filename.",
 )
+@click.option(
+    "--verify-fields",
+    is_flag=True,
+    default=False,
+    help="Connect to Odoo and verify that all CSV columns "
+    "exist on the model before importing.",
+)
 @click.option(
     "--worker", default=1, type=int, help="Number of simultaneous connections."
 )

src/odoo_data_flow/importer.py

@@ -1,18 +1,73 @@
 """This module contains the core logic for importing data into Odoo."""
 
 import ast
+import csv
 import os
 from datetime import datetime
 from typing import Any, Optional
 
 from . import import_threaded
+from .lib import conf_lib
 from .logging_config import log
 
 
+def _verify_import_fields(
+    config: str, model: str, filename: str, separator: str, encoding: str
+) -> bool:
+    """Verify import fields.
+
+    Connects to Odoo and verifies that all columns in the CSV header
+    exist as fields on the target model.
+    """
+    log.info(f"Performing pre-flight check: Verifying fields for model '{model}'...")
+
+    # Step 1: Read the header from the local CSV file
+    try:
+        with open(filename, encoding=encoding) as f:
+            reader = csv.reader(f, delimiter=separator)
+            csv_header = next(reader)
+    except FileNotFoundError:
+        log.error(f"Pre-flight Check Failed: Input file not found at {filename}")
+        return False
+    except Exception as e:
+        log.error(f"Pre-flight Check Failed: Could not read CSV header. Error: {e}")
+        return False
+
+    # Step 2: Get the list of valid fields from the Odoo model
+    try:
+        connection: Any = conf_lib.get_connection_from_config(config_file=config)
+        model_fields_obj = connection.get_model("ir.model.fields")
+        domain = [("model", "=", model)]
+        odoo_fields_data = model_fields_obj.search_read(domain, ["name"])
+        odoo_field_names = {field["name"] for field in odoo_fields_data}
+    except Exception as e:
+        log.error(
+            f"Pre-flight Check Failed: "
+            f"Could not connect to Odoo to get model fields. Error: {e}"
+        )
+        return False
+
+    # Step 3: Compare the lists and find any missing fields
+    missing_fields = [field for field in csv_header if field not in odoo_field_names]
+
+    if missing_fields:
+        log.error(
+            "Pre-flight Check Failed: The following columns in your CSV file "
+            "do not exist on the Odoo model:"
+        )
+        for field in missing_fields:
+            log.error(f"  - '{field}' is not a valid field on model '{model}'")
+        return False
+
+    log.info("Pre-flight Check Successful: All columns are valid fields on the model.")
+    return True
+
+
 def run_import(
     config: str,
     filename: str,
     model: Optional[str] = None,
+    verify_fields: bool = False,
     worker: int = 1,
     batch_size: int = 10,
     skip: int = 0,
@@ -32,6 +87,8 @@ def run_import(
         filename: Path to the source CSV file to import.
         model: The Odoo model to import data into. If not provided, it's inferred
                from the filename.
+        verify_fields: If True, connects to Odoo to verify all CSV columns
+                       exist on the model before starting the import.
         worker: The number of simultaneous connections to use.
         batch_size: The number of records to process in each batch.
         skip: The number of initial lines to skip in the source file.
@@ -50,7 +107,6 @@ def run_import(
     if not final_model:
         base_name = os.path.basename(filename)
         inferred_model = os.path.splitext(base_name)[0].replace("_", ".")
-        # Add a check for invalid inferred names (like hidden files)
         if not inferred_model or inferred_model.startswith("."):
             log.error(
                 "Model not specified and could not be inferred from filename "
@@ -60,6 +116,14 @@ def run_import(
         final_model = inferred_model
         log.info(f"No model provided. Inferred model '{final_model}' from filename.")
 
+    # --- Pre-flight Check ---
+    if verify_fields:
+        if not _verify_import_fields(
+            config, final_model, filename, separator, encoding
+        ):
+            log.error("Aborting import due to failed pre-flight check.")
+            return
+
     try:
         parsed_context = ast.literal_eval(context)
         if not isinstance(parsed_context, dict):
@@ -73,7 +137,6 @@ def run_import(
     ignore_list = ignore.split(",") if ignore else []
 
     file_dir = os.path.dirname(filename)
-
     file_to_process: str
     fail_output_file: str
     is_fail_run: bool

tests/test_importer.py

@@ -1,4 +1,4 @@
-"""Test the high-level import orchestrator."""
+"""Test the high-level import orchestrator, including pre-flight checks."""
 
 from pathlib import Path
 from unittest.mock import MagicMock, patch
@@ -140,3 +140,123 @@ def test_run_import_for_migration(mock_import_data: MagicMock) -> None:
     assert call_kwargs["max_connection"] == 2
     assert call_kwargs["batch_size"] == 50
     assert "tracking_disable" in call_kwargs["context"]
+
+
+@patch("odoo_data_flow.importer.import_threaded.import_data")
+@patch("odoo_data_flow.importer._verify_import_fields")
+def test_run_import_skips_verification_by_default(
+    mock_verify_fields: MagicMock, mock_import_data: MagicMock, tmp_path: Path
+) -> None:
+    """Tests that the field verification step is NOT run if the flag is omitted."""
+    source_file = tmp_path / "data.csv"
+    source_file.write_text("id,name\n1,test")
+
+    run_import(
+        config="dummy.conf",
+        filename=str(source_file),
+        model="res.partner",
+        verify_fields=False,  # Explicitly False
+    )
+
+    mock_verify_fields.assert_not_called()
+    mock_import_data.assert_called_once()
+
+
+@patch("odoo_data_flow.importer.import_threaded.import_data")
+@patch("odoo_data_flow.importer.conf_lib.get_connection_from_config")
+def test_verify_fields_success(
+    mock_get_connection: MagicMock, mock_import_data: MagicMock, tmp_path: Path
+) -> None:
+    """Tests the success path where all CSV columns exist on the Odoo model."""
+    # 1. Setup
+    source_file = tmp_path / "data.csv"
+    source_file.write_text("id,name,email")  # Header for the file to be read
+
+    # Mock the Odoo model object and its return value
+    mock_model_fields_obj = MagicMock()
+    # Simulate Odoo returning a list of valid fields
+    mock_model_fields_obj.search_read.return_value = [
+        {"name": "id"},
+        {"name": "name"},
+        {"name": "email"},
+    ]
+    mock_connection = MagicMock()
+    mock_connection.get_model.return_value = mock_model_fields_obj
+    mock_get_connection.return_value = mock_connection
+
+    # 2. Action
+    run_import(
+        config="dummy.conf",
+        filename=str(source_file),
+        model="res.partner",
+        verify_fields=True,
+        separator=",",  # Use the correct separator for the test file
+    )
+
+    # 3. Assertions
+    # The verification should pass, and the main import function should be called
+    mock_import_data.assert_called_once()
+
+
+@patch("odoo_data_flow.importer.import_threaded.import_data")
+@patch("odoo_data_flow.importer.log.error")
+@patch("odoo_data_flow.importer.conf_lib.get_connection_from_config")
+def test_verify_fields_failure_missing_field(
+    mock_get_connection: MagicMock,
+    mock_log_error: MagicMock,
+    mock_import_data: MagicMock,
+    tmp_path: Path,
+) -> None:
+    """Tests the failure path where a CSV column does not exist on the Odoo model."""
+    # 1. Setup
+    source_file = tmp_path / "data.csv"
+    # This file contains a column that is not on the mocked model below
+    source_file.write_text("id,name,x_studio_legacy_field")
+
+    mock_model_fields_obj = MagicMock()
+    # Simulate Odoo returning only two valid fields
+    mock_model_fields_obj.search_read.return_value = [
+        {"name": "id"},
+        {"name": "name"},
+    ]
+    mock_connection = MagicMock()
+    mock_connection.get_model.return_value = mock_model_fields_obj
+    mock_get_connection.return_value = mock_connection
+
+    # 2. Action
+    run_import(
+        config="dummy.conf",
+        filename=str(source_file),
+        model="res.partner",
+        verify_fields=True,
+        separator=",",
+    )
+
+    # 3. Assertions
+    # An error should be logged, and the main import should NOT be called
+    assert mock_log_error.call_count > 0
+    # Check that the specific error message was one of the logs
+    assert any(
+        "is not a valid field" in call[0][0] for call in mock_log_error.call_args_list
+    )
+    mock_import_data.assert_not_called()
+
+
+@patch("odoo_data_flow.importer.import_threaded.import_data")
+@patch("odoo_data_flow.importer.log.error")
+def test_verify_fields_failure_file_not_found(
+    mock_log_error: MagicMock, mock_import_data: MagicMock
+) -> None:
+    """Tests that verification fails gracefully if the source file is not found."""
+    run_import(
+        config="dummy.conf",
+        filename="non_existent_file.csv",
+        model="res.partner",
+        verify_fields=True,
+    )
+
+    # Assert that the specific error message was logged
+    assert any(
+        "Input file not found" in call[0][0] for call in mock_log_error.call_args_list
+    )
+    mock_import_data.assert_not_called()