Updated engine_kwargs extraction logic and added test

Author: arthurlw

pandas/core/apply.py

@@ -49,6 +49,7 @@
 import pandas.core.common as com
 from pandas.core.construction import ensure_wrapped_if_datetimelike
 from pandas.core.util.numba_ import (
+    extract_numba_options,
     get_jit_arguments,
     prepare_function_arguments,
 )
@@ -224,9 +225,9 @@ def apply(
         if not isinstance(data, np.ndarray):
             if data.empty:
                 return data.copy()  # mimic apply_empty_result()
-            engine_kwargs = (
-                decorator.engine_kwargs if hasattr(decorator, "engine_kwargs") else {}
-            )
+
+            engine_kwargs = extract_numba_options(decorator)
+
             NumbaExecutionEngine.validate_values_for_numba_raw_false(
                 data, get_jit_arguments(engine_kwargs)
             )

pandas/core/frame.py

@@ -10623,7 +10623,6 @@ def apply(
             numba = import_optional_dependency("numba")
             numba_jit = numba.jit(**engine_kwargs or {})
             numba_jit.__pandas_udf__ = NumbaExecutionEngine
-            numba_jit.engine_kwargs = engine_kwargs
             engine = numba_jit
 
         if engine is None or isinstance(engine, str):

pandas/core/util/numba_.py

@@ -148,3 +148,24 @@ def prepare_function_arguments(
 
     args = args[num_required_args:]
     return args, kwargs
+
+
+def extract_numba_options(decorator):
+    """
+    Extract targetoptions from a numba.jit decorator
+    """
+    try:
+        closure = decorator.__closure__
+        if closure is None:
+            return {}
+        freevars = decorator.__code__.co_freevars
+        if "targetoptions" not in freevars:
+            return {}
+        idx = freevars.index("targetoptions")
+        cell = closure[idx]
+        targetoptions = cell.cell_contents
+        if isinstance(targetoptions, dict):
+            return targetoptions
+        return {}
+    except Exception:
+        return {}

pandas/tests/apply/test_numba.py

@@ -2,6 +2,7 @@
 import pytest
 
 from pandas.compat import is_platform_arm
+from pandas.core.util.numba_ import extract_numba_options
 import pandas.util._test_decorators as td
 
 import pandas as pd
@@ -127,3 +128,13 @@ def test_numba_unsupported_dtypes(apply_axis):
         "which is not supported by the numba engine.",
     ):
         df["c"].to_frame().apply(f, engine="numba", axis=apply_axis)
+
+
+@pytest.mark.parametrize("jit_args", [
+    {"parallel": True, "nogil": True},
+    {"parallel": False, "nogil": False},
+])
+def test_extract_numba_options_from_user_decorated_function(jit_args):
+    extracted = extract_numba_options(numba.jit(**jit_args))
+    for k, v in jit_args.items():
+        assert extracted.get(k) == v