Skip to content

Commit a9ddf04

Browse files
committed
Fix GAP libgap segfault in function calls with >3 arguments
This fixes an inconsistent behavior where libgap function calls with more than 3 arguments would sometimes return normal GAP errors and sometimes cause segmentation faults. The root cause was nested GAP_Enter() calls: the main function call used sig_GAP_Enter(), and then make_gap_list() called GAP_Enter() again, causing race conditions in GAP's memory management. The fix replaces GAP_CallFuncList() with GAP_CallFuncArray() and uses C malloc/free for temporary argument arrays instead of creating GAP list objects, eliminating the nested GAP memory management calls. This ensures consistent error handling - invalid calls now always return proper GAP error messages instead of sometimes segfaulting. Fixes: Inconsistent libgap.Sum(*[1,2,3]) behavior (segfault vs GAPError)
1 parent 85c8f1e commit a9ddf04

File tree

1 file changed

+15
-4
lines changed

1 file changed

+15
-4
lines changed

src/sage/libs/gap/element.pyx

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ from sage.cpython.string cimport str_to_bytes, char_to_str
2828
from sage.rings.integer_ring import ZZ
2929
from sage.rings.rational_field import QQ
3030
from sage.rings.real_double import RDF
31+
from libc.stdlib cimport malloc, free
3132

3233
from sage.groups.perm_gps.permgroup_element cimport PermutationGroupElement
3334
from sage.combinat.permutation import Permutation
@@ -2496,11 +2497,12 @@ cdef class GapElement_Function(GapElement):
24962497
hello from the shell
24972498
"""
24982499
cdef Obj result = NULL
2499-
cdef Obj arg_list
25002500
cdef int n = len(args)
25012501
cdef volatile Obj v2
2502+
cdef Obj *arg_array = NULL
2503+
cdef int i
25022504

2503-
if n > 0 and n <= 3:
2505+
if n > 0:
25042506
libgap = self.parent()
25052507
a = [x if isinstance(x, GapElement) else libgap(x) for x in args]
25062508

@@ -2523,8 +2525,17 @@ cdef class GapElement_Function(GapElement):
25232525
(<GapElement>a[1]).value,
25242526
v2)
25252527
else:
2526-
arg_list = make_gap_list(args)
2527-
result = GAP_CallFuncList(self.value, arg_list)
2528+
# Use GAP_CallFuncArray instead of GAP_CallFuncList
2529+
# to avoid creating GAP list objects and nested GAP_Enter calls
2530+
arg_array = <Obj*>malloc(n * sizeof(Obj))
2531+
if arg_array == NULL:
2532+
raise MemoryError("Failed to allocate memory for GAP function arguments")
2533+
2534+
for i in range(n):
2535+
arg_array[i] = (<GapElement>a[i]).value
2536+
2537+
result = GAP_CallFuncArray(self.value, n, arg_array)
2538+
free(arg_array)
25282539
sig_off()
25292540
finally:
25302541
GAP_Leave()

0 commit comments

Comments
 (0)