Trac #30797: Bug in modular sqrt when extend is False and all is True

{{{
sage: GF(103)(-1).sqrt(extend=False, all=True)
...
ValueError: self must be a square
}}}

It should return [] instead.

URL: https://trac.sagemath.org/30797
Reported by: gh-jtnydv25
Ticket author(s): Frédéric Chapoton
Reviewer(s): Travis Scrimshaw

Author: Release Manager

src/sage/rings/finite_rings/integer_mod.pyx

@@ -1047,12 +1047,10 @@ cdef class IntegerMod_abstract(FiniteRingElement):
 
     def sqrt(self, extend=True, all=False):
         r"""
-        Returns square root or square roots of ``self`` modulo
-        `n`.
+        Return square root or square roots of ``self`` modulo `n`.
 
         INPUT:
 
-
         -  ``extend`` - bool (default: ``True``);
            if ``True``, return a square root in an extension ring,
            if necessary. Otherwise, raise a ``ValueError`` if the
@@ -1062,7 +1060,6 @@ cdef class IntegerMod_abstract(FiniteRingElement):
            ``True``, return {all} square roots of self, instead of
            just one.
 
-
         ALGORITHM: Calculates the square roots mod `p` for each of
         the primes `p` dividing the order of the ring, then lifts
         them `p`-adically and uses the CRT to find a square root
@@ -1163,7 +1160,7 @@ cdef class IntegerMod_abstract(FiniteRingElement):
 
         if not self.is_square_c():
             if extend:
-                y = 'sqrt%s'%self
+                y = 'sqrt%s' % self
                 R = self.parent()['x']
                 modulus = R.gen()**2 - R(self)
                 if self._parent.is_field():
@@ -2796,12 +2793,10 @@ cdef class IntegerMod_int(IntegerMod_abstract):
 
     def sqrt(self, extend=True, all=False):
         r"""
-        Returns square root or square roots of ``self`` modulo
-        `n`.
+        Return square root or square roots of ``self`` modulo `n`.
 
         INPUT:
 
-
         -  ``extend`` - bool (default: ``True``);
            if ``True``, return a square root in an extension ring,
            if necessary. Otherwise, raise a ``ValueError`` if the
@@ -2811,7 +2806,6 @@ cdef class IntegerMod_int(IntegerMod_abstract):
            ``True``, return {all} square roots of self, instead of
            just one.
 
-
         ALGORITHM: Calculates the square roots mod `p` for each of
         the primes `p` dividing the order of the ring, then lifts
         them `p`-adically and uses the CRT to find a square root
@@ -2896,6 +2890,13 @@ cdef class IntegerMod_int(IntegerMod_abstract):
             [23, 41, 87, 105]
             sage: [x for x in R if x^2==17]
             [23, 41, 87, 105]
+
+        TESTS:
+
+        Check for :trac:`30797`::
+
+            sage: GF(103)(-1).sqrt(extend=False, all=True)
+            []
         """
         cdef int_fast32_t i, n = self.__modulus.int32
         if n > 100:
@@ -2906,15 +2907,17 @@ cdef class IntegerMod_int(IntegerMod_abstract):
             if jacobi_int(self.ivalue, self.__modulus.int32) == 1:
                 # it's a non-zero square, sqrt(a) = a^(p+1)/4
                 i = mod_pow_int(self.ivalue, (self.__modulus.int32+1)/4, n)
-                if i > n/2:
-                    i = n-i
+                if i > n / 2:
+                    i = n - i
                 if all:
                     return [self._new_c(i), self._new_c(n-i)]
                 else:
                     return self._new_c(i)
             elif self.ivalue == 0:
                 return [self] if all else self
             elif not extend:
+                if all:
+                    return []
                 raise ValueError("self must be a square")
         # Now we use a heuristic to guess whether or not it will
         # be faster to just brute-force search for squares in a c loop...