Improve conjugation test (#6267)

Author: limakzi

lib/grp.gi

@@ -2955,6 +2955,11 @@ end);
 
 InstallMethod(IsConjugate,"subgroups",IsFamFamFam,[IsGroup, IsGroup,IsGroup],
 function(g,x,y)
+  # conjugate subgroups must have the same order
+  if HasSize(x) and HasSize(y) and Size(x) <> Size(y) then
+    return false;
+  fi;
+
   # shortcut for normal subgroups
   if (HasIsNormalInParent(x) and IsNormalInParent(x)
       and CanComputeIsSubset(Parent(x),g) and IsSubset(Parent(x),g))

lib/oprtperm.gi

@@ -1374,6 +1374,16 @@ InstallOtherMethod( RepresentativeActionOp, "permgrp",true, [ IsPermGroup,
     # action on permgroups, use backtrack
     elif act = OnPoints and IsPermGroup( d ) and IsPermGroup( e )  then
 
+      # conjugate subgroups must have the same order
+      if HasSize(d) and HasSize(e) and Size(d) <> Size(e) then
+        return fail;
+      fi;
+
+      # conjugate subgroups must be both abelian or both non-abelian
+      if HasIsAbelian(d) and HasIsAbelian(e) and IsAbelian(d) <> IsAbelian(e) then
+        return fail;
+      fi;
+
       if Size(G)<10^5 or NrMovedPoints(G)<500
         # cyclic is handled special by backtrack
         or IsCyclic(d) or IsCyclic(e) or

tst/testinstall/ConjNatSym.tst

@@ -5,6 +5,17 @@ true
 gap> IsConjugate( SymmetricGroup(5), Group((1,2)), Group((3,4,5)));
 false
 
+# Subgroups with different known orders in a non-natural symmetric group
+# (exercises the RepresentativeActionOp shortcut, not the natural sym method)
+gap> G:= SymmetricGroup( 1000 );;
+gap> H:= Subgroup( G, [ G.1 ] );; Size( H );
+1000
+gap> K:= Subgroup( G, [ G.2 ] );;  Size( K );
+2
+gap> G:= Group( GeneratorsOfGroup( G ) );;
+gap> IsConjugate( G, H, K );
+false
+
 # This runs into the TryNextMethod case
 gap> IsConjugate( SymmetricGroup(200),SymmetricGroup(200), AlternatingGroup(200));
 false