Fix Comm for group automorphisms (#5906)

This used to run into a coset enumeration.

We increase the rank of certain methods for composite maps
to resolve the issue and possibly others.

Author: fingolfin

lib/mapprep.gi

@@ -153,7 +153,7 @@ InstallMethod( CompositionMapping2,
 InstallMethod( IsInjective,
     "for a composition mapping",
     true,
-    [ IsCompositionMappingRep ], 0,
+    [ IsCompositionMappingRep ], SUM_FLAGS,
     function( com )
     if IsInjective( com!.map1 ) and IsInjective( com!.map2 ) then
       return true;
@@ -176,7 +176,7 @@ InstallMethod( IsInjective,
 InstallMethod( IsSingleValued,
     "for a composition mapping",
     true,
-    [ IsCompositionMappingRep ], 0,
+    [ IsCompositionMappingRep ], SUM_FLAGS,
     function( com )
     if IsSingleValued( com!.map1 ) and IsSingleValued( com!.map2 ) then
       return true;
@@ -199,7 +199,7 @@ InstallMethod( IsSingleValued,
 InstallMethod( IsSurjective,
     "for a composition mapping",
     true,
-    [ IsCompositionMappingRep ], 0,
+    [ IsCompositionMappingRep ], SUM_FLAGS,
     function( com )
     if   not IsSurjective( com!.map2 ) then
       return false;
@@ -218,7 +218,7 @@ InstallMethod( IsSurjective,
 InstallMethod( IsTotal,
     "for a composition mapping",
     true,
-    [ IsCompositionMappingRep ], 0,
+    [ IsCompositionMappingRep ], SUM_FLAGS,
     function( com )
     if not IsTotal( com!.map1 ) then
       return false;

tst/testbugfix/2025-01-16-Comm-for-group-automorphisms.tst

@@ -0,0 +1,18 @@
+#@local F,x1,x2,x3,x4,A,u,v,hom
+# The 'IsBijective' and 'Comm's call below used to trigger coset enumeration.
+# See issue https://github.com/gap-system/gap/issues/3898 and also
+# https://github.com/gap-system/gap/issues/5910
+gap> F:=FreeGroup("x1","x2","x3","x4");;
+gap> x1:=F.1;;x2:=F.2;;x3:=F.3;;x4:=F.4;;
+gap> A:=GroupHomomorphismByImages(F,F,[x1,x2,x3,x4],[x1,x2,x2*x3^-1*x2,x4]);;
+gap> u:=(A*A)^A;;
+gap> v:=(A*A)^A;;
+gap> IsBijective(u*v);  # used to hang
+true
+gap> (u*v)(x1);  # verify that computing images works
+x1
+gap> hom:=Comm(u,v);;  # used to hang
+gap> IsGroupHomomorphism(hom);
+true
+gap> hom(x1);  # verify that computing images works
+x1