From 661187414f4f230f66a4c48b44132b04b48f136d Mon Sep 17 00:00:00 2001
From: Max Horn <max@quendi.de>
Date: Tue, 21 Jan 2025 22:42:45 +0100
Subject: [PATCH] Fix Comm for group automorphisms (#5906)

This used to run into a coset enumeration.

We increase the rank of certain methods for composite maps
to resolve the issue and possibly others.
---
 lib/mapprep.gi                                 |  8 ++++----
 ...2025-01-16-Comm-for-group-automorphisms.tst | 18 ++++++++++++++++++
 2 files changed, 22 insertions(+), 4 deletions(-)
 create mode 100644 tst/testbugfix/2025-01-16-Comm-for-group-automorphisms.tst

diff --git a/lib/mapprep.gi b/lib/mapprep.gi
index d812ba11f4..10739a6c18 100644
--- a/lib/mapprep.gi
+++ b/lib/mapprep.gi
@@ -153,7 +153,7 @@ InstallMethod( CompositionMapping2,
 InstallMethod( IsInjective,
     "for a composition mapping",
     true,
-    [ IsCompositionMappingRep ], 0,
+    [ IsCompositionMappingRep ], SUM_FLAGS,
     function( com )
     if IsInjective( com!.map1 ) and IsInjective( com!.map2 ) then
       return true;
@@ -176,7 +176,7 @@ InstallMethod( IsInjective,
 InstallMethod( IsSingleValued,
     "for a composition mapping",
     true,
-    [ IsCompositionMappingRep ], 0,
+    [ IsCompositionMappingRep ], SUM_FLAGS,
     function( com )
     if IsSingleValued( com!.map1 ) and IsSingleValued( com!.map2 ) then
       return true;
@@ -199,7 +199,7 @@ InstallMethod( IsSingleValued,
 InstallMethod( IsSurjective,
     "for a composition mapping",
     true,
-    [ IsCompositionMappingRep ], 0,
+    [ IsCompositionMappingRep ], SUM_FLAGS,
     function( com )
     if   not IsSurjective( com!.map2 ) then
       return false;
@@ -218,7 +218,7 @@ InstallMethod( IsSurjective,
 InstallMethod( IsTotal,
     "for a composition mapping",
     true,
-    [ IsCompositionMappingRep ], 0,
+    [ IsCompositionMappingRep ], SUM_FLAGS,
     function( com )
     if not IsTotal( com!.map1 ) then
       return false;
diff --git a/tst/testbugfix/2025-01-16-Comm-for-group-automorphisms.tst b/tst/testbugfix/2025-01-16-Comm-for-group-automorphisms.tst
new file mode 100644
index 0000000000..e85999578b
--- /dev/null
+++ b/tst/testbugfix/2025-01-16-Comm-for-group-automorphisms.tst
@@ -0,0 +1,18 @@
+#@local F,x1,x2,x3,x4,A,u,v,hom
+# The 'IsBijective' and 'Comm's call below used to trigger coset enumeration.
+# See issue https://github.com/gap-system/gap/issues/3898 and also
+# https://github.com/gap-system/gap/issues/5910
+gap> F:=FreeGroup("x1","x2","x3","x4");;
+gap> x1:=F.1;;x2:=F.2;;x3:=F.3;;x4:=F.4;;
+gap> A:=GroupHomomorphismByImages(F,F,[x1,x2,x3,x4],[x1,x2,x2*x3^-1*x2,x4]);;
+gap> u:=(A*A)^A;;
+gap> v:=(A*A)^A;;
+gap> IsBijective(u*v);  # used to hang
+true
+gap> (u*v)(x1);  # verify that computing images works
+x1
+gap> hom:=Comm(u,v);;  # used to hang
+gap> IsGroupHomomorphism(hom);
+true
+gap> hom(x1);  # verify that computing images works
+x1