release-25.1: roachtest: adding defensive code in ceph/reef test

pkg/cmd/roachtest/tests/s3_microceph.go

@@ -12,11 +12,14 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/cloud/amazon"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test"
+	"github.com/cockroachdb/cockroach/pkg/roachprod/install"
+	"github.com/cockroachdb/cockroach/pkg/util/retry"
 )
 
 // cephDisksScript creates 3 4GB loop devices, e.g. virtual block devices that allows
@@ -126,7 +129,9 @@ func (m cephManager) install(ctx context.Context) {
 		rgwCmd = rgwCmd + ` --ssl-certificate="$(base64 -w0 certs/node.crt)" --ssl-private-key="$(base64 -w0 certs/node.key)"`
 	}
 	m.run(ctx, `starting object gateway`, rgwCmd)
-
+	// We have seen occasional failures in creating users, so we
+	// wait until a read only request succeeds before proceeding.
+	m.checkRGW(ctx)
 	m.run(ctx, `creating backup user`,
 		`sudo radosgw-admin user create --uid=backup --display-name=backup`)
 	m.run(ctx, `add keys to the user`,
@@ -190,3 +195,20 @@ func (m cephManager) run(ctx context.Context, msg string, cmd ...string) {
 	m.c.Run(ctx, option.WithNodes(m.cephNodes), cmd...)
 	m.t.Status(msg, " done")
 }
+
+// checkRGW verifies that the Ceph Object Gateway is up.
+func (m cephManager) checkRGW(ctx context.Context) {
+	m.t.Status("waiting for Ceph Object Gateway...")
+	if err := m.c.RunE(ctx,
+		option.WithNodes(m.cephNodes).
+			WithRetryOpts(retry.Options{
+				InitialBackoff: 2 * time.Second,
+				MaxBackoff:     30 * time.Second,
+				MaxRetries:     10,
+			}).
+			WithShouldRetryFn(func(*install.RunResultDetails) bool { return true }),
+		`sudo radosgw-admin user list`,
+	); err != nil {
+		m.t.Error("failed to connect to Ceph Object Gateway", err)
+	}
+}