roachtest: adding defensive code in ceph/reef test

We have seen sporadic failures in the ceph tests, due to failures
in creating users in the ceph object gateway.

To address this we are adding code to check that the gateway is
up by submitting a read only request, before attempting to
add the user.

Epic: none

Fixes: #148731

Release note: None

Author: sravotto

pkg/cmd/roachtest/tests/s3_microceph.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"net/url"
 	"path/filepath"
+	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/cloud/amazon"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
@@ -124,7 +125,9 @@ func (m cephManager) install(ctx context.Context) {
 		rgwCmd = rgwCmd + ` --ssl-certificate="$(base64 -w0 certs/node.crt)" --ssl-private-key="$(base64 -w0 certs/node.key)"`
 	}
 	m.run(ctx, `starting object gateway`, rgwCmd)
-
+	// We have seen occasional failures in creating users, so we
+	// wait until a read only request succeeds before proceeding.
+	m.checkRGW(ctx)
 	m.run(ctx, `creating backup user`,
 		`sudo radosgw-admin user create --uid=backup --display-name=backup`)
 	m.run(ctx, `add keys to the user`,
@@ -166,3 +169,18 @@ func (m cephManager) run(ctx context.Context, msg string, cmd ...string) {
 	m.c.Run(ctx, option.WithNodes(m.cephNodes), cmd...)
 	m.t.Status(msg, " done")
 }
+
+// checkRGW verifies that the Ceph Object Gateway is up.
+func (m cephManager) checkRGW(ctx context.Context) {
+	m.t.Status("waiting for Ceph Object Gateway...")
+	cmd := `sudo radosgw-admin user list`
+	var err error
+	for i := 0; i < 10; i++ {
+		// Sleep for few seconds, then try the command.
+		time.Sleep(2 * time.Second)
+		if err = m.c.RunE(ctx, option.WithNodes(m.cephNodes), cmd); err == nil {
+			return
+		}
+	}
+	m.t.Error("Ceph Object Gateway not running", err)
+}