Use local rekor and fulcio instances in e2e tests (#3478)

* Clean up e2e test script

- Use -v for clearer output
- All the tests tagged 'e2e' are in ./test, no need to search the whole
  tree
- There's no third_party directory, no need to grep it out
- Clean up services no matter when the script exits

Signed-off-by: Colleen Murphy <[email protected]>

* test: Clean up cosign-secret

Clean up the key pair secret after the test so that the test suite can
be re-run locally if desired. Without this, the secret is set to
'immutable' and can't be overwritten by the next test run.

Signed-off-by: Colleen Murphy <[email protected]>

* Use local rekor and fulcio instances in e2e tests

In 7068357 the e2e tests moved from running on the locally-spun-up
rekor instance to the public instance. This means test signatures are
piling up in the public instance, and the tests may be taking longer
than they need to since they are using an external service.

This change moves back to using the local rekor instance, which the e2e
has still been spinning up even though it has been going unused. Also
now do the same for fulcio.

Signed-off-by: Colleen Murphy <[email protected]>

---------

Signed-off-by: Colleen Murphy <[email protected]>

test/e2e_test.go

@@ -77,8 +77,8 @@ import (
 
 const (
 	serverEnv = "REKOR_SERVER"
-	rekorURL  = "https://rekor.sigstore.dev"
-	fulcioURL = "https://fulcio.sigstore.dev"
+	rekorURL  = "http://127.0.0.1:3000"
+	fulcioURL = "http://127.0.0.1:5555"
 )
 
 var keyPass = []byte("hello")
@@ -1324,6 +1324,11 @@ func TestGenerateKeyPairK8s(t *testing.T) {
 	if v, ok := s.Data["cosign.password"]; !ok || string(v) != password {
 		t.Fatalf("password is incorrect, got %v expected %v", v, "foo")
 	}
+	// Clean up the secret (so tests can be re-run locally)
+	err = client.CoreV1().Secrets(namespace).Delete(ctx, name, metav1.DeleteOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestMultipleSignatures(t *testing.T) {

test/e2e_test.sh

@@ -16,41 +16,58 @@
 
 set -ex
 
-echo "copying rekor repo"
 pushd $HOME
-if [[ ! -d rekor ]]; then
-   git clone https://github.com/sigstore/rekor.git
-else
-   pushd rekor
-   git pull
-   popd
-fi
-cd rekor
 
-echo "starting services"
-docker-compose up -d
-
-count=0
-
-echo -n "waiting up to 60 sec for system to start"
-until [ $(docker-compose ps | grep -c "(healthy)") == 3 ];
-do
-    if [ $count -eq 6 ]; then
-       echo "! timeout reached"
-       exit 1
+echo "downloading service repos"
+for repo in rekor fulcio; do
+    if [[ ! -d $repo ]]; then
+        git clone https://github.com/sigstore/${repo}.git
     else
-       echo -n "."
-       sleep 10
-       let 'count+=1'
+        pushd $repo
+        git pull
+        popd
     fi
 done
 
+echo "starting services"
+export FULCIO_METRICS_PORT=2113
+for repo in rekor fulcio; do
+    pushd $repo
+    docker-compose up -d
+    echo -n "waiting up to 60 sec for system to start"
+    count=0
+    until [ $(docker-compose ps | grep -c "(healthy)") == 3 ];
+    do
+        if [ $count -eq 6 ]; then
+           echo "! timeout reached"
+           exit 1
+        else
+           echo -n "."
+           sleep 10
+           let 'count+=1'
+        fi
+    done
+    popd
+done
+cleanup_services() {
+    echo "cleaning up"
+    for repo in rekor fulcio; do
+        pushd $HOME/$repo
+        docker-compose down
+        popd
+    done
+}
+trap cleanup_services EXIT
+
+curl http://127.0.0.1:3000/api/v1/log/publicKey > rekor.pub
+export SIGSTORE_REKOR_PUBLIC_KEY=$(pwd)/rekor.pub
+
 echo
 echo "running tests"
 
 popd
 go build -o cosign ./cmd/cosign
-go test -tags=e2e -race $(go list ./... | grep -v third_party/)
+go test -tags=e2e -v -race ./test/...
 
 # Test on a private registry
 echo "testing sign/verify/clean on private registry"
@@ -62,6 +79,8 @@ docker run -d -p 5000:5000 --restart always -e REGISTRY_STORAGE_DELETE_ENABLED=t
 export COSIGN_TEST_REPO=localhost:5000
 go test -tags=e2e -v ./test/... -run TestSignVerifyClean
 
+# Use the public instance to verify existing images and manifests
+unset SIGSTORE_REKOR_PUBLIC_KEY
 # Test `cosign dockerfile verify`
 ./cosign dockerfile verify ./test/testdata/single_stage.Dockerfile --certificate-identity https://github.com/distroless/alpine-base/.github/workflows/release.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com
 if (./cosign dockerfile verify ./test/testdata/unsigned_build_stage.Dockerfile --certificate-identity https://github.com/distroless/alpine-base/.github/workflows/release.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com); then false; fi
@@ -80,7 +99,3 @@ if (./cosign manifest verify ./test/testdata/unsigned_manifest.yaml --certificat
 make ko-local
 img="ko.local/cosign:$(git rev-parse HEAD)"
 docker run $img version
-
-echo "cleanup"
-cd $HOME/rekor
-docker-compose down