TMT: add more tests

Signed-off-by: Lokesh Mandvekar <[email protected]>

Author: lsm5

.packit.yaml

@@ -21,7 +21,7 @@ files_to_sync:
     dest: plans/
     delete: true
     mkpath: true
-  - src: systemtest/tmt/
+  - src: tmt/
     dest: test/tmt/
     delete: true
     mkpath: true
@@ -47,9 +47,9 @@ jobs:
   - job: copr_build
     trigger: pull_request
     packages: [skopeo-fedora]
-    notifications: &copr_build_failure_notification
+    notifications: &packit_failure_notification
       failure_comment:
-        message: "Ephemeral COPR build failed. @containers/packit-build please check."
+        message: "Packit jobs failed. @containers/packit-build please check."
     targets: &fedora_copr_targets
       - fedora-all-x86_64
       - fedora-all-aarch64
@@ -62,7 +62,7 @@ jobs:
   - job: copr_build
     trigger: ignore
     packages: [skopeo-eln]
-    notifications: *copr_build_failure_notification
+    notifications: *packit_failure_notification
     targets:
       fedora-eln-x86_64:
         additional_repos:
@@ -76,7 +76,7 @@ jobs:
   - job: copr_build
     trigger: ignore
     packages: [skopeo-centos]
-    notifications: *copr_build_failure_notification
+    notifications: *packit_failure_notification
     targets: &centos_copr_targets
       - centos-stream-9-x86_64
       - centos-stream-9-aarch64
@@ -88,34 +88,83 @@ jobs:
   - job: copr_build
     trigger: commit
     packages: [skopeo-fedora]
-    notifications:
-      failure_comment:
-        message: "podman-next COPR build failed. @containers/packit-build please check."
     branch: main
     owner: rhcontainerbot
     project: podman-next
     enable_net: true
 
-  # Tests on Fedora for main branch
+  # System tests on Fedora
   - job: tests
     trigger: pull_request
     packages: [skopeo-fedora]
-    notifications: &test_failure_notification
-      failure_comment:
-        message: "Tests failed. @containers/packit-build please check."
+    notifications: *packit_failure_notification
     targets: *fedora_copr_targets
-    tf_extra_params:
+    tf_extra_params: &extra_dnf_repos
       environments:
         - artifacts:
           - type: repository-file
             id: https://copr.fedorainfracloud.org/coprs/rhcontainerbot/podman-next/repo/fedora-$releasever/rhcontainerbot-podman-next-fedora-$releasever.repo
+    tmt_plan: "/plans/rpm/system"
+    identifier: "system"
+
+  # ostree-rs-ext tests on Fedora
+  - job: tests
+    trigger: pull_request
+    packages: [skopeo-fedora]
+    notifications: *packit_failure_notification
+    targets: *fedora_copr_targets
+    tf_extra_params: *extra_dnf_repos
+    tmt_plan: "/plans/rpm/ostree-rs-ext"
+    identifier: "ostree-rs-ext"
+
+  # Integration tests on Fedora
+  - job: tests
+    trigger: pull_request
+    skip_build: true
+    packages: [skopeo-fedora]
+    notifications: *packit_failure_notification
+    targets:
+      # x86_6 only because of quay.io/coreos/11bot
+      # See: integration/proxy_test.go
+      - fedora-all
+    tf_extra_params: *extra_dnf_repos
+    tmt_plan: "/plans/no-rpm/integration"
+    identifier: "integration"
+
+  # Unit tests on Fedora
+  - job: tests
+    trigger: pull_request
+    skip_build: true
+    notifications: *packit_failure_notification
+    targets: &fedora_no_rpm_targets
+      - fedora-latest-stable
+    tmt_plan: "/plans/no-rpm/unit"
+    identifier: "unit"
+
+  # Validate test on Fedora
+  - job: tests
+    trigger: pull_request
+    skip_build: true
+    notifications: *packit_failure_notification
+    targets: *fedora_no_rpm_targets
+    tmt_plan: "/plans/no-rpm/validate"
+    identifier: "validate"
+
+  # Test w/ opengpg static linked on Fedora
+  - job: tests
+    trigger: pull_request
+    skip_build: true
+    notifications: *packit_failure_notification
+    targets: *fedora_no_rpm_targets
+    tmt_plan: "/plans/no-rpm/opengpg"
+    identifier: "opengpg"
 
   # Tests on CentOS Stream for main branch
   # Ignore until golang is updated in distro buildroot to go 1.23.3+
   - job: tests
     trigger: ignore
     packages: [skopeo-centos]
-    notifications: *test_failure_notification
+    notifications: *packit_failure_notification
     targets: *centos_copr_targets
     tf_extra_params:
       environments:

Makefile

@@ -203,8 +203,8 @@ test-integration:
 
 
 # Intended for CI, assumed to be running in quay.io/libpod/skopeo_cidev container.
-test-integration-local: bin/skopeo
-	hack/warn-destructive-tests.sh
+# Expects binary in PATH, so better to depend on install-binary target
+test-integration-local: install-binary
 	hack/test-integration.sh
 
 # complicated set of options needed to run podman-in-podman
@@ -220,8 +220,7 @@ test-system:
 	exit $$rc
 
 # Intended for CI, assumed to already be running in quay.io/libpod/skopeo_cidev container.
-test-system-local: bin/skopeo
-	hack/warn-destructive-tests.sh
+test-system-local: $(if $(SKOPEO_BINARY),,bin/skopeo)
 	hack/test-system.sh
 
 test-unit:
@@ -235,7 +234,7 @@ validate:
 test-all-local: validate-local validate-docs test-unit-local
 
 .PHONY: validate-local
-validate-local:
+validate-local: tools
 	hack/validate-git-marks.sh
 	hack/validate-gofmt.sh
 	$(GOBIN)/golangci-lint run --build-tags "${BUILDTAGS}"

hack/test-integration.sh

@@ -1,8 +1,6 @@
 #!/bin/bash
 set -e
 
-make PREFIX=/usr install
-
-echo "cd ./integration;" go test $TESTFLAGS ${BUILDTAGS:+-tags "$BUILDTAGS"}
+echo "cd ./integration;" go test "$TESTFLAGS" ${BUILDTAGS:+-tags "$BUILDTAGS"}
 cd ./integration
-go test $TESTFLAGS ${BUILDTAGS:+-tags "$BUILDTAGS"}
+go test "$TESTFLAGS" ${BUILDTAGS:+-tags "$BUILDTAGS"}

hack/test-system.sh

@@ -37,8 +37,12 @@ EOF
     export CONTAINERS_STORAGE_CONF=/etc/containers/storage.conf
 fi
 
-# Build skopeo, install into /usr/bin
-make PREFIX=/usr install
+if [[ -v SKOPEO_BINARY ]]; then
+    echo "Testing with $SKOPEO_BINARY ..."
+else
+    echo "Testing with $(git rev-parse --show-toplevel)/bin/skopeo ..."
+fi
 
 # Run tests
-SKOPEO_BINARY=/usr/bin/skopeo bats --tap systemtest
+# The skopeo binary used will be at ../bin/skopeo unless set via SKOPEO_BINARY
+bats --tap systemtest

hack/tmt/bats-setup.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+# Install bats
+# https://bats-core.readthedocs.io/en/stable/installation.html
+
+BATS_TMPDIR=$(mktemp -d)
+pushd "$BATS_TMPDIR"
+
+BATS_VERSION=1.12.0
+curl -ssfL https://github.com/bats-core/bats-core/archive/refs/tags/v"$BATS_VERSION".tar.gz | tar -xz
+pushd bats-core-"$BATS_VERSION"
+./install.sh /usr
+popd
+popd

hack/tmt/dnf-repo-setup.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+COPR_REPO_FILE=$(compgen -G "/etc/yum.repos.d/*podman-next*.repo")
+if [[ -n "$COPR_REPO_FILE" ]]; then
+    # shellcheck disable=SC2016
+    sed -i -n '/^priority=/!p;$apriority=1' "${COPR_REPO_FILE}"
+fi
+dnf -y upgrade --allowerasing

integration/registry_test.go

@@ -13,10 +13,17 @@ import (
 )
 
 const (
-	binaryV2        = "registry-v2"
 	binaryV2Schema1 = "registry-v2-schema1"
 )
 
+// getBinaryV2 returns the available registry binary name, preferring "registry-v2" over "registry"
+func getBinaryV2() string {
+	if _, err := exec.LookPath("registry-v2"); err == nil {
+		return "registry-v2"
+	}
+	return "registry"
+}
+
 type testRegistryV2 struct {
 	cmd      *exec.Cmd
 	url      string
@@ -92,7 +99,7 @@ compatibility:
 	if schema1 {
 		cmd = exec.Command(binaryV2Schema1, confPath)
 	} else {
-		cmd = exec.Command(binaryV2, "serve", confPath)
+		cmd = exec.Command(getBinaryV2(), "serve", confPath)
 	}
 
 	consumeAndLogOutputs(t, fmt.Sprintf("registry-%s", url), cmd)

plans/main.fmf

@@ -0,0 +1,25 @@
+discover:
+    how: fmf
+execute:
+    how: tmt
+
+/integration:
+    discover+:
+        test: /tmt/integration
+
+/validate:
+    discover+:
+        test: /tmt/validate
+
+/unit:
+    discover+:
+        test: /tmt/unit
+
+/opengpg:
+    prepare+:
+        - when: distro == centos-stream or distro == rhel
+          how: shell
+          script: bash hack/tmt/bats-setup.sh
+          order: 10
+    discover+:
+        test: /tmt/opengpg

plans/no-rpm.fmf

@@ -0,0 +1,22 @@
+discover:
+    how: fmf
+execute:
+    how: tmt
+prepare:
+    - when: initiator == packit
+      how: shell
+      script: bash hack/tmt/dnf-repo-setup.sh
+      order: 5
+
+/ostree-rs-ext:
+    discover+:
+        test: /tmt/ostree-rs-ext
+
+/system:
+    prepare+:
+        - when: distro == centos-stream or distro == rhel
+          how: shell
+          script: bash hack/tmt/bats-setup.sh
+          order: 10
+    discover+:
+        test: /tmt/system