TMT: add more tests

Signed-off-by: Lokesh Mandvekar <[email protected]>

Author: lsm5

.packit.yaml

@@ -30,9 +30,9 @@ jobs:
   - job: copr_build
     trigger: pull_request
     packages: [skopeo-fedora]
-    notifications: &copr_build_failure_notification
+    notifications: &packit_failure_notification
       failure_comment:
-        message: "Ephemeral COPR build failed. @containers/packit-build please check."
+        message: "Packit jobs failed. @containers/packit-build please check."
     targets: &fedora_copr_targets
       - fedora-all-x86_64
       - fedora-all-aarch64
@@ -45,8 +45,9 @@ jobs:
   - job: copr_build
     trigger: pull_request
     packages: [skopeo-eln]
-    notifications: *copr_build_failure_notification
+    notifications: *packit_failure_notification
     targets: &eln_copr_targets
+    targets:
       fedora-eln-x86_64:
         additional_repos:
           - "https://kojipkgs.fedoraproject.org/repos/eln-build/latest/x86_64/"
@@ -59,7 +60,7 @@ jobs:
   - job: copr_build
     trigger: ignore
     packages: [skopeo-centos]
-    notifications: *copr_build_failure_notification
+    notifications: *packit_failure_notification
     targets: &centos_copr_targets
       - centos-stream-9-x86_64
       - centos-stream-9-aarch64
@@ -71,33 +72,82 @@ jobs:
   - job: copr_build
     trigger: commit
     packages: [skopeo-fedora]
-    notifications:
-      failure_comment:
-        message: "podman-next COPR build failed. @containers/packit-build please check."
     branch: main
     owner: rhcontainerbot
     project: podman-next
     enable_net: true
 
-  # Tests on Fedora for main branch
+  # System tests on Fedora
   - job: tests
     trigger: pull_request
     packages: [skopeo-fedora]
-    notifications: &test_failure_notification
-      failure_comment:
-        message: "Tests failed. @containers/packit-build please check."
+    notifications: *packit_failure_notification
     targets: *fedora_copr_targets
-    tf_extra_params:
+    tf_extra_params: &extra_dnf_repos
       environments:
         - artifacts:
           - type: repository-file
             id: https://copr.fedorainfracloud.org/coprs/rhcontainerbot/podman-next/repo/fedora-$releasever/rhcontainerbot-podman-next-fedora-$releasever.repo
+    tmt_plan: "/plans/rpm/system"
+    identifier: "system"
+
+  # ostree-rs-ext tests on Fedora
+  - job: tests
+    trigger: pull_request
+    packages: [skopeo-fedora]
+    notifications: *packit_failure_notification
+    targets: *fedora_copr_targets
+    tf_extra_params: *extra_dnf_repos
+    tmt_plan: "/plans/rpm/ostree-rs-ext"
+    identifier: "ostree-rs-ext"
+
+  # Integration tests on Fedora
+  - job: tests
+    trigger: pull_request
+    skip_build: true
+    packages: [skopeo-fedora]
+    notifications: *packit_failure_notification
+    targets:
+      # x86_6 only because of quay.io/coreos/11bot
+      # See: integration/proxy_test.go
+      - fedora-all
+    tf_extra_params: *extra_dnf_repos
+    tmt_plan: "/plans/no-rpm/integration"
+    identifier: "integration"
+
+  # Unit tests on Fedora
+  - job: tests
+    trigger: pull_request
+    skip_build: true
+    notifications: *packit_failure_notification
+    targets: &fedora_no_rpm_targets
+      - fedora-latest-stable
+    tmt_plan: "/plans/no-rpm/unit"
+    identifier: "unit"
+
+  # Validate test on Fedora
+  - job: tests
+    trigger: pull_request
+    skip_build: true
+    notifications: *packit_failure_notification
+    targets: *fedora_no_rpm_targets
+    tmt_plan: "/plans/no-rpm/validate"
+    identifier: "validate"
+
+  # Test w/ opengpg static linked on Fedora
+  - job: tests
+    trigger: pull_request
+    skip_build: true
+    notifications: *packit_failure_notification
+    targets: *fedora_no_rpm_targets
+    tmt_plan: "/plans/no-rpm/opengpg"
+    identifier: "opengpg"
 
   # Tests on ELN for main branch
   - job: tests
     trigger: pull_request
     packages: [skopeo-eln]
-    notifications: *test_failure_notification
+    notifications: *packit_failure_notification
     targets: *eln_copr_targets
     tf_extra_params:
       environments:
@@ -110,7 +160,7 @@ jobs:
   - job: tests
     trigger: ignore
     packages: [skopeo-centos]
-    notifications: *test_failure_notification
+    notifications: *packit_failure_notification
     targets: *centos_copr_targets
     tf_extra_params:
       environments:

Makefile

@@ -200,7 +200,6 @@ test-integration:
 	$(CONTAINER_CMD) --security-opt label=disable --cap-add=cap_mknod -v $(CURDIR):$(CONTAINER_GOSRC) -w $(CONTAINER_GOSRC) $(SKOPEO_CIDEV_CONTAINER_FQIN) \
 		$(MAKE) test-integration-local
 
-
 # Primarily intended for CI.
 test-integration-local: bin/skopeo
 	hack/warn-destructive-tests.sh

hack/tmt/bats-setup.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+# Install bats
+# https://bats-core.readthedocs.io/en/stable/installation.html
+
+BATS_TMPDIR=$(mktemp -d)
+pushd "$BATS_TMPDIR"
+
+BATS_VERSION=1.12.0
+curl -ssfL https://github.com/bats-core/bats-core/archive/refs/tags/v"$BATS_VERSION".tar.gz | tar -xz
+pushd bats-core-"$BATS_VERSION"
+./install.sh /usr
+popd
+popd

hack/tmt/dnf-repo-setup.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+COPR_REPO_FILE=$(compgen -G "/etc/yum.repos.d/*podman-next*.repo")
+if [[ -n "$COPR_REPO_FILE" ]]; then
+    # shellcheck disable=SC2016
+    sed -i -n '/^priority=/!p;$apriority=1' "${COPR_REPO_FILE}"
+fi
+dnf -y upgrade --allowerasing

plans/main.fmf

@@ -0,0 +1,25 @@
+discover:
+    how: fmf
+execute:
+    how: tmt
+
+/integration:
+    discover+:
+        test: /tmt/integration
+
+/validate:
+    discover+:
+        test: /tmt/validate
+
+/unit:
+    discover+:
+        test: /tmt/unit
+
+/opengpg:
+    prepare+:
+        - when: distro == centos-stream or distro == rhel
+          how: shell
+          script: bash hack/tmt/bats-setup.sh
+          order: 10
+    discover+:
+        test: /tmt/opengpg

plans/no-rpm.fmf

@@ -0,0 +1,33 @@
+discover:
+    how: fmf
+
+execute:
+    how: tmt
+
+prepare:
+    - when: initiator == packit
+      how: shell
+      script: bash ./hack/tmt/dnf-repo-setup.sh
+      order: 20
+    - name: Disable installing everything from srpm
+      how: install
+      exclude:
+        - ".*"
+    - name: Install the main package
+      how: install
+      package:
+        # skopeo-tests installs skopeo and other deps
+        - skopeo-tests
+
+/ostree-rs-ext:
+    discover+:
+        test: /tmt/ostree-rs-ext
+
+/system:
+    prepare+:
+        - when: distro == centos-stream or distro == rhel
+          how: shell
+          script: bash hack/tmt/bats-setup.sh
+          order: 10
+    discover+:
+        test: /tmt/system

plans/rpm.fmf

@@ -0,0 +1,91 @@
+require:
+    # Some of these are only required for non-rpm tests but it's much simpler
+    # to list them as common deps. Shouldn't be too much of a burden for rpm
+    # jobs.
+    - btrfs-progs-devel
+    - docker-distribution
+    - go-md2man
+    - golang
+    - gpgme-devel
+    - git-core
+    - make
+    - skopeo-tests
+
+environment:
+    BASEDIR: $(git rev-parse --show-toplevel)
+
+/integration:
+    summary: Integration test
+    test: |
+        rpm -q containers-common
+        make -C $BASEDIR test-integration-local
+    duration: 20m
+
+/ostree-rs-ext:
+    /build:
+        summary: ostree-rs-ext build
+        test: bash $BASEDIR/hack/test-ostree.sh build
+
+    /test:
+        summary: ostree-rs-ext test
+        test: |
+            rpm -q ostree
+            bash $BASEDIR/hack/test-ostree.sh test
+        require+:
+            - ostree
+
+/opengpg:
+    # w/ opengpg will only be tested upstream, so we don't care about setting
+    # $RELEASE_TESTING.
+
+    enabled: false
+    adjust:
+        enabled: true
+        when: initiator == packit
+
+    /validate+unit:
+        summary: System test with opengpg
+        test: |
+            make -C $BASEDIR BUILDTAGS+=" containers_image_openpgp" bin/skopeo
+            make -C $BASEDIR test-all-local
+
+    /integration:
+        summary: Integration test with opengpg
+        test: |
+            make -C $BASEDIR BUILDTAGS+=" containers_image_openpgp" bin/skopeo
+            make -C $BASEDIR test-integration-local
+        duration: 30m
+
+    /system:
+        summary: System tests with opengpg
+        test: |
+            make -C $BASEDIR BUILDTAGS+=" containers_image_openpgp" bin/skopeo
+            make -C $BASEDIR test-system-local
+        duration: 30m
+        require+:
+            - bats
+
+/system:
+    summary: System test
+    test: |
+        rpm -q containers-common skopeo
+        make -C $BASEDIR test-system-local
+    environment:
+        SKOPEO_BINARY: /usr/bin/skopeo
+    adjust:
+        - when: initiator != "packit"
+          environment+:
+            RELEASE_TESTING: true
+    duration: 60m
+
+/unit:
+    summary: Unit test
+    test: |
+        make -C $BASEDIR
+        make -C $BASEDIR test-unit-local
+
+/validate:
+    summary: Validate test
+    test: |
+        make -C $BASEDIR
+        make -C $BASEDIR validate-docs validate-local