Merge pull request #5778 from hpcng/master

Merge fixes from master for release 3.7.1

Author: dtrudg

CHANGELOG.md

@@ -9,6 +9,21 @@ _With the release of `v3.0.0`, we're introducing a new changelog format in an at
 
 _The old changelog can be found in the `release-2.6` branch_
 
+# Changes since v3.7.0
+
+## Bug Fixes
+
+  - Accommodate /sys/fs/selinux mount changes on kernel 5.9+.
+  - Fix loop devices file descriptor leak when shared loop devices is
+    enabled.
+  - Use MaxLoopDevices variable from config file in all appropriate
+    locations.
+  - Use -buildmode=default (non pie) on ppc64le to prevent crashes
+    when using plugins.
+  - Remove spurious warning in parseTokenSection()
+  - e2e test fixes for new kernels, new unsquashfs version.
+
+
 # v3.7.0 - [2020-11-24]
 
 ## New features / functionalities
@@ -96,7 +111,7 @@ _The old changelog can be found in the `release-2.6` branch_
   - Increase embedded shell interpreter timeout, to allow slow-running
     environment scripts to complete.
   - Correct buffer handling for key import to allow import from STDIN. 
-  - Reset environment to avoid `LD_LIBRARYPATH` issues when resolving
+  - Reset environment to avoid `LD_LIBRARY_PATH` issues when resolving
     dependencies for the `unsquashfs` sandbox.
   - Fall back to `/sbin/ldconfig` if `ldconfig` on `PATH` fails while
     resolving GPU libraries. Fixes problems on systems using Nix /
@@ -112,7 +127,7 @@ _The old changelog can be found in the `release-2.6` branch_
 
   - A change in Linux kernel 5.9 causes `--fakeroot` builds to fail with a
     `/sys/fs/selinux` remount error. This will be addressed in Singularity
-    v3.5.1.
+    v3.7.1.
 
 
 # v3.6.4 - [2020-10-13]
@@ -162,6 +177,11 @@ Singularity 3.6.3 addresses the following security issues.
     enable arbitrary code execution during the build and/or when the
     built container is run.
 
+  ## Change defaults / behaviours
+
+  - The value for maximum number of loop devices in the config file is now used everywhere
+    instead of redefining this value
+
 ## Bug Fixes
 
   - Add CAP_MKNOD in capability bounding set of RPC to fix issue with

CONTRIBUTORS.md

@@ -41,7 +41,7 @@
     - Hakon Enger <[email protected]>
     - Hugo Meiland <[email protected]>
     - Ian Kaneshiro <[email protected]>, <[email protected]>
-    - Jack Morrison <[email protected]>
+    - Jack Morrison <[email protected]>, <[email protected]>
     - Jacob Chappell <[email protected]>, <[email protected]>
     - Jarrod Johnson <[email protected]>
     - Jason Stover <[email protected]>, <[email protected]>
@@ -82,9 +82,11 @@
     - Shengjing Zhu <[email protected]>
     - Tarcisio Fedrizzi <[email protected]>
     - Thomas Hamel <[email protected]>
+    - Tim Wright <[email protected]>
     - Tru Huynh <[email protected]>
     - Vanessa Sochat <[email protected]>
     - Westley Kurtzer <[email protected]>, <[email protected]>
     - Yannick Cote <[email protected]>, <[email protected]>
     - Yaroslav Halchenko <[email protected]>
     - Onur Yılmaz <[email protected]>
+    - Pranathi Locula <[email protected]>

cmd/internal/cli/build_linux.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	osExec "os/exec"
 	"runtime"
+	"strconv"
 	"strings"
 	"syscall"
 
@@ -67,6 +68,12 @@ func fakerootExec(cmdArgs []string) {
 		sylog.Fatalf("failed to retrieve user information: %s", err)
 	}
 
+	// Append the user's real UID to the environment as _CONTAINERS_ROOTLESS_UID.
+	// This is required in fakeroot builds that may use containers/image 5.7 and above.
+	// https://github.com/containers/image/issues/1066
+	// https://github.com/containers/image/blob/master/internal/rootless/rootless.go
+	os.Setenv("_CONTAINERS_ROOTLESS_UID", strconv.Itoa(os.Getuid()))
+
 	engineConfig := &fakerootConfig.EngineConfig{
 		Args:     args,
 		Envs:     os.Environ(),

e2e/actions/actions.go

@@ -474,7 +474,7 @@ func (c actionTests) STDPipe(t *testing.T) {
 
 // RunFromURI tests min fuctionality for singularity run/exec URI://
 func (c actionTests) RunFromURI(t *testing.T) {
-	e2e.PrepRegistry(t, c.env)
+	e2e.EnsureRegistry(t)
 
 	runScript := "testdata/runscript.sh"
 	bind := fmt.Sprintf("%s:/.singularity.d/runscript", runScript)

e2e/docker/docker.go

@@ -338,7 +338,7 @@ func (c ctx) testDockerRegistry(t *testing.T) {
 	defer cleanup(t)
 	imagePath := filepath.Join(imageDir, "container")
 
-	e2e.PrepRegistry(t, c.env)
+	e2e.EnsureRegistry(t)
 
 	tests := []struct {
 		name string

e2e/imgbuild/imgbuild.go

@@ -49,7 +49,7 @@ func (c imgBuildTests) tempDir(t *testing.T, namespace string) (string, func())
 }
 
 func (c imgBuildTests) buildFrom(t *testing.T) {
-	e2e.PrepRegistry(t, c.env)
+	e2e.EnsureRegistry(t)
 
 	// use a trailing slash in tests for sandbox intentionally to make sure
 	// `singularity build -s /tmp/sand/ docker://alpine` works,

e2e/imgbuild/regressions.go

@@ -308,7 +308,7 @@ func (c *imgBuildTests) issue5166(t *testing.T) {
 }
 
 func (c *imgBuildTests) issue5172(t *testing.T) {
-	e2e.PrepRegistry(t, c.env)
+	e2e.EnsureRegistry(t)
 
 	u := e2e.UserProfile.HostUser(t)
 

e2e/inspect/inspect.go

@@ -70,9 +70,16 @@ func (c ctx) singularityInspect(t *testing.T) {
 
 	require.Command(t, "unsquashfs")
 
-	cmd := exec.Command("unsquashfs", "-d", sandboxImage, squashImage)
+	// First try with -user-xattrs since unsquashfs 4.4 gives an error code if
+	// it can't set system xattrs while rootless.
+	cmd := exec.Command("unsquashfs", "-user-xattrs", "-d", sandboxImage, squashImage)
 	if res := cmd.Run(t); res.Error != nil {
-		t.Fatalf("Unexpected error while running command.\n%s", res)
+		// If we failed, then try without -user-xattrs for older unsquashfs
+		// versions that don't have that flag.
+		cmd := exec.Command("unsquashfs", "-d", sandboxImage, squashImage)
+		if res := cmd.Run(t); res.Error != nil {
+			t.Fatalf("Unexpected error while running command.\n%s", res)
+		}
 	}
 
 	compareLabel := func(label, out string, appName string) func(*testing.T, *inspect.Metadata) {

e2e/internal/e2e/docker.go

@@ -27,8 +27,12 @@ var registrySetup struct {
 	sync.Mutex
 }
 
-// PrepRegistry run a docker registry and push a busybox
-// image and the test image with oras transport.
+// PrepRegistry runs a docker registry and pushes in a busybox image and
+// the test image using the oras transport.
+// This *MUST* be called before any tests using OCI/instances as it
+// temporarily  mounts a shadow instance directory in the test user
+// $HOME that will obscure any instances of concurrent tests, causing
+// them to fail.
 func PrepRegistry(t *testing.T, env TestEnv) {
 	// The docker registry container is only available for amd64 and arm
 	// See: https://hub.docker.com/_/registry?tab=tags
@@ -40,6 +44,8 @@ func PrepRegistry(t *testing.T, env TestEnv) {
 
 	registrySetup.Do(func() {
 
+		t.Log("Preparing docker registry instance.")
+
 		EnsureImage(t, env)
 
 		dockerDefinition := "testdata/Docker_registry.def"
@@ -137,3 +143,15 @@ func KillRegistry(t *testing.T, env TestEnv) {
 		ExpectExit(0),
 	)
 }
+
+// EnsureRegistry fails the current test if the e2e docker registry is not up
+func EnsureRegistry(t *testing.T) {
+	// The docker registry container is only available for amd64 and arm
+	// See: https://hub.docker.com/_/registry?tab=tags
+	// Skip on other architectures
+	require.ArchIn(t, []string{"amd64", "arm64"})
+
+	if registrySetup.up != 1 {
+		t.Fatalf("Registry instance was not setup. e2e.PrepRegistry must be called before this test.")
+	}
+}

e2e/pull/pull.go

@@ -253,7 +253,7 @@ func getImageNameFromURI(imgURI string) string {
 
 func (c *ctx) setup(t *testing.T) {
 	e2e.EnsureImage(t, c.env)
-	e2e.PrepRegistry(t, c.env)
+	e2e.EnsureRegistry(t)
 
 	// setup file and dir to use as invalid images
 	orasInvalidDir, err := ioutil.TempDir(c.env.TestDir, "oras_push_dir-")