Merge pull request #5865 from hpcng/master

dtrudg · web-flow · commit 8f9608432671 · 2021-03-09T08:33:43.000-06:00
master merge for release 3.7.2
diff --git a/cmd/internal/cli/build.go b/cmd/internal/cli/build.go
@@ -24,6 +24,7 @@ import (
 	"github.com/sylabs/singularity/pkg/build/types"
 	"github.com/sylabs/singularity/pkg/build/types/parser"
 	"github.com/sylabs/singularity/pkg/cmdline"
+	"github.com/sylabs/singularity/pkg/image"
 	"github.com/sylabs/singularity/pkg/sylog"
 )
 
@@ -135,7 +136,7 @@ var buildDetachedFlag = cmdline.Flag{
 var buildBuilderFlag = cmdline.Flag{
 	ID:           "buildBuilderFlag",
 	Value:        &buildArgs.builderURL,
-	DefaultValue: endpoint.SCSDefaultBuilderURI,
+	DefaultValue: "",
 	Name:         "builder",
 	Usage:        "remote Build Service URL, setting this implies --remote",
 	EnvKeys:      []string{"BUILDER"},
@@ -145,7 +146,7 @@ var buildBuilderFlag = cmdline.Flag{
 var buildLibraryFlag = cmdline.Flag{
 	ID:           "buildLibraryFlag",
 	Value:        &buildArgs.libraryURL,
-	DefaultValue: endpoint.SCSDefaultLibraryURI,
+	DefaultValue: "",
 	Name:         "library",
 	Usage:        "container Library URL",
 	EnvKeys:      []string{"LIBRARY"},
@@ -299,8 +300,15 @@ func checkBuildTarget(path string) error {
 
 			question := fmt.Sprintf("Build target '%s' already exists and will be deleted during the build process. Do you want to continue? [N/y]", f.Name())
 
-			if isDefFile, _ := parser.IsValidDefinition(abspath); isDefFile {
-				question = fmt.Sprintf("Build target '%s' is a definition file that will be overwritten. Do you still want to overwrite? [N/y]", f.Name())
+			img, err := image.Init(abspath, false)
+			if err != nil {
+				if err != image.ErrUnknownFormat {
+					return fmt.Errorf("while determining '%s' format: %s", f.Name(), err)
+				}
+				// unknown image file format
+				question = fmt.Sprintf("Build target '%s' may be a definition file or a text/binary file that will be overwritten. Do you still want to overwrite it? [N/y]", f.Name())
+			} else {
+				img.File.Close()
 			}
 
 			input, err := interactive.AskYNQuestion("n", question)
diff --git a/cmd/internal/cli/delete.go b/cmd/internal/cli/delete.go
@@ -15,7 +15,6 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/sylabs/singularity/docs"
 	"github.com/sylabs/singularity/internal/app/singularity"
-	"github.com/sylabs/singularity/internal/pkg/remote/endpoint"
 	"github.com/sylabs/singularity/internal/pkg/util/interactive"
 	"github.com/sylabs/singularity/pkg/cmdline"
 	"github.com/sylabs/singularity/pkg/sylog"
@@ -69,7 +68,7 @@ var deleteLibraryURI string
 var deleteLibraryURIFlag = cmdline.Flag{
 	ID:           "deleteLibraryURIFlag",
 	Value:        &deleteLibraryURI,
-	DefaultValue: endpoint.SCSDefaultLibraryURI,
+	DefaultValue: "",
 	Name:         "library",
 	Usage:        "delete images from the provided library",
 	EnvKeys:      []string{"LIBRARY"},
diff --git a/cmd/internal/cli/key.go b/cmd/internal/cli/key.go
@@ -14,15 +14,10 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/sylabs/singularity/docs"
 	"github.com/sylabs/singularity/internal/pkg/buildcfg"
-	"github.com/sylabs/singularity/internal/pkg/remote/endpoint"
 	"github.com/sylabs/singularity/pkg/cmdline"
 	"github.com/sylabs/singularity/pkg/sylog"
 )
 
-const (
-	defaultKeyServer = endpoint.SCSDefaultKeyserverURI
-)
-
 var (
 	keyServerURI        string // -u command line option
 	keySearchLongList   bool   // -l option for long-list
@@ -34,7 +29,7 @@ var (
 var keyServerURIFlag = cmdline.Flag{
 	ID:           "keyServerURIFlag",
 	Value:        &keyServerURI,
-	DefaultValue: defaultKeyServer,
+	DefaultValue: "",
 	Name:         "url",
 	ShortHand:    "u",
 	Usage:        "specify the key server URL",
diff --git a/cmd/internal/cli/pull.go b/cmd/internal/cli/pull.go
@@ -68,7 +68,7 @@ var pullArchFlag = cmdline.Flag{
 var pullLibraryURIFlag = cmdline.Flag{
 	ID:           "pullLibraryURIFlag",
 	Value:        &pullLibraryURI,
-	DefaultValue: endpoint.SCSDefaultLibraryURI,
+	DefaultValue: "",
 	Name:         "library",
 	Usage:        "download images from the provided library",
 	EnvKeys:      []string{"LIBRARY"},
@@ -207,7 +207,7 @@ func pullRun(cmd *cobra.Command, args []string) {
 		if err != nil {
 			sylog.Fatalf("Unable to get library client configuration: %v", err)
 		}
-		co, err := getKeyserverClientOpts(endpoint.SCSDefaultKeyserverURI, endpoint.KeyserverVerifyOp)
+		co, err := getKeyserverClientOpts("", endpoint.KeyserverVerifyOp)
 		if err != nil {
 			sylog.Fatalf("Unable to get keyserver client configuration: %v", err)
 		}
diff --git a/cmd/internal/cli/push.go b/cmd/internal/cli/push.go
@@ -36,7 +36,7 @@ var (
 var pushLibraryURIFlag = cmdline.Flag{
 	ID:           "pushLibraryURIFlag",
 	Value:        &PushLibraryURI,
-	DefaultValue: endpoint.SCSDefaultLibraryURI,
+	DefaultValue: "",
 	Name:         "library",
 	Usage:        "the library to push to",
 	EnvKeys:      []string{"LIBRARY"},
@@ -102,7 +102,7 @@ var PushCmd = &cobra.Command{
 				sylog.Fatalf("Cannot push image to library: %v", remoteWarning)
 			}
 
-			co, err := getKeyserverClientOpts(endpoint.SCSDefaultKeyserverURI, endpoint.KeyserverVerifyOp)
+			co, err := getKeyserverClientOpts("", endpoint.KeyserverVerifyOp)
 			if err != nil {
 				sylog.Fatalf("Unable to get keyserver client configuration: %v", err)
 			}
diff --git a/cmd/internal/cli/search.go b/cmd/internal/cli/search.go
@@ -14,7 +14,6 @@ import (
 	"github.com/sylabs/scs-library-client/client"
 	"github.com/sylabs/singularity/docs"
 	"github.com/sylabs/singularity/internal/pkg/client/library"
-	"github.com/sylabs/singularity/internal/pkg/remote/endpoint"
 	"github.com/sylabs/singularity/pkg/cmdline"
 	"github.com/sylabs/singularity/pkg/sylog"
 )
@@ -32,7 +31,7 @@ var (
 var searchLibraryFlag = cmdline.Flag{
 	ID:           "searchLibraryFlag",
 	Value:        &SearchLibraryURI,
-	DefaultValue: endpoint.SCSDefaultLibraryURI,
+	DefaultValue: "",
 	Name:         "library",
 	Usage:        "URI for library to search",
 	EnvKeys:      []string{"LIBRARY"},
diff --git a/cmd/internal/cli/singularity.go b/cmd/internal/cli/singularity.go
@@ -586,9 +586,10 @@ func CheckRootOrUnpriv(cmd *cobra.Command, args []string) {
 	}
 }
 
+// getKeyServerClientOpts returns client options for keyserver access.
+// A "" value for uri will return client options for the current endpoint.
+// A specified uri will return client options for that keyserver.
 func getKeyserverClientOpts(uri string, op endpoint.KeyserverOp) ([]scskeyclient.Option, error) {
-	isDefault := uri == endpoint.SCSDefaultKeyserverURI
-
 	if currentRemoteEndpoint == nil {
 		var err error
 
@@ -599,19 +600,17 @@ func getKeyserverClientOpts(uri string, op endpoint.KeyserverOp) ([]scskeyclient
 			return nil, fmt.Errorf("unable to load remote configuration: %v", err)
 		}
 	}
-	if currentRemoteEndpoint == endpoint.DefaultEndpointConfig && isDefault {
-		sylog.Warningf("No default remote in use, falling back to default keyserver: %s", uri)
-	}
-	if isDefault {
-		uri = ""
+	if currentRemoteEndpoint == endpoint.DefaultEndpointConfig {
+		sylog.Warningf("No default remote in use, falling back to default keyserver: %s", endpoint.SCSDefaultKeyserverURI)
 	}
 
 	return currentRemoteEndpoint.KeyserverClientOpts(uri, op)
 }
 
+// getLibraryClientConfig returns client config for library server access.
+// A "" value for uri will return client config for the current endpoint.
+// A specified uri will return client options for that library server.
 func getLibraryClientConfig(uri string) (*scslibclient.Config, error) {
-	isDefault := uri == endpoint.SCSDefaultLibraryURI
-
 	if currentRemoteEndpoint == nil {
 		var err error
 
@@ -622,19 +621,17 @@ func getLibraryClientConfig(uri string) (*scslibclient.Config, error) {
 			return nil, fmt.Errorf("unable to load remote configuration: %v", err)
 		}
 	}
-	if currentRemoteEndpoint == endpoint.DefaultEndpointConfig && isDefault {
-		sylog.Warningf("No default remote in use, falling back to default library: %s", uri)
-	}
-	if isDefault {
-		uri = ""
+	if currentRemoteEndpoint == endpoint.DefaultEndpointConfig {
+		sylog.Warningf("No default remote in use, falling back to default library: %s", endpoint.SCSDefaultLibraryURI)
 	}
 
 	return currentRemoteEndpoint.LibraryClientConfig(uri)
 }
 
+// getBuilderClientConfig returns client config for build server access.
+// A "" value for uri will return client config for the current endpoint.
+// A specified uri will return client options for that build server.
 func getBuilderClientConfig(uri string) (*scsbuildclient.Config, error) {
-	isDefault := uri == endpoint.SCSDefaultBuilderURI
-
 	if currentRemoteEndpoint == nil {
 		var err error
 
@@ -645,11 +642,8 @@ func getBuilderClientConfig(uri string) (*scsbuildclient.Config, error) {
 			return nil, fmt.Errorf("unable to load remote configuration: %v", err)
 		}
 	}
-	if currentRemoteEndpoint == endpoint.DefaultEndpointConfig && isDefault {
-		sylog.Warningf("No default remote in use, falling back to default builder: %s", uri)
-	}
-	if isDefault {
-		uri = ""
+	if currentRemoteEndpoint == endpoint.DefaultEndpointConfig {
+		sylog.Warningf("No default remote in use, falling back to default builder: %s", endpoint.SCSDefaultBuilderURI)
 	}
 
 	return currentRemoteEndpoint.BuilderClientConfig(uri)
diff --git a/cmd/internal/cli/verify.go b/cmd/internal/cli/verify.go
@@ -31,7 +31,7 @@ var (
 var verifyServerURIFlag = cmdline.Flag{
 	ID:           "verifyServerURIFlag",
 	Value:        &keyServerURI,
-	DefaultValue: defaultKeyServer,
+	DefaultValue: "",
 	Name:         "url",
 	ShortHand:    "u",
 	Usage:        "specify a URL for a key server",
diff --git a/e2e/pull/pull.go b/e2e/pull/pull.go
@@ -629,6 +629,9 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests {
 
 			t.Run("pull", c.testPullCmd)
 			t.Run("pullDisableCache", c.testPullDisableCacheCmd)
+
+			// Regressions
+			t.Run("issue5808", c.issue5808)
 		}),
 	}
 }
diff --git a/e2e/pull/regressions.go b/e2e/pull/regressions.go
@@ -0,0 +1,72 @@
+// Copyright (c) 2020, Sylabs Inc. All rights reserved.
+// This software is licensed under a 3-clause BSD license. Please consult the
+// LICENSE.md file distributed with the sources of this project regarding your
+// rights to use or distribute this software.
+
+package pull
+
+import (
+	"path"
+	"testing"
+
+	"github.com/sylabs/singularity/e2e/internal/e2e"
+)
+
+// If a remote is set to a different endpoint we should be able to pull
+// with `--library https://library.sylabs.io` from the default Sylabs cloud library.
+func (c ctx) issue5808(t *testing.T) {
+	testEndpoint := "issue5808"
+	testEndpointURI := "https://cloud.staging.sylabs.io"
+	defaultLibraryURI := "https://library.sylabs.io"
+	testImage := "library://sylabs/tests/signed:1.0.0"
+
+	pullDir, cleanup := e2e.MakeTempDir(t, "", "issue5808", "")
+	defer cleanup(t)
+
+	// Add another endpoint
+	argv := []string{"add", "--no-login", testEndpoint, testEndpointURI}
+	c.env.RunSingularity(
+		t,
+		e2e.AsSubtest("remote add"),
+		e2e.WithProfile(e2e.UserProfile),
+		e2e.WithCommand("remote"),
+		e2e.WithArgs(argv...),
+		e2e.ExpectExit(0),
+	)
+	// Remove test remote when we are done here
+	defer func(t *testing.T) {
+		argv := []string{"remove", testEndpoint}
+		c.env.RunSingularity(
+			t,
+			e2e.AsSubtest("remote remove"),
+			e2e.WithProfile(e2e.UserProfile),
+			e2e.WithCommand("remote"),
+			e2e.WithArgs(argv...),
+			e2e.ExpectExit(0),
+		)
+	}(t)
+
+	// Set as default
+	argv = []string{"use", testEndpoint}
+	c.env.RunSingularity(
+		t,
+		e2e.AsSubtest("remote use"),
+		e2e.WithProfile(e2e.UserProfile),
+		e2e.WithCommand("remote"),
+		e2e.WithArgs(argv...),
+		e2e.ExpectExit(0),
+	)
+
+	// Pull a library image
+	dest := path.Join(pullDir, "alpine.sif")
+	argv = []string{"--arch", "amd64", "--library", defaultLibraryURI, dest, testImage}
+	c.env.RunSingularity(
+		t,
+		e2e.AsSubtest("pull"),
+		e2e.WithProfile(e2e.UserProfile),
+		e2e.WithCommand("pull"),
+		e2e.WithArgs(argv...),
+		e2e.ExpectExit(0),
+	)
+
+}
diff --git a/e2e/testdata/Docker_registry.def b/e2e/testdata/Docker_registry.def
@@ -26,7 +26,7 @@ from: registry:2.7.1
     # wait until docker registry is up
     while ! wget -q -O /dev/null 127.0.0.1:5000 ; do sleep 0.5; done
 
-    skopeo --dest-tls-verify=false copy docker://busybox docker://localhost:5000/my-busybox
+    skopeo --dest-tls-verify=false --insecure-policy copy docker://busybox docker://localhost:5000/my-busybox
 
     # e2e PrepRegistry will repeatedly trying to connect to this port
     # giving indication that it can start
diff --git a/go.mod b/go.mod
@@ -11,12 +11,12 @@ require (
 	github.com/buger/jsonparser v1.1.1
 	github.com/bugsnag/bugsnag-go v1.5.1 // indirect
 	github.com/bugsnag/panicwrap v1.2.0 // indirect
-	github.com/containerd/cgroups v0.0.0-20200116170754-a8908713319d
-	github.com/containerd/containerd v1.4.3
-	github.com/containernetworking/cni v0.8.0
-	github.com/containernetworking/plugins v0.9.0
-	github.com/containers/image/v5 v5.9.0
-	github.com/deislabs/oras v0.8.1
+	github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59
+	github.com/containerd/containerd v1.4.4
+	github.com/containernetworking/cni v0.8.1
+	github.com/containernetworking/plugins v0.9.1
+	github.com/containers/image/v5 v5.10.5
+	github.com/deislabs/oras v0.10.0
 	github.com/fatih/color v1.10.0
 	github.com/garyburd/redigo v1.6.0 // indirect
 	github.com/go-log/log v0.2.0
@@ -37,27 +37,33 @@ require (
 	github.com/satori/go.uuid v1.2.0
 	github.com/seccomp/containers-golang v0.6.0
 	github.com/seccomp/libseccomp-golang v0.9.1
-	github.com/spf13/cobra v1.1.1
+	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5
 	github.com/sylabs/json-resp v0.7.0
 	github.com/sylabs/scs-build-client v0.1.5
 	github.com/sylabs/scs-key-client v0.6.0
-	github.com/sylabs/scs-library-client v1.0.2
+	github.com/sylabs/scs-library-client v1.0.3
 	github.com/sylabs/sif v1.2.1
 	github.com/vbauerster/mpb/v4 v4.12.2
+	github.com/vbauerster/mpb/v6 v6.0.2
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 // indirect
 	github.com/yvasiyarov/gorelic v0.0.6 // indirect
 	github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 // indirect
 	go.opencensus.io v0.22.2 // indirect
-	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
+	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad
 	golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect
-	golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637
+	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools/v3 v3.0.3
-	mvdan.cc/sh/v3 v3.2.1
-	rsc.io/letsencrypt v0.0.3 // indirect
+	mvdan.cc/sh/v3 v3.2.2
 )
 
-replace golang.org/x/crypto => github.com/sylabs/golang-x-crypto v0.0.0-20181006204705-4bce89e8e9a9
+replace (
+	// These are required for oras v0.9.0
+	github.com/docker/distribution => github.com/docker/distribution v0.0.0-20191216044856-a8371794149d
+	github.com/docker/docker => github.com/moby/moby v17.12.0-ce-rc1.0.20200618181300-9dc6525e6118+incompatible
+
+	golang.org/x/crypto => github.com/sylabs/golang-x-crypto v0.0.0-20181006204705-4bce89e8e9a9
+)
diff --git a/go.sum b/go.sum
diff --git a/internal/pkg/client/progress.go b/internal/pkg/client/progress.go
diff --git a/pkg/util/loop/loop_linux_test.go b/pkg/util/loop/loop_linux_test.go

Original file line number	Diff line number	Diff line change
`@@ -586,9 +586,10 @@ func CheckRootOrUnpriv(cmd *cobra.Command, args []string) {`
`586`	`586`	`}`
`587`	`587`	`}`
`588`	`588`
	`589`	`+// getKeyServerClientOpts returns client options for keyserver access.`
	`590`	`+// A "" value for uri will return client options for the current endpoint.`
	`591`	`+// A specified uri will return client options for that keyserver.`
`589`	`592`	`func getKeyserverClientOpts(uri string, op endpoint.KeyserverOp) ([]scskeyclient.Option, error) {`
`590`		`- isDefault := uri == endpoint.SCSDefaultKeyserverURI`
`591`		`-`
`592`	`593`	`if currentRemoteEndpoint == nil {`
`593`	`594`	`var err error`
`594`	`595`
`@@ -599,19 +600,17 @@ func getKeyserverClientOpts(uri string, op endpoint.KeyserverOp) ([]scskeyclient`
`599`	`600`	`return nil, fmt.Errorf("unable to load remote configuration: %v", err)`
`600`	`601`	`}`
`601`	`602`	`}`
`602`		`- if currentRemoteEndpoint == endpoint.DefaultEndpointConfig && isDefault {`
`603`		`- sylog.Warningf("No default remote in use, falling back to default keyserver: %s", uri)`
`604`		`- }`
`605`		`- if isDefault {`
`606`		`- uri = ""`
	`603`	`+ if currentRemoteEndpoint == endpoint.DefaultEndpointConfig {`
	`604`	`+ sylog.Warningf("No default remote in use, falling back to default keyserver: %s", endpoint.SCSDefaultKeyserverURI)`
`607`	`605`	`}`
`608`	`606`
`609`	`607`	`return currentRemoteEndpoint.KeyserverClientOpts(uri, op)`
`610`	`608`	`}`
`611`	`609`
	`610`	`+// getLibraryClientConfig returns client config for library server access.`
	`611`	`+// A "" value for uri will return client config for the current endpoint.`
	`612`	`+// A specified uri will return client options for that library server.`
`612`	`613`	`func getLibraryClientConfig(uri string) (*scslibclient.Config, error) {`
`613`		`- isDefault := uri == endpoint.SCSDefaultLibraryURI`
`614`		`-`
`615`	`614`	`if currentRemoteEndpoint == nil {`
`616`	`615`	`var err error`
`617`	`616`
`@@ -622,19 +621,17 @@ func getLibraryClientConfig(uri string) (*scslibclient.Config, error) {`
`622`	`621`	`return nil, fmt.Errorf("unable to load remote configuration: %v", err)`
`623`	`622`	`}`
`624`	`623`	`}`
`625`		`- if currentRemoteEndpoint == endpoint.DefaultEndpointConfig && isDefault {`
`626`		`- sylog.Warningf("No default remote in use, falling back to default library: %s", uri)`
`627`		`- }`
`628`		`- if isDefault {`
`629`		`- uri = ""`
	`624`	`+ if currentRemoteEndpoint == endpoint.DefaultEndpointConfig {`
	`625`	`+ sylog.Warningf("No default remote in use, falling back to default library: %s", endpoint.SCSDefaultLibraryURI)`
`630`	`626`	`}`
`631`	`627`
`632`	`628`	`return currentRemoteEndpoint.LibraryClientConfig(uri)`
`633`	`629`	`}`
`634`	`630`
	`631`	`+// getBuilderClientConfig returns client config for build server access.`
	`632`	`+// A "" value for uri will return client config for the current endpoint.`
	`633`	`+// A specified uri will return client options for that build server.`
`635`	`634`	`func getBuilderClientConfig(uri string) (*scsbuildclient.Config, error) {`
`636`		`- isDefault := uri == endpoint.SCSDefaultBuilderURI`
`637`		`-`
`638`	`635`	`if currentRemoteEndpoint == nil {`
`639`	`636`	`var err error`
`640`	`637`
`@@ -645,11 +642,8 @@ func getBuilderClientConfig(uri string) (*scsbuildclient.Config, error) {`
`645`	`642`	`return nil, fmt.Errorf("unable to load remote configuration: %v", err)`
`646`	`643`	`}`
`647`	`644`	`}`
`648`		`- if currentRemoteEndpoint == endpoint.DefaultEndpointConfig && isDefault {`
`649`		`- sylog.Warningf("No default remote in use, falling back to default builder: %s", uri)`
`650`		`- }`
`651`		`- if isDefault {`
`652`		`- uri = ""`
	`645`	`+ if currentRemoteEndpoint == endpoint.DefaultEndpointConfig {`
	`646`	`+ sylog.Warningf("No default remote in use, falling back to default builder: %s", endpoint.SCSDefaultBuilderURI)`
`653`	`647`	`}`
`654`	`648`
`655`	`649`	`return currentRemoteEndpoint.BuilderClientConfig(uri)`
Original file line number	Diff line number	Diff line change
`@@ -629,6 +629,9 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests {`
`629`	`629`
`630`	`630`	`t.Run("pull", c.testPullCmd)`
`631`	`631`	`t.Run("pullDisableCache", c.testPullDisableCacheCmd)`
	`632`	`+`
	`633`	`+ // Regressions`
	`634`	`+ t.Run("issue5808", c.issue5808)`
`632`	`635`	`}),`
`633`	`636`	`}`
`634`	`637`	`}`