nydusify: small improvements for mount & check subcommands

- Add `--prefetch` option for enabling full image data prefetch. - Support `HTTP_PROXY` / `HTTPS_PROXY` env for enabling proxy for nydusd. - Change nydusd log level to `warn` for mount & check subcommands. Signed-off-by: Yan Song <[email protected]>
dragonflyoss · Aug 28, 2024 · 3eb5c7b · 3eb5c7b
1 parent 52ed07b
commit 3eb5c7b
Show file tree

Hide file tree

Showing 6 changed files with 79 additions and 42 deletions.
diff --git a/contrib/nydusify/cmd/nydusify.go b/contrib/nydusify/cmd/nydusify.go
@@ -24,7 +24,6 @@ import (
 	"github.com/urfave/cli/v2"
 
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker"
-	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker/rule"
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/chunkdict/generator"
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/committer"
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/converter"
@@ -796,7 +795,12 @@ func main() {
 					Usage:     "Json configuration file for storage backend",
 					EnvVars:   []string{"BACKEND_CONFIG_FILE"},
 				},
-
+				&cli.BoolFlag{
+					Name:    "prefetch",
+					Value:   false,
+					Usage:   "Enable full image data prefetch",
+					EnvVars: []string{"PREFETCH"},
+				},
 				&cli.StringFlag{
 					Name:    "mount-path",
 					Value:   "./image-fs",
@@ -836,13 +840,11 @@ func main() {
 						return err
 					}
 
-					backendConfigStruct, err := rule.NewRegistryBackendConfig(parsed)
+					backendConfigStruct, err := utils.NewRegistryBackendConfig(parsed, c.Bool("target-insecure"))
 					if err != nil {
 						return errors.Wrap(err, "parse registry backend configuration")
 					}
 
-					backendConfigStruct.SkipVerify = c.Bool("target-insecure")
-
 					bytes, err := json.Marshal(backendConfigStruct)
 					if err != nil {
 						return errors.Wrap(err, "marshal registry backend configuration")
@@ -865,6 +867,7 @@ func main() {
 					BackendType:    backendType,
 					BackendConfig:  backendConfig,
 					ExpectedArch:   arch,
+					Prefetch:       c.Bool("prefetch"),
 				})
 				if err != nil {
 					return err

diff --git a/contrib/nydusify/pkg/checker/checker.go b/contrib/nydusify/pkg/checker/checker.go
@@ -168,6 +168,7 @@ func (checker *Checker) check(ctx context.Context) error {
 			TargetInsecure:  checker.TargetInsecure,
 			PlainHTTP:       checker.targetParser.Remote.IsWithHTTP(),
 			NydusdConfig: tool.NydusdConfig{
+				EnablePrefetch: true,
 				NydusdPath:     checker.NydusdPath,
 				BackendType:    checker.BackendType,
 				BackendConfig:  checker.BackendConfig,

diff --git a/contrib/nydusify/pkg/checker/rule/filesystem.go b/contrib/nydusify/pkg/checker/rule/filesystem.go
@@ -6,7 +6,6 @@ package rule
 
 import (
 	"context"
-	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
@@ -16,7 +15,6 @@ import (
 	"syscall"
 
 	"github.com/distribution/reference"
-	dockerconfig "github.com/docker/cli/cli/config"
 
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker/tool"
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/parser"
@@ -221,28 +219,6 @@ func (rule *FilesystemRule) mountSourceImage() (*tool.Image, error) {
 	return image, nil
 }
 
-func NewRegistryBackendConfig(parsed reference.Named) (RegistryBackendConfig, error) {
-
-	backendConfig := RegistryBackendConfig{
-		Scheme: "https",
-		Host:   reference.Domain(parsed),
-		Repo:   reference.Path(parsed),
-	}
-
-	config := dockerconfig.LoadDefaultConfigFile(os.Stderr)
-	authConfig, err := config.GetAuthConfig(backendConfig.Host)
-	if err != nil {
-		return backendConfig, errors.Wrap(err, "get docker registry auth config")
-	}
-	var auth string
-	if authConfig.Username != "" && authConfig.Password != "" {
-		auth = base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", authConfig.Username, authConfig.Password)))
-	}
-	backendConfig.Auth = auth
-
-	return backendConfig, nil
-}
-
 func (rule *FilesystemRule) mountNydusImage() (*tool.Nydusd, error) {
 	logrus.Infof("Mounting Nydus image to %s", rule.NydusdConfig.MountPath)
 
@@ -263,7 +239,7 @@ func (rule *FilesystemRule) mountNydusImage() (*tool.Nydusd, error) {
 		rule.NydusdConfig.BackendType = "registry"
 
 		if rule.NydusdConfig.BackendConfig == "" {
-			backendConfig, err := NewRegistryBackendConfig(parsed)
+			backendConfig, err := utils.NewRegistryBackendConfig(parsed, rule.TargetInsecure)
 			if err != nil {
 				return nil, errors.Wrap(err, "failed to parse backend configuration")
 			}

diff --git a/contrib/nydusify/pkg/checker/tool/nydusd.go b/contrib/nydusify/pkg/checker/tool/nydusd.go
@@ -76,12 +76,10 @@ func makeConfig(conf NydusdConfig) error {
 	if conf.BackendType == "" {
 		conf.BackendType = "localfs"
 		conf.BackendConfig = `{"dir": "/fake"}`
-		conf.EnablePrefetch = false
 	} else {
 		if conf.BackendConfig == "" {
 			return errors.Errorf("empty backend configuration string")
 		}
-		conf.EnablePrefetch = true
 	}
 	if err := tpl.Execute(&ret, conf); err != nil {
 		return errors.New("failed to prepare configuration file for Nydusd")
@@ -176,7 +174,7 @@ func (nydusd *Nydusd) Mount() error {
 		"--apisock",
 		nydusd.APISockPath,
 		"--log-level",
-		"error",
+		"warn",
 	}
 
 	cmd := exec.Command(nydusd.NydusdPath, args...)

diff --git a/contrib/nydusify/pkg/utils/backend.go b/contrib/nydusify/pkg/utils/backend.go
@@ -0,0 +1,57 @@
+package utils
+
+import (
+	"encoding/base64"
+	"fmt"
+	"os"
+
+	"github.com/distribution/reference"
+	dockerconfig "github.com/docker/cli/cli/config"
+	"github.com/pkg/errors"
+)
+
+type RegistryBackendConfig struct {
+	Scheme     string             `json:"scheme"`
+	Host       string             `json:"host"`
+	Repo       string             `json:"repo"`
+	Auth       string             `json:"auth,omitempty"`
+	SkipVerify bool               `json:"skip_verify,omitempty"`
+	Proxy      BackendProxyConfig `json:"proxy"`
+}
+
+type BackendProxyConfig struct {
+	URL      string `json:"url"`
+	Fallback bool   `json:"fallback"`
+	PingURL  string `json:"ping_url"`
+}
+
+func NewRegistryBackendConfig(parsed reference.Named, insecure bool) (RegistryBackendConfig, error) {
+	proxyURL := os.Getenv("HTTP_PROXY")
+	if proxyURL == "" {
+		proxyURL = os.Getenv("HTTPS_PROXY")
+	}
+
+	backendConfig := RegistryBackendConfig{
+		Scheme:     "https",
+		Host:       reference.Domain(parsed),
+		Repo:       reference.Path(parsed),
+		SkipVerify: insecure,
+		Proxy: BackendProxyConfig{
+			URL:      proxyURL,
+			Fallback: true,
+		},
+	}
+
+	config := dockerconfig.LoadDefaultConfigFile(os.Stderr)
+	authConfig, err := config.GetAuthConfig(backendConfig.Host)
+	if err != nil {
+		return backendConfig, errors.Wrap(err, "get docker registry auth config")
+	}
+	var auth string
+	if authConfig.Username != "" && authConfig.Password != "" {
+		auth = base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", authConfig.Username, authConfig.Password)))
+	}
+	backendConfig.Auth = auth
+
+	return backendConfig, nil
+}
diff --git a/contrib/nydusify/pkg/viewer/viewer.go b/contrib/nydusify/pkg/viewer/viewer.go
@@ -37,6 +37,7 @@ type Opt struct {
 	BackendConfig string
 	ExpectedArch  string
 	FsVersion     string
+	Prefetch      bool
 }
 
 // fsViewer provides complete view of file system in nydus image
@@ -63,15 +64,16 @@ func New(opt Opt) (*FsViewer, error) {
 	mode := "cached"
 
 	nydusdConfig := tool.NydusdConfig{
-		NydusdPath:    opt.NydusdPath,
-		BackendType:   opt.BackendType,
-		BackendConfig: opt.BackendConfig,
-		BootstrapPath: filepath.Join(opt.WorkDir, "nydus_bootstrap"),
-		ConfigPath:    filepath.Join(opt.WorkDir, "fs/nydusd_config.json"),
-		BlobCacheDir:  filepath.Join(opt.WorkDir, "fs/nydus_blobs"),
-		MountPath:     opt.MountPath,
-		APISockPath:   filepath.Join(opt.WorkDir, "fs/nydus_api.sock"),
-		Mode:          mode,
+		EnablePrefetch: opt.Prefetch,
+		NydusdPath:     opt.NydusdPath,
+		BackendType:    opt.BackendType,
+		BackendConfig:  opt.BackendConfig,
+		BootstrapPath:  filepath.Join(opt.WorkDir, "nydus_bootstrap"),
+		ConfigPath:     filepath.Join(opt.WorkDir, "fs/nydusd_config.json"),
+		BlobCacheDir:   filepath.Join(opt.WorkDir, "fs/nydus_blobs"),
+		MountPath:      opt.MountPath,
+		APISockPath:    filepath.Join(opt.WorkDir, "fs/nydus_api.sock"),
+		Mode:           mode,
 	}
 
 	fsViewer := &FsViewer{