From 56693d4860f794b1fbe11b8091806572f4edfaf2 Mon Sep 17 00:00:00 2001 From: Sascha Grunert Date: Tue, 29 Sep 2020 13:55:48 +0200 Subject: [PATCH] Add `krel anago push` subcommand Target of this subcommand is to completely replace the push logic in anago. For now we only stage the local artifacts with it to verify that everything works as intended. Signed-off-by: Sascha Grunert --- anago | 5 +- cmd/krel/cmd/anago/anago.go | 8 --- cmd/krel/cmd/anago/push.go | 68 ++++++++++++++++++++ pkg/release/push.go | 121 ++++++++++++++++++++++-------------- pkg/release/release.go | 84 ++++++++++++++----------- pkg/release/release_test.go | 55 +++++++--------- pkg/util/common.go | 46 +++++++++----- pkg/util/common_test.go | 30 +++------ 8 files changed, 256 insertions(+), 161 deletions(-) create mode 100644 cmd/krel/cmd/anago/push.go diff --git a/anago b/anago index e4fef64b328..75920fb7a98 100755 --- a/anago +++ b/anago @@ -1334,8 +1334,9 @@ push_all_artifacts () { if [[ -z "$STAGED_LOCATION" ]]; then # Locally Stage the release artifacts in build directory (gcs-stage) - common::runstep release::gcs::locally_stage_release_artifacts \ - $version $BUILD_OUTPUT-$version || return 1 + logrun -v krel anago push \ + --version "$version" --build-dir "$BUILD_OUTPUT-$version" \ + || return 1 fi # The full release stage case diff --git a/cmd/krel/cmd/anago/anago.go b/cmd/krel/cmd/anago/anago.go index 405533df501..8bf6a5de522 100644 --- a/cmd/krel/cmd/anago/anago.go +++ b/cmd/krel/cmd/anago/anago.go @@ -82,14 +82,6 @@ func init() { release.DefaultBaseDir, "", ) - - for _, f := range []string{ - "release-type", - } { - if err := AnagoCmd.MarkPersistentFlagRequired(f); err != nil { - logrus.Fatalf("Unable to set %q flag as required: %v", f, err) - } - } } // runAnago is the function invoked by 'krel anago', responsible for submitting release jobs to GCB diff --git a/cmd/krel/cmd/anago/push.go b/cmd/krel/cmd/anago/push.go new file mode 100644 index 00000000000..09cb1279d1f --- /dev/null +++ b/cmd/krel/cmd/anago/push.go @@ -0,0 +1,68 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package anago + +import ( + "github.com/pkg/errors" + "github.com/spf13/cobra" + + "k8s.io/release/pkg/release" +) + +// pushCmd represents the subcommand for `krel anago push` +var pushCmd = &cobra.Command{ + Use: "push", + Short: "Push release artifacts into the Google Cloud", + Long: `krel anago push + +This subcommand can be used to push the release artifacts to the Google Cloud. +It's only indented to be used from anago, which means the command might be +removed in future releases again when anago goes end of life. +`, + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, args []string) error { + return errors.Wrap(runPush(pushOpts, version), "run krel anago push") + }, +} + +var ( + pushOpts = &release.PushBuildOptions{} + version string +) + +func init() { + pushCmd.PersistentFlags().StringVar( + &version, + "version", + "", + "version to be used", + ) + + pushCmd.PersistentFlags().StringVar( + &pushOpts.BuildDir, + "build-dir", + "", + "build artifact directory of the release", + ) + + AnagoCmd.AddCommand(pushCmd) +} + +func runPush(opts *release.PushBuildOptions, version string) error { + return release.NewPushBuild(opts).StageLocalArtifacts(version) +} diff --git a/pkg/release/push.go b/pkg/release/push.go index 02c6652912f..256b23dd835 100644 --- a/pkg/release/push.go +++ b/pkg/release/push.go @@ -87,27 +87,27 @@ type stageFile struct { var gcpStageFiles = []stageFile{ { srcPath: filepath.Join(GCEPath, "configure-vm.sh"), - dstPath: filepath.Join(GCSStagePath, "extra/gce"), + dstPath: "extra/gce/configure-vm.sh", required: false, }, { srcPath: filepath.Join(GCIPath, "node.yaml"), - dstPath: filepath.Join(GCSStagePath, "extra/gce"), + dstPath: "extra/gce/node.yaml", required: true, }, { srcPath: filepath.Join(GCIPath, "master.yaml"), - dstPath: filepath.Join(GCSStagePath, "extra/gce"), + dstPath: "extra/gce/master.yaml", required: true, }, { srcPath: filepath.Join(GCIPath, "configure.sh"), - dstPath: filepath.Join(GCSStagePath, "extra/gce"), + dstPath: "extra/gce/configure.sh", required: true, }, { srcPath: filepath.Join(GCIPath, "shutdown.sh"), - dstPath: filepath.Join(GCSStagePath, "extra/gce"), + dstPath: "extra/gce/shutdown.sh", required: false, }, } @@ -115,27 +115,27 @@ var gcpStageFiles = []stageFile{ var windowsStageFiles = []stageFile{ { srcPath: filepath.Join(WindowsLocalPath, "configure.ps1"), - dstPath: WindowsGCSPath, + dstPath: "extra/gce/windows/configure.ps1", required: true, }, { srcPath: filepath.Join(WindowsLocalPath, "common.psm1"), - dstPath: WindowsGCSPath, + dstPath: "extra/gce/windows/common.psm1", required: true, }, { srcPath: filepath.Join(WindowsLocalPath, "k8s-node-setup.psm1"), - dstPath: WindowsGCSPath, + dstPath: "extra/gce/windows/k8s-node-setup.psm1", required: true, }, { srcPath: filepath.Join(WindowsLocalPath, "testonly/install-ssh.psm1"), - dstPath: WindowsGCSPath, + dstPath: "extra/gce/windows/install-ssh.psm1", required: true, }, { srcPath: filepath.Join(WindowsLocalPath, "testonly/user-profile.psm1"), - dstPath: WindowsGCSPath, + dstPath: "extra/gce/windows/user-profile.psm1", required: true, }, } @@ -151,15 +151,6 @@ func (p *PushBuild) Push() error { if err != nil { return errors.Wrap(err, "find latest version") } - - if p.opts.CI && IsDirtyBuild(latest) { - return errors.New("refusing to push dirty build with --ci flag given") - } - - if p.opts.VersionSuffix != "" { - latest += "-" + p.opts.VersionSuffix - } - logrus.Infof("Latest version is %s", latest) client, err := storage.NewClient(context.Background()) @@ -283,65 +274,103 @@ func (p *PushBuild) findLatestVersion() (latestVersion string, err error) { valid, err := IsValidReleaseBuild(latestVersion) if err != nil { - return "", errors.Wrap(err, "determine if release build version is valid") + return "", errors.Wrap( + err, "determine if release build version is valid", + ) } if !valid { - return "", errors.Errorf("build version %s is not valid for release", latestVersion) + return "", errors.Errorf( + "build version %s is not valid for release", latestVersion, + ) + } + + if p.opts.CI && IsDirtyBuild(latestVersion) { + return "", errors.Errorf( + "refusing to push dirty build %s with --ci flag given", + latestVersion, + ) } + + if p.opts.VersionSuffix != "" { + latestVersion += "-" + p.opts.VersionSuffix + } + return latestVersion, nil } // StageLocalArtifacts locally stages the release artifacts // was releaselib.sh: release::gcs::locally_stage_release_artifacts func (p *PushBuild) StageLocalArtifacts(version string) error { - if err := util.RemoveAndReplaceDir( - filepath.Join(p.opts.BuildDir, GCSStagePath), - ); err != nil { + logrus.Info("Staging local artifacts") + stageDir := filepath.Join(p.opts.BuildDir, GCSStagePath, version) + + logrus.Infof("Cleaning staging dir %s", stageDir) + if err := util.RemoveAndReplaceDir(stageDir); err != nil { return errors.Wrap(err, "remove and replace GCS staging directory") } // Copy release tarballs to local GCS staging directory for push + logrus.Info("Copying release tarballs") if err := util.CopyDirContentsLocal( - filepath.Join(p.opts.BuildDir, ReleaseTarsPath), - filepath.Join(p.opts.BuildDir, GCSStagePath), + filepath.Join(p.opts.BuildDir, ReleaseTarsPath), stageDir, ); err != nil { return errors.Wrap(err, "copy source directory into destination") } // Copy helpful GCP scripts to local GCS staging directory for push - for _, file := range gcpStageFiles { - if err := util.CopyFileLocal( - filepath.Join(p.opts.BuildDir, file.srcPath), - filepath.Join(p.opts.BuildDir, file.dstPath), - file.required, - ); err != nil { - return errors.Wrap(err, "copy GCP stage files") - } + logrus.Info("Copying GCP stage files") + if err := p.copyStageFiles(stageDir, gcpStageFiles); err != nil { + return errors.Wrapf(err, "copy GCP stage files") } // Copy helpful Windows scripts to local GCS staging directory for push - for _, file := range windowsStageFiles { - if err := util.CopyFileLocal( - filepath.Join(p.opts.BuildDir, file.srcPath), - filepath.Join(p.opts.BuildDir, file.dstPath), - file.required, - ); err != nil { - return errors.Wrap(err, "copy Windows stage files") - } + logrus.Info("Copying Windows stage files") + if err := p.copyStageFiles(stageDir, windowsStageFiles); err != nil { + return errors.Wrapf(err, "copy Windows stage files") } - // Copy the "naked" binaries to GCS. This is useful for install scripts - // that download the binaries directly and don't need tars. + // Copy the plain binaries to GCS. This is useful for install scripts that + // download the binaries directly and don't need tars. + logrus.Info("Copying plain binaries") if err := CopyBinaries( filepath.Join(p.opts.BuildDir, ReleaseStagePath), + stageDir, ); err != nil { return errors.Wrap(err, "stage binaries") } // Write the release checksums - gcsStagePath := filepath.Join(p.opts.BuildDir, GCSStagePath, version) - if err := WriteChecksums(gcsStagePath); err != nil { + logrus.Info("Writing checksums") + if err := WriteChecksums(stageDir); err != nil { return errors.Wrap(err, "write checksums") } return nil } + +// copyStageFiles takes the staging dir and copies each file of `files` into +// it. It also ensures that the base dir exists before copying the file (if the +// file is `required`). +func (p *PushBuild) copyStageFiles(stageDir string, files []stageFile) error { + for _, file := range files { + dstPath := filepath.Join(stageDir, file.dstPath) + + if file.required { + if err := os.MkdirAll( + filepath.Dir(dstPath), os.FileMode(0o755), + ); err != nil { + return errors.Wrapf( + err, "create destination path %s", file.dstPath, + ) + } + } + + if err := util.CopyFileLocal( + filepath.Join(p.opts.BuildDir, file.srcPath), + dstPath, file.required, + ); err != nil { + return errors.Wrapf(err, "copy stage file") + } + } + + return nil +} diff --git a/pkg/release/release.go b/pkg/release/release.go index 51fe7920e42..dad47d06cd6 100644 --- a/pkg/release/release.go +++ b/pkg/release/release.go @@ -86,10 +86,6 @@ const ( // WindowsLocalPath is the directory where Windows GCE scripts are created. WindowsLocalPath = ReleaseStagePath + "/full/kubernetes/cluster/gce/windows" - // WindowsGCSPath is the directory where Windoes GCE scripts are staged - // before push to GCS. - WindowsGCSPath = "gcs-stage/extra/gce/windows" - // ProductionBucket is the default bucket for Kubernetes releases ProductionBucket = "kubernetes-release" @@ -322,54 +318,60 @@ func GetOCIManifest(tarPath string) (*ocispec.Manifest, error) { } // CopyBinaries takes the provided `rootPath` and copies the binaries sorted by -// their platform into the pre-defined `$PWD/bin/$PLATFORM` directories. -func CopyBinaries(rootPath string) error { +// their platform into the `targetPath`. +func CopyBinaries(rootPath, targetPath string) error { platformsPath := filepath.Join(rootPath, "client") - platforms, err := ioutil.ReadDir(platformsPath) + platformsAndArches, err := ioutil.ReadDir(platformsPath) if err != nil { return errors.Wrapf(err, "retrieve platforms from %s", platformsPath) } - cwd, err := os.Getwd() - if err != nil { - return errors.Wrap(err, "get current working directory") - } - - for _, platform := range platforms { - if !platform.IsDir() { + for _, platformArch := range platformsAndArches { + if !platformArch.IsDir() { logrus.Warnf( - "Skipping platform %q because it's not a directory", - platform.Name(), + "Skipping platform and arch %q because it's not a directory", + platformArch.Name(), ) continue } - logrus.Infof("Copying binaries for %s platform", platform.Name()) + split := strings.Split(platformArch.Name(), "-") + if len(split) != 2 { + return errors.Errorf( + "expected `platform-arch` format for %s", platformArch.Name(), + ) + } + + platform := split[0] + arch := split[1] + logrus.Infof( + "Copying binaries for %s platform on %s arch", platform, arch, + ) src := filepath.Join( - rootPath, "client", platform.Name(), "kubernetes/client/bin", + rootPath, "client", platformArch.Name(), "kubernetes", "client", "bin", ) - dst := filepath.Join(cwd, "bin", platform.Name()) // We assume here the "server package" is a superset of the "client // package" - serverSrc := filepath.Join(rootPath, "server", platform.Name()) + serverSrc := filepath.Join(rootPath, "server", platformArch.Name()) if util.Exists(serverSrc) { - logrus.Infof("Server sources found in %s, copying them", serverSrc) - src = filepath.Join(serverSrc, "kubernetes/server/bin") + logrus.Infof("Server source found in %s, copying them", serverSrc) + src = filepath.Join(serverSrc, "kubernetes", "server", "bin") } - logrus.Infof("Copying binaries from %s to %s", src, dst) + dst := filepath.Join(targetPath, "bin", platform, arch) + logrus.Infof("Copying server binaries from %s to %s", src, dst) if err := util.CopyDirContentsLocal(src, dst); err != nil { return errors.Wrapf(err, "copy server binaries from %s to %s", src, dst, ) } - // Copy node binaries if they exist and this isn't a 'server' platform. - nodeSrc := filepath.Join(rootPath, "node", platform.Name()) + // Copy node binaries if they exist and this isn't a 'server' platform + nodeSrc := filepath.Join(rootPath, "node", platformArch.Name()) if !util.Exists(serverSrc) && util.Exists(nodeSrc) { - src = filepath.Join(nodeSrc, "kubernetes/node/bin") + src = filepath.Join(nodeSrc, "kubernetes", "node", "bin") logrus.Infof("Copying node binaries from %s to %s", src, dst) if err := util.CopyDirContentsLocal(src, dst); err != nil { @@ -456,10 +458,6 @@ func WriteChecksums(rootPath string) error { } logrus.Infof("Hashing files in %s", rootPath) - files, err := ioutil.ReadDir(rootPath) - if err != nil { - return errors.Wrapf(err, "reading files in %s", rootPath) - } writeSHAFile := func(fileName string, hasher hash.Hash) error { sha, err := fileToHash(fileName, hasher) @@ -474,16 +472,26 @@ func WriteChecksums(rootPath string) error { ) } - for _, file := range files { - fullFilePath := filepath.Join(rootPath, file.Name()) + if err := filepath.Walk(rootPath, + func(path string, file os.FileInfo, err error) error { + if err != nil { + return err + } + if file.IsDir() { + return nil + } - if err := writeSHAFile(fullFilePath, sha256.New()); err != nil { - return errors.Wrapf(err, "write %s.sha256", file.Name()) - } + if err := writeSHAFile(path, sha256.New()); err != nil { + return errors.Wrapf(err, "write %s.sha256", file.Name()) + } - if err := writeSHAFile(fullFilePath, sha512.New()); err != nil { - return errors.Wrapf(err, "write %s.sha512", file.Name()) - } + if err := writeSHAFile(path, sha512.New()); err != nil { + return errors.Wrapf(err, "write %s.sha512", file.Name()) + } + return nil + }, + ); err != nil { + return errors.Wrapf(err, "traversing root path %s", rootPath) } return nil diff --git a/pkg/release/release_test.go b/pkg/release/release_test.go index 4b1d9ad7214..d83707b2f02 100644 --- a/pkg/release/release_test.go +++ b/pkg/release/release_test.go @@ -589,17 +589,9 @@ func TestGetOCIManifest(t *testing.T) { } func TestCopyBinaries(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - - testDir, err := ioutil.TempDir("", "test-copy-binaries-") - require.Nil(t, err) - require.Nil(t, os.Chdir(testDir)) - defer os.RemoveAll(testDir) - for _, tc := range []struct { prepare func() (rootPath string, cleanup func()) - validate func(error) + validate func(error, string) }{ { // success client prepare: func() (string, func()) { @@ -607,9 +599,9 @@ func TestCopyBinaries(t *testing.T) { require.Nil(t, err) binDir := filepath.Join( - tempDir, "client/linux/kubernetes/client/bin", + tempDir, "client/linux-amd64/kubernetes/client/bin", ) - require.Nil(t, os.MkdirAll(binDir, os.FileMode(0o644))) + require.Nil(t, os.MkdirAll(binDir, os.FileMode(0o755))) for _, f := range []string{"1", "2", "3"} { _, err = os.Create(filepath.Join(binDir, f)) @@ -620,10 +612,10 @@ func TestCopyBinaries(t *testing.T) { require.Nil(t, os.RemoveAll(tempDir)) } }, - validate: func(err error) { + validate: func(err error, testDir string) { require.Nil(t, err) - binDir := filepath.Join(testDir, "bin/linux") + binDir := filepath.Join(testDir, "bin/linux/amd64") require.FileExists(t, filepath.Join(binDir, "1")) require.FileExists(t, filepath.Join(binDir, "2")) require.FileExists(t, filepath.Join(binDir, "3")) @@ -638,7 +630,7 @@ func TestCopyBinaries(t *testing.T) { require.Nil(t, err) require.Nil(t, os.MkdirAll(filepath.Join( - tempDir, "client/linux/kubernetes/client/bin", + tempDir, "client/linux-amd64/kubernetes/client/bin", ), os.FileMode(0o755))) _, err = os.Create(filepath.Join(tempDir, "client/some-file")) @@ -648,7 +640,7 @@ func TestCopyBinaries(t *testing.T) { require.Nil(t, os.RemoveAll(tempDir)) } }, - validate: func(err error) { require.Nil(t, err) }, + validate: func(err error, _ string) { require.Nil(t, err) }, }, { // success server prepare: func() (string, func()) { @@ -656,17 +648,17 @@ func TestCopyBinaries(t *testing.T) { require.Nil(t, err) require.Nil(t, os.MkdirAll(filepath.Join( - tempDir, "client/linux/kubernetes/client/bin", + tempDir, "client/linux-amd64/kubernetes/client/bin", ), os.FileMode(0o755))) require.Nil(t, os.MkdirAll(filepath.Join( - tempDir, "server/linux/kubernetes/server/bin", + tempDir, "server/linux-amd64/kubernetes/server/bin", ), os.FileMode(0o755))) return tempDir, func() { require.Nil(t, os.RemoveAll(tempDir)) } }, - validate: func(err error) { require.Nil(t, err) }, + validate: func(err error, _ string) { require.Nil(t, err) }, }, { // success node prepare: func() (string, func()) { @@ -674,17 +666,17 @@ func TestCopyBinaries(t *testing.T) { require.Nil(t, err) require.Nil(t, os.MkdirAll(filepath.Join( - tempDir, "client/linux/kubernetes/client/bin", + tempDir, "client/linux-amd64/kubernetes/client/bin", ), os.FileMode(0o755))) require.Nil(t, os.MkdirAll(filepath.Join( - tempDir, "node/linux/kubernetes/node/bin", + tempDir, "node/linux-amd64/kubernetes/node/bin", ), os.FileMode(0o755))) return tempDir, func() { require.Nil(t, os.RemoveAll(tempDir)) } }, - validate: func(err error) { require.Nil(t, err) }, + validate: func(err error, _ string) { require.Nil(t, err) }, }, { // failure wrong server dir prepare: func() (string, func()) { @@ -692,17 +684,17 @@ func TestCopyBinaries(t *testing.T) { require.Nil(t, err) require.Nil(t, os.MkdirAll(filepath.Join( - tempDir, "client/linux/kubernetes/client/bin", + tempDir, "client/linux-amd64/kubernetes/client/bin", ), os.FileMode(0o755))) require.Nil(t, os.MkdirAll(filepath.Join( - tempDir, "server/linux/kubernetes/wrong/bin", + tempDir, "server/linux-amd64/kubernetes/wrong/bin", ), os.FileMode(0o755))) return tempDir, func() { require.Nil(t, os.RemoveAll(tempDir)) } }, - validate: func(err error) { require.NotNil(t, err) }, + validate: func(err error, _ string) { require.NotNil(t, err) }, }, { // failure wrong node dir prepare: func() (string, func()) { @@ -710,35 +702,34 @@ func TestCopyBinaries(t *testing.T) { require.Nil(t, err) require.Nil(t, os.MkdirAll(filepath.Join( - tempDir, "client/linux/kubernetes/client/bin", + tempDir, "client/linux-amd64/kubernetes/client/bin", ), os.FileMode(0o755))) require.Nil(t, os.MkdirAll(filepath.Join( - tempDir, "node/linux/kubernetes/wrong/bin", + tempDir, "node/linux-amd64/kubernetes/wrong/bin", ), os.FileMode(0o755))) return tempDir, func() { require.Nil(t, os.RemoveAll(tempDir)) } }, - validate: func(err error) { require.NotNil(t, err) }, + validate: func(err error, _ string) { require.NotNil(t, err) }, }, { // empty dirs should error prepare: func() (string, func()) { return "", func() {} }, - validate: func(err error) { require.NotNil(t, err) }, + validate: func(err error, _ string) { require.NotNil(t, err) }, }, } { // Given rootPath, cleanup := tc.prepare() + stageDir := filepath.Join(rootPath, "stage") // When - err := CopyBinaries(rootPath) + err := CopyBinaries(rootPath, stageDir) // Then - tc.validate(err) + tc.validate(err, stageDir) cleanup() } - - require.Nil(t, os.Chdir(cwd)) } func TestWriteChecksums(t *testing.T) { diff --git a/pkg/util/common.go b/pkg/util/common.go index 7a8794b1fad..9cc4fc4b62c 100644 --- a/pkg/util/common.go +++ b/pkg/util/common.go @@ -442,45 +442,58 @@ func SemverToTagString(tag semver.Version) string { // CopyFileLocal copies a local file from one local location to another. func CopyFileLocal(src, dst string, required bool) error { + logrus.Infof("Trying to copy file %s to %s (required: %v)", src, dst, required) srcStat, err := os.Stat(src) if err != nil && required { - return err + return errors.Wrapf( + err, "source %s is required but does not exist", src, + ) } if os.IsNotExist(err) && !required { + logrus.Infof( + "File %s does not exist but is also not required", + filepath.Base(src), + ) return nil } if !srcStat.Mode().IsRegular() { - return errors.New("cannot copy non-regular file: IsRegular reports whether m describes a regular file. That is, it tests that no mode type bits are set") + return errors.New("cannot copy non-regular file: IsRegular reports " + + "whether m describes a regular file. That is, it tests that no " + + "mode type bits are set") } source, err := os.Open(src) if err != nil { - return err + return errors.Wrapf(err, "open source file %s", src) } defer source.Close() destination, err := os.Create(dst) if err != nil { - return err + return errors.Wrapf(err, "create destination file %s", dst) } defer destination.Close() - _, err = io.Copy(destination, source) - return err + if _, err := io.Copy(destination, source); err != nil { + return errors.Wrapf(err, "copy source %s to destination %s", src, dst) + } + logrus.Infof("Copied %s", filepath.Base(dst)) + return nil } // CopyDirContentsLocal copies local directory contents from one local location // to another. func CopyDirContentsLocal(src, dst string) error { + logrus.Infof("Trying to copy dir %s to %s", src, dst) // If initial destination does not exist create it. if _, err := os.Stat(dst); err != nil { - if err := os.MkdirAll(dst, os.FileMode(0755)); err != nil { - return errors.Wrapf(err, "Unable to create directory at path %s", dst) + if err := os.MkdirAll(dst, os.FileMode(0o755)); err != nil { + return errors.Wrapf(err, "create destination directory %s", dst) } } files, err := ioutil.ReadDir(src) if err != nil { - return err + return errors.Wrapf(err, "reading source dir %s", src) } for _, file := range files { srcPath := filepath.Join(src, file.Name()) @@ -488,35 +501,38 @@ func CopyDirContentsLocal(src, dst string) error { fileInfo, err := os.Stat(srcPath) if err != nil { - return err + return errors.Wrapf(err, "stat source path %s", srcPath) } switch fileInfo.Mode() & os.ModeType { case os.ModeDir: if !Exists(dstPath) { if err := os.MkdirAll(dstPath, os.FileMode(0755)); err != nil { - return err + return errors.Wrapf(err, "creating destination dir %s", dstPath) } } if err := CopyDirContentsLocal(srcPath, dstPath); err != nil { - return err + return errors.Wrapf(err, "copy %s to %s", srcPath, dstPath) } default: if err := CopyFileLocal(srcPath, dstPath, false); err != nil { - return err + return errors.Wrapf(err, "copy %s to %s", srcPath, dstPath) } } } + logrus.Info("Done") return nil } // RemoveAndReplaceDir removes a directory and its contents then recreates it. func RemoveAndReplaceDir(path string) error { + logrus.Infof("Removing %s", path) if err := os.RemoveAll(path); err != nil { - return err + return errors.Wrapf(err, "remove %s", path) } + logrus.Infof("Creating %s", path) if err := os.MkdirAll(path, os.FileMode(0755)); err != nil { - return err + return errors.Wrapf(err, "create %s", path) } return nil } diff --git a/pkg/util/common_test.go b/pkg/util/common_test.go index 94a5dc667b5..f8dd9b3cc4d 100644 --- a/pkg/util/common_test.go +++ b/pkg/util/common_test.go @@ -23,7 +23,6 @@ import ( "io/ioutil" "os" "path/filepath" - "syscall" "testing" "time" @@ -267,12 +266,9 @@ func TestCopyFile(t *testing.T) { dst string required bool } - type want struct { - err error - } cases := map[string]struct { - args args - want want + args args + shouldError bool }{ "CopyFileSuccess": { args: args{ @@ -280,9 +276,7 @@ func TestCopyFile(t *testing.T) { dst: dstFileOnePath, required: true, }, - want: want{ - err: nil, - }, + shouldError: false, }, "CopyFileNotExistNotIgnore": { args: args{ @@ -290,13 +284,7 @@ func TestCopyFile(t *testing.T) { dst: dstFileOnePath, required: true, }, - want: want{ - err: &os.PathError{ - Op: "stat", - Path: "path/does/not/exit", - Err: syscall.ENOENT, - }, - }, + shouldError: true, }, "CopyFileNotExistIgnore": { args: args{ @@ -304,16 +292,18 @@ func TestCopyFile(t *testing.T) { dst: dstFileOnePath, required: false, }, - want: want{ - err: nil, - }, + shouldError: false, }, } for name, tc := range cases { t.Run(name, func(t *testing.T) { copyErr := CopyFileLocal(tc.args.src, tc.args.dst, tc.args.required) - require.IsType(t, tc.want.err, copyErr) + if tc.shouldError { + require.NotNil(t, copyErr) + } else { + require.Nil(t, copyErr) + } if copyErr == nil { _, err := os.Stat(filepath.Join(tc.args.dst)) if err != nil && tc.args.required {