feat: always copy sources to the source folder (#470)

* feat: always copy sources to the source folder
No real benefit in adding an extra step to copy
sources. UI already copies sources anyway.
* feat: write all the new files in the source/ folder to a file
* fix: install script should try to install plugins on prereleases as well

Signed-off-by: Harikrishnan Balagopal <[email protected]>

go.mod

@@ -7,6 +7,7 @@ require (
 	code.cloudfoundry.org/cli v7.1.0+incompatible
 	github.com/AlecAivazis/survey/v2 v2.2.3
 	github.com/Masterminds/semver/v3 v3.1.1
+	github.com/a8m/tree v0.0.0-20210115125333-10a5fd5b637d
 	github.com/cloudfoundry/bosh-cli v6.4.1+incompatible
 	github.com/cloudfoundry/bosh-utils v0.0.0-20210327100132-d1d58eaab188 // indirect
 	github.com/cppforlife/go-patch v0.2.0 // indirect
@@ -21,6 +22,7 @@ require (
 	github.com/mikefarah/yq/v4 v4.4.1
 	github.com/moby/buildkit v0.7.2
 	github.com/openshift/api v0.0.0-20200930075302-db52bc4ef99f // release-4.6
+	github.com/otiai10/copy v1.0.2
 	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
 	github.com/pkg/errors v0.9.1
 	github.com/qri-io/starlib v0.4.2

go.sum

@@ -254,6 +254,8 @@ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrU
 github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
+github.com/a8m/tree v0.0.0-20210115125333-10a5fd5b637d h1:4E8RufAN3UQ/weB6AnQ4y5miZCO0Yco8ZdGId41WuQs=
+github.com/a8m/tree v0.0.0-20210115125333-10a5fd5b637d/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg=
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5 h1:rFw4nCn9iMW+Vajsk51NtYIcwSTkXr+JGrMd36kTDJw=
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
 github.com/agnivade/levenshtein v1.0.1 h1:3oJU7J3FGFmyhn8KHjmVaZCN5hxTr7GxgRue+sxIXdQ=
@@ -2109,9 +2111,6 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v0.0.0-20161030231247-84d19640f6a7 h1:lrPFiLn7iuI5BPhrGUtOINABGYcgSNM2nSB4QteC3kI=
 github.com/xeipuuv/gojsonschema v0.0.0-20161030231247-84d19640f6a7/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
-github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
-github.com/xeipuuv/gojsonschema v1.1.0 h1:ngVtJC9TY/lg0AA/1k48FYhBrhRoFlEmWzsehpNAaZg=
-github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
 github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=

internal/move2kube/translator.go

@@ -80,7 +80,6 @@ func Translate(plan plantypes.Plan, outputPath string, qadisablecli bool, transf
 	}
 	log.Debugf("Total storages customized : %d", len(customizedIR.Storages))
 
-	customizedIR.AddCopySources = !qadisablecli
 	if err := transform.Transform(customizedIR, outputPath, transformPaths); err != nil {
 		log.Fatalf("Error occurred while running the customizers. Error: %q", err)
 	}

internal/transformer/k8stransformer.go

@@ -54,7 +54,6 @@ type K8sTransformer struct {
 	Name                            string
 	IgnoreUnsupportedKinds          bool
 	ExposedServicePaths             map[string]string
-	AddCopySources                  bool
 }
 
 // NewK8sTransformer creates a new instance of K8sTransformer
@@ -97,7 +96,6 @@ func (kt *K8sTransformer) Transform(ir irtypes.IR) error {
 	kt.reorderParameterizedObjects()
 
 	kt.RootDir = ir.RootDir
-	kt.AddCopySources = ir.AddCopySources
 
 	for _, service := range ir.Services {
 		if service.HasValidAnnotation(common.ExposeSelector) {
@@ -183,7 +181,7 @@ func (kt *K8sTransformer) WriteObjects(outputPath string, transformPaths []strin
 	}
 
 	// source/
-	areNewImagesCreated := writeContainers(kt.Containers, outputPath, kt.RootDir, kt.Values.RegistryURL, kt.Values.RegistryNamespace, kt.AddCopySources)
+	areNewImagesCreated := writeContainers(kt.Containers, outputPath, kt.RootDir, kt.Values.RegistryURL, kt.Values.RegistryNamespace)
 
 	// deploy/helm/ and scripts/deployhelm.sh
 	if err := kt.generateHelmMetadata(outputPath, kt.Values); err != nil {
@@ -215,7 +213,7 @@ func (kt *K8sTransformer) WriteObjects(outputPath string, transformPaths []strin
 	}
 
 	// README.md
-	kt.writeReadMe(kt.Name, areNewImagesCreated, kt.AddCopySources, outputPath)
+	kt.writeReadMe(kt.Name, areNewImagesCreated, outputPath)
 
 	return nil
 }
@@ -373,15 +371,13 @@ func (kt *K8sTransformer) generateKustomize(kustomizePath string, transformPaths
 	return kustomize.GenerateKustomize(kustomizePath, filenames, fixedConvertedObjs, fixedConvertedParamObjs)
 }
 
-func (kt *K8sTransformer) writeReadMe(project string, areNewImages bool, addCopySources bool, outpath string) {
+func (kt *K8sTransformer) writeReadMe(project string, areNewImages bool, outpath string) {
 	err := common.WriteTemplateToFile(templates.K8sReadme_md, struct {
 		Project        string
 		NewImages      bool
-		AddCopySources bool
 	}{
 		Project:        project,
 		NewImages:      areNewImages,
-		AddCopySources: addCopySources,
 	}, filepath.Join(outpath, "README.md"), common.DefaultFilePermission)
 	if err != nil {
 		log.Errorf("Unable to write readme : %s", err)

internal/transformer/knativetransformer.go

@@ -40,7 +40,6 @@ type KnativeTransformer struct {
 	TargetClusterSpec      collecttypes.ClusterMetadataSpec
 	Name                   string
 	IgnoreUnsupportedKinds bool
-	AddCopySources         bool
 }
 
 // Transform translates intermediate representation to destination objects
@@ -55,7 +54,6 @@ func (kt *KnativeTransformer) Transform(ir irtypes.IR) error {
 	kt.IgnoreUnsupportedKinds = ir.Kubernetes.IgnoreUnsupportedKinds
 	kt.TransformedObjects = convertIRToObjects(irtypes.NewEnhancedIRFromIR(ir), kt.getAPIResources())
 	kt.RootDir = ir.RootDir
-	kt.AddCopySources = ir.AddCopySources
 	log.Debugf("Total transformed objects : %d", len(kt.TransformedObjects))
 
 	return nil

internal/transformer/templates/K8sReadme.md

@@ -12,9 +12,6 @@ Prerequisites
 Next Steps
 ----------
 {{- if .NewImages }}
-{{- if .AddCopySources}}
-* Copy the source directory into the "./source/" folder for packaging as containers using "./scripts/copysources.sh <SRC_DIR>"
-{{- end }}
 * Build your images using "./scripts/buildimages.sh"
 * Push images to registry "./scripts/pushimages.sh <REGISTRY_URL> <REGISTRY_NAMESPACE>"
 {{- end}}

internal/transformer/transformer.go

@@ -17,14 +17,18 @@ limitations under the License.
 package transform
 
 import (
+	"bytes"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
 	"reflect"
 	"strings"
 
+	"github.com/a8m/tree"
+	"github.com/a8m/tree/ostree"
 	"github.com/konveyor/move2kube/internal/apiresource"
 	"github.com/konveyor/move2kube/internal/common"
 	"github.com/konveyor/move2kube/internal/k8sschema"
@@ -37,6 +41,7 @@ import (
 	"github.com/konveyor/move2kube/internal/transformer/transformations"
 	irtypes "github.com/konveyor/move2kube/internal/types"
 	collecttypes "github.com/konveyor/move2kube/types/collection"
+	"github.com/otiai10/copy"
 	log "github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -86,12 +91,11 @@ func convertIRToObjects(ir irtypes.EnhancedIR, apis []apiresource.IAPIResource)
 }
 
 // writeContainers returns true if any scripts were written
-func writeContainers(containers []irtypes.Container, outputPath, rootDir, registryURL, registryNamespace string, addCopySources bool) bool {
-	containersDir := common.SourceDir
-	containersPath := path.Join(outputPath, containersDir)
-	log.Debugf("containersPath: %s", containersPath)
-	if err := os.MkdirAll(containersPath, common.DefaultDirectoryPermission); err != nil {
-		log.Errorf("Unable to create directory %s : %s", containersPath, err)
+func writeContainers(containers []irtypes.Container, outputPath, rootDir, registryURL, registryNamespace string) bool {
+	sourcePath := filepath.Join(outputPath, common.SourceDir)
+	log.Debugf("containersPath: %s", sourcePath)
+	if err := os.MkdirAll(sourcePath, common.DefaultDirectoryPermission); err != nil {
+		log.Errorf("Unable to create directory %s : %s", sourcePath, err)
 	}
 	scriptsPath := path.Join(outputPath, common.ScriptsDir)
 	if err := os.MkdirAll(scriptsPath, common.DefaultDirectoryPermission); err != nil {
@@ -112,20 +116,20 @@ func writeContainers(containers []irtypes.Container, outputPath, rootDir, regist
 		log.Debugf("New Container : %s", container.ImageNames[0])
 		dockerImages = append(dockerImages, container.ImageNames...)
 		for relPath, filecontents := range container.NewFiles {
-			writepath := filepath.Join(containersPath, relPath)
-			directory := filepath.Dir(writepath)
+			writePath := filepath.Join(sourcePath, relPath)
+			directory := filepath.Dir(writePath)
 			if err := os.MkdirAll(directory, common.DefaultDirectoryPermission); err != nil {
 				log.Errorf("Unable to create directory %s : %s", directory, err)
 				continue
 			}
 			fileperm := common.DefaultFilePermission
-			if filepath.Ext(writepath) == ".sh" {
+			if filepath.Ext(writePath) == ".sh" {
 				fileperm = common.DefaultExecutablePermission
-				buildScripts = append(buildScripts, filepath.Join(containersDir, relPath))
+				buildScripts = append(buildScripts, filepath.Join(common.SourceDir, relPath))
 			}
-			log.Debugf("Writing at %s", writepath)
-			if err := ioutil.WriteFile(writepath, []byte(filecontents), fileperm); err != nil {
-				log.Warnf("Error writing file at %s : %s", writepath, err)
+			log.Debugf("Writing at %s", writePath)
+			if err := ioutil.WriteFile(writePath, []byte(filecontents), fileperm); err != nil {
+				log.Warnf("Error writing to file at path %s Error: %q", writePath, err)
 			}
 		}
 	}
@@ -141,6 +145,27 @@ func writeContainers(containers []irtypes.Container, outputPath, rootDir, regist
 			log.Errorf("Unable to create manual image : %s", err)
 		}
 	}
+
+	{
+		// write out the list of new files as a directory tree
+		treeBytes := bytes.Buffer{}
+		treeBufBytes := io.Writer(&treeBytes)
+		sourceTree := tree.New(sourcePath)
+		opts := &tree.Options{
+			Fs:      new(ostree.FS),
+			OutFile: treeBufBytes,
+		}
+		numDir, numFiles := sourceTree.Visit(opts)
+		log.Debugf("Visiting files in source/ . Found %d directories and %d files", numDir, numFiles)
+		sourceTree.Print(opts)
+		log.Debugf("%s", treeBytes.String())
+		newFiles := common.SourceDir + "/\n" + strings.Join(strings.Split(treeBytes.String(), "\n")[1:], "\n") // remove the first line containing source directory path
+		newFilesTextPath := filepath.Join(outputPath, "newfiles.txt")
+		if err := ioutil.WriteFile(newFilesTextPath, []byte(newFiles), common.DefaultFilePermission); err != nil {
+			log.Errorf("Faled to create a file at path %s . Error: %q", newFilesTextPath, err)
+		}
+	}
+
 	if len(buildScripts) > 0 {
 		buildScriptMap := map[string]string{}
 		for _, value := range buildScripts {
@@ -150,27 +175,16 @@ func writeContainers(containers []irtypes.Container, outputPath, rootDir, regist
 		log.Debugf("buildscripts %s", buildScripts)
 		log.Debugf("buildScriptMap %s", buildScriptMap)
 		writepath := filepath.Join(scriptsPath, "buildimages.sh")
-		err := common.WriteTemplateToFile(templates.Buildimages_sh, buildScriptMap, writepath, common.DefaultExecutablePermission)
-		if err != nil {
+		if err := common.WriteTemplateToFile(templates.Buildimages_sh, buildScriptMap, writepath, common.DefaultExecutablePermission); err != nil {
 			log.Errorf("Unable to create script to build images : %s", err)
 		}
-		if addCopySources {
-			relRootDir, err := filepath.Rel(outputPath, rootDir)
-			if err != nil {
-				log.Errorf("Failed to make the root directory path %q relative to the output directory %q Error %q", rootDir, outputPath, err)
-				relRootDir = rootDir
-			}
-			writepath = filepath.Join(scriptsPath, "copysources.sh")
-			err = common.WriteTemplateToFile(templates.CopySources_sh, struct {
-				RelRootDir string
-				Dst        string
-			}{
-				RelRootDir: relRootDir,
-				Dst:        containersDir,
-			}, writepath, common.DefaultExecutablePermission)
-			if err != nil {
-				log.Errorf("Unable to create script to build images : %s", err)
-			}
+
+		// copy all the sources into source/
+		sourcePath := filepath.Join(outputPath, common.SourceDir)
+		if err := os.MkdirAll(sourcePath, common.DefaultDirectoryPermission); err != nil {
+			log.Errorf("Failed to create the source directory at path %s . Error: %q", sourcePath, err)
+		} else if err := copy.Copy(rootDir, sourcePath); err != nil {
+			log.Errorf("Failed to copy the sources over to the folder at path %s Error: %q", sourcePath, err)
 		}
 	}
 	if len(dockerImages) > 0 {

internal/types/ir.go

@@ -49,8 +49,6 @@ type IR struct {
 	Values outputtypes.HelmValues
 
 	IngressTLSSecretName string
-
-	AddCopySources bool
 }
 
 // EnhancedIR is IR with extra data specific to API resource sets

scripts/install.sh

@@ -46,6 +46,19 @@ HAS_OPENSSL="$(command -v openssl >/dev/null && echo true || echo false)"
 HAS_SHA256SUM="$(command -v sha256sum >/dev/null && echo true || echo false)"
 HAS_MOVE2KUBE="$(command -v "$BINARY_NAME" >/dev/null && echo true || echo false)"
 
+isURLExist() {
+    if [ "$#" -ne 1 ]; then
+        echo 'isURLExist needs exactly 1 arg: the url to check'
+        echo "actual args: $*"
+        exit 1
+    fi
+    if [ "$HAS_CURL" = 'true' ]; then
+        curl -fsS --head "$1" >/dev/null 2>&1
+        return
+    fi
+    wget -q --spider "$1"
+}
+
 download() {
     if [ "$#" -ne 2 ]; then
         echo 'download needs exactly 2 args: the url to download and the output path'
@@ -244,7 +257,7 @@ runAsRoot() {
 testVersion() {
     set +e
     if ! command -v "$BINARY_NAME" >/dev/null; then
-        echo "$BINARY_NAME not found. Is $MOVE2KUBE_INSTALL_DIR on your "'$PATH?'
+        echo "$BINARY_NAME not found. Is $MOVE2KUBE_INSTALL_DIR on your PATH?"
         exit 1
     fi
     set -e
@@ -283,6 +296,10 @@ askBeforeInstallingKubectlPlugins() {
 downloadAndInstallPlugin() {
     local plugin_dist="kubectl-translate-$MOVE2KUBE_TAG-$OS-$ARCH.tar.gz"
     local plugin_download_url="https://github.com/konveyor/move2kube/releases/download/$MOVE2KUBE_TAG/$plugin_dist"
+    if ! isURLExist "$plugin_download_url"; then
+        echo 'Failed to find the plugins for this version. Skipping plugin installation.'
+        return 0
+    fi
     local plugin_tmp_file="$MOVE2KUBE_TMP_ROOT/$plugin_dist"
     if [ "$VERIFY_CHECKSUM" = 'true' ]; then
         downloadAndVerifyChecksum "$plugin_download_url" "$plugin_tmp_file"
@@ -331,15 +348,10 @@ main() {
     if ! checkMove2KubeInstalledVersion; then
         downloadMove2Kube
         installMove2Kube
-        if [[ "$MOVE2KUBE_TAG" = *-* ]]; then
-            echo "$MOVE2KUBE_TAG is a prerelease. Plugins are available only for stable releases."
+        if askBeforeInstallingKubectlPlugins; then
+            downloadAndInstallPlugin
         else
-            echo "$MOVE2KUBE_TAG is a stable release."
-            if askBeforeInstallingKubectlPlugins; then
-                downloadAndInstallPlugin
-            else
-                echo 'Failed to get confirmation. Not installing kubectl plugins.'
-            fi
+            echo 'Failed to get confirmation. Not installing kubectl plugins.'
         fi
     fi
     testVersion