Merge branch 'main' into API-46

Author: mike-hoang

pkg/devfile/parser/parse.go

@@ -400,24 +400,14 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D
 func parseFromRegistry(importReference v1.ImportReference, resolveCtx *resolutionContextTree, tool resolverTools) (d DevfileObj, err error) {
 	id := importReference.Id
 	registryURL := importReference.RegistryUrl
-	destDir := "."
-
-	// contains the downloaded stack from the registry
-	tempDir, _ := ioutil.TempDir("", "")
-	defer os.RemoveAll(tempDir)
+	destDir := d.Ctx.GetAbsPath()
 
 	if registryURL != "" {
-		err = getStackFromRegistry(id, registryURL, tempDir, destDir)
+		devfileContent, err := getDevfileFromRegistry(id, registryURL, importReference.Version)
 		if err != nil {
 			return DevfileObj{}, err
 		}
-		devfileContent, err := getDevfileFromDir(tempDir)
-		if err != nil {
-			devfileContent, err = getDevfileFromRegistry(id, registryURL, importReference.Version)
-			if err != nil {
-				return DevfileObj{}, err
-			}
-		}
+		getStackFromRegistry(id, registryURL, destDir)
 		d.Ctx, err = devfileCtx.NewByteContentDevfileCtx(devfileContent)
 		if err != nil {
 			return d, errors.Wrap(err, "failed to set devfile content from bytes")
@@ -428,14 +418,8 @@ func parseFromRegistry(importReference v1.ImportReference, resolveCtx *resolutio
 
 	} else if tool.registryURLs != nil {
 		for _, registryURL := range tool.registryURLs {
-			err = getStackFromRegistry(id, registryURL, tempDir, destDir)
-			if err != nil {
-				return DevfileObj{}, err
-			}
-			devfileContent, err := getDevfileFromDir(tempDir)
-			if err != nil {
-				devfileContent, err = getDevfileFromRegistry(id, registryURL, importReference.Version)
-			}
+			devfileContent, err := getDevfileFromRegistry(id, registryURL, importReference.Version)
+			getStackFromRegistry(id, registryURL, destDir)
 			if devfileContent != nil && err == nil {
 				d.Ctx, err = devfileCtx.NewByteContentDevfileCtx(devfileContent)
 				if err != nil {
@@ -454,24 +438,6 @@ func parseFromRegistry(importReference v1.ImportReference, resolveCtx *resolutio
 	return DevfileObj{}, fmt.Errorf("failed to get id: %s from registry URLs provided", id)
 }
 
-func getStackFromRegistry(id, registryURL, stackDir, destDir string) error {
-	if !strings.HasPrefix(registryURL, "http://") && !strings.HasPrefix(registryURL, "https://") {
-		return fmt.Errorf("the provided registryURL: %s is not a valid URL", registryURL)
-	}
-	registryLibrary.PullStackFromRegistry(registryURL, id, stackDir, registryLibrary.RegistryOptions{})
-	util.CopyAllDirFiles(stackDir, destDir)
-
-	return nil
-}
-
-func getDevfileFromDir(dir string) ([]byte, error) {
-	devfileContent, err := ioutil.ReadFile(path.Join(dir, "devfile.yaml"))
-	if err != nil {
-		return nil, fmt.Errorf("failed to get devfile from stack registry")
-	}
-	return devfileContent, err
-}
-
 func getDevfileFromRegistry(id, registryURL, version string) ([]byte, error) {
 	if !strings.HasPrefix(registryURL, "http://") && !strings.HasPrefix(registryURL, "https://") {
 		return nil, fmt.Errorf("the provided registryURL: %s is not a valid URL", registryURL)
@@ -482,6 +448,15 @@ func getDevfileFromRegistry(id, registryURL, version string) ([]byte, error) {
 	return util.HTTPGetRequest(param, 0)
 }
 
+func getStackFromRegistry(id, registryURL, destDir string) {
+	stackDir, _ := ioutil.TempDir(os.TempDir(), fmt.Sprintf("stack-%s", id))
+	defer os.RemoveAll(stackDir)
+
+	registryLibrary.PullStackFromRegistry(registryURL, id, stackDir, registryLibrary.RegistryOptions{})
+	util.CopyAllDirFiles(stackDir, destDir)
+	return
+}
+
 func parseFromKubeCRD(importReference v1.ImportReference, resolveCtx *resolutionContextTree, tool resolverTools) (d DevfileObj, err error) {
 
 	if tool.k8sClient == nil || tool.context == nil {

pkg/devfile/parser/parse_test.go

@@ -20,7 +20,6 @@ import (
 	"net/http/httptest"
 	"os"
 	"path"
-	"path/filepath"
 	"reflect"
 	"sigs.k8s.io/yaml"
 	"strings"
@@ -4318,52 +4317,6 @@ func Test_parseFromRegistry(t *testing.T) {
 	}
 }
 
-func Test_getDevfileFromDir(t *testing.T) {
-	tempDir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Errorf("Failed to create temp dir: %s, error: %v", tempDir, err)
-	}
-	defer os.RemoveAll(tempDir)
-
-	devfile := filepath.Join(tempDir, "devfile.yaml")
-	if err := ioutil.WriteFile(devfile, []byte(""), 0666); err != nil {
-		t.Errorf("Failed to create temp devfile, error: %v", err)
-	}
-
-	missingDevfileDir := "no/devfile/here"
-
-	tests := []struct {
-		name    string
-		dir     string
-		wantErr bool
-	}{
-		{
-			name:    "should be able to get devfile from dir",
-			dir:     tempDir,
-			wantErr: false,
-		},
-		{
-			name:    "should fail if directory doesn't have devfile",
-			dir:     missingDevfileDir,
-			wantErr: true,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			gotErr := false
-			_, err = getDevfileFromDir(tt.dir)
-			if err != nil {
-				gotErr = true
-			}
-
-			if !reflect.DeepEqual(gotErr, tt.wantErr) {
-				t.Errorf("Got error: %t, want error: %t", gotErr, tt.wantErr)
-			}
-		})
-	}
-}
-
 func Test_parseFromKubeCRD(t *testing.T) {
 	const (
 		namespace  = "default"

pkg/util/util.go

@@ -421,7 +421,11 @@ func FetchResourceQuantity(resourceType corev1.ResourceName, min string, max str
 
 // CheckPathExists checks if a path exists or not
 func CheckPathExists(path string) bool {
-	if _, err := os.Stat(path); !os.IsNotExist(err) {
+	return checkPathExistsOnFS(path, filesystem.DefaultFs{})
+}
+
+func checkPathExistsOnFS(path string, fs filesystem.Filesystem) bool {
+	if _, err := fs.Stat(path); !os.IsNotExist(err) {
 		// path to file does exist
 		return true
 	}
@@ -1156,10 +1160,14 @@ func CopyFile(srcPath string, dstPath string, info os.FileInfo) error {
 }
 
 // CopyAllDirFiles recursively copies a source directory to a destination directory
-func CopyAllDirFiles(srcDir string, destDir string) error {
+func CopyAllDirFiles(srcDir, destDir string) error {
+	return copyAllDirFilesOnFS(srcDir, destDir, filesystem.DefaultFs{})
+}
+
+func copyAllDirFilesOnFS(srcDir, destDir string, fs filesystem.Filesystem) error {
 	var info os.FileInfo
 
-	files, err := ioutil.ReadDir(srcDir)
+	files, err := fs.ReadDir(srcDir)
 	if err != nil {
 		return errors.Wrapf(err, "failed reading dir %v", srcDir)
 	}
@@ -1169,22 +1177,22 @@ func CopyAllDirFiles(srcDir string, destDir string) error {
 		destPath := path.Join(destDir, file.Name())
 
 		if file.IsDir() {
-			if info, err = os.Stat(srcPath); err != nil {
+			if info, err = fs.Stat(srcPath); err != nil {
 				return err
 			}
-			if err = os.MkdirAll(destPath, info.Mode()); err != nil {
+			if err = fs.MkdirAll(destPath, info.Mode()); err != nil {
 				return err
 			}
-			if err = CopyAllDirFiles(srcPath, destPath); err != nil {
+			if err = copyAllDirFilesOnFS(srcPath, destPath, fs); err != nil {
 				return err
 			}
 		} else {
 			if file.Name() == "devfile.yaml" {
 				continue
 			}
 			// Only copy files that do not exist in the destination directory
-			if _, err := os.Stat(destPath); errors.Is(err, os.ErrNotExist) {
-				if err := CopyFile(srcPath, destPath, file); err != nil {
+			if !checkPathExistsOnFS(destPath, fs) {
+				if err := copyFileOnFs(srcPath, destPath, fs); err != nil {
 					return errors.Wrapf(err, "failed to copy %s to %s", srcPath, destPath)
 				}
 			}
@@ -1193,6 +1201,40 @@ func CopyAllDirFiles(srcDir string, destDir string) error {
 	return nil
 }
 
+// copied from: https://github.com/devfile/registry-support/blob/main/index/generator/library/util.go
+func copyFileOnFs(src, dst string, fs filesystem.Filesystem) error {
+	var err error
+	var srcinfo os.FileInfo
+
+	srcfd, err := fs.Open(src)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if e := srcfd.Close(); e != nil {
+			fmt.Printf("err occurred while closing file: %v", e)
+		}
+	}()
+
+	dstfd, err := fs.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if e := dstfd.Close(); e != nil {
+			fmt.Printf("err occurred while closing file: %v", e)
+		}
+	}()
+
+	if _, err = io.Copy(dstfd, srcfd); err != nil {
+		return err
+	}
+	if srcinfo, err = fs.Stat(src); err != nil {
+		return err
+	}
+	return fs.Chmod(dst, srcinfo.Mode())
+}
+
 // PathEqual compare the paths to determine if they are equal
 func PathEqual(firstPath string, secondPath string) bool {
 	firstAbsPath, _ := GetAbsPath(firstPath)