From 0636a5ae8cb71367f64208c9688adf971f834a66 Mon Sep 17 00:00:00 2001 From: Stephanie Date: Wed, 17 Mar 2021 13:52:08 -0400 Subject: [PATCH 1/4] support .devfile.yaml and parse from a path Signed-off-by: Stephanie --- pkg/devfile/parser/context/context.go | 33 ++++++++++++++++++++++++--- pkg/util/util.go | 4 ++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/pkg/devfile/parser/context/context.go b/pkg/devfile/parser/context/context.go index 6dd99de5..b94508eb 100644 --- a/pkg/devfile/parser/context/context.go +++ b/pkg/devfile/parser/context/context.go @@ -3,6 +3,10 @@ package parser import ( "fmt" "net/url" + "os" + "path" + "path/filepath" + "strings" "github.com/devfile/library/pkg/testingutil/filesystem" "github.com/devfile/library/pkg/util" @@ -69,7 +73,17 @@ func (d *DevfileCtx) populateDevfile() (err error) { // Populate fills the DevfileCtx struct with relevant context info func (d *DevfileCtx) Populate() (err error) { - + if !strings.HasSuffix(d.relPath, ".yaml"){ + if _, err := os.Stat(filepath.Join(d.relPath, "devfile.yaml")); os.IsNotExist(err) { + if _, err := os.Stat(filepath.Join(d.relPath, ".devfile.yaml")); os.IsNotExist(err) { + return fmt.Errorf("the provided path is not a valid yaml filepath, and no devfile.yaml or .devfile.yaml under provided path: %s", d.relPath) + } else { + d.relPath = filepath.Join(d.relPath, ".devfile.yaml") + } + } else { + d.relPath = filepath.Join(d.relPath, "devfile.yaml") + } + } if err := d.SetAbsPath(); err != nil { return err } @@ -90,11 +104,24 @@ func (d *DevfileCtx) Populate() (err error) { // PopulateFromURL fills the DevfileCtx struct with relevant context info func (d *DevfileCtx) PopulateFromURL() (err error) { - - _, err = url.ParseRequestURI(d.url) + u, err := url.ParseRequestURI(d.url) if err != nil { return err } + if !strings.HasSuffix(d.url, ".yaml"){ + u.Path = path.Join(u.Path,"devfile.yaml") + param := util.HTTPRequestParams{ + URL: u.String(), + } + if _, err = util.HTTPGetRequest(param, 0); err != nil { + u.Path = path.Join(u.Path, ".devfile.yaml") + param.URL = u.String() + if _, err = util.HTTPGetRequest(param, 0); err != nil { + return fmt.Errorf("the provided url is not a valid yaml filepath, and no devfile.yaml or .devfile.yaml under provided url: %s", d.url) + } + } + d.url = u.String() + } if d.uriMap == nil { d.uriMap = make(map[string]bool) } diff --git a/pkg/util/util.go b/pkg/util/util.go index e8f4b2cd..0adb4849 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -1030,6 +1030,10 @@ func DownloadFileInMemory(url string) ([]byte, error) { if err != nil { return nil, err } + // We have a non 1xx / 2xx status, return an error + if (resp.StatusCode - 300) > 0 { + return nil, errors.Errorf("fail to retrive %s: %s", url, http.StatusText(resp.StatusCode)) + } defer resp.Body.Close() return ioutil.ReadAll(resp.Body) From 82cb43ac4705767f807aed68265fef2bc2c6fbc7 Mon Sep 17 00:00:00 2001 From: Stephanie Date: Wed, 17 Mar 2021 14:03:16 -0400 Subject: [PATCH 2/4] resolve unit test failure Signed-off-by: Stephanie --- pkg/devfile/parser/context/context.go | 20 ++++++++++---------- pkg/devfile/parser/parse_test.go | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/devfile/parser/context/context.go b/pkg/devfile/parser/context/context.go index b94508eb..be281682 100644 --- a/pkg/devfile/parser/context/context.go +++ b/pkg/devfile/parser/context/context.go @@ -73,16 +73,16 @@ func (d *DevfileCtx) populateDevfile() (err error) { // Populate fills the DevfileCtx struct with relevant context info func (d *DevfileCtx) Populate() (err error) { - if !strings.HasSuffix(d.relPath, ".yaml"){ - if _, err := os.Stat(filepath.Join(d.relPath, "devfile.yaml")); os.IsNotExist(err) { - if _, err := os.Stat(filepath.Join(d.relPath, ".devfile.yaml")); os.IsNotExist(err) { - return fmt.Errorf("the provided path is not a valid yaml filepath, and no devfile.yaml or .devfile.yaml under provided path: %s", d.relPath) - } else { - d.relPath = filepath.Join(d.relPath, ".devfile.yaml") - } + if !strings.HasSuffix(d.relPath, ".yaml") { + if _, err := os.Stat(filepath.Join(d.relPath, "devfile.yaml")); os.IsNotExist(err) { + if _, err := os.Stat(filepath.Join(d.relPath, ".devfile.yaml")); os.IsNotExist(err) { + return fmt.Errorf("the provided path is not a valid yaml filepath, and no devfile.yaml or .devfile.yaml under provided path: %s", d.relPath) } else { - d.relPath = filepath.Join(d.relPath, "devfile.yaml") + d.relPath = filepath.Join(d.relPath, ".devfile.yaml") } + } else { + d.relPath = filepath.Join(d.relPath, "devfile.yaml") + } } if err := d.SetAbsPath(); err != nil { return err @@ -108,8 +108,8 @@ func (d *DevfileCtx) PopulateFromURL() (err error) { if err != nil { return err } - if !strings.HasSuffix(d.url, ".yaml"){ - u.Path = path.Join(u.Path,"devfile.yaml") + if !strings.HasSuffix(d.url, ".yaml") { + u.Path = path.Join(u.Path, "devfile.yaml") param := util.HTTPRequestParams{ URL: u.String(), } diff --git a/pkg/devfile/parser/parse_test.go b/pkg/devfile/parser/parse_test.go index 9e72c7f5..160a7cda 100644 --- a/pkg/devfile/parser/parse_test.go +++ b/pkg/devfile/parser/parse_test.go @@ -2390,7 +2390,7 @@ func Test_parseParentAndPlugin_RecursivelyReference_withMultipleURI(t *testing.T defer testServer3.Close() t.Run("it should error out if URI is recursively referenced with multiple references", func(t *testing.T) { err := parseParentAndPlugin(devFileObj) - expectedErr := fmt.Sprintf("URI %v%v is recursively referenced", httpPrefix, uri1) + expectedErr := fmt.Sprintf("URI %v%v/devfile.yaml is recursively referenced", httpPrefix, uri1) // Unexpected error if err == nil || !reflect.DeepEqual(expectedErr, err.Error()) { t.Errorf("Test_parseParentAndPlugin_RecursivelyReference_withMultipleURI() unexpected error = %v", err) From be2a24e50d68d1be817d91931c161739db175aee Mon Sep 17 00:00:00 2001 From: Stephanie Date: Thu, 18 Mar 2021 17:53:05 -0400 Subject: [PATCH 3/4] update based on suggestion Signed-off-by: Stephanie --- pkg/devfile/parser/context/content.go | 2 +- pkg/devfile/parser/context/context.go | 14 +++++--------- pkg/util/util.go | 4 ++-- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/pkg/devfile/parser/context/content.go b/pkg/devfile/parser/context/content.go index e9d81739..7f98dcaa 100644 --- a/pkg/devfile/parser/context/content.go +++ b/pkg/devfile/parser/context/content.go @@ -54,7 +54,7 @@ func (d *DevfileCtx) SetDevfileContent() error { if d.url != "" { data, err = util.DownloadFileInMemory(d.url) if err != nil { - return errors.Wrap(err, "error getting parent info from url") + return errors.Wrap(err, "error getting devfile info from url") } } else if d.absPath != "" { // Read devfile diff --git a/pkg/devfile/parser/context/context.go b/pkg/devfile/parser/context/context.go index be281682..327fc433 100644 --- a/pkg/devfile/parser/context/context.go +++ b/pkg/devfile/parser/context/context.go @@ -76,7 +76,7 @@ func (d *DevfileCtx) Populate() (err error) { if !strings.HasSuffix(d.relPath, ".yaml") { if _, err := os.Stat(filepath.Join(d.relPath, "devfile.yaml")); os.IsNotExist(err) { if _, err := os.Stat(filepath.Join(d.relPath, ".devfile.yaml")); os.IsNotExist(err) { - return fmt.Errorf("the provided path is not a valid yaml filepath, and no devfile.yaml or .devfile.yaml under provided path: %s", d.relPath) + return fmt.Errorf("the provided path is not a valid yaml filepath, and devfile.yaml or .devfile.yaml not found in the provided path : %s", d.relPath) } else { d.relPath = filepath.Join(d.relPath, ".devfile.yaml") } @@ -110,14 +110,10 @@ func (d *DevfileCtx) PopulateFromURL() (err error) { } if !strings.HasSuffix(d.url, ".yaml") { u.Path = path.Join(u.Path, "devfile.yaml") - param := util.HTTPRequestParams{ - URL: u.String(), - } - if _, err = util.HTTPGetRequest(param, 0); err != nil { - u.Path = path.Join(u.Path, ".devfile.yaml") - param.URL = u.String() - if _, err = util.HTTPGetRequest(param, 0); err != nil { - return fmt.Errorf("the provided url is not a valid yaml filepath, and no devfile.yaml or .devfile.yaml under provided url: %s", d.url) + if _, err = util.DownloadFileInMemory(u.String()); err != nil { + u.Path = path.Join(path.Dir(u.Path), ".devfile.yaml") + if _, err = util.DownloadFileInMemory(u.String()); err != nil { + return fmt.Errorf("the provided url is not a valid yaml filepath, and devfile.yaml or .devfile.yaml not found in the provided path : %s", d.url) } } d.url = u.String() diff --git a/pkg/util/util.go b/pkg/util/util.go index 0adb4849..b18d9360 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -768,7 +768,7 @@ func HTTPGetRequest(request HTTPRequestParams, cacheFor int) ([]byte, error) { // We have a non 1xx / 2xx status, return an error if (resp.StatusCode - 300) > 0 { - return nil, errors.Errorf("fail to retrive %s: %s", request.URL, http.StatusText(resp.StatusCode)) + return nil, errors.Errorf("fail to retrive %s, status code: %s", request.URL, http.StatusText(resp.StatusCode)) } // Process http response @@ -1032,7 +1032,7 @@ func DownloadFileInMemory(url string) ([]byte, error) { } // We have a non 1xx / 2xx status, return an error if (resp.StatusCode - 300) > 0 { - return nil, errors.Errorf("fail to retrive %s: %s", url, http.StatusText(resp.StatusCode)) + return nil, errors.Errorf("fail to retrive %s, status code: %s", url, http.StatusText(resp.StatusCode)) } defer resp.Body.Close() From d770d3425657305a2c610f8bfd7427078f58b4ff Mon Sep 17 00:00:00 2001 From: Stephanie Date: Fri, 19 Mar 2021 12:08:52 -0400 Subject: [PATCH 4/4] edit error messages Signed-off-by: Stephanie --- pkg/util/util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/util/util.go b/pkg/util/util.go index b18d9360..82425894 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -768,7 +768,7 @@ func HTTPGetRequest(request HTTPRequestParams, cacheFor int) ([]byte, error) { // We have a non 1xx / 2xx status, return an error if (resp.StatusCode - 300) > 0 { - return nil, errors.Errorf("fail to retrive %s, status code: %s", request.URL, http.StatusText(resp.StatusCode)) + return nil, errors.Errorf("fail to retrive %s, %v: %s", request.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) } // Process http response @@ -1032,7 +1032,7 @@ func DownloadFileInMemory(url string) ([]byte, error) { } // We have a non 1xx / 2xx status, return an error if (resp.StatusCode - 300) > 0 { - return nil, errors.Errorf("fail to retrive %s, status code: %s", url, http.StatusText(resp.StatusCode)) + return nil, errors.Errorf("fail to retrive %s, %v: %s", url, resp.StatusCode, http.StatusText(resp.StatusCode)) } defer resp.Body.Close()