Skip to content

Gate downloading of git resources #174

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jul 7, 2023
Merged

Gate downloading of git resources #174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a3c3381
Gate download git resources
maysunfaisal Jun 29, 2023
adab2b0
Update parsing from uri git tests
maysunfaisal Jun 30, 2023
cff5c26
Update readme
maysunfaisal Jul 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,14 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g
...
```

10. By default, the library downloads the Git repository resources associated with the Git URL that is mentioned in a devfile uri field. To turn off the download, pass in the `DownloadGitResources` property in the parser argument
```go
downloadGitResources := false
parserArgs := parser.ParserArgs{
DownloadGitResources: &downloadGitResources,
}
```

## Projects using devfile/library

The following projects are consuming this library as a Golang dependency
Expand Down
35 changes: 24 additions & 11 deletions pkg/devfile/parser/parse.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,16 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/devfile/library/v2/pkg/git"
"github.com/hashicorp/go-multierror"
"io/ioutil"
"net/url"
"os"
"path"
"reflect"
"strings"

"github.com/devfile/library/v2/pkg/git"
"github.com/hashicorp/go-multierror"

"github.com/devfile/api/v2/pkg/attributes"
devfileCtx "github.com/devfile/library/v2/pkg/devfile/parser/context"
"github.com/devfile/library/v2/pkg/devfile/parser/data"
Expand Down Expand Up @@ -167,6 +168,8 @@ type ParserArgs struct {
// ImageNamesAsSelector sets the information that will be used to handle image names as selectors when parsing the Devfile.
// Not setting this field or setting it to nil disables the logic of handling image names as selectors.
ImageNamesAsSelector *ImageSelectorArgs
// DownloadGitResources downloads the resources from Git repository if true
DownloadGitResources *bool
}

// ImageSelectorArgs defines the structure to leverage for using image names as selectors after parsing the Devfile.
Expand Down Expand Up @@ -214,12 +217,18 @@ func ParseDevfile(args ParserArgs) (d DevfileObj, err error) {
d.Ctx.SetToken(args.Token)
}

downloadGitResources := true
if args.DownloadGitResources != nil {
downloadGitResources = *args.DownloadGitResources
}

tool := resolverTools{
defaultNamespace: args.DefaultNamespace,
registryURLs: args.RegistryURLs,
context: args.Context,
k8sClient: args.K8sClient,
httpTimeout: args.HTTPTimeout,
defaultNamespace: args.DefaultNamespace,
registryURLs: args.RegistryURLs,
context: args.Context,
k8sClient: args.K8sClient,
httpTimeout: args.HTTPTimeout,
downloadGitResources: downloadGitResources,
}

flattenedDevfile := true
Expand Down Expand Up @@ -273,6 +282,8 @@ type resolverTools struct {
k8sClient client.Client
// httpTimeout is the timeout value in seconds passed in from the client.
httpTimeout *int
// downloadGitResources downloads the resources from Git repository if true
downloadGitResources bool
}

func populateAndParseDevfile(d DevfileObj, resolveCtx *resolutionContextTree, tool resolverTools, flattenedDevfile bool) (DevfileObj, error) {
Expand Down Expand Up @@ -522,10 +533,12 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D
d.Ctx.SetToken(token)
}

destDir := path.Dir(curDevfileCtx.GetAbsPath())
err = downloadGitRepoResources(newUri, destDir, tool.httpTimeout, token)
if err != nil {
return DevfileObj{}, err
if tool.downloadGitResources {
destDir := path.Dir(curDevfileCtx.GetAbsPath())
err = downloadGitRepoResources(newUri, destDir, tool.httpTimeout, token)
if err != nil {
return DevfileObj{}, err
}
}
}
importReference.Uri = newUri
Expand Down
84 changes: 53 additions & 31 deletions pkg/devfile/parser/parse_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
"bytes"
"context"
"fmt"
"github.com/devfile/library/v2/pkg/util"
"io/ioutil"
"net"
"net/http"
Expand All @@ -31,6 +30,8 @@ import (
"strings"
"testing"

"github.com/devfile/library/v2/pkg/util"

v1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
"github.com/devfile/library/v2/pkg/git"

Expand Down Expand Up @@ -4235,16 +4236,17 @@ func Test_parseFromURI_GitProviders(t *testing.T) {
invalidDevfilePathError := "error getting devfile from url: failed to retrieve*"

tests := []struct {
name string
url string
gitUrl *git.GitUrl
token string
destDir string
importReference v1.ImportReference
wantDevFile DevfileObj
wantError *string
wantResources []string
wantResourceContent []byte
name string
url string
gitUrl *git.GitUrl
token string
destDir string
importReference v1.ImportReference
wantDevFile DevfileObj
wantError *string
wantResources []string
wantResourceContent []byte
downloadGitResources bool
}{
{
name: "private parent devfile",
Expand All @@ -4256,9 +4258,10 @@ func Test_parseFromURI_GitProviders(t *testing.T) {
Uri: server.URL,
},
},
wantDevFile: minimalDevfile,
wantResources: []string{"resource.file"},
wantResourceContent: []byte("private repo\ngit switched"),
wantDevFile: minimalDevfile,
wantResources: []string{"resource.file"},
wantResourceContent: []byte("private repo\ngit switched"),
downloadGitResources: true,
},
{
name: "public parent devfile",
Expand All @@ -4270,9 +4273,22 @@ func Test_parseFromURI_GitProviders(t *testing.T) {
Uri: server.URL,
},
},
wantDevFile: minimalDevfile,
wantResources: []string{"resource.file"},
wantResourceContent: []byte("public repo\ngit switched"),
wantDevFile: minimalDevfile,
wantResources: []string{"resource.file"},
wantResourceContent: []byte("public repo\ngit switched"),
downloadGitResources: true,
},
{
name: "public parent devfile with download turned off",
url: validUrl,
gitUrl: validGitUrl,
importReference: v1.ImportReference{
ImportReferenceUnion: v1.ImportReferenceUnion{
Uri: server.URL,
},
},
wantDevFile: minimalDevfile,
downloadGitResources: false,
},
{
// a valid parent url must contain a revision
Expand All @@ -4293,8 +4309,9 @@ func Test_parseFromURI_GitProviders(t *testing.T) {
Uri: server.URL,
},
},
wantError: &invalidDevfilePathError,
wantResources: []string{},
wantError: &invalidDevfilePathError,
wantResources: []string{},
downloadGitResources: true,
},
{
name: "public parent devfile that is not from a git provider",
Expand All @@ -4306,8 +4323,9 @@ func Test_parseFromURI_GitProviders(t *testing.T) {
Uri: server.URL,
},
},
wantDevFile: minimalDevfile,
wantResources: []string{},
wantDevFile: minimalDevfile,
wantResources: []string{},
downloadGitResources: true,
},
{
name: "public parent devfile with no devfile path",
Expand All @@ -4325,8 +4343,9 @@ func Test_parseFromURI_GitProviders(t *testing.T) {
Uri: server.URL,
},
},
wantError: &invalidDevfilePathError,
wantResources: []string{},
wantError: &invalidDevfilePathError,
wantResources: []string{},
downloadGitResources: true,
},
{
name: "public parent devfile with invalid devfile path",
Expand All @@ -4346,8 +4365,9 @@ func Test_parseFromURI_GitProviders(t *testing.T) {
Uri: server.URL,
},
},
wantError: &invalidDevfilePathError,
wantResources: []string{},
wantError: &invalidDevfilePathError,
wantResources: []string{},
downloadGitResources: true,
},
{
name: "private parent devfile with invalid token",
Expand All @@ -4359,8 +4379,9 @@ func Test_parseFromURI_GitProviders(t *testing.T) {
Uri: server.URL,
},
},
wantError: &invalidTokenError,
wantResources: []string{},
wantError: &invalidTokenError,
wantResources: []string{},
downloadGitResources: true,
},
{
name: "private parent devfile with invalid revision",
Expand All @@ -4380,8 +4401,9 @@ func Test_parseFromURI_GitProviders(t *testing.T) {
Uri: server.URL,
},
},
wantError: &invalidGitSwitchError,
wantResources: []string{},
wantError: &invalidGitSwitchError,
wantResources: []string{},
downloadGitResources: true,
},
}

Expand All @@ -4396,13 +4418,13 @@ func Test_parseFromURI_GitProviders(t *testing.T) {

// tt.gitUrl is the parent devfile URL
downloadGitRepoResources = mockDownloadGitRepoResources(tt.url, tt.gitUrl, tt.token)
got, err := parseFromURI(tt.importReference, curDevfileContext, &resolutionContextTree{}, resolverTools{})
got, err := parseFromURI(tt.importReference, curDevfileContext, &resolutionContextTree{}, resolverTools{downloadGitResources: tt.downloadGitResources})

// validate even if we want an error; check that no files are copied to destDir
validateGitResourceFunctions(t, tt.wantResources, tt.wantResourceContent, destDir)

if (err != nil) != (tt.wantError != nil) {
t.Errorf("Unexpected error: %v, wantErr %v", err, tt.wantError)
t.Errorf("Unexpected error: %v, wantErr %v", err, *tt.wantError)
} else if err == nil && !reflect.DeepEqual(got.Data, tt.wantDevFile.Data) {
t.Errorf("Wanted: %v, got: %v, difference at %v", tt.wantDevFile, got, pretty.Compare(tt.wantDevFile, got))
} else if err != nil {
Expand Down