Skip to content

Add library package for resolving devfile parents and plugins #74

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,23 @@ require (
github.com/fatih/color v1.7.0
github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
github.com/gobwas/glob v0.2.3
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 // indirect
github.com/google/go-cmp v0.4.0
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348
github.com/mattn/go-colorable v0.1.2 // indirect
github.com/mattn/go-isatty v0.0.12 // indirect
github.com/openshift/api v0.0.0-20200930075302-db52bc4ef99f
github.com/openshift/api v3.9.0+incompatible
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should stick with openshift/api version 4.6 or higher. I had to explicitly update to a newer version because openshift/console and openshift/odo uses newer versions and there is a breaking dependency change when going from version 3.9 to 4.0 and above

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I'm not sure why this got pinned to 3.9.0 -- I assume it's Go's automatic "upgrading" (3.9 is bigger than 0.0?)

I'll fix this.

github.com/pkg/errors v0.9.1
github.com/spf13/afero v1.2.2
github.com/stretchr/testify v1.6.1
github.com/xeipuuv/gojsonschema v1.2.0
k8s.io/api v0.19.0
k8s.io/apimachinery v0.19.0
golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 // indirect
google.golang.org/protobuf v1.24.0 // indirect
k8s.io/api v0.18.6
k8s.io/apimachinery v0.18.6
k8s.io/klog v1.0.0
sigs.k8s.io/controller-runtime v0.6.3
sigs.k8s.io/yaml v1.2.0
)
57 changes: 25 additions & 32 deletions go.sum

Large diffs are not rendered by default.

39 changes: 39 additions & 0 deletions pkg/flatten/annotate.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package flatten

import (
dw "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
"github.com/devfile/api/v2/pkg/attributes"
)

const (
ImportSourceAttribute = "library.devfile.io/imported-by"
)

// AddSourceAttributesForTemplate adds an attribute 'library.devfile.io/imported-by=<plugin-name>' to all elements of
// a plugin that support attributes.
func AddSourceAttributesForTemplate(sourceID string, template *dw.DevWorkspaceTemplateSpec) {
for idx, component := range template.Components {
if component.Attributes == nil {
template.Components[idx].Attributes = attributes.Attributes{}
}
template.Components[idx].Attributes.PutString(ImportSourceAttribute, sourceID)
}
for idx, command := range template.Commands {
if command.Attributes == nil {
template.Commands[idx].Attributes = attributes.Attributes{}
}
template.Commands[idx].Attributes.PutString(ImportSourceAttribute, sourceID)
}
for idx, project := range template.Projects {
if project.Attributes == nil {
template.Projects[idx].Attributes = attributes.Attributes{}
}
template.Projects[idx].Attributes.PutString(ImportSourceAttribute, sourceID)
}
for idx, project := range template.StarterProjects {
if project.Attributes == nil {
template.StarterProjects[idx].Attributes = attributes.Attributes{}
}
template.StarterProjects[idx].Attributes.PutString(ImportSourceAttribute, sourceID)
}
}
27 changes: 27 additions & 0 deletions pkg/flatten/common.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
//
// Copyright (c) 2019-2021 Red Hat, Inc.
// This program and the accompanying materials are made
// available under the terms of the Eclipse Public License 2.0
// which is available at https://www.eclipse.org/legal/epl-2.0/
//
// SPDX-License-Identifier: EPL-2.0
//
// Contributors:
// Red Hat, Inc. - initial API and implementation
//

package flatten

import devfile "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"

func DevWorkspaceIsFlattened(devworkspace *devfile.DevWorkspaceTemplateSpec) bool {
if devworkspace.Parent != nil {
return false
}
for _, component := range devworkspace.Components {
if component.Plugin != nil {
return false
}
}
return true
}
268 changes: 268 additions & 0 deletions pkg/flatten/flatten.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,268 @@
//
// Copyright (c) 2019-2021 Red Hat, Inc.
// This program and the accompanying materials are made
// available under the terms of the Eclipse Public License 2.0
// which is available at https://www.eclipse.org/legal/epl-2.0/
//
// SPDX-License-Identifier: EPL-2.0
//
// Contributors:
// Red Hat, Inc. - initial API and implementation
//

package flatten

import (
"context"
"fmt"
"net/url"
"path"

devfile "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
"github.com/devfile/api/v2/pkg/utils/overriding"
"github.com/devfile/library/pkg/flatten/network"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)

// ResolverTools contains required structs and data for resolving remote components of a devfile (plugins and parents)
type ResolverTools struct {
// DefaultNamespace is the default namespace to use for resolving Kubernetes ImportReferences that do not include one
DefaultNamespace string
// DefaultRegistryURL is the default registry URL to use when a component specifies an id but not registryURL
DefaultRegistryURL string
// Context is the context used for making Kubernetes or HTTP requests
Context context.Context
// K8sClient is the Kubernetes client instance used for interacting with a cluster
K8sClient client.Client
// HttpClient is the HTTP client used for making network requests when resolving plugins or parents.
HttpClient network.HTTPGetter
}

// ResolveDevWorkspace takes a DevWorkspaceTemplateSpec and returns a "resolved" version of it -- i.e. one where all plugins and parents
// are inlined as components.
// TODO:
// - Implement flattening for DevWorkspace parents
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see resolveParentComponent func, isn't it already done?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Forgot to remove the TODO after implementing :)

func ResolveDevWorkspace(workspace *devfile.DevWorkspaceTemplateSpec, tooling ResolverTools) (*devfile.DevWorkspaceTemplateSpec, error) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume it may simplify testing for clients if such functionality would be provided with interface. Like DevWorkspaceResolver.
Then here we would not need to pass resolverTooling which is good thing as well I think, instead we'll have something like:

r := resolver.NewWithTooling(tooling)
r.resolveDevWorkspace(tooling)

But it's not a strong suggestion, just assumption.

resolutionCtx := &resolutionContextTree{}
resolvedDW, err := recursiveResolve(workspace, tooling, resolutionCtx)
if err != nil {
return nil, err
}
return resolvedDW, nil
}

// recursiveResolve recursively resolves plugins and parents until the result contains no parents or plugin components.
// This is a recursive function, where resolveCtx is used to build a tree of resolved components. This is used to avoid
// plugin or parent import cycles.
func recursiveResolve(workspace *devfile.DevWorkspaceTemplateSpec, tooling ResolverTools, resolveCtx *resolutionContextTree) (*devfile.DevWorkspaceTemplateSpec, error) {
if DevWorkspaceIsFlattened(workspace) {
return workspace.DeepCopy(), nil
}
resolvedParent := &devfile.DevWorkspaceTemplateSpecContent{}
if workspace.Parent != nil {
resolvedParentSpec, err := resolveParentComponent(workspace.Parent, tooling)
if err != nil {
return nil, err
}
if !DevWorkspaceIsFlattened(resolvedParentSpec) {
// TODO: implemenent this
return nil, fmt.Errorf("parents containing plugins or parents are not supported")
}
AddSourceAttributesForTemplate("parent", resolvedParentSpec)
resolvedParent = &resolvedParentSpec.DevWorkspaceTemplateSpecContent
}

resolvedContent := &devfile.DevWorkspaceTemplateSpecContent{}
resolvedContent.Projects = workspace.Projects
resolvedContent.StarterProjects = workspace.StarterProjects
resolvedContent.Commands = workspace.Commands
resolvedContent.Events = workspace.Events

var pluginSpecContents []*devfile.DevWorkspaceTemplateSpecContent
for _, component := range workspace.Components {
if component.Plugin == nil {
// No action necessary
resolvedContent.Components = append(resolvedContent.Components, component)
} else {
pluginComponent, err := resolvePluginComponent(component.Name, component.Plugin, tooling)
if err != nil {
return nil, err
}
newCtx := resolveCtx.addPlugin(component.Name, component.Plugin)
if err := newCtx.hasCycle(); err != nil {
return nil, err
}

resolvedPlugin, err := recursiveResolve(pluginComponent, tooling, newCtx)
if err != nil {
return nil, err
}

AddSourceAttributesForTemplate(component.Name, resolvedPlugin)
pluginSpecContents = append(pluginSpecContents, &resolvedPlugin.DevWorkspaceTemplateSpecContent)
}
}

resolvedContent, err := overriding.MergeDevWorkspaceTemplateSpec(resolvedContent, resolvedParent, pluginSpecContents...)
if err != nil {
return nil, fmt.Errorf("failed to merge DevWorkspace parents/plugins: %w", err)
}

return &devfile.DevWorkspaceTemplateSpec{
DevWorkspaceTemplateSpecContent: *resolvedContent,
}, nil
}

// resolveParentComponent resolves the parent DevWorkspaceTemplateSpec that a parent reference refers to.
func resolveParentComponent(parent *devfile.Parent, tooling ResolverTools) (resolvedParent *devfile.DevWorkspaceTemplateSpec, err error) {
switch {
case parent.Kubernetes != nil:
// Search in default namespace if namespace ref is unset
if parent.Kubernetes.Namespace == "" {
parent.Kubernetes.Namespace = tooling.DefaultNamespace
}
Comment on lines +122 to +125
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

probably isnt reqd since this check and assignment are done in resolveElementByKubernetesImport(), same for plugins

resolvedParent, err = resolveElementByKubernetesImport("parent", parent.Kubernetes, tooling)
case parent.Uri != "":
resolvedParent, err = resolveElementByURI("parent", parent.Uri, tooling)
case parent.Id != "":
resolvedParent, err = resolveElementById("parent", parent.Id, parent.RegistryUrl, tooling)
default:
err = fmt.Errorf("devfile parent does not define any resources")
}
if err != nil {
return nil, err
}
if parent.Components != nil || parent.Commands != nil || parent.Projects != nil || parent.StarterProjects != nil {
overrideSpec, err := overriding.OverrideDevWorkspaceTemplateSpec(&resolvedParent.DevWorkspaceTemplateSpecContent, parent.ParentOverrides)

if err != nil {
return nil, err
}
resolvedParent.DevWorkspaceTemplateSpecContent = *overrideSpec
}
return resolvedParent, nil
}

// resolvePluginComponent resolves the DevWorkspaceTemplateSpec that a plugin component refers to. The name parameter is
// used to construct meaningful error messages (e.g. issue resolving plugin 'name')
func resolvePluginComponent(
name string,
plugin *devfile.PluginComponent,
tooling ResolverTools) (resolvedPlugin *devfile.DevWorkspaceTemplateSpec, err error) {
switch {
case plugin.Kubernetes != nil:
// Search in default namespace if namespace ref is unset
if plugin.Kubernetes.Namespace == "" {
plugin.Kubernetes.Namespace = tooling.DefaultNamespace
}
resolvedPlugin, err = resolveElementByKubernetesImport(name, plugin.Kubernetes, tooling)
case plugin.Uri != "":
resolvedPlugin, err = resolveElementByURI(name, plugin.Uri, tooling)
case plugin.Id != "":
resolvedPlugin, err = resolveElementById(name, plugin.Id, plugin.RegistryUrl, tooling)
default:
err = fmt.Errorf("plugin %s does not define any resources", name)
}
if err != nil {
return nil, err
}

if plugin.Components != nil || plugin.Commands != nil {
overrideSpec, err := overriding.OverrideDevWorkspaceTemplateSpec(&resolvedPlugin.DevWorkspaceTemplateSpecContent, devfile.PluginOverrides{
Components: plugin.Components,
Commands: plugin.Commands,
})

if err != nil {
return nil, err
}
resolvedPlugin.DevWorkspaceTemplateSpecContent = *overrideSpec
}
return resolvedPlugin, nil
}

// resolveElementByKubernetesImport resolves a plugin specified by a Kubernetes reference.
// The name parameter is used to construct meaningful error messages (e.g. issue resolving plugin 'name')
func resolveElementByKubernetesImport(
name string,
kubeReference *devfile.KubernetesCustomResourceImportReference,
tools ResolverTools) (resolvedPlugin *devfile.DevWorkspaceTemplateSpec, err error) {

if tools.K8sClient == nil {
return nil, fmt.Errorf("cannot resolve resources by kubernetes reference: no kubernetes client provided")
}

namespace := kubeReference.Namespace
if namespace == "" {
if tools.DefaultNamespace == "" {
return nil, fmt.Errorf("'%s' specifies a kubernetes reference without namespace and a default is not provided", name)
}
namespace = tools.DefaultNamespace
}

var dwTemplate devfile.DevWorkspaceTemplate
namespacedName := types.NamespacedName{
Name: kubeReference.Name,
Namespace: namespace,
}
err = tools.K8sClient.Get(tools.Context, namespacedName, &dwTemplate)
if err != nil {
if errors.IsNotFound(err) {
return nil, fmt.Errorf("plugin for component %s not found", name)
}
return nil, fmt.Errorf("failed to retrieve plugin referenced by kubernetes name and namespace '%s': %w", name, err)
}
return &dwTemplate.Spec, nil
}

// resolveElementById resolves a component specified by ID and registry URL. The name parameter is used to
// construct meaningful error messages (e.g. issue resolving plugin 'name'). When registry URL is empty,
// the DefaultRegistryURL from tools is used.
func resolveElementById(
name string,
id string,
registryUrl string,
tools ResolverTools) (resolvedPlugin *devfile.DevWorkspaceTemplateSpec, err error) {

if tools.HttpClient == nil {
return nil, fmt.Errorf("cannot resolve resources by id: no HTTP client provided")
}

if registryUrl == "" {
if tools.DefaultRegistryURL == "" {
return nil, fmt.Errorf("'%s' specifies id but has no registryUrl and a default is not provided", name)
}
registryUrl = tools.DefaultRegistryURL
}
pluginURL, err := url.Parse(registryUrl)
if err != nil {
return nil, fmt.Errorf("failed to parse registry URL for component %s: %w", name, err)
}
pluginURL.Path = path.Join(pluginURL.Path, id)

dwt, err := network.FetchDevWorkspaceTemplate(pluginURL.String(), tools.HttpClient)
if err != nil {
return nil, fmt.Errorf("failed to resolve component %s from registry %s: %w", name, registryUrl, err)
}
return dwt, nil
}

// resolveElementByURI resolves a plugin defined by URI. The name parameter is used to construct meaningful
// error messages (e.g. issue resolving plugin 'name')
func resolveElementByURI(
name string,
uri string,
tools ResolverTools) (resolvedPlugin *devfile.DevWorkspaceTemplateSpec, err error) {

if tools.HttpClient == nil {
return nil, fmt.Errorf("cannot resolve resources by URI: no HTTP client provided")
}

dwt, err := network.FetchDevWorkspaceTemplate(uri, tools.HttpClient)
if err != nil {
return nil, fmt.Errorf("failed to resolve component %s by URI: %w", name, err)
}
return dwt, nil
}
Loading