make eval to use kpt container & exec runtime (#1898)

* change eval to use kpt container runtime

* make eval to use kpt exec runtime

Author: Shell32-Natsu

e2e/testdata/fn-eval/missing-fn-config/.expected/config.yaml

@@ -15,4 +15,4 @@
 testType: eval
 exitCode: 1
 image: gcr.io/kpt-fn/set-namespace:v0.1
-stdErr: "failed to configure function: input namespace cannot be emptyerror: exit status 1"
+stdErr: "failed to configure function: input namespace cannot be empty"

e2e/testdata/fn-eval/missing-fn-image/.expected/config.yaml

@@ -17,5 +17,4 @@ exitCode: 1
 image: gcr.io/kpt-fn/dne # non-existing image
 args:
   namespace: staging
-stdErr: "Failed to fetch \"latest\" from request \"/v2/kpt-fn/dne/manifests/latest\""
-
+stdErr: 'failed to check function image existence: function image "gcr.io/kpt-fn/dne" doesn''t exist'

internal/fnruntime/container.go

@@ -27,6 +27,7 @@ import (
 	"github.com/GoogleContainerTools/kpt/internal/errors"
 	"github.com/GoogleContainerTools/kpt/internal/printer"
 	"github.com/GoogleContainerTools/kpt/internal/types"
+	"sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil"
 )
 
 // containerNetworkName is a type for network name used in container
@@ -43,6 +44,7 @@ const (
 // function such as network access.
 type ContainerFnPermission struct {
 	AllowNetwork bool
+	AllowMount   bool
 }
 
 // ContainerFnWrapper wraps the real function filter, prints
@@ -75,6 +77,15 @@ type ContainerFn struct {
 	// The default value is 5 minutes.
 	Timeout time.Duration
 	Perm    ContainerFnPermission
+	// UIDGID is the os User ID and Group ID that will be
+	// used to run the container in format userId:groupId.
+	// If it's empty, "nobody" will be used.
+	UIDGID string
+	// StorageMounts are the storage or directories to mount
+	// into the container
+	StorageMounts []runtimeutil.StorageMount
+	// Env is a slice of env string that will be exposed to container
+	Env []string
 }
 
 // Run runs the container function using docker runtime.
@@ -116,6 +127,10 @@ func (f *ContainerFn) getDockerCmd() (*exec.Cmd, context.CancelFunc) {
 	if f.Perm.AllowNetwork {
 		network = networkNameHost
 	}
+	uidgid := "nobody"
+	if f.UIDGID != "" {
+		uidgid = f.UIDGID
+	}
 
 	args := []string{
 		"run", "--rm", "-i",
@@ -125,8 +140,14 @@ func (f *ContainerFn) getDockerCmd() (*exec.Cmd, context.CancelFunc) {
 		// to stderr. We don't need this once we support structured
 		// results.
 		"-e", "LOG_TO_STDERR=true",
+		"--user", uidgid,
 		"--security-opt=no-new-privileges",
 	}
+	for _, storageMount := range f.StorageMounts {
+		args = append(args, "--mount", storageMount.String())
+	}
+	args = append(args,
+		runtimeutil.NewContainerEnvFromStringSlice(f.Env).GetDockerFlags()...)
 	args = append(args, f.Image)
 	// setup container run timeout
 	timeout := defaultTimeout

internal/fnruntime/exec.go

@@ -0,0 +1,73 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fnruntime
+
+import (
+	"bytes"
+	"context"
+	goerrors "errors"
+	"fmt"
+	"io"
+	"os/exec"
+	"time"
+
+	"github.com/GoogleContainerTools/kpt/internal/errors"
+	"github.com/GoogleContainerTools/kpt/internal/printer"
+)
+
+type ExecFn struct {
+	// Path is the os specific path to the executable
+	// file. It can be relative or absolute.
+	Path string
+	// Args are the arguments to the executable
+	Args []string
+	// Container function will be killed after this timeour.
+	// The default value is 5 minutes.
+	Timeout time.Duration
+}
+
+// Run runs the executable file which reads the input from r and
+// writes the output to w.
+func (f *ExecFn) Run(r io.Reader, w io.Writer) error {
+	// setup exec run timeout
+	timeout := defaultTimeout
+	if f.Timeout != 0 {
+		timeout = f.Timeout
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, f.Path, f.Args...)
+
+	errSink := bytes.Buffer{}
+	cmd.Stdin = r
+	cmd.Stdout = w
+	cmd.Stderr = &errSink
+
+	if err := cmd.Run(); err != nil {
+		var exitErr *exec.ExitError
+		if goerrors.As(err, &exitErr) {
+			return &errors.FnExecError{
+				OriginalErr:    exitErr,
+				ExitCode:       exitErr.ExitCode(),
+				Stderr:         errSink.String(),
+				TruncateOutput: printer.TruncateOutput,
+			}
+		}
+		return fmt.Errorf("unexpected function error: %w", err)
+	}
+
+	return nil
+}

thirdparty/kyaml/runfn/runfn.go

@@ -16,14 +16,12 @@ import (
 	"sync/atomic"
 
 	"sigs.k8s.io/kustomize/kyaml/errors"
-	"sigs.k8s.io/kustomize/kyaml/fn/runtime/container"
-	"sigs.k8s.io/kustomize/kyaml/fn/runtime/exec"
 	"sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil"
-	"sigs.k8s.io/kustomize/kyaml/fn/runtime/starlark"
 	"sigs.k8s.io/kustomize/kyaml/kio"
 	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 
+	"github.com/GoogleContainerTools/kpt/internal/fnruntime"
 	"github.com/GoogleContainerTools/kpt/internal/pkg"
 	"github.com/GoogleContainerTools/kpt/internal/types"
 	"github.com/GoogleContainerTools/kpt/pkg/api/kptfile/v1alpha2"
@@ -218,26 +216,7 @@ func (r RunFns) runFunctions(
 		Outputs:               outputs,
 		ContinueOnEmptyResult: r.ContinueOnEmptyResult,
 	}
-	if r.LogSteps {
-		err = pipeline.ExecuteWithCallback(func(op kio.Filter) {
-			var identifier string
-
-			switch filter := op.(type) {
-			case *container.Filter:
-				identifier = filter.Image
-			case *exec.Filter:
-				identifier = filter.Path
-			case *starlark.Filter:
-				identifier = filter.String()
-			default:
-				identifier = "unknown-type function"
-			}
-
-			_, _ = fmt.Fprintf(r.LogWriter, "Running %s\n", identifier)
-		})
-	} else {
-		err = pipeline.Execute()
-	}
+	err = pipeline.Execute()
 	if err != nil {
 		return err
 	}
@@ -446,28 +425,38 @@ func (r *RunFns) defaultFnFilterProvider(spec runtimeutil.FunctionSpec, fnConfig
 		if err != nil {
 			return nil, err
 		}
-		c := container.NewContainer(
-			runtimeutil.ContainerSpec{
-				Image:         spec.Container.Image,
-				Network:       spec.Container.Network,
-				StorageMounts: r.StorageMounts,
-				Env:           spec.Container.Env,
+		c := &fnruntime.ContainerFn{
+			Path:          r.uniquePath,
+			Image:         spec.Container.Image,
+			UIDGID:        uidgid,
+			StorageMounts: r.StorageMounts,
+			Env:           spec.Container.Env,
+			Perm: fnruntime.ContainerFnPermission{
+				AllowNetwork: spec.Container.Network,
+				// mounts are always from CLI flags so we allow
+				// them by default for eval
+				AllowMount: true,
 			},
-			uidgid,
-		)
-		cf := &c
-		cf.Exec.FunctionConfig = fnConfig
-		cf.Exec.ResultsFile = resultsFile
-		cf.Exec.DeferFailure = spec.DeferFailure
+		}
+		cf := &runtimeutil.FunctionFilter{
+			Run:            c.Run,
+			FunctionConfig: fnConfig,
+			DeferFailure:   spec.DeferFailure,
+			ResultsFile:    resultsFile,
+		}
 		return cf, nil
 	}
 
 	if spec.Exec.Path != "" {
-		ef := &exec.Filter{Path: spec.Exec.Path}
-
-		ef.FunctionConfig = fnConfig
-		ef.ResultsFile = resultsFile
-		ef.DeferFailure = spec.DeferFailure
+		e := &fnruntime.ExecFn{
+			Path: spec.Exec.Path,
+		}
+		ef := &runtimeutil.FunctionFilter{
+			Run:            e.Run,
+			FunctionConfig: fnConfig,
+			DeferFailure:   spec.DeferFailure,
+			ResultsFile:    resultsFile,
+		}
 		return ef, nil
 	}