e2e: use podman-remote in story_openshift scenarios

Involves moving podman command code (test side) from test/integration
to test/extended. Now it can be shared between e2e and integration
tests. This only affects `test/integration/podman_test.go` on the
integration side.

Former feature file tags @story_registry, @story_health, and
@story_marketplace now tag respective scenarios inside
@story_openshift scope.

Author: Jakub Sliacan

test/e2e/features/story_openshift.feature

@@ -1,13 +1,13 @@
 @story_openshift
-Feature: 3 Openshift stories
+Feature: 4 Openshift stories
 
 	Background:
 		Given ensuring CRC cluster is running succeeds
 		And ensuring user is logged in succeeds
 
 	# End-to-end health check
 
-	@darwin @linux @windows @startstop @testdata
+	@darwin @linux @windows @startstop @testdata @story_health
 	Scenario: Overall cluster health
 		#Given ensuring CRC cluster is running succeeds
 		#And ensuring user is logged in succeeds
@@ -38,8 +38,8 @@ Feature: 3 Openshift stories
 
 	# Local image to image-registry feature
 
-	@darwin @linux @windows
-	Scenario: Mirror image to OpenShift image registry (via oc registry login)
+	@darwin @linux @windows @story_registry
+	Scenario: Mirror image to OpenShift image registry
 		# mirror
 		When executing "oc new-project testproj-img" succeeds
 		And  executing "oc registry login --insecure=true" succeeds
@@ -56,14 +56,11 @@ Feature: 3 Openshift stories
 		# cleanup
 		And executing "oc delete project testproj-img" succeeds
 
-	@linux
-	Scenario: Create local image, push to registry, deploy (via podman login)
-		Given checking that CRC is running
-		And executing "podman pull quay.io/centos7/httpd-24-centos7" succeeds
-		When executing "oc new-project testproj-img" succeeds
-		And executing "podman login -u kubeadmin -p $(oc whoami -t) default-route-openshift-image-registry.apps-crc.testing --tls-verify=false" succeeds
-		And executing "podman tag quay.io/centos7/httpd-24-centos7 default-route-openshift-image-registry.apps-crc.testing/testproj-img/hello:test" succeeds
-		And executing "podman push default-route-openshift-image-registry.apps-crc.testing/testproj-img/hello:test --tls-verify=false" succeeds
+	@darwin @linux @windows @local-img-to-registry @testdata @story_registry
+	Scenario: Pull image locally, push to registry, deploy
+		Given podman command is available
+		And executing "oc new-project testproj-img" succeeds
+		When pulling image "quay.io/centos7/httpd-24-centos7", logging in, and pushing local image to internal registry succeeds
 		And executing "oc apply -f hello.yaml" succeeds
 		When executing "oc rollout status deployment hello" succeeds
 		Then stdout should contain "successfully rolled out"
@@ -75,7 +72,7 @@ Feature: 3 Openshift stories
 
 	# Operator from marketplace
 
-	@darwin @linux @windows @testdata
+	@darwin @linux @windows @testdata @story_marketplace
 	Scenario: Install new operator
 		When executing "oc apply -f redis-sub.yaml" succeeds
 		Then with up to "20" retries with wait period of "30s" command "oc get csv" output matches ".*redis-operator\.(.*)Succeeded$"

test/e2e/testsuite/testsuite.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/crc-org/crc/pkg/crc/constants"
 	"github.com/crc-org/crc/pkg/crc/preset"
+	"github.com/crc-org/crc/test/extended/crc/cmd"
 	crcCmd "github.com/crc-org/crc/test/extended/crc/cmd"
 	"github.com/crc-org/crc/test/extended/util"
 	"github.com/cucumber/godog"
@@ -324,12 +325,18 @@ func InitializeScenario(s *godog.ScenarioContext) {
 		ExecuteCommandWithExpectedExitStatus)
 	s.Step(`^execut(?:e|ing) single crc (.*) command (.*)$`,
 		ExecuteSingleCommandWithExpectedExitStatus)
+	s.Step(`^execut(?:e|ing) podman command (.*) (succeeds|fails)$`,
+		ExecutingPodmanCommandSucceedsFails)
 	s.Step(`^ensuring CRC cluster is running (succeeds|fails)$`,
 		EnsureCRCIsRunningSucceedsOrFails)
-	s.Step(`ensuring user is logged in (succeeds|fails)`,
+	s.Step(`^ensuring user is logged in (succeeds|fails)`,
 		EnsureUserIsLoggedIntoClusterSucceedsOrFails)
+	s.Step(`^podman command is available$`,
+		PodmanCommandIsAvailable)
 	s.Step(`^deleting a pod (succeeds|fails)$`,
 		DeletingPodSucceedsOrFails)
+	s.Step(`^pulling image "(.*)", logging in, and pushing local image to internal registry succeeds$`,
+		PullLoginTagPushImageSucceeds)
 
 	// CRC file operations
 	s.Step(`^file "([^"]*)" exists in CRC home folder$`,
@@ -663,3 +670,73 @@ func DeletingPodSucceedsOrFails(expected string) error {
 	}
 	return err
 }
+
+func PodmanCommandIsAvailable() error {
+
+	// Do what 'eval $(crc podman-env) would do
+	path := os.ExpandEnv("${HOME}/.crc/bin/oc:$PATH")
+	csshk := os.ExpandEnv("${HOME}/.crc/machines/crc/id_ecdsa")
+	dh := os.ExpandEnv("unix:///${HOME}/.crc/machines/crc/docker.sock")
+	ch := "ssh://[email protected]:2222/run/user/1000/podman/podman.sock"
+	if runtime.GOOS == "windows" {
+		userHomeDir, _ := os.UserHomeDir()
+		unexpandedPath := filepath.Join(userHomeDir, ".crc/bin/oc;${PATH}")
+		path = os.ExpandEnv(unexpandedPath)
+		csshk = filepath.Join(userHomeDir, ".crc/machines/crc/id_ecdsa")
+		dh = "npipe:////./pipe/rc-podman"
+	}
+	if runtime.GOOS == "linux" {
+		ch = "ssh://[email protected]:22/run/user/1000/podman/podman.sock"
+	}
+
+	os.Setenv("PATH", path)
+	os.Setenv("CONTAINER_SSHKEY", csshk)
+	os.Setenv("CONTAINER_HOST", ch)
+	os.Setenv("DOCKER_HOST", dh)
+
+	return nil
+
+}
+
+func ExecutingPodmanCommandSucceedsFails(command string, expected string) error {
+
+	var err error
+	if expected == "succeeds" {
+		_, err = cmd.RunPodmanExpectSuccess(strings.Split(command[1:len(command)-1], " ")...)
+	} else if expected == "fails" {
+		_, err = cmd.RunPodmanExpectFail(strings.Split(command[1:len(command)-1], " ")...)
+	}
+
+	return err
+}
+
+func PullLoginTagPushImageSucceeds(image string) error {
+	_, err := cmd.RunPodmanExpectSuccess("pull", image)
+	if err != nil {
+		return err
+	}
+
+	err = util.ExecuteCommand("oc whoami -t")
+	if err != nil {
+		return err
+	}
+
+	token := util.GetLastCommandOutput("stdout")
+	fmt.Println(token)
+	_, err = cmd.RunPodmanExpectSuccess("login", "-u", "kubeadmin", "-p", token, "default-route-openshift-image-registry.apps-crc.testing", "--tls-verify=false") // $(oc whoami -t)
+	if err != nil {
+		return err
+	}
+
+	_, err = cmd.RunPodmanExpectSuccess("tag", "quay.io/centos7/httpd-24-centos7", "default-route-openshift-image-registry.apps-crc.testing/testproj-img/hello:test")
+	if err != nil {
+		return err
+	}
+
+	_, err = cmd.RunPodmanExpectSuccess("push", "default-route-openshift-image-registry.apps-crc.testing/testproj-img/hello:test", "--tls-verify=false")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

test/extended/crc/cmd/cmd.go

@@ -1,13 +1,18 @@
 package cmd
 
 import (
+	"bytes"
 	"fmt"
+	"io"
 	"os/exec"
 	"runtime"
 	"strings"
+	"syscall"
+	"time"
 
 	"github.com/crc-org/crc/pkg/crc/logging"
 	"github.com/crc-org/crc/test/extended/util"
+	"github.com/sirupsen/logrus"
 )
 
 const (
@@ -189,3 +194,157 @@ func (c Command) ExecuteSingleWithExpectedExit(expectedExit string) error {
 	}
 	return fmt.Errorf("%s is a valid expected exit status", expectedExit)
 }
+
+// PODMAN
+
+// PodmanBuilder is used to build, customize, and execute a podman-remote command.
+type PodmanBuilder struct {
+	cmd     *exec.Cmd
+	timeout <-chan time.Time
+}
+
+// NewPodmanCommand returns a PodmanBuilder for running CRC.
+func NewPodmanCommand(args ...string) *PodmanBuilder {
+
+	cmd := exec.Command("podman", args...)
+
+	switch runtime.GOOS {
+	case "linux":
+		cmd = exec.Command("podman-remote", args...)
+	case "windows":
+		cmd = exec.Command("podman.exe", args...)
+	}
+
+	return &PodmanBuilder{
+		cmd: cmd,
+	}
+}
+
+// WithTimeout sets the given timeout and returns itself.
+func (b *PodmanBuilder) WithTimeout(t <-chan time.Time) *PodmanBuilder {
+	b.timeout = t
+	return b
+}
+
+// WithStdinData sets the given data to stdin and returns itself.
+func (b PodmanBuilder) WithStdinData(data string) *PodmanBuilder {
+	b.cmd.Stdin = strings.NewReader(data)
+	return &b
+}
+
+// WithStdinReader sets the given reader and returns itself.
+func (b PodmanBuilder) WithStdinReader(reader io.Reader) *PodmanBuilder {
+	b.cmd.Stdin = reader
+	return &b
+}
+
+// ExecOrDie runs the executable or dies if error occurs.
+func (b PodmanBuilder) ExecOrDie() (string, error) {
+	stdout, err := b.Exec()
+	return stdout, err
+}
+
+// ExecOrDieWithLogs runs the executable or dies if error occurs.
+func (b PodmanBuilder) ExecOrDieWithLogs() (string, string, error) {
+	stdout, stderr, err := b.ExecWithFullOutput()
+	return stdout, stderr, err
+}
+
+// Exec runs the executable.
+func (b PodmanBuilder) Exec() (string, error) {
+	stdout, _, err := b.ExecWithFullOutput()
+	return stdout, err
+}
+
+// ExecWithFullOutput runs the executable and returns the stdout and stderr.
+func (b PodmanBuilder) ExecWithFullOutput() (string, string, error) {
+	return Exec(b.cmd, b.timeout)
+}
+
+func Exec(cmd *exec.Cmd, timeout <-chan time.Time) (string, string, error) {
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout, cmd.Stderr = &stdout, &stderr
+
+	logrus.Infof("Running '%s %s'", cmd.Path, strings.Join(cmd.Args[1:], " ")) // skip arg[0] as it is printed separately
+	if err := cmd.Start(); err != nil {
+		return "", "", fmt.Errorf("error starting %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, cmd.Stdout, cmd.Stderr, err)
+	}
+	errCh := make(chan error, 1)
+	go func() {
+		errCh <- cmd.Wait()
+	}()
+	select {
+	case err := <-errCh:
+		if err != nil {
+			var rc = 127
+			if ee, ok := err.(*exec.ExitError); ok {
+				rc = int(ee.Sys().(syscall.WaitStatus).ExitStatus())
+				logrus.Infof("rc: %d", rc)
+			}
+			return stdout.String(), stderr.String(), CodeExitError{
+				Err:  fmt.Errorf("error running %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, cmd.Stdout, cmd.Stderr, err),
+				Code: rc,
+			}
+		}
+	case <-timeout:
+		_ = cmd.Process.Kill()
+		return "", "", fmt.Errorf("timed out waiting for command %v:\nCommand stdout:\n%v\nstderr:\n%v", cmd, cmd.Stdout, cmd.Stderr)
+	}
+	logrus.Infof("stderr: %q", stderr.String())
+	logrus.Infof("stdout: %q", stdout.String())
+	return stdout.String(), stderr.String(), nil
+}
+
+// RunPodmanExpectSuccess is a convenience wrapper over podman-remote
+func RunPodmanExpectSuccess(args ...string) (string, error) {
+	return NewPodmanCommand(args...).ExecOrDie()
+}
+
+// RunPodmanExpectFail is a convenience wrapper over PodmanBuilder
+// if err != nil: return stderr, nil
+// if err == nil: return stdout, err
+func RunPodmanExpectFail(args ...string) (string, error) {
+	stdout, stderr, err := NewPodmanCommand(args...).ExecWithFullOutput()
+
+	if err == nil {
+		err = fmt.Errorf("Expected error but exited without error")
+		return stdout, err
+	}
+
+	return stderr, nil
+}
+
+// ExitError is an interface that presents an API similar to os.ProcessState, which is
+// what ExitError from os/exec is.  This is designed to make testing a bit easier and
+// probably loses some of the cross-platform properties of the underlying library.
+type ExitError interface {
+	String() string
+	Error() string
+	Exited() bool
+	ExitStatus() int
+}
+
+// CodeExitError is an implementation of ExitError consisting of an error object
+// and an exit code (the upper bits of os.exec.ExitStatus).
+type CodeExitError struct {
+	Err  error
+	Code int
+}
+
+var _ ExitError = CodeExitError{}
+
+func (e CodeExitError) Error() string {
+	return e.Err.Error()
+}
+
+func (e CodeExitError) String() string {
+	return e.Err.Error()
+}
+
+func (e CodeExitError) Exited() bool {
+	return true
+}
+
+func (e CodeExitError) ExitStatus() int {
+	return e.Code
+}

test/extended/util/shell.go

@@ -304,6 +304,10 @@ func CommandReturnShouldNotContainContent(commandField string, notexpected *mess
 	return CompareExpectedWithActualNotContains(notexpected.Content, shell.GetLastCmdOutput(commandField))
 }
 
+func GetLastCommandOutput(commandField string) string {
+	return shell.GetLastCmdOutput(commandField)
+}
+
 func CommandReturnShouldBeEmpty(commandField string) error {
 	return CompareExpectedWithActualEquals("", shell.GetLastCmdOutput(commandField))
 }

test/integration/podman_test.go

@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 	"runtime"
 
+	"github.com/crc-org/crc/test/extended/crc/cmd"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -49,15 +50,19 @@ var _ = Describe("podman preset", Label("podman-preset"), func() {
 		})
 
 		It("version", func() {
-			Expect(RunPodmanExpectSuccess("version")).Should(MatchRegexp(`Version:[\s]*\d+\.\d+\.\d+`))
+			out, err := cmd.RunPodmanExpectSuccess("version")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(out).Should(MatchRegexp(`Version:[\s]*\d+\.\d+\.\d+`))
 		})
 
 		It("pull image", func() {
-			RunPodmanExpectSuccess("pull", "fedora")
+			_, err := cmd.RunPodmanExpectSuccess("pull", "fedora")
+			Expect(err).NotTo(HaveOccurred())
 		})
 
 		It("run image", func() {
-			RunPodmanExpectSuccess("run", "fedora")
+			_, err := cmd.RunPodmanExpectSuccess("run", "fedora")
+			Expect(err).NotTo(HaveOccurred())
 		})
 
 		It("cleanup CRC", func() {