Start In-Cluster Git Server For e2e Tests (#2823)

* Start In-Cluster Git Server For e2e Tests
* Create git-server Deployment resource
* Enable serving empty repository by default
* Wait for Service to be ready

Author: martinmaly

porch/apiserver/pkg/e2e/suite.go

@@ -22,6 +22,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
@@ -32,9 +33,11 @@ import (
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/object"
 	"gopkg.in/yaml.v2"
+	appsv1 "k8s.io/api/apps/v1"
 	coreapi "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/rest"
 	aggregatorv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -153,8 +156,7 @@ func (t *TestSuite) CreateGitRepo() GitConfig {
 		return createLocalGitServer(t.T)
 	} else {
 		// Deploy Git server via k8s client.
-		t.Fatal("Creatig git server on k8s not yet supported.")
-		return GitConfig{}
+		return t.createInClusterGitServer()
 	}
 }
 
@@ -247,6 +249,7 @@ func createClientScheme(t *testing.T) *runtime.Scheme {
 		configapi.AddToScheme,
 		coreapi.AddToScheme,
 		aggregatorv1.AddToScheme,
+		appsv1.AddToScheme,
 	}) {
 		if err := api(scheme); err != nil {
 			t.Fatalf("Failed to initialize test k8s api client")
@@ -356,3 +359,187 @@ func createInitialCommit(t *testing.T, repo *gogit.Repository) {
 		t.Fatalf("Failed to set refs/heads/main to commit sha %s", commitHash)
 	}
 }
+
+func inferGitServerImage(porchImage string) string {
+	slash := strings.LastIndex(porchImage, "/")
+	repo := porchImage[:slash+1]
+	image := porchImage[slash+1:]
+	colon := strings.LastIndex(image, ":")
+	tag := image[colon+1:]
+
+	return repo + "git-server:" + tag
+}
+
+func (t *TestSuite) createInClusterGitServer() GitConfig {
+	ctx := context.TODO()
+
+	// Determine git-server image name. Use the same container registry and tag as the Porch server,
+	// replacing base image name with `git-server`. TODO: Make configurable?
+
+	var porch appsv1.Deployment
+	t.GetF(ctx, client.ObjectKey{
+		Namespace: "porch-system",
+		Name:      "porch-server",
+	}, &porch)
+
+	gitImage := inferGitServerImage(porch.Spec.Template.Spec.Containers[0].Image)
+
+	var replicas int32 = 1
+	var selector = strings.ReplaceAll(t.Name(), "/", "_")
+
+	t.CreateF(ctx, &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "git-server",
+			Namespace: t.namespace,
+			Annotations: map[string]string{
+				"kpt.dev/porch-test": t.Name(),
+			},
+		},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: &replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"git-server": selector,
+				},
+			},
+			Template: coreapi.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"git-server": selector,
+					},
+				},
+				Spec: coreapi.PodSpec{
+					Containers: []coreapi.Container{
+						{
+							Name:  "git-server",
+							Image: gitImage,
+							Args:  []string{},
+							Ports: []coreapi.ContainerPort{
+								{
+									ContainerPort: 8080,
+									Protocol:      coreapi.ProtocolTCP,
+								},
+							},
+							ImagePullPolicy: coreapi.PullIfNotPresent,
+						},
+					},
+				},
+			},
+		},
+	})
+
+	t.Cleanup(func() {
+		t.DeleteE(ctx, &appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "git-server",
+				Namespace: t.namespace,
+			},
+		})
+	})
+
+	t.CreateF(ctx, &coreapi.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "git-server-service",
+			Namespace: t.namespace,
+			Annotations: map[string]string{
+				"kpt.dev/porch-test": t.Name(),
+			},
+		},
+		Spec: coreapi.ServiceSpec{
+			Ports: []coreapi.ServicePort{
+				{
+					Protocol: coreapi.ProtocolTCP,
+					Port:     8080,
+					TargetPort: intstr.IntOrString{
+						Type:   intstr.Int,
+						IntVal: 8080,
+					},
+				},
+			},
+			Selector: map[string]string{
+				"git-server": selector,
+			},
+		},
+	})
+
+	t.Cleanup(func() {
+		t.DeleteE(ctx, &coreapi.Service{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "git-server-service",
+				Namespace: t.namespace,
+			},
+		})
+	})
+
+	t.Logf("Waiting for git-server to start ...")
+
+	// Wait a minute for git server to start up.
+	giveUp := time.Now().Add(time.Minute)
+
+	for {
+		time.Sleep(5 * time.Second)
+
+		var server appsv1.Deployment
+		t.GetF(ctx, client.ObjectKey{
+			Namespace: t.namespace,
+			Name:      "git-server",
+		}, &server)
+		if server.Status.AvailableReplicas > 0 {
+			t.Logf("git server is up")
+			break
+		}
+
+		if time.Now().After(giveUp) {
+			t.Fatalf("git server failed to start: %s", &server)
+			return GitConfig{}
+		}
+	}
+
+	t.Logf("Waiting for git-serever-service to be ready ...")
+
+	// Check the Endpoint resource for readiness
+	giveUp = time.Now().Add(time.Minute)
+
+	for {
+		time.Sleep(5 * time.Second)
+
+		var endpoint coreapi.Endpoints
+		err := t.client.Get(ctx, client.ObjectKey{
+			Namespace: t.namespace,
+			Name:      "git-server-service",
+		}, &endpoint)
+
+		if err == nil && endpointIsReady(&endpoint) {
+			t.Logf("git-server-service is ready")
+			break
+		}
+
+		if time.Now().After(giveUp) {
+			t.Fatalf("git-server0-service not ready on time: %s", &endpoint)
+			return GitConfig{}
+		}
+	}
+
+	return GitConfig{
+		Repo:      fmt.Sprintf("http://git-server-service.%s.svc.cluster.local:8080", t.namespace),
+		Branch:    "main",
+		Directory: "/",
+	}
+}
+
+func endpointIsReady(endpoints *coreapi.Endpoints) bool {
+	if len(endpoints.Subsets) == 0 {
+		return false
+	}
+	for _, s := range endpoints.Subsets {
+		if len(s.Addresses) == 0 {
+			return false
+		}
+		for _, a := range s.Addresses {
+			if a.IP == "" {
+				return false
+			}
+		}
+	}
+	return true
+}

porch/engine/pkg/engine/clone_test.go

@@ -99,7 +99,7 @@ func createRepoWithContents(t *testing.T, contentDir string) *gogit.Repository {
 
 	main := plumbing.NewHashReference(plumbing.ReferenceName("refs/heads/main"), hash)
 	if err := repo.Storer.SetReference(main); err != nil {
-		t.Fatalf("Failed to set refs/heads/main to commit sha %s", hash)
+		t.Fatalf("Failed to set refs/heads/main to commit sha %s: %v", hash, err)
 	}
 	head := plumbing.NewSymbolicReference(plumbing.HEAD, "refs/heads/main")
 	if err := repo.Storer.SetReference(head); err != nil {

porch/repository/pkg/git/testing.go

@@ -151,7 +151,7 @@ func (s *GitServer) serveGitInfoRefs(w http.ResponseWriter, r *http.Request) err
 		switch ref.Type() {
 		case plumbing.SymbolicReference:
 			if r, err := s.repo.Reference(ref.Name(), true); err != nil {
-				klog.Warningf("Skippling unresolvable symbolic reference %q: %w", ref.Name(), err)
+				klog.Warningf("Skipping unresolvable symbolic reference %q: %v", ref.Name(), err)
 				return nil
 			} else {
 				resolved = r

porch/test/git/main.go

@@ -18,13 +18,17 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
 	"os/signal"
+	"time"
 
 	"github.com/GoogleContainerTools/kpt/porch/repository/pkg/git"
 	gogit "github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/object"
 	"k8s.io/klog/v2"
 )
 
@@ -41,11 +45,22 @@ func main() {
 }
 
 func run(dirs []string) error {
-	if len(dirs) != 1 {
-		return fmt.Errorf("expected one path to Git directory to serve. Got %d", len(dirs))
-	}
+	var dir string
+
+	switch len(dirs) {
+	case 0:
+		var err error
+		dir, err = ioutil.TempDir("", "repo-*")
+		if err != nil {
+			return fmt.Errorf("failed to create temporary directory for git repository: %w", err)
+		}
 
-	dir := dirs[0]
+	case 1:
+		dir = dirs[0]
+
+	default:
+		return fmt.Errorf("can server only one git repository, not %d", len(dirs))
+	}
 
 	var repo *gogit.Repository
 	var err error
@@ -59,6 +74,12 @@ func run(dirs []string) error {
 		if err != nil {
 			return fmt.Errorf("failed to initialize git repository %q: %w", dir, err)
 		}
+		if err := createEmptyCommit(repo); err != nil {
+			return err
+		}
+
+		// Delete go-git default branch
+		_ = repo.Storer.RemoveReference(plumbing.Master)
 	}
 
 	server, err := git.NewGitServer(repo)
@@ -87,3 +108,54 @@ func run(dirs []string) error {
 
 	return nil
 }
+
+func createEmptyCommit(repo *gogit.Repository) error {
+	store := repo.Storer
+	// Create first commit using empty tree.
+	emptyTree := object.Tree{}
+	encodedTree := store.NewEncodedObject()
+	if err := emptyTree.Encode(encodedTree); err != nil {
+		return fmt.Errorf("failed to encode initial empty commit tree: %w", err)
+	}
+
+	treeHash, err := store.SetEncodedObject(encodedTree)
+	if err != nil {
+		return fmt.Errorf("failed to create initial empty commit tree: %w", err)
+	}
+
+	sig := object.Signature{
+		Name:  "Git Server",
+		Email: "[email protected]",
+		When:  time.Now(),
+	}
+
+	commit := object.Commit{
+		Author:       sig,
+		Committer:    sig,
+		Message:      "Empty Commit",
+		TreeHash:     treeHash,
+		ParentHashes: []plumbing.Hash{}, // No parents
+	}
+
+	encodedCommit := store.NewEncodedObject()
+	if err := commit.Encode(encodedCommit); err != nil {
+		return fmt.Errorf("failed to encode initial empty commit: %w", err)
+	}
+
+	commitHash, err := store.SetEncodedObject(encodedCommit)
+	if err != nil {
+		return fmt.Errorf("failed to create initial empty commit: %w", err)
+	}
+
+	main := plumbing.NewHashReference(plumbing.ReferenceName("refs/heads/main"), commitHash)
+	if err := repo.Storer.SetReference(main); err != nil {
+		return fmt.Errorf("failed to set refs/heads/main to commit sha %s: %w", commitHash, err)
+	}
+
+	head := plumbing.NewSymbolicReference(plumbing.HEAD, "refs/heads/main")
+	if err := repo.Storer.SetReference(head); err != nil {
+		return fmt.Errorf("failed to set HEAD to refs/heads/main: %w", err)
+	}
+
+	return nil
+}