feat: implement filter and score

Dockerfile

@@ -0,0 +1,27 @@
+# Build the manager binary
+FROM golang:1.17 as builder
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+# cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN go mod download
+
+# Copy the go source
+COPY cmd/scheduler/main.go main.go
+COPY apis/ apis/
+COPY pkg/ pkg/
+
+# Build
+ARG TARGETARCH
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on go build -o manager main.go
+
+# Use distroless as minimal base image to package the manager binary
+# Refer to https://github.com/GoogleContainerTools/distroless for more details
+FROM --platform=${TARGETPLATFORM:-linux/amd64} gcr.io/distroless/static:nonroot
+WORKDIR /
+COPY --from=builder /workspace/manager .
+
+ENTRYPOINT ["/manager"]

Makefile

@@ -1,3 +1,8 @@
+# TODO(aramase) use the github registry to publish the image
+REGISTRY ?= aramase
+IMAGE_NAME := placement-policy
+IMAGE_VERSION ?= v0.1.0
+
 # Directories
 ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 BIN_DIR := $(abspath $(ROOT_DIR)/bin)
@@ -118,3 +123,28 @@ integration-test: install-etcd autogen manager manifests
 .PHONY: e2e-test
 e2e-test: 
 	go test -tags=e2e -v ./test/e2e
+
+## --------------------------------------
+## Images
+## --------------------------------------
+
+OUTPUT_TYPE ?= type=registry
+BUILDX_BUILDER_NAME ?= img-builder
+QEMU_VERSION ?= 5.2.0-2
+
+.PHONY: docker-buildx-builder
+docker-buildx-builder:
+	@if ! docker buildx ls | grep $(BUILDX_BUILDER_NAME); then \
+		docker run --rm --privileged multiarch/qemu-user-static:$(QEMU_VERSION) --reset -p yes; \
+		docker buildx create --name $(BUILDX_BUILDER_NAME) --use; \
+		docker buildx inspect $(BUILDX_BUILDER_NAME) --bootstrap; \
+	fi
+
+.PHONY: docker-build
+docker-build: docker-buildx-builder
+	docker buildx build \
+		--file Dockerfile \
+		--output=$(OUTPUT_TYPE) \
+		--platform="linux/amd64" \
+		--pull \
+		--tag $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION) .

apis/v1alpha1/placementpolicy_types.go

@@ -22,6 +22,11 @@ const (
 	ActionMust Action = "Must"
 	// ActionMustNot means the pods must not be placed on the node
 	ActionMustNot Action = "MustNot"
+
+	// PlacementPolicyAnnotationKey is the annotation key for placement policy
+	PlacementPolicyAnnotationKey = "placement-policy.x-k8s.io/policy-name"
+	// PlacementPolicyPreferenceAnnotationKey is the annotation key for placement policy node preference
+	PlacementPolicyPreferenceAnnotationKey = "placement-policy.x-k8s.io/node-preference-matching-labels"
 )
 
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.

examples/pod.yml

@@ -5,7 +5,7 @@ metadata:
   labels:
     app: nginx
 spec:
-  replicas: 5
+  replicas: 10
   selector:
     matchLabels:
       app: nginx

examples/v1alpha_placementpolicy.yml → examples/v1alpha1_placementpolicy.yml

@@ -4,7 +4,7 @@ metadata:
   name: pp-one
 spec:
   weight: 100
-  enforcementMode: Strict #BestEffort
+  enforcementMode: Strict
   podSelector:
     matchLabels:
       app: nginx

manifest_staging/charts/placement-policy-scheduler-plugins/templates/configmap.yaml

@@ -14,9 +14,15 @@ data:
     profiles:
     - schedulerName: placement-policy-plugins-scheduler
       plugins:
+        preScore:
+          enabled:
+          - name: placementpolicy
         score:
           enabled:
           - name: placementpolicy
+        preFilter:
+          enabled:
+          - name: placementpolicy
         filter:
           enabled:
           - name: placementpolicy

manifest_staging/deploy/kube-scheduler-configuration.yml

@@ -195,9 +195,15 @@ data:
     profiles:
     - schedulerName: placement-policy-plugins-scheduler
       plugins:
+        preScore:
+          enabled:
+          - name: placementpolicy
         score:
           enabled:
           - name: placementpolicy
+        preFilter:
+          enabled:
+          - name: placementpolicy
         filter:
           enabled:
           - name: placementpolicy
@@ -307,7 +313,7 @@ spec:
         - --config=/etc/schedulerconfig/scheduler-config.yaml
         - --authentication-kubeconfig=/etc/kubernetes/scheduler.conf
         - --authorization-kubeconfig=/etc/kubernetes/scheduler.conf
-        image: helayoty/placement-policy:v0.1.0
+        image: aramase/placement-policy:v0.1.0
         name: pp-plugins-scheduler
         securityContext:
           privileged: true

pkg/plugins/placementpolicy/core/core.go

@@ -0,0 +1,110 @@
+package core
+
+import (
+	"context"
+	"sort"
+
+	"github.com/Azure/placement-policy-scheduler-plugins/apis/v1alpha1"
+	ppclientset "github.com/Azure/placement-policy-scheduler-plugins/pkg/client/clientset/versioned"
+	ppinformers "github.com/Azure/placement-policy-scheduler-plugins/pkg/client/informers/externalversions/apis/v1alpha1"
+	pplisters "github.com/Azure/placement-policy-scheduler-plugins/pkg/client/listers/apis/v1alpha1"
+	"github.com/Azure/placement-policy-scheduler-plugins/pkg/utils"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/kubernetes"
+	corelisters "k8s.io/client-go/listers/core/v1"
+	"k8s.io/kubernetes/pkg/scheduler/framework"
+)
+
+// Manager defines the interfaces for PlacementPolicy management.
+type Manager interface {
+	GetPlacementPolicyForPod(context.Context, *corev1.Pod) (*v1alpha1.PlacementPolicy, error)
+	GetPodsWithLabels(context.Context, map[string]string) ([]*corev1.Pod, error)
+	AnnotatePod(context.Context, *corev1.Pod, *v1alpha1.PlacementPolicy, bool) (*corev1.Pod, error)
+	GetPlacementPolicy(context.Context, string, string) (*v1alpha1.PlacementPolicy, error)
+}
+
+type PlacementPolicyManager struct {
+	// client is a clientset for the kube API server.
+	client kubernetes.Interface
+	// client is a placementPolicy client
+	ppClient ppclientset.Interface
+	// podLister is pod lister
+	podLister corelisters.PodLister
+	// snapshotSharedLister is pod shared list
+	snapshotSharedLister framework.SharedLister
+	// ppLister is placementPolicy lister
+	ppLister pplisters.PlacementPolicyLister
+}
+
+func NewPlacementPolicyManager(
+	client kubernetes.Interface,
+	ppClient ppclientset.Interface,
+	snapshotSharedLister framework.SharedLister,
+	ppInformer ppinformers.PlacementPolicyInformer,
+	podLister corelisters.PodLister) *PlacementPolicyManager {
+	return &PlacementPolicyManager{
+		client:               client,
+		ppClient:             ppClient,
+		snapshotSharedLister: snapshotSharedLister,
+		ppLister:             ppInformer.Lister(),
+		podLister:            podLister,
+	}
+}
+
+// GetPlacementPolicyForPod returns the placement policy for the given pod
+func (m *PlacementPolicyManager) GetPlacementPolicyForPod(ctx context.Context, pod *corev1.Pod) (*v1alpha1.PlacementPolicy, error) {
+	ppList, err := m.ppLister.PlacementPolicies(pod.Namespace).List(labels.Everything())
+	if err != nil {
+		return nil, err
+	}
+	// filter the placement policy list based on the pod's labels
+	ppList = m.filterPlacementPolicyList(ppList, pod)
+	if len(ppList) == 0 {
+		return nil, nil
+	}
+	if len(ppList) > 1 {
+		// if there are multiple placement policies, sort them by weight and return the first one
+		sort.Sort(sort.Reverse(ByWeight(ppList)))
+	}
+
+	return ppList[0], nil
+}
+
+func (m *PlacementPolicyManager) GetPodsWithLabels(ctx context.Context, podLabels map[string]string) ([]*corev1.Pod, error) {
+	return m.podLister.List(labels.Set(podLabels).AsSelector())
+}
+
+// AnnotatePod annotates the pod with the placement policy.
+func (m *PlacementPolicyManager) AnnotatePod(ctx context.Context, pod *corev1.Pod, pp *v1alpha1.PlacementPolicy, preferredNodeWithMatchingLabels bool) (*corev1.Pod, error) {
+	annotations := map[string]string{}
+	if pod.Annotations != nil {
+		annotations = pod.Annotations
+	}
+
+	preference := "false"
+	if preferredNodeWithMatchingLabels {
+		preference = "true"
+	}
+	annotations[v1alpha1.PlacementPolicyAnnotationKey] = pp.Name
+	annotations[v1alpha1.PlacementPolicyPreferenceAnnotationKey] = preference
+	pod.Annotations = annotations
+	return m.client.CoreV1().Pods(pod.Namespace).Update(ctx, pod, metav1.UpdateOptions{})
+}
+
+func (m *PlacementPolicyManager) GetPlacementPolicy(ctx context.Context, namespace, name string) (*v1alpha1.PlacementPolicy, error) {
+	return m.ppLister.PlacementPolicies(namespace).Get(name)
+}
+
+func (m *PlacementPolicyManager) filterPlacementPolicyList(ppList []*v1alpha1.PlacementPolicy, pod *corev1.Pod) []*v1alpha1.PlacementPolicy {
+	var filteredPPList []*v1alpha1.PlacementPolicy
+	for _, pp := range ppList {
+		labels := pp.Spec.PodSelector.MatchLabels
+		if utils.HasMatchingLabels(pod.Labels, labels) {
+			filteredPPList = append(filteredPPList, pp)
+		}
+	}
+	return filteredPPList
+}

pkg/plugins/placementpolicy/core/sort.go

@@ -0,0 +1,15 @@
+package core
+
+import "github.com/Azure/placement-policy-scheduler-plugins/apis/v1alpha1"
+
+type ByWeight []*v1alpha1.PlacementPolicy
+
+func (a ByWeight) Len() int { return len(a) }
+
+func (a ByWeight) Swap(i, j int) {
+	a[i], a[j] = a[j], a[i]
+}
+
+func (a ByWeight) Less(i, j int) bool {
+	return a[i].Spec.Weight > a[j].Spec.Weight
+}