Add pod eviction logic for scheduler preemption

Add Preempt to scheduler interface
Add preemption to the scheduling workflow
Minor changes to the scheduler integration test library

plugin/pkg/scheduler/algorithm/predicates/error.go

@@ -25,6 +25,11 @@ import (
 var (
 	// The predicateName tries to be consistent as the predicate name used in DefaultAlgorithmProvider defined in
 	// defaults.go (which tend to be stable for backward compatibility)
+
+	// NOTE: If you add a new predicate failure error for a predicate that can never
+	// be made to pass by removing pods, or you change an existing predicate so that
+	// it can never be made to pass by removing pods, you need to add the predicate
+	// failure error in nodesWherePreemptionMightHelp() in scheduler/core/generic_scheduler.go
 	ErrDiskConflict              = newPredicateFailureError("NoDiskConflict")
 	ErrVolumeZoneConflict        = newPredicateFailureError("NoVolumeZoneConflict")
 	ErrNodeSelectorNotMatch      = newPredicateFailureError("MatchNodeSelector")

plugin/pkg/scheduler/algorithm/predicates/metadata.go

@@ -40,8 +40,8 @@ type matchingPodAntiAffinityTerm struct {
 	node *v1.Node
 }
 
-// NOTE: When new fields are added/removed or logic is changed, please make sure
-// that RemovePod and AddPod functions are updated to work with the new changes.
+// NOTE: When new fields are added/removed or logic is changed, please make sure that
+// RemovePod, AddPod, and ShallowCopy functions are updated to work with the new changes.
 type predicateMetadata struct {
 	pod           *v1.Pod
 	podBestEffort bool
@@ -172,13 +172,17 @@ func (meta *predicateMetadata) ShallowCopy() algorithm.PredicateMetadata {
 		podRequest:           meta.podRequest,
 		serviceAffinityInUse: meta.serviceAffinityInUse,
 	}
+	newPredMeta.podPorts = map[int]bool{}
 	for k, v := range meta.podPorts {
 		newPredMeta.podPorts[k] = v
 	}
+	newPredMeta.matchingAntiAffinityTerms = map[string][]matchingPodAntiAffinityTerm{}
 	for k, v := range meta.matchingAntiAffinityTerms {
 		newPredMeta.matchingAntiAffinityTerms[k] = append([]matchingPodAntiAffinityTerm(nil), v...)
 	}
-	newPredMeta.serviceAffinityMatchingPodServices = append([]*v1.Service(nil), meta.serviceAffinityMatchingPodServices...)
-	newPredMeta.serviceAffinityMatchingPodList = append([]*v1.Pod(nil), meta.serviceAffinityMatchingPodList...)
+	newPredMeta.serviceAffinityMatchingPodServices = append([]*v1.Service(nil),
+		meta.serviceAffinityMatchingPodServices...)
+	newPredMeta.serviceAffinityMatchingPodList = append([]*v1.Pod(nil),
+		meta.serviceAffinityMatchingPodList...)
 	return (algorithm.PredicateMetadata)(newPredMeta)
 }

plugin/pkg/scheduler/algorithm/predicates/metadata_test.go

@@ -355,3 +355,46 @@ func TestPredicateMetadata_AddRemovePod(t *testing.T) {
 		}
 	}
 }
+
+// TestPredicateMetadata_ShallowCopy tests the ShallowCopy function. It is based
+// on the idea that shallow-copy should produce an object that is deep-equal to the original
+// object.
+func TestPredicateMetadata_ShallowCopy(t *testing.T) {
+	source := predicateMetadata{
+		pod: &v1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "test",
+				Namespace: "testns",
+			},
+		},
+		podBestEffort: true,
+		podRequest: &schedulercache.Resource{
+			MilliCPU:         1000,
+			Memory:           300,
+			AllowedPodNumber: 4,
+		},
+		podPorts: map[int]bool{1234: true, 456: false},
+		matchingAntiAffinityTerms: map[string][]matchingPodAntiAffinityTerm{
+			"term1": {
+				{
+					term: &v1.PodAffinityTerm{TopologyKey: "node"},
+					node: &v1.Node{
+						ObjectMeta: metav1.ObjectMeta{Name: "machine1"},
+					},
+				},
+			},
+		},
+		serviceAffinityInUse: true,
+		serviceAffinityMatchingPodList: []*v1.Pod{
+			{ObjectMeta: metav1.ObjectMeta{Name: "pod1"}},
+			{ObjectMeta: metav1.ObjectMeta{Name: "pod2"}},
+		},
+		serviceAffinityMatchingPodServices: []*v1.Service{
+			{ObjectMeta: metav1.ObjectMeta{Name: "service1"}},
+		},
+	}
+
+	if !reflect.DeepEqual(source.ShallowCopy().(*predicateMetadata), &source) {
+		t.Errorf("Copy is not equal to source!")
+	}
+}

plugin/pkg/scheduler/algorithm/predicates/predicates.go

@@ -45,6 +45,10 @@ import (
 	"github.com/golang/glog"
 )
 
+const (
+	MatchInterPodAffinity = "MatchInterPodAffinity"
+)
+
 // NodeInfo: Other types for predicate functions...
 type NodeInfo interface {
 	GetNodeInfo(nodeID string) (*v1.Node, error)

plugin/pkg/scheduler/algorithm/scheduler_interface.go

@@ -47,6 +47,10 @@ type SchedulerExtender interface {
 // onto machines.
 type ScheduleAlgorithm interface {
 	Schedule(*v1.Pod, NodeLister) (selectedMachine string, err error)
+	// Preempt receives scheduling errors for a pod and tries to create room for
+	// the pod by preempting lower priority pods if possible.
+	// It returns the node where preemption happened, a list of preempted pods, and error if any.
+	Preempt(*v1.Pod, NodeLister, error) (selectedNode *v1.Node, preemptedPods []*v1.Pod, err error)
 	// Predicates() returns a pointer to a map of predicate functions. This is
 	// exposed for testing.
 	Predicates() map[string]FitPredicate

plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go

@@ -155,7 +155,7 @@ func defaultPredicates() sets.String {
 		),
 		// Fit is determined by inter-pod affinity.
 		factory.RegisterFitPredicateFactory(
-			"MatchInterPodAffinity",
+			predicates.MatchInterPodAffinity,
 			func(args factory.PluginFactoryArgs) algorithm.FitPredicate {
 				return predicates.NewPodAffinityPredicate(args.NodeInfo, args.PodLister)
 			},

plugin/pkg/scheduler/core/extender_test.go

@@ -183,6 +183,8 @@ func (f *FakeExtender) IsBinder() bool {
 	return true
 }
 
+var _ algorithm.SchedulerExtender = &FakeExtender{}
+
 func TestGenericSchedulerWithExtenders(t *testing.T) {
 	tests := []struct {
 		name                 string