topology-aware: implement strict hint and CPU isolation preferences.

cmd/plugins/topology-aware/policy/mocks_test.go

@@ -433,6 +433,13 @@ func (m *mockContainer) GetEffectiveAnnotation(key string) (string, bool) {
 	}
 	return pod.GetEffectiveAnnotation(key, m.name)
 }
+func (m *mockContainer) QueryEffectiveAnnotation(key string) (string, cache.AnnotationScope, bool) {
+	pod, ok := m.GetPod()
+	if !ok {
+		return "", cache.UnscopedAnnotation, false
+	}
+	return pod.QueryEffectiveAnnotation(key, m.name)
+}
 func (m *mockContainer) EvalKey(string) interface{} {
 	panic("unimplemented")
 }
@@ -461,6 +468,9 @@ func (m *mockContainer) InsertMount(*cache.Mount) {
 func (m *mockContainer) GetTopologyHints() topology.Hints {
 	return topology.Hints{}
 }
+func (m *mockContainer) StrictTopologyHints() bool {
+	return false
+}
 func (m *mockContainer) SetCPUShares(int64) {
 }
 func (m *mockContainer) SetCPUPeriod(int64) {
@@ -650,6 +660,16 @@ func (m *mockPod) GetEffectiveAnnotation(key, container string) (string, bool) {
 	v, ok := m.annotations[key]
 	return v, ok
 }
+func (m *mockPod) QueryEffectiveAnnotation(key, container string) (string, cache.AnnotationScope, bool) {
+	if v, ok := m.annotations[key+"/container."+container]; ok {
+		return v, cache.ContainerScopedAnnotation, true
+	}
+	if v, ok := m.annotations[key+"/pod"]; ok {
+		return v, cache.PodScopedAnnotation, true
+	}
+	v, ok := m.annotations[key]
+	return v, cache.UnscopedAnnotation, ok
+}
 func (m *mockPod) GetContainerAffinity(string) ([]*cache.Affinity, error) {
 	panic("unimplemented")
 }

cmd/plugins/topology-aware/policy/pod-preferences.go

@@ -47,6 +47,8 @@ const (
 	keySchedulingClass = "scheduling-class." + kubernetes.ResmgrKeyNamespace
 	// effective annotation key for isolated CPU preference
 	preferIsolatedCPUsKey = "prefer-isolated-cpus" + "." + kubernetes.ResmgrKeyNamespace
+	// effective annotation key for strict isolated CPU preference
+	strictPreferIsolatedCPUsKey = "require-isolated-cpus" + "." + kubernetes.ResmgrKeyNamespace
 	// effective annotation key for shared CPU preference
 	preferSharedCPUsKey = "prefer-shared-cpus" + "." + kubernetes.ResmgrKeyNamespace
 	// effective annotation key for memory type preference
@@ -129,10 +131,21 @@ func boolConfigPreference(ptr *bool) (bool, prefKind) {
 // for the given container. If an effective annotation is not found, it uses
 // the global configuration for isolated CPU preference.
 func isolatedCPUsPreference(pod cache.Pod, container cache.Container) (bool, prefKind) {
-	key := preferIsolatedCPUsKey
-	value, ok := pod.GetEffectiveAnnotation(key, container.GetName())
-	if !ok {
+	sKey := strictPreferIsolatedCPUsKey
+	sVal, sScope, sOk := pod.QueryEffectiveAnnotation(sKey, container.GetName())
+
+	pKey := preferIsolatedCPUsKey
+	pVal, pScope, pOk := pod.QueryEffectiveAnnotation(pKey, container.GetName())
+
+	key, value := "", ""
+
+	switch {
+	case !sOk && !pOk:
 		return boolConfigPreference(opt.PreferIsolated)
+	case (sOk && !pOk) || sScope <= pScope:
+		key, value = sKey, sVal
+	default: // case (!sOk && pOk) || pScope < sScope:
+		key, value = pKey, pVal
 	}
 
 	preference, err := strconv.ParseBool(value)
@@ -147,6 +160,14 @@ func isolatedCPUsPreference(pod cache.Pod, container cache.Container) (bool, pre
 	return preference, prefAnnotated
 }
 
+// strictIsolatedCPUsPreference returns true if isolated CPU allocation is required.
+func strictIsolatedCPUsPreference(pod cache.Pod, container cache.Container) bool {
+	_, sScope, sOk := pod.QueryEffectiveAnnotation(strictPreferIsolatedCPUsKey, container.GetName())
+	_, pScope, pOk := pod.QueryEffectiveAnnotation(preferIsolatedCPUsKey, container.GetName())
+
+	return (sOk && !pOk) || (sOk && pOk && sScope <= pScope)
+}
+
 // sharedCPUsPreference returns whether shared CPU allocation is preferred for
 // the given container. If an effective annotation is not found, it uses the
 // global configuration for shared CPU preference.

cmd/plugins/topology-aware/policy/resources.go

@@ -29,6 +29,7 @@ import (
 	"github.com/containers/nri-plugins/pkg/kubernetes"
 	"github.com/containers/nri-plugins/pkg/resmgr/cache"
 	libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory"
+	topoutil "github.com/containers/nri-plugins/pkg/utils/topology"
 )
 
 type (
@@ -374,6 +375,11 @@ func (cs *supply) Allocate(r Request, o *libmem.Offer) (Grant, map[string]libmem
 		return nil, nil, err
 	}
 
+	if err := r.(*request).verifyStrictPreferences(grant); err != nil {
+		cs.ReleaseCPU(grant)
+		return nil, nil, err
+	}
+
 	zone, updates, err := o.Commit()
 	if err != nil {
 		cs.ReleaseCPU(grant)
@@ -871,6 +877,75 @@ func (cr *request) ColdStart() time.Duration {
 	return cr.coldStart
 }
 
+func (cr *request) verifyStrictPreferences(g Grant) error {
+	if err := cr.verifyStrictTopologyHints(g); err != nil {
+		return err
+	}
+	if err := cr.verifyStrictCPUPreferences(g); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (cr *request) verifyStrictTopologyHints(g Grant) error {
+	if !cr.GetContainer().StrictTopologyHints() {
+		return nil
+	}
+
+	for _, h := range cr.GetContainer().GetTopologyHints() {
+		hint := topoutil.NewHint(g.GetCPUNode().System(), h)
+
+		if g.SharedPortion() > 0 {
+			if cpus := hint.MisalignedCPUSet(g.SharedCPUs()); cpus.Size() > 0 {
+				return policyError("granted shared CPUs %q fail strict hint %v",
+					cpus.String(), h)
+			}
+		}
+
+		if g.ReservedPortion() > 0 {
+			if cpus := hint.MisalignedCPUSet(g.ReservedCPUs()); cpus.Size() > 0 {
+				return policyError("granted reserved CPUs %q fail strict hint %v",
+					cpus.String(), h)
+			}
+		}
+
+		if excl := g.ExclusiveCPUs(); excl.Size() > 0 {
+			if cpus := hint.MisalignedCPUSet(excl); cpus.Size() > 0 {
+				return policyError("granted exclusive CPUs %q fail strict hint %v",
+					cpus.String(), h)
+			}
+		}
+
+		if mems := hint.MisalignedMems(g.GetMemoryZone()); mems.Size() > 0 {
+			return policyError("granted memory zones %s fail strict hint %v",
+				mems.String(), h)
+		}
+	}
+
+	return nil
+}
+
+func (cr *request) verifyStrictCPUPreferences(g Grant) error {
+	full := cr.FullCPUs()
+
+	if full < 1 || !cr.Isolate() {
+		return nil
+	}
+
+	ctr := cr.GetContainer()
+	pod, _ := ctr.GetPod()
+	if !strictIsolatedCPUsPreference(pod, ctr) {
+		return nil
+	}
+
+	if cpus := g.IsolatedCPUs(); cpus.Size() < full {
+		return policyError("granted isolated CPUs %q less than requested %d",
+			cpus.String(), full)
+	}
+
+	return nil
+}
+
 // Score collects data for scoring this supply wrt. the given request.
 func (cs *supply) GetScore(req Request) Score {
 	score := &score{

docs/resource-policy/policy/topology-aware.md

@@ -462,6 +462,13 @@ metadata:
 These Pod annotations have no effect on containers which are not eligible for
 exclusive allocation.
 
+The `require-isolated-cpus.resource-policy.nri.io` annotation key can be
+used instead of `prefer-isolated-cpus` to require isolated CPUs for eligible
+containers. It is syntactically identical to `prefer-isolate-cpus`, but the
+semantics are strict. If the exclusively allocated CPUs for such a container
+are not isolated, the policy will fail the creation of the container with an
+error.
+
 ### Preferred Topology Level for Burstable Containers Without CPU Limit
 
 CPU-unlimited burstable containers are by default preferred to allocate to a
@@ -849,7 +856,22 @@ the individually picked resources. If picking resources by hints fails for any
 of the devices, the policy falls back to picking resource from the pool without
 considering device hints.
 
+**Strict Topology Hints**
+
+Containers can be annotated for strict topology hints using the
+`strict.topologyhints.resource-policy.nri.io` annotation. For instance the
+following annotation select strict topology hints for container `highprio`:
+
+```yaml
+...
+  metadata:
+    annotations:
+      strict.topologyhints.resource-policy.nri.io/container.highprio: "true"
+...
+```
 
+If the allocated CPU and memory leave any topology hint unsatisfied for such
+a container, the policy will fail container creation with an error.
 
 ### Container Affinity and Anti-Affinity
 

pkg/resmgr/cache/cache.go

@@ -59,6 +59,14 @@ const (
 	// TopologyHintsKey can be used to opt out from automatic topology hint generation.
 	TopologyHintsKey = "topologyhints" + "." + kubernetes.ResmgrKeyNamespace
 
+	// TestTopologyHintsKey can be used to annotate fake topology hints for testing.
+	TestTopologyHintsKey = "test." + TopologyHintsKey
+
+	// StrictTopologyHintsKey can be used to force strict interpretation of topology
+	// hints. An unsatisfied strict topology hint should prevent the creation of the
+	// container.
+	StrictTopologyHintsKey = "strict." + TopologyHintsKey
+
 	// PreserveCpuKey means that CPU resources should not be touched.
 	PreserveCpuKey = "cpu.preserve." + kubernetes.ResmgrKeyNamespace
 	// PreserveMemoryKey means that memory resources should not be touched.
@@ -74,6 +82,21 @@ const (
 	AnnotatedResourcesKey = kubernetes.AnnotatedResourcesKey
 )
 
+// AnnotationScope denotes the scope of a queried effective annotation.
+type AnnotationScope int
+
+const (
+	// ContainerScopedAnnotation indicates a container scoped
+	// annotation using the $key/container.$container key syntax.
+	ContainerScopedAnnotation AnnotationScope = iota
+	// PodScopedAnnotation indicates a pod scoped annotation
+	// using the $key/pod key syntax.
+	PodScopedAnnotation
+	// UnscopedAnnotation indicates a unscoped annotation
+	// using the plain $key key syntax.
+	UnscopedAnnotation
+)
+
 // PodState is the pod state in the runtime.
 type PodState int32
 
@@ -118,6 +141,10 @@ type Pod interface {
 	// and return the value of the first key found.
 	GetEffectiveAnnotation(key, container string) (string, bool)
 
+	// QueryEffectiveAnnotation is like GetEffectiveAnnotation but also returns
+	// the scope of the found annotation.
+	QueryEffectiveAnnotation(key, container string) (string, AnnotationScope, bool)
+
 	// GetPodResources returns the pod resources for this pod, waiting for any
 	// pending fetch to complete or a timeout.
 	GetPodResources() *podresapi.PodResources
@@ -221,6 +248,10 @@ type Container interface {
 	// GetEffectiveAnnotation returns the effective annotation for the container from the pod.
 	GetEffectiveAnnotation(key string) (string, bool)
 
+	// QueryEffectiveAnnotation is like GetEffectiveAnnotation but also returns
+	// the scope of the found annotation.
+	QueryEffectiveAnnotation(key string) (string, AnnotationScope, bool)
+
 	// Containers can be subject for expression evaluation.
 	resmgr.Evaluable
 
@@ -249,6 +280,8 @@ type Container interface {
 
 	// Get any attached topology hints.
 	GetTopologyHints() topology.Hints
+	// StrictTopologyHints returns true if hints should be strict.
+	StrictTopologyHints() bool
 
 	// SetCPUShares sets the CFS CPU shares of the container.
 	SetCPUShares(int64)

pkg/resmgr/cache/container.go

@@ -31,6 +31,7 @@ import (
 	"github.com/containers/nri-plugins/pkg/kubernetes"
 	libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory"
 	"github.com/containers/nri-plugins/pkg/topology"
+	"github.com/containers/nri-plugins/pkg/utils"
 
 	nri "github.com/containerd/nri/pkg/api"
 	v1 "k8s.io/api/core/v1"
@@ -195,6 +196,7 @@ func (c *container) generateTopologyHints() {
 		mountHints       = true
 		deviceHints      = true
 		podResourceHints = true
+		testHints        = utils.TestAPIsEnabled()
 	)
 
 	if preference, ok := c.GetEffectiveAnnotation(TopologyHintsKey); ok {
@@ -274,6 +276,23 @@ func (c *container) generateTopologyHints() {
 	} else {
 		log.Info("automatic topology hint generation disabled for pod resources")
 	}
+
+	if testHints {
+		hints := map[string]topology.Hint{}
+		if value, ok := c.GetEffectiveAnnotation(TestTopologyHintsKey); ok {
+			if err := yaml.Unmarshal([]byte(value), &hints); err != nil {
+				log.Error("failed to parse test topology hints annotation for %s: %v",
+					c.PrettyName(), err)
+			} else {
+				for p, h := range hints {
+					h.Provider = p
+					hints[h.Provider] = h
+					log.Info("%s: injected test topology hint %v", c.PrettyName(), h)
+				}
+				c.TopologyHints = topology.MergeTopologyHints(c.TopologyHints, hints)
+			}
+		}
+	}
 }
 
 func isReadOnlyMount(m *nri.Mount) bool {
@@ -505,6 +524,14 @@ func (c *container) GetEffectiveAnnotation(key string) (string, bool) {
 	return pod.GetEffectiveAnnotation(key, c.GetName())
 }
 
+func (c *container) QueryEffectiveAnnotation(key string) (string, AnnotationScope, bool) {
+	pod, ok := c.GetPod()
+	if !ok {
+		return "", UnscopedAnnotation, false
+	}
+	return pod.QueryEffectiveAnnotation(key, c.GetName())
+}
+
 func (c *container) GetResourceRequirements() v1.ResourceRequirements {
 	return c.Requirements
 }
@@ -613,6 +640,22 @@ func (c *container) GetTopologyHints() topology.Hints {
 	return c.TopologyHints
 }
 
+func (c *container) StrictTopologyHints() bool {
+	value, ok := c.GetEffectiveAnnotation(StrictTopologyHintsKey)
+	if !ok {
+		return false
+	}
+
+	strict, err := strconv.ParseBool(value)
+	if err != nil {
+		log.Error("%s: invalid strict topology hints annotation (%q, %q): %v",
+			c.PrettyName(), StrictTopologyHintsKey, value, err)
+		return false
+	}
+
+	return strict
+}
+
 func (c *container) getPendingRequest() interface{} {
 	if c.request == nil {
 		if c.GetState() == ContainerStateCreating {

pkg/resmgr/cache/pod.go

@@ -121,15 +121,20 @@ func (p *pod) GetResmgrAnnotation(key string) (string, bool) {
 }
 
 func (p *pod) GetEffectiveAnnotation(key, container string) (string, bool) {
+	value, _, ok := p.QueryEffectiveAnnotation(key, container)
+	return value, ok
+}
+
+func (p *pod) QueryEffectiveAnnotation(key, container string) (string, AnnotationScope, bool) {
 	annotations := p.Pod.GetAnnotations()
 	if v, ok := annotations[key+"/container."+container]; ok {
-		return v, true
+		return v, ContainerScopedAnnotation, true
 	}
 	if v, ok := annotations[key+"/pod"]; ok {
-		return v, true
+		return v, PodScopedAnnotation, true
 	}
 	v, ok := annotations[key]
-	return v, ok
+	return v, UnscopedAnnotation, ok
 }
 
 func (p *pod) GetQOSClass() v1.PodQOSClass {