diff --git a/pkg/apis/core/types.go b/pkg/apis/core/types.go index 21ffb64dec5a1..50e6b0897218a 100644 --- a/pkg/apis/core/types.go +++ b/pkg/apis/core/types.go @@ -2495,6 +2495,8 @@ type ResourceRequirements struct { // +featureGate=DynamicResourceAllocation // +optional Claims []ResourceClaim + // Add mustKeepCPUs + MustKeepCPUs string } // VolumeResourceRequirements describes the storage resource requirements for a volume. diff --git a/pkg/kubelet/cm/cpumanager/cpu_assignment.go b/pkg/kubelet/cm/cpumanager/cpu_assignment.go index 27afabefa423f..d473cf27a021d 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_assignment.go +++ b/pkg/kubelet/cm/cpumanager/cpu_assignment.go @@ -95,6 +95,11 @@ type numaOrSocketsFirstFuncs interface { sortAvailableNUMANodes() []int sortAvailableSockets() []int sortAvailableCores() []int + takeFullFirstLevelForResize() + takeFullSecondLevelForResize() + sortAvailableNUMANodesForResize() []int + sortAvailableSocketsForResize() []int + sortAvailableCoresForResize() []int } type numaFirst struct{ acc *cpuAccumulator } @@ -204,8 +209,145 @@ func (s *socketsFirst) sortAvailableCores() []int { return result } +// If NUMA nodes are higher in the memory hierarchy than sockets, then we take +// from the set of NUMA Nodes as the first level for resize. +func (n *numaFirst) takeFullFirstLevelForResize() { + n.acc.takeRemainCpusForFullNUMANodes() +} + +// If NUMA nodes are higher in the memory hierarchy than sockets, then we take +// from the set of sockets as the second level for resize. +func (n *numaFirst) takeFullSecondLevelForResize() { + n.acc.takeRemainCpusForFullSockets() +} + +// If NUMA nodes are higher in the memory hierarchy than sockets, then return the available NUMA nodes +// which have allocated CPUs to Container. +func (n *numaFirst) sortAvailableNUMANodesForResize() []int { + allocatedNumaNodesSet := n.acc.resultDetails.NUMANodes() + availableNumaNodesSet := n.acc.details.NUMANodes() + numas := allocatedNumaNodesSet.Intersection(availableNumaNodesSet).UnsortedList() + n.acc.sort(numas, n.acc.details.CPUsInNUMANodes) + return numas +} + +// If NUMA nodes are higher in the memory hierarchy than sockets, +// Firstly, pull the socket which are allocated CPUs to the Container +// Secondly, pull the other sockets which are not allocated CPUs to the Container, but contains in the NUMA node which are allocated CPUs to the Container +func (n *numaFirst) sortAvailableSocketsForResize() []int { + var result []int + + // Sort allocated sockets + allocatedSocketsSet := n.acc.resultDetails.Sockets() + availableSocketsSet := n.acc.details.Sockets() + allocatedSockets := allocatedSocketsSet.Intersection(availableSocketsSet).UnsortedList() + n.acc.sort(allocatedSockets, n.acc.details.CPUsInSockets) + result = append(result, allocatedSockets...) + + // Sort the sockets in allocated numa node, but not allocated CPU on these sockets + for _, numa := range n.sortAvailableNUMANodesForResize() { + socketSet := n.acc.details.SocketsInNUMANodes(numa) + sockets := socketSet.Difference(allocatedSocketsSet).UnsortedList() + n.acc.sort(sockets, n.acc.details.CPUsInSockets) + result = append(result, sockets...) + } + return result +} + +// If NUMA nodes are higher in the memory hierarchy than sockets, +// Firstly, pull the cores which are allocated CPUs to the Container +// Secondly, pull the other cores which are not allocated CPUs to the Container, but contains in the NUMA node which are allocated CPUs to the Container +func (n *numaFirst) sortAvailableCoresForResize() []int { + var result []int + + // Sort allocated cores + allocatedCoresSet := n.acc.resultDetails.Cores() + availableCoresSet := n.acc.details.Cores() + allocatedCores := allocatedCoresSet.Intersection(availableCoresSet).UnsortedList() + n.acc.sort(allocatedCores, n.acc.details.CPUsInCores) + result = append(result, allocatedCores...) + + // Sort the cores in allocated sockets, and allocated numa, but not allocated CPU on these sockets and numa + for _, socket := range n.acc.sortAvailableSocketsForResize() { + coresSet := n.acc.details.CoresInSockets(socket) + cores := coresSet.Difference(allocatedCoresSet).UnsortedList() + n.acc.sort(cores, n.acc.details.CPUsInCores) + result = append(result, cores...) + } + return result +} + +// If sockets are higher in the memory hierarchy than NUMA nodes, then we take +// from the set of NUMA Nodes as the first level for resize. +func (s *socketsFirst) takeFullFirstLevelForResize() { + s.acc.takeRemainCpusForFullSockets() +} + +// If sockets are higher in the memory hierarchy than NUMA nodes, then we take +// from the set of sockets as the second level for resize. +func (s *socketsFirst) takeFullSecondLevelForResize() { + s.acc.takeRemainCpusForFullNUMANodes() +} + +// If sockets are higher in the memory hierarchy than NUMA nodes, +// Firstly, pull the NUMA nodes which are allocated CPUs to the Container +// Secondly, pull the other NUMA nodes which are not allocated CPUs to the Container, but contains in the sockets which are allocated CPUs to the Container +func (s *socketsFirst) sortAvailableNUMANodesForResize() []int { + var result []int + + // Sort allocated sockets + allocatedNUMANodesSet := s.acc.resultDetails.NUMANodes() + availableNUMANodesSet := s.acc.details.NUMANodes() + allocatedNUMANodes := allocatedNUMANodesSet.Intersection(availableNUMANodesSet).UnsortedList() + s.acc.sort(allocatedNUMANodes, s.acc.details.CPUsInNUMANodes) + result = append(result, allocatedNUMANodes...) + + // Sort the sockets in allocated numa node, but not allocated CPU on these sockets + for _, socket := range s.sortAvailableSocketsForResize() { + NUMANodesSet := s.acc.details.NUMANodesInSockets(socket) + NUMANodes := NUMANodesSet.Difference(allocatedNUMANodesSet).UnsortedList() + s.acc.sort(NUMANodes, s.acc.details.CPUsInNUMANodes) + result = append(result, NUMANodes...) + } + return result +} + +// If sockets are higher in the memory hierarchy than NUMA nodes, then return the available sockets +// which have allocated CPUs to Container. +func (s *socketsFirst) sortAvailableSocketsForResize() []int { + allocatedSocketsSet := s.acc.resultDetails.Sockets() + availableSocketsSet := s.acc.details.Sockets() + sockets := allocatedSocketsSet.Intersection(availableSocketsSet).UnsortedList() + s.acc.sort(sockets, s.acc.details.CPUsInSockets) + return sockets +} + +// If sockets are higher in the memory hierarchy than NUMA nodes, +// Firstly, pull the cores which are allocated CPUs to the Container +// Secondly, pull the other cores which are not allocated CPUs to the Container, but contains in the socket which are allocated CPUs to the Container +func (s *socketsFirst) sortAvailableCoresForResize() []int { + var result []int + + // Sort allocated cores + allocatedCoresSet := s.acc.resultDetails.Cores() + availableCoresSet := s.acc.details.Cores() + allocatedCores := allocatedCoresSet.Intersection(availableCoresSet).UnsortedList() + s.acc.sort(allocatedCores, s.acc.details.CPUsInCores) + result = append(result, allocatedCores...) + + // Sort the cores in allocated sockets, and allocated numa, but not allocated CPU on these sockets and numa + for _, NUMANode := range s.acc.sortAvailableNUMANodesForResize() { + coresSet := s.acc.details.CoresInNUMANodes(NUMANode) + cores := coresSet.Difference(allocatedCoresSet).UnsortedList() + s.acc.sort(cores, s.acc.details.CPUsInCores) + result = append(result, cores...) + } + return result +} + type availableCPUSorter interface { sort() []int + sortForResize() []int } type sortCPUsPacked struct{ acc *cpuAccumulator } @@ -222,6 +364,14 @@ func (s sortCPUsSpread) sort() []int { return s.acc.sortAvailableCPUsSpread() } +func (s sortCPUsPacked) sortForResize() []int { + return s.acc.sortAvailableCPUsPackedForResize() +} + +func (s sortCPUsSpread) sortForResize() []int { + return s.acc.sortAvailableCPUsSpreadForResize() +} + // CPUSortingStrategy describes the CPU sorting solution within the socket scope. // Using topoDualSocketHT (12 CPUs, 2 sockets, 6 cores) as an example: // @@ -282,6 +432,9 @@ type cpuAccumulator struct { // cardinality equal to the total number of CPUs to accumulate. result cpuset.CPUSet + // `resultDetails` is the set of allocated CPUs in `result` + resultDetails topology.CPUDetails + numaOrSocketsFirst numaOrSocketsFirstFuncs // availableCPUSorter is used to control the cpu sorting result. @@ -291,12 +444,50 @@ type cpuAccumulator struct { availableCPUSorter availableCPUSorter } -func newCPUAccumulator(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, numCPUs int, cpuSortingStrategy CPUSortingStrategy) *cpuAccumulator { +func newCPUAccumulator(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, numCPUs int, cpuSortingStrategy CPUSortingStrategy, reusableCPUsForResize *cpuset.CPUSet, mustKeepCPUsForScaleDown *cpuset.CPUSet) *cpuAccumulator { acc := &cpuAccumulator{ topo: topo, details: topo.CPUDetails.KeepOnly(availableCPUs), numCPUsNeeded: numCPUs, result: cpuset.New(), + resultDetails: topo.CPUDetails.KeepOnly(cpuset.New()), + } + + if reusableCPUsForResize != nil { + if !reusableCPUsForResize.IsEmpty() { + // Increase of CPU resources ( scale up ) + // Take existing from allocated + // CPUs + if numCPUs > reusableCPUsForResize.Size() { + // scale up ... + acc.take(reusableCPUsForResize.Clone()) + } + + // Decrease of CPU resources ( scale down ) + // Take delta from allocated CPUs, if mustKeepCPUsForScaleDown + // is not nil, use explicetely those. If it is nil + // take delta starting from lowest CoreId of CPUs ( TODO esotsal, perhaps not needed). + if numCPUs < reusableCPUsForResize.Size() { + if mustKeepCPUsForScaleDown != nil { + // If explicetely CPUs to keep + // during scale down is given ( this requires + // addition in container[].resources ... which + // could be possible to patch ? Esotsal Note This means + // modifying API code + if !(mustKeepCPUsForScaleDown.Intersection(reusableCPUsForResize.Clone())).IsEmpty() { + acc.take(mustKeepCPUsForScaleDown.Clone()) + } else { + return acc + } + } + } + + if numCPUs == reusableCPUsForResize.Size() { + // nothing to do return as is + acc.take(reusableCPUsForResize.Clone()) + return acc + } + } } if topo.NumSockets >= topo.NumNUMANodes { @@ -387,6 +578,21 @@ func (a *cpuAccumulator) freeCPUs() []int { return a.availableCPUSorter.sort() } +// Return true if this numa only allocated CPUs for this Container +func (a *cpuAccumulator) isFullNUMANodeForResize(numaID int) bool { + return a.resultDetails.CPUsInNUMANodes(numaID).Size()+a.details.CPUsInNUMANodes(numaID).Size() == a.topo.CPUDetails.CPUsInNUMANodes(numaID).Size() +} + +// Return true if this Socket only allocated CPUs for this Container +func (a *cpuAccumulator) isFullSocketForResize(socketID int) bool { + return a.resultDetails.CPUsInSockets(socketID).Size()+a.details.CPUsInSockets(socketID).Size() == a.topo.CPUsPerSocket() +} + +// return true if this Socket only allocated CPUs for this Container +func (a *cpuAccumulator) isFullCoreForResize(coreID int) bool { + return a.resultDetails.CPUsInCores(coreID).Size()+a.details.CPUsInCores(coreID).Size() == a.topo.CPUsPerCore() +} + // Sorts the provided list of NUMA nodes/sockets/cores/cpus referenced in 'ids' // by the number of available CPUs contained within them (smallest to largest). // The 'getCPU()' parameter defines the function that should be called to @@ -516,8 +722,108 @@ func (a *cpuAccumulator) sortAvailableCPUsSpread() []int { return result } +// Sort all NUMA nodes with at least one free CPU. +// +// If NUMA nodes are higher than sockets in the memory hierarchy, they are sorted by ascending number +// of free CPUs that they contain. "higher than sockets in the memory hierarchy" means that NUMA nodes +// contain a bigger number of CPUs (free and busy) than sockets, or equivalently that each NUMA node +// contains more than one socket. +// +// If instead NUMA nodes are lower in the memory hierarchy than sockets, they are sorted as follows. +// First part, sort the NUMA nodes which contains the CPUs allocated to Container. and these NUMA nodes +// are sorted by number of free CPUs that they contain. +// Second part, sort the NUMA nodes contained in the sockets which contains the CPUs allocated to Container, +// but exclude the NUMA nodes in first part. these NUMA nodes sorted by the rule as below +// +// First, they are sorted by number of free CPUs in the sockets that contain them. Then, for each +// socket they are sorted by number of free CPUs that they contain. The order is always ascending. +func (a *cpuAccumulator) sortAvailableNUMANodesForResize() []int { + return a.numaOrSocketsFirst.sortAvailableNUMANodesForResize() +} + +// Sort all sockets with at least one free CPU. +// +// If sockets are higher than NUMA nodes in the memory hierarchy, they are sorted by ascending number +// of free CPUs that they contain. "higher than NUMA nodes in the memory hierarchy" means that +// sockets contain a bigger number of CPUs (free and busy) than NUMA nodes, or equivalently that each +// socket contains more than one NUMA node. +// +// If instead sockets are lower in the memory hierarchy than NUMA nodes, they are sorted as follows. +// First part, sort the sockets which contains the CPUs allocated to Container. and these sockets +// are sorted by number of free CPUs that they contain. +// Second part, sort the sockets contained in the NUMA nodes which contains the CPUs allocated to Container, +// but exclude the sockets in first part. these sockets sorted by the rule as below +// +// First, they are sorted by number of free CPUs in the NUMA nodes that contain them. Then, for each +// NUMA node they are sorted by number of free CPUs that they contain. The order is always ascending. +func (a *cpuAccumulator) sortAvailableSocketsForResize() []int { + return a.numaOrSocketsFirst.sortAvailableSocketsForResize() +} + +// Sort all cores with at least one free CPU. +// +// If sockets are higher in the memory hierarchy than NUMA nodes, meaning that sockets contain a +// bigger number of CPUs (free and busy) than NUMA nodes, or equivalently that each socket contains +// more than one NUMA node, the cores are sorted as follows. +// First part, sort the cores which contains the CPUs allocated to Container. and these cores +// are sorted by number of free CPUs that they contain. +// Second part, sort the cores contained in the NUMA nodes which contains the CPUs allocated to Container, +// but exclude the cores in first part. these cores sorted by the rule as below +// First, they are sorted by number of +// free CPUs that their sockets contain. Then, for each socket, the cores in it are sorted by number +// of free CPUs that their NUMA nodes contain. Then, for each NUMA node, the cores in it are sorted +// by number of free CPUs that they contain. The order is always ascending. + +// If instead NUMA nodes are higher in the memory hierarchy than sockets, the sorting happens in the +// same way as described in the previous paragraph. +func (a *cpuAccumulator) sortAvailableCoresForResize() []int { + return a.numaOrSocketsFirst.sortAvailableCoresForResize() +} + +// Sort all free CPUs. +// +// If sockets are higher in the memory hierarchy than NUMA nodes, meaning that sockets contain a +// bigger number of CPUs (free and busy) than NUMA nodes, or equivalently that each socket contains +// more than one NUMA node, the CPUs are sorted as follows. +// First part, sort the cores which contains the CPUs allocated to Container. and these cores +// are sorted by number of free CPUs that they contain. for each core, the CPUs in it are +// sorted by numerical ID. +// Second part, sort the cores contained in the NUMA nodes which contains the CPUs allocated to Container, +// but exclude the cores in first part. these cores sorted by the rule as below +// First, they are sorted by number of +// free CPUs that their sockets contain. Then, for each socket, the CPUs in it are sorted by number +// of free CPUs that their NUMA nodes contain. Then, for each NUMA node, the CPUs in it are sorted +// by number of free CPUs that their cores contain. Finally, for each core, the CPUs in it are +// sorted by numerical ID. The order is always ascending. +// +// If instead NUMA nodes are higher in the memory hierarchy than sockets, the sorting happens in the +// same way as described in the previous paragraph. +func (a *cpuAccumulator) sortAvailableCPUsPackedForResize() []int { + var result []int + for _, core := range a.sortAvailableCoresForResize() { + cpus := a.details.CPUsInCores(core).UnsortedList() + sort.Ints(cpus) + result = append(result, cpus...) + } + return result +} + +// Sort all available CPUs: +// - First by core using sortAvailableSocketsForResize(). +// - Then within each socket, sort cpus directly using the sort() algorithm defined above. +func (a *cpuAccumulator) sortAvailableCPUsSpreadForResize() []int { + var result []int + for _, socket := range a.sortAvailableSocketsForResize() { + cpus := a.details.CPUsInSockets(socket).UnsortedList() + sort.Ints(cpus) + result = append(result, cpus...) + } + return result +} + func (a *cpuAccumulator) take(cpus cpuset.CPUSet) { a.result = a.result.Union(cpus) + a.resultDetails = a.topo.CPUDetails.KeepOnly(a.result) a.details = a.details.KeepOnly(a.details.CPUs().Difference(a.result)) a.numCPUsNeeded -= cpus.Size() } @@ -621,6 +927,55 @@ func (a *cpuAccumulator) takeRemainingCPUs() { } } +func (a *cpuAccumulator) takeRemainCpusForFullNUMANodes() { + for _, numa := range a.sortAvailableNUMANodesForResize() { + if a.isFullNUMANodeForResize(numa) { + cpusInNUMANode := a.details.CPUsInNUMANodes(numa) + if !a.needsAtLeast(cpusInNUMANode.Size()) { + continue + } + klog.V(4).InfoS("takeRemainCpusForFullNUMANodes: claiming NUMA node", "numa", numa, "cpusInNUMANode", cpusInNUMANode) + a.take(cpusInNUMANode) + } + } +} + +func (a *cpuAccumulator) takeRemainCpusForFullSockets() { + for _, socket := range a.sortAvailableSocketsForResize() { + if a.isFullSocketForResize(socket) { + cpusInSocket := a.details.CPUsInSockets(socket) + if !a.needsAtLeast(cpusInSocket.Size()) { + continue + } + klog.V(4).InfoS("takeRemainCpusForFullSockets: claiming Socket", "socket", socket, "cpusInSocket", cpusInSocket) + a.take(cpusInSocket) + } + } +} + +func (a *cpuAccumulator) takeRemainCpusForFullCores() { + for _, core := range a.sortAvailableCoresForResize() { + if a.isFullCoreForResize(core) { + cpusInCore := a.details.CPUsInCores(core) + if !a.needsAtLeast(cpusInCore.Size()) { + continue + } + klog.V(4).InfoS("takeRemainCpusForFullCores: claiming Core", "core", core, "cpusInCore", cpusInCore) + a.take(cpusInCore) + } + } +} + +func (a *cpuAccumulator) takeRemainingCPUsForResize() { + for _, cpu := range a.availableCPUSorter.sortForResize() { + klog.V(4).InfoS("takeRemainingCPUsForResize: claiming CPU", "cpu", cpu) + a.take(cpuset.New(cpu)) + if a.isSatisfied() { + return + } + } +} + // rangeNUMANodesNeededToSatisfy returns minimum and maximum (in this order) number of NUMA nodes // needed to satisfy the cpuAccumulator's goal of accumulating `a.numCPUsNeeded` CPUs, assuming that // CPU groups have size given by the `cpuGroupSize` argument. @@ -747,24 +1102,40 @@ func (a *cpuAccumulator) iterateCombinations(n []int, k int, f func([]int) LoopC // the least amount of free CPUs to the one with the highest amount of free CPUs (i.e. in ascending // order of free CPUs). For any NUMA node, the cores are selected from the ones in the socket with // the least amount of free CPUs to the one with the highest amount of free CPUs. -func takeByTopologyNUMAPacked(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, numCPUs int, cpuSortingStrategy CPUSortingStrategy, preferAlignByUncoreCache bool) (cpuset.CPUSet, error) { - acc := newCPUAccumulator(topo, availableCPUs, numCPUs, cpuSortingStrategy) +func takeByTopologyNUMAPacked(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, numCPUs int, cpuSortingStrategy CPUSortingStrategy, preferAlignByUncoreCache bool, reusableCPUsForResize *cpuset.CPUSet, mustKeepCPUsForScaleDown *cpuset.CPUSet) (cpuset.CPUSet, error) { + + // If the number of CPUs requested to be retained is not a subset + // of reusableCPUs, then we fail early + if reusableCPUsForResize != nil && mustKeepCPUsForScaleDown != nil { + if (mustKeepCPUsForScaleDown.Intersection(reusableCPUsForResize.Clone())).IsEmpty() { + return cpuset.New(), fmt.Errorf("requested CPUs to be retained %s are not a subset of reusable CPUs %s", mustKeepCPUsForScaleDown.String(), reusableCPUsForResize.String()) + } + } + + acc := newCPUAccumulator(topo, availableCPUs, numCPUs, cpuSortingStrategy, reusableCPUsForResize, mustKeepCPUsForScaleDown) if acc.isSatisfied() { return acc.result, nil } if acc.isFailed() { return cpuset.New(), fmt.Errorf("not enough cpus available to satisfy request: requested=%d, available=%d", numCPUs, availableCPUs.Size()) } - // Algorithm: topology-aware best-fit // 1. Acquire whole NUMA nodes and sockets, if available and the container // requires at least a NUMA node or socket's-worth of CPUs. If NUMA // Nodes map to 1 or more sockets, pull from NUMA nodes first. // Otherwise pull from sockets first. + acc.numaOrSocketsFirst.takeFullFirstLevelForResize() + if acc.isSatisfied() { + return acc.result, nil + } acc.numaOrSocketsFirst.takeFullFirstLevel() if acc.isSatisfied() { return acc.result, nil } + acc.numaOrSocketsFirst.takeFullSecondLevelForResize() + if acc.isSatisfied() { + return acc.result, nil + } acc.numaOrSocketsFirst.takeFullSecondLevel() if acc.isSatisfied() { return acc.result, nil @@ -784,6 +1155,10 @@ func takeByTopologyNUMAPacked(topo *topology.CPUTopology, availableCPUs cpuset.C // a core's-worth of CPUs. // If `CPUSortingStrategySpread` is specified, skip taking the whole core. if cpuSortingStrategy != CPUSortingStrategySpread { + acc.takeRemainCpusForFullCores() + if acc.isSatisfied() { + return acc.result, nil + } acc.takeFullCores() if acc.isSatisfied() { return acc.result, nil @@ -793,6 +1168,10 @@ func takeByTopologyNUMAPacked(topo *topology.CPUTopology, availableCPUs cpuset.C // 4. Acquire single threads, preferring to fill partially-allocated cores // on the same sockets as the whole cores we have already taken in this // allocation. + acc.takeRemainingCPUsForResize() + if acc.isSatisfied() { + return acc.result, nil + } acc.takeRemainingCPUs() if acc.isSatisfied() { return acc.result, nil @@ -864,32 +1243,51 @@ func takeByTopologyNUMAPacked(topo *topology.CPUTopology, availableCPUs cpuset.C // of size 'cpuGroupSize' according to the algorithm described above. This is // important, for example, to ensure that all CPUs (i.e. all hyperthreads) from // a single core are allocated together. -func takeByTopologyNUMADistributed(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, numCPUs int, cpuGroupSize int, cpuSortingStrategy CPUSortingStrategy) (cpuset.CPUSet, error) { +func takeByTopologyNUMADistributed(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, numCPUs int, cpuGroupSize int, cpuSortingStrategy CPUSortingStrategy, reusableCPUsForResize *cpuset.CPUSet, mustKeepCPUsForScaleDown *cpuset.CPUSet) (cpuset.CPUSet, error) { // If the number of CPUs requested cannot be handed out in chunks of // 'cpuGroupSize', then we just call out the packing algorithm since we // can't distribute CPUs in this chunk size. // PreferAlignByUncoreCache feature not implemented here yet and set to false. // Support for PreferAlignByUncoreCache to be done at beta release. if (numCPUs % cpuGroupSize) != 0 { - return takeByTopologyNUMAPacked(topo, availableCPUs, numCPUs, cpuSortingStrategy, false) + return takeByTopologyNUMAPacked(topo, availableCPUs, numCPUs, cpuSortingStrategy, false, reusableCPUsForResize, mustKeepCPUsForScaleDown) + } + + // If the number of CPUs requested to be retained is not a subset + // of reusableCPUs, then we fail early + if reusableCPUsForResize != nil && mustKeepCPUsForScaleDown != nil { + if (mustKeepCPUsForScaleDown.Intersection(reusableCPUsForResize.Clone())).IsEmpty() { + return cpuset.New(), fmt.Errorf("requested CPUs to be retained %s are not a subset of reusable CPUs %s", mustKeepCPUsForScaleDown.String(), reusableCPUsForResize.String()) + } } // Otherwise build an accumulator to start allocating CPUs from. - acc := newCPUAccumulator(topo, availableCPUs, numCPUs, cpuSortingStrategy) + acc := newCPUAccumulator(topo, availableCPUs, numCPUs, cpuSortingStrategy, nil, mustKeepCPUsForScaleDown) if acc.isSatisfied() { return acc.result, nil } if acc.isFailed() { return cpuset.New(), fmt.Errorf("not enough cpus available to satisfy request: requested=%d, available=%d", numCPUs, availableCPUs.Size()) } - // Get the list of NUMA nodes represented by the set of CPUs in 'availableCPUs'. numas := acc.sortAvailableNUMANodes() + reusableCPUsForResizeDetail := acc.topo.CPUDetails.KeepOnly(cpuset.New()) + allocatedCPUsNumber := 0 + if reusableCPUsForResize != nil { + reusableCPUsForResizeDetail = acc.topo.CPUDetails.KeepOnly(*reusableCPUsForResize) + allocatedCPUsNumber = reusableCPUsForResize.Size() + } + allocatedNumas := reusableCPUsForResizeDetail.NUMANodes() + allocatedCPUPerNuma := make(mapIntInt, len(numas)) + for _, numa := range numas { + allocatedCPUPerNuma[numa] = reusableCPUsForResizeDetail.CPUsInNUMANodes(numa).Size() + } // Calculate the minimum and maximum possible number of NUMA nodes that // could satisfy this request. This is used to optimize how many iterations // of the loop we need to go through below. minNUMAs, maxNUMAs := acc.rangeNUMANodesNeededToSatisfy(cpuGroupSize) + minNUMAs = max(minNUMAs, allocatedNumas.Size()) // Try combinations of 1,2,3,... NUMA nodes until we find a combination // where we can evenly distribute CPUs across them. To optimize things, we @@ -909,10 +1307,16 @@ func takeByTopologyNUMADistributed(topo *topology.CPUTopology, availableCPUs cpu return Break } + // Check if the 'allocatedNumas' CPU set is a subset of the 'comboSet' + comboSet := cpuset.New(combo...) + if !allocatedNumas.IsSubsetOf(comboSet) { + return Continue + } + // Check that this combination of NUMA nodes has enough CPUs to // satisfy the allocation overall. cpus := acc.details.CPUsInNUMANodes(combo...) - if cpus.Size() < numCPUs { + if (cpus.Size() + allocatedCPUsNumber) < numCPUs { return Continue } @@ -920,7 +1324,7 @@ func takeByTopologyNUMADistributed(topo *topology.CPUTopology, availableCPUs cpu // 'cpuGroupSize' across the NUMA nodes in this combo. numCPUGroups := 0 for _, numa := range combo { - numCPUGroups += (acc.details.CPUsInNUMANodes(numa).Size() / cpuGroupSize) + numCPUGroups += ((acc.details.CPUsInNUMANodes(numa).Size() + allocatedCPUPerNuma[numa]) / cpuGroupSize) } if (numCPUGroups * cpuGroupSize) < numCPUs { return Continue @@ -932,7 +1336,10 @@ func takeByTopologyNUMADistributed(topo *topology.CPUTopology, availableCPUs cpu distribution := (numCPUs / len(combo) / cpuGroupSize) * cpuGroupSize for _, numa := range combo { cpus := acc.details.CPUsInNUMANodes(numa) - if cpus.Size() < distribution { + if (cpus.Size() + allocatedCPUPerNuma[numa]) < distribution { + return Continue + } + if allocatedCPUPerNuma[numa] > distribution { return Continue } } @@ -947,7 +1354,7 @@ func takeByTopologyNUMADistributed(topo *topology.CPUTopology, availableCPUs cpu availableAfterAllocation[numa] = acc.details.CPUsInNUMANodes(numa).Size() } for _, numa := range combo { - availableAfterAllocation[numa] -= distribution + availableAfterAllocation[numa] -= (distribution - allocatedCPUPerNuma[numa]) } // Check if there are any remaining CPUs to distribute across the @@ -1054,7 +1461,8 @@ func takeByTopologyNUMADistributed(topo *topology.CPUTopology, availableCPUs cpu // size 'cpuGroupSize' from 'bestCombo'. distribution := (numCPUs / len(bestCombo) / cpuGroupSize) * cpuGroupSize for _, numa := range bestCombo { - cpus, _ := takeByTopologyNUMAPacked(acc.topo, acc.details.CPUsInNUMANodes(numa), distribution, cpuSortingStrategy, false) + reusableCPUsPerNumaForResize := reusableCPUsForResizeDetail.CPUsInNUMANodes(numa) + cpus, _ := takeByTopologyNUMAPacked(acc.topo, acc.details.CPUsInNUMANodes(numa), distribution, cpuSortingStrategy, false, &reusableCPUsPerNumaForResize, mustKeepCPUsForScaleDown) acc.take(cpus) } @@ -1069,7 +1477,7 @@ func takeByTopologyNUMADistributed(topo *topology.CPUTopology, availableCPUs cpu if acc.details.CPUsInNUMANodes(numa).Size() < cpuGroupSize { continue } - cpus, _ := takeByTopologyNUMAPacked(acc.topo, acc.details.CPUsInNUMANodes(numa), cpuGroupSize, cpuSortingStrategy, false) + cpus, _ := takeByTopologyNUMAPacked(acc.topo, acc.details.CPUsInNUMANodes(numa), cpuGroupSize, cpuSortingStrategy, false, nil, mustKeepCPUsForScaleDown) acc.take(cpus) remainder -= cpuGroupSize } @@ -1093,5 +1501,5 @@ func takeByTopologyNUMADistributed(topo *topology.CPUTopology, availableCPUs cpu // If we never found a combination of NUMA nodes that we could properly // distribute CPUs across, fall back to the packing algorithm. - return takeByTopologyNUMAPacked(topo, availableCPUs, numCPUs, cpuSortingStrategy, false) + return takeByTopologyNUMAPacked(topo, availableCPUs, numCPUs, cpuSortingStrategy, false, reusableCPUsForResize, mustKeepCPUsForScaleDown) } diff --git a/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go b/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go index 241bfe611b359..7ab86f54361bb 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go +++ b/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go @@ -114,7 +114,7 @@ func TestCPUAccumulatorFreeSockets(t *testing.T) { for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { - acc := newCPUAccumulator(tc.topo, tc.availableCPUs, 0, CPUSortingStrategyPacked) + acc := newCPUAccumulator(tc.topo, tc.availableCPUs, 0, CPUSortingStrategyPacked, nil, nil) result := acc.freeSockets() sort.Ints(result) if !reflect.DeepEqual(result, tc.expect) { @@ -214,7 +214,7 @@ func TestCPUAccumulatorFreeNUMANodes(t *testing.T) { for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { - acc := newCPUAccumulator(tc.topo, tc.availableCPUs, 0, CPUSortingStrategyPacked) + acc := newCPUAccumulator(tc.topo, tc.availableCPUs, 0, CPUSortingStrategyPacked, nil, nil) result := acc.freeNUMANodes() if !reflect.DeepEqual(result, tc.expect) { t.Errorf("expected %v to equal %v", result, tc.expect) @@ -263,7 +263,7 @@ func TestCPUAccumulatorFreeSocketsAndNUMANodes(t *testing.T) { for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { - acc := newCPUAccumulator(tc.topo, tc.availableCPUs, 0, CPUSortingStrategyPacked) + acc := newCPUAccumulator(tc.topo, tc.availableCPUs, 0, CPUSortingStrategyPacked, nil, nil) resultNUMANodes := acc.freeNUMANodes() if !reflect.DeepEqual(resultNUMANodes, tc.expectNUMANodes) { t.Errorf("expected NUMA Nodes %v to equal %v", resultNUMANodes, tc.expectNUMANodes) @@ -335,7 +335,7 @@ func TestCPUAccumulatorFreeCores(t *testing.T) { for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { - acc := newCPUAccumulator(tc.topo, tc.availableCPUs, 0, CPUSortingStrategyPacked) + acc := newCPUAccumulator(tc.topo, tc.availableCPUs, 0, CPUSortingStrategyPacked, nil, nil) result := acc.freeCores() if !reflect.DeepEqual(result, tc.expect) { t.Errorf("expected %v to equal %v", result, tc.expect) @@ -391,7 +391,7 @@ func TestCPUAccumulatorFreeCPUs(t *testing.T) { for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { - acc := newCPUAccumulator(tc.topo, tc.availableCPUs, 0, CPUSortingStrategyPacked) + acc := newCPUAccumulator(tc.topo, tc.availableCPUs, 0, CPUSortingStrategyPacked, nil, nil) result := acc.freeCPUs() if !reflect.DeepEqual(result, tc.expect) { t.Errorf("expected %v to equal %v", result, tc.expect) @@ -477,7 +477,7 @@ func TestCPUAccumulatorTake(t *testing.T) { for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { - acc := newCPUAccumulator(tc.topo, tc.availableCPUs, tc.numCPUs, CPUSortingStrategyPacked) + acc := newCPUAccumulator(tc.topo, tc.availableCPUs, tc.numCPUs, CPUSortingStrategyPacked, nil, nil) totalTaken := 0 for _, cpus := range tc.takeCPUs { acc.take(cpus) @@ -750,7 +750,7 @@ func TestTakeByTopologyNUMAPacked(t *testing.T) { strategy = CPUSortingStrategySpread } - result, err := takeByTopologyNUMAPacked(tc.topo, tc.availableCPUs, tc.numCPUs, strategy, tc.opts.PreferAlignByUncoreCacheOption) + result, err := takeByTopologyNUMAPacked(tc.topo, tc.availableCPUs, tc.numCPUs, strategy, tc.opts.PreferAlignByUncoreCacheOption, nil, nil) if tc.expErr != "" && err != nil && err.Error() != tc.expErr { t.Errorf("expected error to be [%v] but it was [%v]", tc.expErr, err) } @@ -851,7 +851,7 @@ func TestTakeByTopologyWithSpreadPhysicalCPUsPreferredOption(t *testing.T) { if tc.opts.DistributeCPUsAcrossCores { strategy = CPUSortingStrategySpread } - result, err := takeByTopologyNUMAPacked(tc.topo, tc.availableCPUs, tc.numCPUs, strategy, tc.opts.PreferAlignByUncoreCacheOption) + result, err := takeByTopologyNUMAPacked(tc.topo, tc.availableCPUs, tc.numCPUs, strategy, tc.opts.PreferAlignByUncoreCacheOption, nil, nil) if tc.expErr != "" && err.Error() != tc.expErr { t.Errorf("testCase %q failed, expected error to be [%v] but it was [%v]", tc.description, tc.expErr, err) } @@ -1053,7 +1053,474 @@ func TestTakeByTopologyNUMADistributed(t *testing.T) { for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { - result, err := takeByTopologyNUMADistributed(tc.topo, tc.availableCPUs, tc.numCPUs, tc.cpuGroupSize, CPUSortingStrategyPacked) + result, err := takeByTopologyNUMADistributed(tc.topo, tc.availableCPUs, tc.numCPUs, tc.cpuGroupSize, CPUSortingStrategyPacked, nil, nil) + if err != nil { + if tc.expErr == "" { + t.Errorf("unexpected error [%v]", err) + } + if tc.expErr != "" && err.Error() != tc.expErr { + t.Errorf("expected error to be [%v] but it was [%v]", tc.expErr, err) + } + return + } + if !result.Equals(tc.expResult) { + t.Errorf("expected result [%s] to equal [%s]", result, tc.expResult) + } + }) + } +} + +type takeByTopologyTestCaseForResize struct { + description string + topo *topology.CPUTopology + opts StaticPolicyOptions + availableCPUs cpuset.CPUSet + reusableCPUs cpuset.CPUSet + numCPUs int + expErr string + expResult cpuset.CPUSet +} + +func commonTakeByTopologyTestCasesForResize(t *testing.T) []takeByTopologyTestCaseForResize { + return []takeByTopologyTestCaseForResize{ + { + "Allocated 1 CPUs, and take 1 cpus from single socket with HT", + topoSingleSocketHT, + StaticPolicyOptions{}, + mustParseCPUSet(t, "1-7"), + cpuset.New(0), + 1, + "", + cpuset.New(0), + }, + { + "Allocated 1 CPU, and take 2 cpu from single socket with HT", + topoSingleSocketHT, + StaticPolicyOptions{}, + mustParseCPUSet(t, "1-7"), + cpuset.New(0), + 2, + "", + cpuset.New(0, 4), + }, + { + "Allocated 1 CPU, and take 2 cpu from single socket with HT, some cpus are taken, no sibling CPU of allocated CPU", + topoSingleSocketHT, + StaticPolicyOptions{}, + mustParseCPUSet(t, "1,3,5,6,7"), + cpuset.New(0), + 2, + "", + cpuset.New(0, 6), + }, + { + "Allocated 1 CPU, and take 3 cpu from single socket with HT, some cpus are taken, no sibling CPU of allocated CPU", + topoSingleSocketHT, + StaticPolicyOptions{}, + mustParseCPUSet(t, "1,3,5,6,7"), + cpuset.New(0), + 3, + "", + cpuset.New(0, 1, 5), + }, + { + "Allocated 1 CPU, and take all cpu from single socket with HT", + topoSingleSocketHT, + StaticPolicyOptions{}, + mustParseCPUSet(t, "1-7"), + cpuset.New(0), + 8, + "", + mustParseCPUSet(t, "0-7"), + }, + { + "Allocated 1 CPU, take a core from dual socket with HT", + topoDualSocketHT, + StaticPolicyOptions{}, + mustParseCPUSet(t, "0-10"), + cpuset.New(11), + 2, + "", + cpuset.New(5, 11), + }, + { + "Allocated 1 CPU, take a socket of cpus from dual socket with HT", + topoDualSocketHT, + StaticPolicyOptions{}, + mustParseCPUSet(t, "0-10"), + cpuset.New(11), + 6, + "", + cpuset.New(1, 3, 5, 7, 9, 11), + }, + { + "Allocated 1 CPU, take a socket of cpus and 1 core of CPU from dual socket with HT", + topoDualSocketHT, + StaticPolicyOptions{}, + mustParseCPUSet(t, "0-10"), + cpuset.New(11), + 8, + "", + cpuset.New(0, 1, 3, 5, 6, 7, 9, 11), + }, + { + "Allocated 1 CPU, take a socket of cpus from dual socket with multi-numa-per-socket with HT", + topoDualSocketMultiNumaPerSocketHT, + StaticPolicyOptions{}, + mustParseCPUSet(t, "0-38,40-79"), + cpuset.New(39), + 40, + "", + mustParseCPUSet(t, "20-39,60-79"), + }, + { + "Allocated 1 CPU, take a NUMA node of cpus from dual socket with multi-numa-per-socket with HT", + topoDualSocketMultiNumaPerSocketHT, + StaticPolicyOptions{}, + mustParseCPUSet(t, "0-38,40-79"), + cpuset.New(39), + 20, + "", + mustParseCPUSet(t, "30-39,70-79"), + }, + { + "Allocated 2 CPUs, take a socket and a NUMA node of cpus from dual socket with multi-numa-per-socket with HT", + topoDualSocketMultiNumaPerSocketHT, + StaticPolicyOptions{}, + mustParseCPUSet(t, "0-38,40-58,60-79"), + cpuset.New(39, 59), + 60, + "", + mustParseCPUSet(t, "0-19,30-59,70-79"), + }, + { + "Allocated 1 CPU, take NUMA nodes of cpus from dual socket with multi-numa-per-socket with HT, the NUMA node with allocated CPUs already taken some CPUs", + topoDualSocketMultiNumaPerSocketHT, + StaticPolicyOptions{}, + mustParseCPUSet(t, "0-38,40-69"), + cpuset.New(39), + 40, + "", + mustParseCPUSet(t, "0-9,20-29,39-48,60-69"), + }, + { + "Allocated 1 CPU, take NUMA nodes of cpus from dual socket with multi-numa-per-socket with HT, the NUMA node with allocated CPUs already taken more CPUs", + topoDualSocketMultiNumaPerSocketHT, + StaticPolicyOptions{}, + mustParseCPUSet(t, "9,30-38,49"), + cpuset.New(), + 1, + "", + mustParseCPUSet(t, "9"), + }, + { + "Allocated 1 CPU, take NUMA nodes of cpus and 1 CPU from dual socket with multi-numa-per-socket with HT, the NUMA node with allocated CPUs already taken some CPUs", + topoDualSocketMultiNumaPerSocketHT, + StaticPolicyOptions{}, + mustParseCPUSet(t, "0-38,40-69"), + cpuset.New(39), + 41, + "", + mustParseCPUSet(t, "0-19,39-59"), + }, + { + "Allocated 1 CPUs, take a socket of cpus from single socket with HT, 3 cpus", + topoSingleSocketHT, + StaticPolicyOptions{DistributeCPUsAcrossCores: true}, + mustParseCPUSet(t, "0-6"), + cpuset.New(7), + 3, + "", + mustParseCPUSet(t, "0,1,7"), + }, + { + "Allocated 1 CPUs, take a socket of cpus from dual socket with HT, 3 cpus", + topoDualSocketHT, + StaticPolicyOptions{DistributeCPUsAcrossCores: true}, + mustParseCPUSet(t, "0-10"), + cpuset.New(11), + 3, + "", + mustParseCPUSet(t, "1,3,11"), + }, + { + "Allocated 1 CPUs, take a socket of cpus from dual socket with HT, 6 cpus", + topoDualSocketHT, + StaticPolicyOptions{DistributeCPUsAcrossCores: true}, + mustParseCPUSet(t, "0-10"), + cpuset.New(11), + 6, + "", + mustParseCPUSet(t, "1,3,5,7,9,11"), + }, + { + "Allocated 1 CPUs, take a socket of cpus from dual socket with HT, 8 cpus", + topoDualSocketHT, + StaticPolicyOptions{DistributeCPUsAcrossCores: true}, + mustParseCPUSet(t, "0-10"), + cpuset.New(11), + 8, + "", + mustParseCPUSet(t, "0,1,2,3,5,7,9,11"), + }, + { + "Allocated 1 CPUs, take a socket of cpus from dual socket without HT, 2 cpus", + topoDualSocketNoHT, + StaticPolicyOptions{DistributeCPUsAcrossCores: true}, + mustParseCPUSet(t, "0-6"), + cpuset.New(7), + 2, + "", + mustParseCPUSet(t, "4,7"), + }, + { + "Allocated 1 CPUs, take a socket of cpus from dual socket with multi numa per socket and HT, 8 cpus", + topoDualSocketMultiNumaPerSocketHT, + StaticPolicyOptions{DistributeCPUsAcrossCores: true}, + mustParseCPUSet(t, "0-38,40-79"), + cpuset.New(39), + 8, + "", + mustParseCPUSet(t, "20-26,39"), + }, + { + "Allocated 1 CPU, take NUMA nodes of cpus from dual socket with multi-numa-per-socket with HT, the NUMA node with allocated CPUs already taken some CPUs", + topoDualSocketMultiNumaPerSocketHT, + StaticPolicyOptions{DistributeCPUsAcrossCores: true}, + mustParseCPUSet(t, "0-38,40-69"), + cpuset.New(39), + 40, + "", + mustParseCPUSet(t, "0-9,20-39,60-69"), + }, + { + "Allocated 1 CPUs, take a socket of cpus from quad socket four way with HT, 12 cpus", + topoQuadSocketFourWayHT, + StaticPolicyOptions{DistributeCPUsAcrossCores: true}, + mustParseCPUSet(t, "0-59,61-287"), + cpuset.New(60), + 8, + "", + mustParseCPUSet(t, "3,4,11,12,15,16,23,60"), + }, + } +} + +func TestTakeByTopologyNUMAPackedForResize(t *testing.T) { + testCases := commonTakeByTopologyTestCasesForResize(t) + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + strategy := CPUSortingStrategyPacked + if tc.opts.DistributeCPUsAcrossCores { + strategy = CPUSortingStrategySpread + } + + result, err := takeByTopologyNUMAPacked(tc.topo, tc.availableCPUs, tc.numCPUs, strategy, tc.opts.PreferAlignByUncoreCacheOption, &tc.reusableCPUs, nil) + + if tc.expErr != "" && err != nil && err.Error() != tc.expErr { + t.Errorf("expected error to be [%v] but it was [%v]", tc.expErr, err) + } + if !result.Equals(tc.expResult) { + t.Errorf("expected result [%s] to equal [%s]", result, tc.expResult) + } + }) + } +} + +type takeByTopologyExtendedTestCaseForResize struct { + description string + topo *topology.CPUTopology + availableCPUs cpuset.CPUSet + reusableCPUs cpuset.CPUSet + numCPUs int + cpuGroupSize int + expErr string + expResult cpuset.CPUSet +} + +func commonTakeByTopologyExtendedTestCasesForResize(t *testing.T) []takeByTopologyExtendedTestCaseForResize { + return []takeByTopologyExtendedTestCaseForResize{ + { + "Allocated 1 CPUs, allocate 4 full cores with 2 distributed across each NUMA node", + topoDualSocketHT, + mustParseCPUSet(t, "0-10"), + cpuset.New(11), + 8, + 1, + "", + mustParseCPUSet(t, "0,6,2,8,1,7,5,11"), + }, + { + "Allocated 8 CPUs, allocate 32 full cores with 8 distributed across each NUMA node", + topoDualSocketMultiNumaPerSocketHT, + mustParseCPUSet(t, "0-35,40-75"), + mustParseCPUSet(t, "36-39,76-79"), + 64, + 1, + "", + mustParseCPUSet(t, "0-7,10-17,20-27,30-33,36-39,40-47,50-57,60-67,70-73,76-79"), + }, + { + "Allocated 2 CPUs, allocate 8 full cores with 2 distributed across each NUMA node", + topoDualSocketMultiNumaPerSocketHT, + mustParseCPUSet(t, "2,10-12,20-22,30-32,40-41,50-51,60-61,70-71"), + mustParseCPUSet(t, "0,1"), + 16, + 1, + "", + mustParseCPUSet(t, "0-1,10-11,20-21,30-31,40-41,50-51,60-61,70-71"), + }, + { + "Allocated 1 CPUs, take 1 cpu from dual socket with HT - core from Socket 0", + topoDualSocketHT, + mustParseCPUSet(t, "0-10"), + mustParseCPUSet(t, "11"), + 1, + 1, + "", + mustParseCPUSet(t, "11"), + }, + { + "Allocated 1 CPUs, take 2 cpu from dual socket with HT - core from Socket 0", + topoDualSocketHT, + mustParseCPUSet(t, "0-10"), + mustParseCPUSet(t, "11"), + 2, + 1, + "", + mustParseCPUSet(t, "5,11"), + }, + { + "Allocated 2 CPUs, allocate 31 full cores with 15 CPUs distributed across each NUMA node and 1 CPU spilling over to each of NUMA 0, 1", + topoDualSocketMultiNumaPerSocketHT, + mustParseCPUSet(t, "2-79"), + mustParseCPUSet(t, "0,1"), + 62, + 1, + "", + mustParseCPUSet(t, "0-7,10-17,20-27,30-37,40-47,50-57,60-66,70-76"), + }, + { + "Allocated 2 CPUs, allocate 31 full cores with 14 CPUs distributed across each NUMA node and 2 CPUs spilling over to each of NUMA 0, 1, 2 (cpuGroupSize 2)", + topoDualSocketMultiNumaPerSocketHT, + mustParseCPUSet(t, "2-79"), + mustParseCPUSet(t, "0,1"), + 62, + 2, + "", + mustParseCPUSet(t, "0-7,10-17,20-27,30-36,40-47,50-57,60-67,70-76"), + }, + { + "Allocated 2 CPUs, allocate 31 full cores with 15 CPUs distributed across each NUMA node and 1 CPU spilling over to each of NUMA 2, 3 (to keep balance)", + topoDualSocketMultiNumaPerSocketHT, + mustParseCPUSet(t, "2-8,10-18,20-39,40-48,50-58,60-79"), + mustParseCPUSet(t, "0,1"), + 62, + 1, + "", + mustParseCPUSet(t, "0-7,10-17,20-27,30-37,40-46,50-56,60-67,70-77"), + }, + { + "Allocated 2 CPUs, allocate 31 full cores with 14 CPUs distributed across each NUMA node and 2 CPUs spilling over to each of NUMA 0, 2, 3 (to keep balance with cpuGroupSize 2)", + topoDualSocketMultiNumaPerSocketHT, + mustParseCPUSet(t, "2-8,10-18,20-39,40-48,50-58,60-79"), + mustParseCPUSet(t, "0,1"), + 62, + 2, + "", + mustParseCPUSet(t, "0-7,10-16,20-27,30-37,40-47,50-56,60-67,70-77"), + }, + { + "Allocated 4 CPUs, ensure bestRemainder chosen with NUMA nodes that have enough CPUs to satisfy the request", + topoDualSocketMultiNumaPerSocketHT, + mustParseCPUSet(t, "10-13,20-23,30-36,40-43,50-53,60-63,70-76"), + mustParseCPUSet(t, "0-3"), + 34, + 1, + "", + mustParseCPUSet(t, "0-3,10-13,20-23,30-34,40-43,50-53,60-63,70-74"), + }, + { + "Allocated 4 CPUs, ensure previous failure encountered on live machine has been fixed (1/1)", + topoDualSocketMultiNumaPerSocketHTLarge, + mustParseCPUSet(t, "0,128,30,31,158,159,47,171-175,62,63,190,191,75-79,203-207,94,96,222,223,101-111,229-239,126,127,254,255"), + mustParseCPUSet(t, "43-46"), + 28, + 1, + "", + mustParseCPUSet(t, "43-47,75-79,96,101-105,171-174,203-206,229-232"), + }, + { + "Allocated 14 CPUs, allocate 24 full cores with 8 distributed across the first 3 NUMA nodes", + topoDualSocketMultiNumaPerSocketHT, + mustParseCPUSet(t, "8-39,48-79"), + mustParseCPUSet(t, "0-7,40-47"), + 48, + 1, + "", + mustParseCPUSet(t, "0-7,10-17,20-27,40-47,50-57,60-67"), + }, + { + "Allocated 20 CPUs, allocated CPUs in numa0 is bigger than distribute CPUs, allocated CPUs by takeByTopologyNUMAPacked", + topoDualSocketMultiNumaPerSocketHT, + mustParseCPUSet(t, "10-39,50-79"), + mustParseCPUSet(t, "0-9,40-49"), + 48, + 1, + "", + mustParseCPUSet(t, "0-23,40-63"), + }, + { + "Allocated 12 CPUs, allocate 24 full cores with 8 distributed across the first 3 NUMA nodes (taking all but 2 from the first NUMA node)", + topoDualSocketMultiNumaPerSocketHT, + mustParseCPUSet(t, "8-29,32-39,48-69,72-79"), + mustParseCPUSet(t, "1-7,41-47"), + 48, + 1, + "", + mustParseCPUSet(t, "1-8,10-17,20-27,41-48,50-57,60-67"), + }, + { + "Allocated 10 CPUs, allocate 24 full cores with 8 distributed across the first 3 NUMA nodes (even though all 8 could be allocated from the first NUMA node)", + topoDualSocketMultiNumaPerSocketHT, + mustParseCPUSet(t, "2-29,31-39,42-69,71-79"), + mustParseCPUSet(t, "2-7,42-47"), + 48, + 1, + "", + mustParseCPUSet(t, "2-9,10-17,20-27,42-49,50-57,60-67"), + }, + { + "Allocated 2 CPUs, allocate 13 full cores distributed across the 2 NUMA nodes", + topoDualSocketMultiNumaPerSocketHT, + mustParseCPUSet(t, "0-29,31-69,71-79"), + mustParseCPUSet(t, "30,70"), + 26, + 1, + "", + mustParseCPUSet(t, "20-26,30-36,60-65,70-75"), + }, + { + "Allocated 2 CPUs, allocate 13 full cores distributed across the 2 NUMA nodes (cpuGroupSize 2)", + topoDualSocketMultiNumaPerSocketHT, + mustParseCPUSet(t, "0-29,31-69,71-79"), + mustParseCPUSet(t, "30,70"), + 26, + 2, + "", + mustParseCPUSet(t, "20-25,30-36,60-65,70-76"), + }, + } +} + +func TestTakeByTopologyNUMADistributedForResize(t *testing.T) { + testCases := commonTakeByTopologyExtendedTestCasesForResize(t) + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + + result, err := takeByTopologyNUMADistributed(tc.topo, tc.availableCPUs, tc.numCPUs, tc.cpuGroupSize, CPUSortingStrategyPacked, &tc.reusableCPUs, nil) if err != nil { if tc.expErr == "" { t.Errorf("unexpected error [%v]", err) diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager.go b/pkg/kubelet/cm/cpumanager/cpu_manager.go index 8b59ba6712190..d6a66b5facb24 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_manager.go +++ b/pkg/kubelet/cm/cpumanager/cpu_manager.go @@ -275,6 +275,9 @@ func (m *manager) AddContainer(pod *v1.Pod, container *v1.Container, containerID if cset, exists := m.state.GetCPUSet(string(pod.UID), container.Name); exists { m.lastUpdateState.SetCPUSet(string(pod.UID), container.Name, cset) } + if cset, exists := m.state.GetPromisedCPUSet(string(pod.UID), container.Name); exists { + m.lastUpdateState.SetPromisedCPUSet(string(pod.UID), container.Name, cset) + } m.containerMap.Add(string(pod.UID), container.Name, containerID) } @@ -482,6 +485,7 @@ func (m *manager) reconcileState() (success []reconciledContainer, failure []rec continue } m.lastUpdateState.SetCPUSet(string(pod.UID), container.Name, cset) + m.lastUpdateState.SetPromisedCPUSet(string(pod.UID), container.Name, cset) } success = append(success, reconciledContainer{pod.Name, container.Name, containerID}) } diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go index 6c3af2dc3f36c..64f179de84608 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go +++ b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go @@ -46,6 +46,7 @@ import ( ) type mockState struct { + promised state.ContainerCPUAssignments assignments state.ContainerCPUAssignments defaultCPUSet cpuset.CPUSet } @@ -55,6 +56,11 @@ func (s *mockState) GetCPUSet(podUID string, containerName string) (cpuset.CPUSe return res.Clone(), ok } +func (s *mockState) GetPromisedCPUSet(podUID string, containerName string) (cpuset.CPUSet, bool) { + res, ok := s.promised[podUID][containerName] + return res.Clone(), ok +} + func (s *mockState) GetDefaultCPUSet() cpuset.CPUSet { return s.defaultCPUSet.Clone() } @@ -66,6 +72,13 @@ func (s *mockState) GetCPUSetOrDefault(podUID string, containerName string) cpus return s.GetDefaultCPUSet() } +func (s *mockState) SetPromisedCPUSet(podUID string, containerName string, cset cpuset.CPUSet) { + if _, exists := s.promised[podUID]; !exists { + s.promised[podUID] = make(map[string]cpuset.CPUSet) + } + s.promised[podUID][containerName] = cset +} + func (s *mockState) SetCPUSet(podUID string, containerName string, cset cpuset.CPUSet) { if _, exists := s.assignments[podUID]; !exists { s.assignments[podUID] = make(map[string]cpuset.CPUSet) @@ -82,11 +95,16 @@ func (s *mockState) Delete(podUID string, containerName string) { if len(s.assignments[podUID]) == 0 { delete(s.assignments, podUID) } + delete(s.promised[podUID], containerName) + if len(s.promised[podUID]) == 0 { + delete(s.promised, podUID) + } } func (s *mockState) ClearState() { s.defaultCPUSet = cpuset.CPUSet{} s.assignments = make(state.ContainerCPUAssignments) + s.promised = make(state.ContainerCPUAssignments) } func (s *mockState) SetCPUAssignments(a state.ContainerCPUAssignments) { @@ -97,6 +115,14 @@ func (s *mockState) GetCPUAssignments() state.ContainerCPUAssignments { return s.assignments.Clone() } +func (s *mockState) SetCPUPromised(a state.ContainerCPUAssignments) { + s.promised = a.Clone() +} + +func (s *mockState) GetCPUPromised() state.ContainerCPUAssignments { + return s.promised.Clone() +} + type mockPolicy struct { err error } @@ -167,6 +193,7 @@ func makePod(podUID, containerName, cpuRequest, cpuLimit string) *v1.Pod { } pod.UID = types.UID(podUID) + pod.Name = podUID pod.Spec.Containers[0].Name = containerName return pod @@ -320,6 +347,7 @@ func TestCPUManagerAdd(t *testing.T) { mgr := &manager{ policy: testCase.policy, state: &mockState{ + promised: state.ContainerCPUAssignments{}, assignments: state.ContainerCPUAssignments{}, defaultCPUSet: cpuset.New(1, 2, 3, 4), }, @@ -544,6 +572,7 @@ func TestCPUManagerAddWithInitContainers(t *testing.T) { policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.New(), topologymanager.NewFakeManager(), nil) mockState := &mockState{ + promised: testCase.stAssignments, assignments: testCase.stAssignments, defaultCPUSet: testCase.stDefaultCPUSet, } @@ -736,6 +765,7 @@ func TestCPUManagerRemove(t *testing.T) { err: nil, }, state: &mockState{ + promised: state.ContainerCPUAssignments{}, assignments: state.ContainerCPUAssignments{}, defaultCPUSet: cpuset.New(), }, @@ -1232,6 +1262,7 @@ func TestReconcileState(t *testing.T) { mgr := &manager{ policy: testCase.policy, state: &mockState{ + promised: testCase.stAssignments, assignments: testCase.stAssignments, defaultCPUSet: testCase.stDefaultCPUSet, }, @@ -1340,6 +1371,7 @@ func TestCPUManagerAddWithResvList(t *testing.T) { mgr := &manager{ policy: testCase.policy, state: &mockState{ + promised: state.ContainerCPUAssignments{}, assignments: state.ContainerCPUAssignments{}, defaultCPUSet: cpuset.New(0, 1, 2, 3), }, @@ -1489,6 +1521,7 @@ func TestCPUManagerGetAllocatableCPUs(t *testing.T) { policy: testCase.policy, activePods: func() []*v1.Pod { return nil }, state: &mockState{ + promised: state.ContainerCPUAssignments{}, assignments: state.ContainerCPUAssignments{}, defaultCPUSet: cpuset.New(0, 1, 2, 3), }, diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go index 28591c5baf14c..859c83391281e 100644 --- a/pkg/kubelet/cm/cpumanager/policy_static.go +++ b/pkg/kubelet/cm/cpumanager/policy_static.go @@ -31,6 +31,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" "k8s.io/kubernetes/pkg/kubelet/metrics" + "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/utils/cpuset" ) @@ -64,6 +65,88 @@ func (e SMTAlignmentError) Type() string { return ErrorSMTAlignment } +// prohibitedCPUAllocationError represents an error due to an +// attempt to reduce container exclusively allocated +// pool below container exclusively promised pool +// allocated when container was created. +type prohibitedCPUAllocationError struct { + RequestedCPUs string + AllocatedCPUs string + PromisedCPUs int + GuaranteedCPUs int +} + +func (e prohibitedCPUAllocationError) Error() string { + return fmt.Sprintf("prohibitedCPUAllocation Error: Skip resize, Not allowed to reduce container exclusively allocated pool below promised, (requested CPUs = %s, allocated CPUs = %s, promised CPUs = %d, guaranteed CPUs = %d)", e.RequestedCPUs, e.AllocatedCPUs, e.PromisedCPUs, e.GuaranteedCPUs) +} + +// Type returns human-readable type of this error. +// Used in the HandlePodResourcesResize to populate Failure reason +func (e prohibitedCPUAllocationError) Type() string { + return types.ErrorProhibitedCPUAllocation +} + +// inconsistentCPUAllocationError represents an error due to an +// attempt to either move a container from exclusively allocated +// pool to shared pool or move a container from shared pool to +// exclusively allocated pool. +type inconsistentCPUAllocationError struct { + RequestedCPUs string + AllocatedCPUs string + Shared2Exclusive bool +} + +func (e inconsistentCPUAllocationError) Error() string { + if e.RequestedCPUs == e.AllocatedCPUs { + return fmt.Sprintf("inconsistentCPUAllocation Error: Skip resize, nothing to be done, (requested CPUs = %s equal to allocated CPUs = %s)", e.RequestedCPUs, e.AllocatedCPUs) + } + if e.Shared2Exclusive { + return fmt.Sprintf("inconsistentCPUAllocation Error: Not allowed to move a container from shared pool to exclusively allocated pool, (requested CPUs = %s, allocated CPUs = %s)", e.RequestedCPUs, e.AllocatedCPUs) + } else { + return fmt.Sprintf("inconsistentCPUAllocation Error: Not allowed to move a container from exclusively allocated pool to shared pool, not allowed (requested CPUs = %s, allocated CPUs = %s)", e.RequestedCPUs, e.AllocatedCPUs) + } +} + +// Type returns human-readable type of this error. +// Used in the HandlePodResourcesResize to populate Failure reason +func (e inconsistentCPUAllocationError) Type() string { + return types.ErrorInconsistentCPUAllocation +} + +// getPromisedCPUSetError represents an error due to a +// failed attempt to GetPromisedCPUSet from state +type getPromisedCPUSetError struct { + PodUID string + ContainerName string +} + +func (e getPromisedCPUSetError) Error() string { + return fmt.Sprintf("getPromisedCPUSet Error: Skip resize, unable to get PromisedCPUSet, nothing to be done, (podUID = %s, containerName %s)", e.PodUID, e.ContainerName) +} + +// Type returns human-readable type of this error. +// Used in the HandlePodResourcesResize to populate Failure reason +func (e getPromisedCPUSetError) Type() string { + return types.ErrorGetPromisedCPUSet +} + +// getCPUSetError represents an error due to a +// failed attempt to GetCPUSet from state +type getCPUSetError struct { + PodUID string + ContainerName string +} + +func (e getCPUSetError) Error() string { + return fmt.Sprintf("getCPUSet Error: Skip resize, unable to get CPUSet, nothing to be done, (podUID = %s, containerName %s)", e.PodUID, e.ContainerName) +} + +// Type returns human-readable type of this error. +// Used in the HandlePodResourcesResize to populate Failure reason +func (e getCPUSetError) Type() string { + return types.ErrorGetCPUSet +} + // staticPolicy is a CPU manager policy that does not change CPU // assignments for exclusively pinned guaranteed containers after the main // container process starts. @@ -117,6 +200,8 @@ type staticPolicy struct { affinity topologymanager.Store // set of CPUs to reuse across allocations in a pod cpusToReuse map[string]cpuset.CPUSet + // set of CPUs to reuse during pod resize + cpusToReuseDuringResize map[string]cpuset.CPUSet // options allow to fine-tune the behaviour of the policy options StaticPolicyOptions // we compute this value multiple time, and it's not supposed to change @@ -144,11 +229,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv klog.InfoS("Static policy created with configuration", "options", opts, "cpuGroupSize", cpuGroupSize) policy := &staticPolicy{ - topology: topology, - affinity: affinity, - cpusToReuse: make(map[string]cpuset.CPUSet), - options: opts, - cpuGroupSize: cpuGroupSize, + topology: topology, + affinity: affinity, + cpusToReuse: make(map[string]cpuset.CPUSet), + options: opts, + cpuGroupSize: cpuGroupSize, + cpusToReuseDuringResize: make(map[string]cpuset.CPUSet), } allCPUs := topology.CPUDetails.CPUs() @@ -161,7 +247,7 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv // // For example: Given a system with 8 CPUs available and HT enabled, // if numReservedCPUs=2, then reserved={0,4} - reserved, _ = policy.takeByTopology(allCPUs, numReservedCPUs) + reserved, _ = policy.takeByTopology(allCPUs, numReservedCPUs, nil, nil) } if reserved.Size() != numReservedCPUs { @@ -315,6 +401,15 @@ func (p *staticPolicy) updateCPUsToReuse(pod *v1.Pod, container *v1.Container, c func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Container) (rerr error) { numCPUs := p.guaranteedCPUs(pod, container) + if utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling) { + // During a pod resize, handle corner cases + err := p.validateInPlacePodVerticalScaling(s, pod, container) + if err != nil { + klog.ErrorS(err, "Static policy: Unable to resize allocated CPUs", "pod", klog.KObj(pod), "containerName", container.Name, "numCPUs", numCPUs) + return err + } + } + if numCPUs == 0 { // container belongs in the shared pool (nothing to do; use default cpuset) return nil @@ -358,6 +453,12 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai availablePhysicalCPUs := p.GetAvailablePhysicalCPUs(s).Size() + if utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling) { + if cs, ok := podutil.GetContainerStatus(pod.Status.ContainerStatuses, container.Name); ok { + cpuAllocatedQuantity := cs.AllocatedResources[v1.ResourceCPU] + availablePhysicalCPUs += int(cpuAllocatedQuantity.Value()) + } + } // It's legal to reserve CPUs which are not core siblings. In this case the CPU allocator can descend to single cores // when picking CPUs. This will void the guarantee of FullPhysicalCPUsOnly. To prevent this, we need to additionally consider // all the core siblings of the reserved CPUs as unavailable when computing the free CPUs, before to start the actual allocation. @@ -371,10 +472,60 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai } } } - if cset, ok := s.GetCPUSet(string(pod.UID), container.Name); ok { - p.updateCPUsToReuse(pod, container, cset) - klog.InfoS("Static policy: container already present in state, skipping", "pod", klog.KObj(pod), "containerName", container.Name) - return nil + if cpuset, ok := s.GetCPUSet(string(pod.UID), container.Name); ok { + if utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScalingExclusiveCPUs) { + if utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling) { + klog.InfoS("Static policy: container already present in state, attempting InPlacePodVerticalScaling", "pod", klog.KObj(pod), "containerName", container.Name) + if cpusInUseByPodContainerToResize, ok := s.GetCPUSet(string(pod.UID), container.Name); ok { + // Call Topology Manager to get the aligned socket affinity across all hint providers. + hint := p.affinity.GetAffinity(string(pod.UID), container.Name) + klog.InfoS("Topology Affinity", "pod", klog.KObj(pod), "containerName", container.Name, "affinity", hint) + // Attempt new allocation ( reusing allocated CPUs ) according to the NUMA affinity contained in the hint + // Since NUMA affinity container in the hint is unmutable already allocated CPUs pass the criteria + if mustKeepCPUsForResize, ok := s.GetPromisedCPUSet(string(pod.UID), container.Name); ok { + mustKeepCPUsBaseFromContainer := p.GetMustKeepCPUs(container, cpuset) + if mustKeepCPUsBasePerCpuUsage != nil { + if mustKeepCPUsForResize.IsSubsetOf(mustKeepCPUsBaseFromContainer) && mustKeepCPUsBaseFromContainer.Size() < numCPUs { + mustKeepCPUsForResize = mustKeepCPUsBaseFromContainer + } + } + newallocatedcpuset, err := p.allocateCPUs(s, numCPUs, hint.NUMANodeAffinity, p.cpusToReuse[string(pod.UID)], &cpusInUseByPodContainerToResize, &mustKeepCPUsForResize) + if err != nil { + klog.ErrorS(err, "Static policy: Unable to allocate new CPUs", "pod", klog.KObj(pod), "containerName", container.Name, "numCPUs", numCPUs) + return err + } + // Allocation successful, update the current state + s.SetCPUSet(string(pod.UID), container.Name, newallocatedcpuset.CPUs) + p.updateCPUsToReuse(pod, container, newallocatedcpuset.CPUs) + // Updated state to the checkpoint file will be stored during + // the reconcile loop. TODO is this a problem? I don't believe + // because if kubelet will be terminated now, anyhow it will be + // needed the state to be cleaned up, an error will appear requiring + // the node to be drained. I think we are safe. All computations are + // using state_mem and not the checkpoint. + return nil + } else { + return getPromisedCPUSetError{ + PodUID: string(pod.UID), + ContainerName: container.Name, + } + } + } else { + return getCPUSetError{ + PodUID: string(pod.UID), + ContainerName: container.Name, + } + } + } else { + p.updateCPUsToReuse(pod, container, cpuset) + klog.InfoS("Static policy: InPlacePodVerticalScaling alognside CPU Static policy requires InPlacePodVerticalScaling to be enabled, skipping pod resize") + return nil + } + } else { + p.updateCPUsToReuse(pod, container, cpuset) + klog.InfoS("Static policy: container already present in state, skipping", "pod", klog.KObj(pod), "containerName", container.Name) + return nil + } } // Call Topology Manager to get the aligned socket affinity across all hint providers. @@ -382,13 +533,14 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai klog.InfoS("Topology Affinity", "pod", klog.KObj(pod), "containerName", container.Name, "affinity", hint) // Allocate CPUs according to the NUMA affinity contained in the hint. - cpuAllocation, err := p.allocateCPUs(s, numCPUs, hint.NUMANodeAffinity, p.cpusToReuse[string(pod.UID)]) + cpuAllocation, err := p.allocateCPUs(s, numCPUs, hint.NUMANodeAffinity, p.cpusToReuse[string(pod.UID)], nil, nil) if err != nil { klog.ErrorS(err, "Unable to allocate CPUs", "pod", klog.KObj(pod), "containerName", container.Name, "numCPUs", numCPUs) return err } s.SetCPUSet(string(pod.UID), container.Name, cpuAllocation.CPUs) + s.SetPromisedCPUSet(string(pod.UID), container.Name, cpuAllocation.CPUs) p.updateCPUsToReuse(pod, container, cpuAllocation.CPUs) p.updateMetricsOnAllocate(s, cpuAllocation) @@ -396,6 +548,30 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai return nil } +func (p *staticPolicy) GetMustKeepCPUs(container *v1.Container, oldCpuset cpuset.CPUSet) *cpuset.CPUSet { + mustKeepCPUs := cpuset.New() + klog.InfoS("GetMustKeepCPUs", "container.Resources.MustKeepCPUs", container.Resources.MustKeepCPUs) + ResourcesMustKeepCPUs, err := cpuset.Parse(container.Resources.MustKeepCPUs) + if err == nil && ResourcesMustKeepCPUs.Size() != 0 { + mustKeepCPUs = oldCpuset.Intersection(ResourcesMustKeepCPUs) + } + klog.InfoS("mustKeepCPUs ", "is", mustKeepCPUs) + if p.options.FullPhysicalCPUsOnly { + // mustKeepCPUs must be aligned to the physical core + if (mustKeepCPUs.Size() % 2) != 0 { + return nil + } + mustKeepCPUsDetail := p.topology.CPUDetails.KeepOnly(mustKeepCPUs) + mustKeepCPUsDetailCores := mustKeepCPUsDetail.Cores() + if (mustKeepCPUs.Size() / mustKeepCPUsDetailCores.Size()) != p.cpuGroupSize { + klog.InfoS("mustKeepCPUs is nil") + return nil + } + } + klog.InfoS("GetMustKeepCPUs", "mustKeepCPUs", mustKeepCPUs) + return &mustKeepCPUs +} + // getAssignedCPUsOfSiblings returns assigned cpus of given container's siblings(all containers other than the given container) in the given pod `podUID`. func getAssignedCPUsOfSiblings(s state.State, podUID string, containerName string) cpuset.CPUSet { assignments := s.GetCPUAssignments() @@ -423,10 +599,18 @@ func (p *staticPolicy) RemoveContainer(s state.State, podUID string, containerNa return nil } -func (p *staticPolicy) allocateCPUs(s state.State, numCPUs int, numaAffinity bitmask.BitMask, reusableCPUs cpuset.CPUSet) (topology.Allocation, error) { +func (p *staticPolicy) allocateCPUs(s state.State, numCPUs int, numaAffinity bitmask.BitMask, reusableCPUs cpuset.CPUSet, reusableCPUsForResize *cpuset.CPUSet, mustKeepCPUsForResize *cpuset.CPUSet) (topology.Allocation, error) { klog.InfoS("AllocateCPUs", "numCPUs", numCPUs, "socket", numaAffinity) - - allocatableCPUs := p.GetAvailableCPUs(s).Union(reusableCPUs) + allocatableCPUs := cpuset.New() + if reusableCPUsForResize != nil { + if numCPUs >= reusableCPUsForResize.Size() { + allocatableCPUs = allocatableCPUs.Union(p.GetAvailableCPUs(s).Union(reusableCPUsForResize.Clone())) + } else if numCPUs < reusableCPUsForResize.Size() { + allocatableCPUs = reusableCPUsForResize.Clone() + } + } else { + allocatableCPUs = allocatableCPUs.Union(p.GetAvailableCPUs(s).Union(reusableCPUs)) + } // If there are aligned CPUs in numaAffinity, attempt to take those first. result := topology.EmptyAllocation() @@ -438,7 +622,7 @@ func (p *staticPolicy) allocateCPUs(s state.State, numCPUs int, numaAffinity bit numAlignedToAlloc = numCPUs } - allocatedCPUs, err := p.takeByTopology(alignedCPUs, numAlignedToAlloc) + allocatedCPUs, err := p.takeByTopology(alignedCPUs, numAlignedToAlloc, reusableCPUsForResize, mustKeepCPUsForResize) if err != nil { return topology.EmptyAllocation(), err } @@ -447,7 +631,7 @@ func (p *staticPolicy) allocateCPUs(s state.State, numCPUs int, numaAffinity bit } // Get any remaining CPUs from what's leftover after attempting to grab aligned ones. - remainingCPUs, err := p.takeByTopology(allocatableCPUs.Difference(result.CPUs), numCPUs-result.CPUs.Size()) + remainingCPUs, err := p.takeByTopology(allocatableCPUs.Difference(result.CPUs), numCPUs-result.CPUs.Size(), reusableCPUsForResize, mustKeepCPUsForResize) if err != nil { return topology.EmptyAllocation(), err } @@ -455,7 +639,17 @@ func (p *staticPolicy) allocateCPUs(s state.State, numCPUs int, numaAffinity bit result.Aligned = p.topology.CheckAlignment(result.CPUs) // Remove allocated CPUs from the shared CPUSet. - s.SetDefaultCPUSet(s.GetDefaultCPUSet().Difference(result.CPUs)) + if reusableCPUsForResize != nil { + if reusableCPUsForResize.Size() < result.CPUs.Size() { + // Scale up or creation has been performed + s.SetDefaultCPUSet(s.GetDefaultCPUSet().Difference(result.CPUs)) + } else if reusableCPUsForResize.Size() > result.CPUs.Size() { + // Scale down has been performed + s.SetDefaultCPUSet(s.GetDefaultCPUSet().Union(reusableCPUsForResize.Difference(result.CPUs))) + } + } else { + s.SetDefaultCPUSet(s.GetDefaultCPUSet().Difference(result.CPUs)) + } klog.InfoS("AllocateCPUs", "result", result.String()) return result, nil @@ -468,10 +662,6 @@ func (p *staticPolicy) guaranteedCPUs(pod *v1.Pod, container *v1.Container) int return 0 } cpuQuantity := container.Resources.Requests[v1.ResourceCPU] - // In-place pod resize feature makes Container.Resources field mutable for CPU & memory. - // AllocatedResources holds the value of Container.Resources.Requests when the pod was admitted. - // We should return this value because this is what kubelet agreed to allocate for the container - // and the value configured with runtime. if utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling) { containerStatuses := pod.Status.ContainerStatuses if podutil.IsRestartableInitContainer(container) { @@ -479,8 +669,15 @@ func (p *staticPolicy) guaranteedCPUs(pod *v1.Pod, container *v1.Container) int containerStatuses = append(containerStatuses, pod.Status.InitContainerStatuses...) } } - if cs, ok := podutil.GetContainerStatus(containerStatuses, container.Name); ok { - cpuQuantity = cs.AllocatedResources[v1.ResourceCPU] + if utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScalingExclusiveCPUs) { + klog.InfoS("InPlacePodVerticalScaling alognside CPU Static policy implemented, allowing pod resize") + } else { + // InPlacePodVerticalScaling alognside CPU Static policy is disabled by default + // We should return this value because this is what kubelet agreed to allocate for the container + // and the value configured with runtime. + if cs, ok := podutil.GetContainerStatus(containerStatuses, container.Name); ok { + cpuQuantity = cs.AllocatedResources[v1.ResourceCPU] + } } } cpuValue := cpuQuantity.Value() @@ -528,7 +725,7 @@ func (p *staticPolicy) podGuaranteedCPUs(pod *v1.Pod) int { return requestedByLongRunningContainers } -func (p *staticPolicy) takeByTopology(availableCPUs cpuset.CPUSet, numCPUs int) (cpuset.CPUSet, error) { +func (p *staticPolicy) takeByTopology(availableCPUs cpuset.CPUSet, numCPUs int, reusableCPUsForResize *cpuset.CPUSet, mustKeepCPUsForScaleDown *cpuset.CPUSet) (cpuset.CPUSet, error) { cpuSortingStrategy := CPUSortingStrategyPacked if p.options.DistributeCPUsAcrossCores { cpuSortingStrategy = CPUSortingStrategySpread @@ -539,10 +736,9 @@ func (p *staticPolicy) takeByTopology(availableCPUs cpuset.CPUSet, numCPUs int) if p.options.FullPhysicalCPUsOnly { cpuGroupSize = p.cpuGroupSize } - return takeByTopologyNUMADistributed(p.topology, availableCPUs, numCPUs, cpuGroupSize, cpuSortingStrategy) + return takeByTopologyNUMADistributed(p.topology, availableCPUs, numCPUs, cpuGroupSize, cpuSortingStrategy, reusableCPUsForResize, mustKeepCPUsForScaleDown) } - - return takeByTopologyNUMAPacked(p.topology, availableCPUs, numCPUs, cpuSortingStrategy, p.options.PreferAlignByUncoreCacheOption) + return takeByTopologyNUMAPacked(p.topology, availableCPUs, numCPUs, cpuSortingStrategy, p.options.PreferAlignByUncoreCacheOption, reusableCPUsForResize, mustKeepCPUsForScaleDown) } func (p *staticPolicy) GetTopologyHints(s state.State, pod *v1.Pod, container *v1.Container) map[string][]topologymanager.TopologyHint { @@ -557,23 +753,25 @@ func (p *staticPolicy) GetTopologyHints(s state.State, pod *v1.Pod, container *v return nil } - // Short circuit to regenerate the same hints if there are already - // guaranteed CPUs allocated to the Container. This might happen after a - // kubelet restart, for example. - if allocated, exists := s.GetCPUSet(string(pod.UID), container.Name); exists { - if allocated.Size() != requested { - klog.InfoS("CPUs already allocated to container with different number than request", "pod", klog.KObj(pod), "containerName", container.Name, "requestedSize", requested, "allocatedSize", allocated.Size()) - // An empty list of hints will be treated as a preference that cannot be satisfied. - // In definition of hints this is equal to: TopologyHint[NUMANodeAffinity: nil, Preferred: false]. - // For all but the best-effort policy, the Topology Manager will throw a pod-admission error. + if !utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScalingExclusiveCPUs) || !utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling) { + // Short circuit to regenerate the same hints if there are already + // guaranteed CPUs allocated to the Container. This might happen after a + // kubelet restart, for example. + if allocated, exists := s.GetCPUSet(string(pod.UID), container.Name); exists { + if allocated.Size() != requested { + klog.InfoS("CPUs already allocated to container with different number than request", "pod", klog.KObj(pod), "containerName", container.Name, "requestedSize", requested, "allocatedSize", allocated.Size()) + // An empty list of hints will be treated as a preference that cannot be satisfied. + // In definition of hints this is equal to: TopologyHint[NUMANodeAffinity: nil, Preferred: false]. + // For all but the best-effort policy, the Topology Manager will throw a pod-admission error. + return map[string][]topologymanager.TopologyHint{ + string(v1.ResourceCPU): {}, + } + } + klog.InfoS("Regenerating TopologyHints for CPUs already allocated", "pod", klog.KObj(pod), "containerName", container.Name) return map[string][]topologymanager.TopologyHint{ - string(v1.ResourceCPU): {}, + string(v1.ResourceCPU): p.generateCPUTopologyHints(allocated, cpuset.CPUSet{}, requested), } } - klog.InfoS("Regenerating TopologyHints for CPUs already allocated", "pod", klog.KObj(pod), "containerName", container.Name) - return map[string][]topologymanager.TopologyHint{ - string(v1.ResourceCPU): p.generateCPUTopologyHints(allocated, cpuset.CPUSet{}, requested), - } } // Get a list of available CPUs. @@ -613,11 +811,13 @@ func (p *staticPolicy) GetPodTopologyHints(s state.State, pod *v1.Pod) map[strin if allocated, exists := s.GetCPUSet(string(pod.UID), container.Name); exists { if allocated.Size() != requestedByContainer { klog.InfoS("CPUs already allocated to container with different number than request", "pod", klog.KObj(pod), "containerName", container.Name, "allocatedSize", requested, "requestedByContainer", requestedByContainer, "allocatedSize", allocated.Size()) - // An empty list of hints will be treated as a preference that cannot be satisfied. - // In definition of hints this is equal to: TopologyHint[NUMANodeAffinity: nil, Preferred: false]. - // For all but the best-effort policy, the Topology Manager will throw a pod-admission error. - return map[string][]topologymanager.TopologyHint{ - string(v1.ResourceCPU): {}, + if !utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScalingExclusiveCPUs) || !utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling) { + // An empty list of hints will be treated as a preference that cannot be satisfied. + // In definition of hints this is equal to: TopologyHint[NUMANodeAffinity: nil, Preferred: false]. + // For all but the best-effort policy, the Topology Manager will throw a pod-admission error. + return map[string][]topologymanager.TopologyHint{ + string(v1.ResourceCPU): {}, + } } } // A set of CPUs already assigned to containers in this pod @@ -814,3 +1014,61 @@ func updateAllocationPerNUMAMetric(topo *topology.CPUTopology, allocatedCPUs cpu metrics.CPUManagerAllocationPerNUMA.WithLabelValues(strconv.Itoa(numaNode)).Set(float64(count)) } } + +func (p *staticPolicy) validateInPlacePodVerticalScaling(s state.State, pod *v1.Pod, container *v1.Container) error { + + if v1qos.GetPodQOS(pod) != v1.PodQOSGuaranteed { + return nil + } + cpuQuantity := container.Resources.Requests[v1.ResourceCPU] + if cs, ok := podutil.GetContainerStatus(pod.Status.ContainerStatuses, container.Name); ok { + allocatedCPUQuantity := cs.AllocatedResources[v1.ResourceCPU] + if allocatedCPUQuantity.Value() > 0 { + if allocatedCPUQuantity.Value()*1000 == allocatedCPUQuantity.MilliValue() { + // container belongs in exclusive pool + if cpuQuantity.Value()*1000 != cpuQuantity.MilliValue() { + // container move to shared pool not allowed + return inconsistentCPUAllocationError{ + RequestedCPUs: cpuQuantity.String(), + AllocatedCPUs: allocatedCPUQuantity.String(), + Shared2Exclusive: false, + } + } + if mustKeepCPUsPromised, ok := s.GetPromisedCPUSet(string(pod.UID), container.Name); ok { + numCPUs := p.guaranteedCPUs(pod, container) + promisedCPUsQuantity := mustKeepCPUsPromised.Size() + if promisedCPUsQuantity > numCPUs { + return prohibitedCPUAllocationError{ + RequestedCPUs: cpuQuantity.String(), + AllocatedCPUs: allocatedCPUQuantity.String(), + PromisedCPUs: promisedCPUsQuantity, + GuaranteedCPUs: numCPUs, + } + } + } else { + return getPromisedCPUSetError{ + PodUID: string(pod.UID), + ContainerName: container.Name, + } + } + } else if cpuQuantity.Value()*1000 == cpuQuantity.MilliValue() { + // container belongs in shared pool + // container move to exclusive pool not allowed + return inconsistentCPUAllocationError{ + RequestedCPUs: cpuQuantity.String(), + AllocatedCPUs: allocatedCPUQuantity.String(), + Shared2Exclusive: true, + } + } + } else if cpuQuantity.Value()*1000 == cpuQuantity.MilliValue() { + // container belongs in shared pool + // container move to exclusive pool not allowed + return inconsistentCPUAllocationError{ + RequestedCPUs: cpuQuantity.String(), + AllocatedCPUs: allocatedCPUQuantity.String(), + Shared2Exclusive: true, + } + } + } + return nil +} diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go index db3a3649b5607..4421326922271 100644 --- a/pkg/kubelet/cm/cpumanager/policy_static_test.go +++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go @@ -22,6 +22,7 @@ import ( "testing" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" @@ -41,9 +42,14 @@ type staticPolicyTest struct { podUID string options map[string]string containerName string + stPromised state.ContainerCPUAssignments stAssignments state.ContainerCPUAssignments stDefaultCPUSet cpuset.CPUSet pod *v1.Pod + qosClass v1.PodQOSClass + podAllocated string + resizeLimit string + resizeRequest string topologyHint *topologymanager.TopologyHint expErr error expCPUAlloc bool @@ -60,6 +66,7 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest { podUID: spt.podUID, options: spt.options, // accessed in read-only containerName: spt.containerName, + stPromised: spt.stAssignments.Clone(), stAssignments: spt.stAssignments.Clone(), stDefaultCPUSet: spt.stDefaultCPUSet.Clone(), pod: spt.pod, // accessed in read-only @@ -87,6 +94,11 @@ func TestStaticPolicyStart(t *testing.T) { { description: "non-corrupted state", topo: topoDualSocketHT, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "0": cpuset.New(0), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "0": cpuset.New(0), @@ -99,6 +111,7 @@ func TestStaticPolicyStart(t *testing.T) { description: "empty cpuset", topo: topoDualSocketHT, numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(), expCSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11), @@ -107,6 +120,7 @@ func TestStaticPolicyStart(t *testing.T) { description: "reserved cores 0 & 6 are not present in available cpuset", topo: topoDualSocketHT, numReservedCPUs: 2, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 1), expErr: fmt.Errorf("not all reserved cpus: \"0,6\" are present in defaultCpuSet: \"0-1\""), @@ -116,6 +130,7 @@ func TestStaticPolicyStart(t *testing.T) { topo: topoDualSocketHT, numReservedCPUs: 2, options: map[string]string{StrictCPUReservationOption: "true"}, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 1), expErr: fmt.Errorf("some of strictly reserved cpus: \"0\" are present in defaultCpuSet: \"0-1\""), @@ -123,6 +138,11 @@ func TestStaticPolicyStart(t *testing.T) { { description: "assigned core 2 is still present in available cpuset", topo: topoDualSocketHT, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "0": cpuset.New(0, 1, 2), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "0": cpuset.New(0, 1, 2), @@ -135,6 +155,11 @@ func TestStaticPolicyStart(t *testing.T) { description: "assigned core 2 is still present in available cpuset (StrictCPUReservationOption)", topo: topoDualSocketHT, options: map[string]string{StrictCPUReservationOption: "true"}, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "0": cpuset.New(0, 1, 2), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "0": cpuset.New(0, 1, 2), @@ -146,6 +171,12 @@ func TestStaticPolicyStart(t *testing.T) { { description: "core 12 is not present in topology but is in state cpuset", topo: topoDualSocketHT, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "0": cpuset.New(0, 1, 2), + "1": cpuset.New(3, 4), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "0": cpuset.New(0, 1, 2), @@ -158,6 +189,12 @@ func TestStaticPolicyStart(t *testing.T) { { description: "core 11 is present in topology but is not in state cpuset", topo: topoDualSocketHT, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "0": cpuset.New(0, 1, 2), + "1": cpuset.New(3, 4), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "0": cpuset.New(0, 1, 2), @@ -176,6 +213,7 @@ func TestStaticPolicyStart(t *testing.T) { } policy := p.(*staticPolicy) st := &mockState{ + promised: testCase.stPromised, assignments: testCase.stAssignments, defaultCPUSet: testCase.stDefaultCPUSet, } @@ -229,6 +267,11 @@ func TestStaticPolicyAdd(t *testing.T) { description: "GuPodMultipleCores, SingleSocketHT, ExpectAllocOneCore", topo: topoSingleSocketHT, numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer100": cpuset.New(2, 3, 6, 7), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.New(2, 3, 6, 7), @@ -244,6 +287,11 @@ func TestStaticPolicyAdd(t *testing.T) { description: "GuPodMultipleCores, DualSocketHT, ExpectAllocOneSocket", topo: topoDualSocketHT, numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer100": cpuset.New(2), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.New(2), @@ -259,6 +307,11 @@ func TestStaticPolicyAdd(t *testing.T) { description: "GuPodMultipleCores, DualSocketHT, ExpectAllocThreeCores", topo: topoDualSocketHT, numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer100": cpuset.New(1, 5), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.New(1, 5), @@ -274,6 +327,11 @@ func TestStaticPolicyAdd(t *testing.T) { description: "GuPodMultipleCores, DualSocketNoHT, ExpectAllocOneSocket", topo: topoDualSocketNoHT, numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer100": cpuset.New(), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.New(), @@ -289,6 +347,11 @@ func TestStaticPolicyAdd(t *testing.T) { description: "GuPodMultipleCores, DualSocketNoHT, ExpectAllocFourCores", topo: topoDualSocketNoHT, numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer100": cpuset.New(4, 5), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.New(4, 5), @@ -304,6 +367,11 @@ func TestStaticPolicyAdd(t *testing.T) { description: "GuPodMultipleCores, DualSocketHT, ExpectAllocOneSocketOneCore", topo: topoDualSocketHT, numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer100": cpuset.New(2), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.New(2), @@ -319,6 +387,7 @@ func TestStaticPolicyAdd(t *testing.T) { description: "NonGuPod, SingleSocketHT, NoAlloc", topo: topoSingleSocketHT, numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), pod: makePod("fakePod", "fakeContainer1", "1000m", "2000m"), @@ -330,6 +399,7 @@ func TestStaticPolicyAdd(t *testing.T) { description: "GuPodNonIntegerCore, SingleSocketHT, NoAlloc", topo: topoSingleSocketHT, numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), pod: makePod("fakePod", "fakeContainer4", "977m", "977m"), @@ -343,6 +413,11 @@ func TestStaticPolicyAdd(t *testing.T) { // Expect all CPUs from Socket 0. description: "GuPodMultipleCores, topoQuadSocketFourWayHT, ExpectAllocSock0", topo: topoQuadSocketFourWayHT, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer100": cpuset.New(3, 11, 4, 5, 6, 7), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.New(3, 11, 4, 5, 6, 7), @@ -359,6 +434,12 @@ func TestStaticPolicyAdd(t *testing.T) { // Expect CPUs from the 2 full cores available from the three Sockets. description: "GuPodMultipleCores, topoQuadSocketFourWayHT, ExpectAllocAllFullCoresFromThreeSockets", topo: topoQuadSocketFourWayHT, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer100": largeTopoCPUSet.Difference(cpuset.New(1, 25, 13, 38, 2, 9, 11, 35, 23, 48, 12, 51, + 53, 173, 113, 233, 54, 61)), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": largeTopoCPUSet.Difference(cpuset.New(1, 25, 13, 38, 2, 9, 11, 35, 23, 48, 12, 51, @@ -376,6 +457,12 @@ func TestStaticPolicyAdd(t *testing.T) { // Expect all CPUs from Socket 1 and the hyper-threads from the full core. description: "GuPodMultipleCores, topoQuadSocketFourWayHT, ExpectAllocAllSock1+FullCore", topo: topoQuadSocketFourWayHT, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer100": largeTopoCPUSet.Difference(largeTopoSock1CPUSet.Union(cpuset.New(10, 34, 22, 47, 53, + 173, 61, 181, 108, 228, 115, 235))), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": largeTopoCPUSet.Difference(largeTopoSock1CPUSet.Union(cpuset.New(10, 34, 22, 47, 53, @@ -397,6 +484,7 @@ func TestStaticPolicyAdd(t *testing.T) { description: "GuPodSingleCore, SingleSocketHT, ExpectAllocOneCPU", topo: topoSingleSocketHT, numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"), @@ -409,6 +497,11 @@ func TestStaticPolicyAdd(t *testing.T) { // Expect allocation of all the CPUs from the partial cores. description: "GuPodMultipleCores, topoQuadSocketFourWayHT, ExpectAllocCPUs", topo: topoQuadSocketFourWayHT, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer100": largeTopoCPUSet.Difference(cpuset.New(10, 11, 53, 37, 55, 67, 52)), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": largeTopoCPUSet.Difference(cpuset.New(10, 11, 53, 37, 55, 67, 52)), @@ -424,6 +517,7 @@ func TestStaticPolicyAdd(t *testing.T) { description: "GuPodSingleCore, SingleSocketHT, ExpectError", topo: topoSingleSocketHT, numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"), @@ -435,21 +529,31 @@ func TestStaticPolicyAdd(t *testing.T) { description: "GuPodMultipleCores, SingleSocketHT, ExpectSameAllocation", topo: topoSingleSocketHT, numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer3": cpuset.New(1, 2, 5, 6), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ - "fakeContainer3": cpuset.New(2, 3, 6, 7), + "fakeContainer3": cpuset.New(1, 2, 5, 6), }, }, - stDefaultCPUSet: cpuset.New(0, 1, 4, 5), + stDefaultCPUSet: cpuset.New(0, 3, 4, 7), pod: makePod("fakePod", "fakeContainer3", "4000m", "4000m"), expErr: nil, expCPUAlloc: true, - expCSet: cpuset.New(2, 3, 6, 7), + expCSet: cpuset.New(1, 2, 5, 6), }, { description: "GuPodMultipleCores, DualSocketHT, NoAllocExpectError", topo: topoDualSocketHT, numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer100": cpuset.New(1, 2, 3), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.New(1, 2, 3), @@ -465,6 +569,11 @@ func TestStaticPolicyAdd(t *testing.T) { description: "GuPodMultipleCores, SingleSocketHT, NoAllocExpectError", topo: topoSingleSocketHT, numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer100": cpuset.New(1, 2, 3, 4, 5, 6), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.New(1, 2, 3, 4, 5, 6), @@ -482,6 +591,11 @@ func TestStaticPolicyAdd(t *testing.T) { // Error is expected since available CPUs are less than the request. description: "GuPodMultipleCores, topoQuadSocketFourWayHT, NoAlloc", topo: topoQuadSocketFourWayHT, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer100": largeTopoCPUSet.Difference(cpuset.New(10, 11, 53, 37, 55, 67, 52)), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": largeTopoCPUSet.Difference(cpuset.New(10, 11, 53, 37, 55, 67, 52)), @@ -504,6 +618,7 @@ func TestStaticPolicyAdd(t *testing.T) { FullPCPUsOnlyOption: "true", }, numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"), @@ -519,6 +634,7 @@ func TestStaticPolicyAdd(t *testing.T) { FullPCPUsOnlyOption: "true", }, numReservedCPUs: 8, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: largeTopoCPUSet, pod: makePod("fakePod", "fakeContainer15", "15000m", "15000m"), @@ -534,6 +650,7 @@ func TestStaticPolicyAdd(t *testing.T) { }, numReservedCPUs: 2, reservedCPUs: newCPUSetPtr(1, 6), + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 2, 3, 4, 5, 7, 8, 9, 10, 11), pod: makePod("fakePod", "fakeContainerBug113537_1", "10000m", "10000m"), @@ -548,6 +665,7 @@ func TestStaticPolicyAdd(t *testing.T) { FullPCPUsOnlyOption: "true", }, numReservedCPUs: 2, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 7, 8, 9, 10, 11), pod: makePod("fakePod", "fakeContainerBug113537_2", "10000m", "10000m"), @@ -563,6 +681,7 @@ func TestStaticPolicyAdd(t *testing.T) { }, numReservedCPUs: 2, reservedCPUs: newCPUSetPtr(0, 6), + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 7, 8, 9, 10, 11), pod: makePod("fakePod", "fakeContainerBug113537_2", "10000m", "10000m"), @@ -571,6 +690,141 @@ func TestStaticPolicyAdd(t *testing.T) { expCSet: cpuset.New(1, 2, 3, 4, 5, 7, 8, 9, 10, 11), }, } + + // testcases for podResize + podResizeTestCases := []staticPolicyTest{ + { + description: "podResize GuPodMultipleCores, SingleSocketHT, ExpectSameAllocation", + topo: topoSingleSocketHT, + numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer3": cpuset.New(1, 2, 5, 6), + }, + }, + stAssignments: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer3": cpuset.New(1, 2, 5, 6), + }, + }, + stDefaultCPUSet: cpuset.New(0, 3, 4, 7), + pod: makePod("fakePod", "fakeContainer3", "4000m", "4000m"), + expErr: nil, + expCPUAlloc: true, + expCSet: cpuset.New(1, 2, 5, 6), + }, + { + description: "podResize GuPodSingleCore, SingleSocketHT, ExpectAllocOneCPU", + topo: topoSingleSocketHT, + options: map[string]string{ + FullPCPUsOnlyOption: "true", + }, + numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer3": cpuset.New(1, 5), + }, + }, + stAssignments: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer3": cpuset.New(1, 5), + }, + }, + stDefaultCPUSet: cpuset.New(0, 2, 3, 4, 6, 7), + pod: makePod("fakePod", "fakeContainer3", "4000m", "4000m"), + expErr: nil, + expCPUAlloc: true, + expCSet: cpuset.New(1, 5), + }, + { + description: "podResize GuPodSingleCore, SingleSocketHT, ExpectAllocOneCPU", + topo: topoSingleSocketHT, + options: map[string]string{ + FullPCPUsOnlyOption: "true", + }, + numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer3": cpuset.New(1, 5), + }, + }, + stAssignments: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer3": cpuset.New(1, 5), + }, + }, + stDefaultCPUSet: cpuset.New(0, 2, 3, 4, 6, 7), + pod: makePod("fakePod", "fakeContainer3", "2000m", "2000m"), + expErr: nil, + expCPUAlloc: true, + expCSet: cpuset.New(1, 5), + }, + { + description: "podResize", + topo: topoSingleSocketHT, + options: map[string]string{ + FullPCPUsOnlyOption: "true", + }, + numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer3": cpuset.New(1, 5), + }, + }, + stAssignments: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer3": cpuset.New(1, 5), + }, + }, + stDefaultCPUSet: cpuset.New(0, 2, 3, 4, 6, 7), + pod: makePod("fakePod", "fakeContainer3", "100m", "100m"), + //expErr: inconsistentCPUAllocationError{RequestedCPUs: "0", AllocatedCPUs: "2"}, + expErr: nil, + expCPUAlloc: true, + expCSet: cpuset.New(1, 5), + }, + { + description: "podResize", + topo: topoSingleSocketHT, + options: map[string]string{ + FullPCPUsOnlyOption: "false", + }, + numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + pod: makePod("fakePod", "fakeContainer3", "1000m", "1000m"), + //expErr: inconsistentCPUAllocationError{RequestedCPUs: "0", AllocatedCPUs: "2"}, + expErr: nil, + expCPUAlloc: true, + expCSet: cpuset.New(4), + }, + { + description: "podResize", + topo: topoSingleSocketHT, + options: map[string]string{ + FullPCPUsOnlyOption: "true", + }, + numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer3": cpuset.New(1, 5), + }, + }, + stAssignments: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer3": cpuset.New(1, 5), + }, + }, + stDefaultCPUSet: cpuset.New(0, 2, 3, 4, 6, 7), + pod: makePod("fakePod", "fakeContainer3", "100m", "100m"), + //expErr: inconsistentCPUAllocationError{RequestedCPUs: "0", AllocatedCPUs: "2"}, + expErr: nil, + expCPUAlloc: true, + expCSet: cpuset.New(1, 5), + }, + } + newNUMAAffinity := func(bits ...int) bitmask.BitMask { affinity, _ := bitmask.NewBitMask(bits...) return affinity @@ -583,6 +837,7 @@ func TestStaticPolicyAdd(t *testing.T) { AlignBySocketOption: "true", }, numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(2, 11, 21, 22), pod: makePod("fakePod", "fakeContainer2", "2000m", "2000m"), @@ -598,6 +853,7 @@ func TestStaticPolicyAdd(t *testing.T) { AlignBySocketOption: "false", }, numReservedCPUs: 1, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(2, 11, 21, 22), pod: makePod("fakePod", "fakeContainer2", "2000m", "2000m"), @@ -631,6 +887,9 @@ func TestStaticPolicyAdd(t *testing.T) { for _, testCase := range alignBySocketOptionTestCases { runStaticPolicyTestCaseWithFeatureGate(t, testCase) } + for _, testCase := range podResizeTestCases { + runStaticPolicyTestCaseWithFeatureGateAlongsideInPlacePodVerticalScaling(t, testCase) + } } func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) { @@ -648,6 +907,7 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) { } st := &mockState{ + promised: testCase.stPromised, assignments: testCase.stAssignments, defaultCPUSet: testCase.stDefaultCPUSet, } @@ -691,6 +951,12 @@ func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyT runStaticPolicyTestCase(t, testCase) } +func runStaticPolicyTestCaseWithFeatureGateAlongsideInPlacePodVerticalScaling(t *testing.T, testCase staticPolicyTest) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.CPUManagerPolicyAlphaOptions, true) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.InPlacePodVerticalScaling, true) + runStaticPolicyTestCase(t, testCase) +} + func TestStaticPolicyReuseCPUs(t *testing.T) { testCases := []struct { staticPolicyTest @@ -707,6 +973,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { []struct{ request, limit string }{ {"2000m", "2000m"}}), // 0, 4 containerName: "initContainer-0", + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), }, @@ -722,6 +989,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { } st := &mockState{ + promised: testCase.stAssignments, assignments: testCase.stAssignments, defaultCPUSet: testCase.stDefaultCPUSet, } @@ -750,6 +1018,307 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { } } +func TestStaticPolicyPodResizeCPUsSingleContainerPod(t *testing.T) { + testCases := []struct { + staticPolicyTest + expAllocErr error + expCSetAfterAlloc cpuset.CPUSet + expCSetAfterResize cpuset.CPUSet + expCSetAfterResizeSize int + expCSetAfterRemove cpuset.CPUSet + }{ + { + staticPolicyTest: staticPolicyTest{ + description: "SingleSocketHT, PodResize, Container in exclusively allocated pool, Increase allocated CPUs", + topo: topoSingleSocketHT, + pod: makeMultiContainerPodWithOptions( + nil, + []*containerOptions{ + {request: "2000m", limit: "2000m", restartPolicy: v1.ContainerRestartPolicy("Never")}}, // 0, 4 + ), + qosClass: v1.PodQOSGuaranteed, + podAllocated: "2000m", + resizeLimit: "4000m", + resizeRequest: "4000m", + containerName: "appContainer-0", + stPromised: state.ContainerCPUAssignments{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + expCSetAfterAlloc: cpuset.New(1, 2, 3, 5, 6, 7), + expCSetAfterResize: cpuset.New(1, 2, 3, 5, 6, 7), + expCSetAfterRemove: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + { + staticPolicyTest: staticPolicyTest{ + description: "SingleSocketHT, PodResize, Container in exclusively allocated pool, Keep same allocated CPUs", + topo: topoSingleSocketHT, + pod: makeMultiContainerPodWithOptions( + nil, + []*containerOptions{ + {request: "2000m", limit: "2000m", restartPolicy: v1.ContainerRestartPolicy("Never")}}, // 0, 4 + ), + qosClass: v1.PodQOSGuaranteed, + podAllocated: "2000m", + resizeLimit: "2000m", + resizeRequest: "2000m", + containerName: "appContainer-0", + stPromised: state.ContainerCPUAssignments{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + expAllocErr: inconsistentCPUAllocationError{RequestedCPUs: "2", AllocatedCPUs: "2", Shared2Exclusive: false}, + expCSetAfterAlloc: cpuset.New(1, 2, 3, 5, 6, 7), + expCSetAfterResize: cpuset.New(1, 2, 3, 5, 6, 7), + expCSetAfterRemove: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + { + staticPolicyTest: staticPolicyTest{ + description: "SingleSocketHT, PodResize, Container in exclusively allocated pool, Decrease allocated CPUs", + topo: topoSingleSocketHT, + pod: makeMultiContainerPodWithOptions( + nil, + []*containerOptions{ + {request: "4000m", limit: "4000m", restartPolicy: v1.ContainerRestartPolicy("Never")}}, // 0-1, 4-5 + ), + qosClass: v1.PodQOSGuaranteed, + podAllocated: "4000m", + resizeLimit: "2000m", + resizeRequest: "2000m", + containerName: "appContainer-0", + stPromised: state.ContainerCPUAssignments{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + expCSetAfterAlloc: cpuset.New(2, 3, 6, 7), + expCSetAfterResizeSize: 4, + expCSetAfterRemove: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + { + staticPolicyTest: staticPolicyTest{ + description: "SingleSocketHT, PodResize, Container in shared pool with more than one core, Attempt to move to exclusively allocated pool", + topo: topoSingleSocketHT, + pod: makeMultiContainerPodWithOptions( + nil, + []*containerOptions{ + {request: "2100m", limit: "2100m", restartPolicy: v1.ContainerRestartPolicy("Never")}}, // 0-7 + ), + qosClass: v1.PodQOSGuaranteed, + podAllocated: "2100m", + resizeLimit: "2000m", + resizeRequest: "2000m", + containerName: "appContainer-0", + stPromised: state.ContainerCPUAssignments{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + expAllocErr: inconsistentCPUAllocationError{RequestedCPUs: "2", AllocatedCPUs: "2100m", Shared2Exclusive: true}, + expCSetAfterAlloc: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + expCSetAfterResize: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + expCSetAfterRemove: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + { + staticPolicyTest: staticPolicyTest{ + description: "SingleSocketHT, PodResize, Container in shared pool, Increase CPU and keep in shared pool", + topo: topoSingleSocketHT, + pod: makeMultiContainerPodWithOptions( + nil, + []*containerOptions{ + {request: "100m", limit: "100m", restartPolicy: v1.ContainerRestartPolicy("Never")}}, // 0-7 + ), + qosClass: v1.PodQOSGuaranteed, + podAllocated: "100m", + resizeLimit: "200m", + resizeRequest: "200m", + containerName: "appContainer-0", + stPromised: state.ContainerCPUAssignments{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + expCSetAfterAlloc: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + expCSetAfterResize: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + expCSetAfterRemove: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + { + staticPolicyTest: staticPolicyTest{ + description: "SingleSocketHT, PodResize, Container in shared pool, Increase CPU and keep in shared pool", + topo: topoSingleSocketHT, + pod: makeMultiContainerPodWithOptions( + nil, + []*containerOptions{ + {request: "1100m", limit: "1100m", restartPolicy: v1.ContainerRestartPolicy("Never")}}, // 0-7 + ), + qosClass: v1.PodQOSGuaranteed, + podAllocated: "1100m", + resizeLimit: "1200m", + resizeRequest: "1200m", + containerName: "appContainer-0", + stPromised: state.ContainerCPUAssignments{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + expCSetAfterAlloc: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + expCSetAfterResize: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + expCSetAfterRemove: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + { + staticPolicyTest: staticPolicyTest{ + description: "SingleSocketHT, PodResize, Container in shared pool with less than one core, Decrease CPU and keep in shared pool", + topo: topoSingleSocketHT, + pod: makeMultiContainerPodWithOptions( + nil, + []*containerOptions{ + {request: "200m", limit: "200m", restartPolicy: v1.ContainerRestartPolicy("Never")}}, // 0-7 + ), + qosClass: v1.PodQOSGuaranteed, + podAllocated: "200m", + resizeLimit: "100m", + resizeRequest: "100m", + containerName: "appContainer-0", + stPromised: state.ContainerCPUAssignments{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + expCSetAfterAlloc: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + expCSetAfterResize: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + expCSetAfterRemove: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + { + staticPolicyTest: staticPolicyTest{ + description: "SingleSocketHT, PodResize, Container in shared pool with more than one core, Decrease CPU and keep in shared pool", + topo: topoSingleSocketHT, + pod: makeMultiContainerPodWithOptions( + nil, + []*containerOptions{ + {request: "1200m", limit: "1200m", restartPolicy: v1.ContainerRestartPolicy("Never")}}, // 0-7 + ), + qosClass: v1.PodQOSGuaranteed, + podAllocated: "1200m", + resizeLimit: "1100m", + resizeRequest: "1100m", + containerName: "appContainer-0", + stPromised: state.ContainerCPUAssignments{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + expCSetAfterAlloc: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + expCSetAfterResize: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + expCSetAfterRemove: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + { + staticPolicyTest: staticPolicyTest{ + description: "SingleSocketHT, PodResize, Container in exclusively allocated pool, Move to shared pool", + topo: topoSingleSocketHT, + pod: makeMultiContainerPodWithOptions( + nil, + []*containerOptions{ + {request: "2000m", limit: "2000m", restartPolicy: v1.ContainerRestartPolicy("Never")}}, // 0-1, 4-5 + ), + qosClass: v1.PodQOSGuaranteed, + podAllocated: "2000m", + resizeLimit: "1500m", + resizeRequest: "1500m", + containerName: "appContainer-0", + stPromised: state.ContainerCPUAssignments{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + expAllocErr: inconsistentCPUAllocationError{RequestedCPUs: "1500m", AllocatedCPUs: "2", Shared2Exclusive: false}, + expCSetAfterAlloc: cpuset.New(1, 2, 3, 5, 6, 7), + expCSetAfterResize: cpuset.New(1, 2, 3, 5, 6, 7), + expCSetAfterRemove: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + } + + for _, testCase := range testCases { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.CPUManagerPolicyAlphaOptions, true) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.InPlacePodVerticalScaling, true) + t.Run(testCase.description, func(t *testing.T) { + + policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.New(), topologymanager.NewFakeManager(), nil) + + st := &mockState{ + promised: testCase.stAssignments, + assignments: testCase.stAssignments, + defaultCPUSet: testCase.stDefaultCPUSet, + } + pod := testCase.pod + pod.Status.QOSClass = testCase.qosClass + + // allocate + for _, container := range append(pod.Spec.InitContainers, pod.Spec.Containers...) { + err := policy.Allocate(st, pod, &container) + if err != nil { + t.Errorf("StaticPolicy Allocate() error (%v). expected no error but got %v", + testCase.description, err) + } + } + if !reflect.DeepEqual(st.defaultCPUSet, testCase.expCSetAfterAlloc) { + t.Errorf("StaticPolicy Allocate() error (%v) before pod resize. expected default cpuset %v but got %v", + testCase.description, testCase.expCSetAfterAlloc, st.defaultCPUSet) + } + + // resize + pod.Status.ContainerStatuses = []v1.ContainerStatus{ + { + Name: testCase.containerName, + AllocatedResources: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(testCase.podAllocated), + }, + }, + } + pod.Spec.Containers[0].Resources = v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse(testCase.resizeLimit), + }, + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse(testCase.resizeRequest), + }, + } + podResized := pod + for _, container := range append(podResized.Spec.InitContainers, podResized.Spec.Containers...) { + err := policy.Allocate(st, podResized, &container) + if err != nil { + if !reflect.DeepEqual(err, testCase.expAllocErr) { + t.Errorf("StaticPolicy Allocate() error (%v), expected error: %v but got: %v", + testCase.description, testCase.expAllocErr, err) + } + } + } + if testCase.expCSetAfterResizeSize > 0 { + // expCSetAfterResizeSize is used when testing scale down because allocated CPUs are not deterministic, + // since size of defaultCPUSet is deterministic and also interesection with expected allocation + // should not be nill. < ====== TODO esotsal + if !reflect.DeepEqual(st.defaultCPUSet.Size(), testCase.expCSetAfterResizeSize) { + t.Errorf("StaticPolicy Allocate() error (%v) after pod resize. expected default cpuset size equal to %v but got %v", + testCase.description, testCase.expCSetAfterResizeSize, st.defaultCPUSet.Size()) + } + } else { + if !reflect.DeepEqual(st.defaultCPUSet, testCase.expCSetAfterResize) { + t.Errorf("StaticPolicy Allocate() error (%v) after pod resize. expected default cpuset %v but got %v", + testCase.description, testCase.expCSetAfterResize, st.defaultCPUSet) + } + } + + // remove + err := policy.RemoveContainer(st, string(pod.UID), testCase.containerName) + if err != nil { + t.Errorf("StaticPolicy RemoveContainer() error (%v) after pod resize. expected no error but got %v", + testCase.description, err) + } + + if !reflect.DeepEqual(st.defaultCPUSet, testCase.expCSetAfterRemove) { + t.Errorf("StaticPolicy RemoveContainer() error (%v) after pod resize. expected default cpuset %v but got %v", + testCase.description, testCase.expCSetAfterRemove, st.defaultCPUSet) + } + if _, found := st.assignments[string(pod.UID)][testCase.containerName]; found { + t.Errorf("StaticPolicy RemoveContainer() error (%v) after pod resize. expected (pod %v, container %v) not be in assignments %v", + testCase.description, testCase.podUID, testCase.containerName, st.assignments) + } + }) + } +} + func TestStaticPolicyDoNotReuseCPUs(t *testing.T) { testCases := []struct { staticPolicyTest @@ -764,6 +1333,7 @@ func TestStaticPolicyDoNotReuseCPUs(t *testing.T) { {request: "4000m", limit: "4000m", restartPolicy: v1.ContainerRestartPolicyAlways}}, // 0, 1, 4, 5 []*containerOptions{ {request: "2000m", limit: "2000m"}}), // 2, 6 + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), }, @@ -778,6 +1348,7 @@ func TestStaticPolicyDoNotReuseCPUs(t *testing.T) { } st := &mockState{ + promised: testCase.stPromised, assignments: testCase.stAssignments, defaultCPUSet: testCase.stDefaultCPUSet, } @@ -798,6 +1369,307 @@ func TestStaticPolicyDoNotReuseCPUs(t *testing.T) { } } +func TestStaticPolicyPodResizeCPUsMultiContainerPod(t *testing.T) { + testCases := []struct { + staticPolicyTest + containerName2 string + expAllocErr error + expCSetAfterAlloc cpuset.CPUSet + expCSetAfterResize cpuset.CPUSet + expCSetAfterResizeSize int + expCSetAfterRemove cpuset.CPUSet + }{ + { + staticPolicyTest: staticPolicyTest{ + description: "SingleSocketHT, PodResize, Containers in exclusively allocated pool, Increase appContainer-0 allocated CPUs", + topo: topoSingleSocketHT, + pod: makeMultiContainerPodWithOptions( + nil, + []*containerOptions{ + {request: "2000m", limit: "2000m", restartPolicy: v1.ContainerRestartPolicy("Never")}, // 0, 4 + {request: "2000m", limit: "2000m", restartPolicy: v1.ContainerRestartPolicy("Never")}}, // 1, 5 + ), + qosClass: v1.PodQOSGuaranteed, + podAllocated: "2000m", + resizeLimit: "4000m", + resizeRequest: "4000m", + containerName: "appContainer-0", + stPromised: state.ContainerCPUAssignments{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + containerName2: "appContainer-1", + expCSetAfterAlloc: cpuset.New(2, 3, 6, 7), + expCSetAfterResize: cpuset.New(2, 3, 6, 7), + expCSetAfterRemove: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + { + staticPolicyTest: staticPolicyTest{ + description: "SingleSocketHT, PodResize, Containers in exclusively allocated pool, Keep same allocated CPUs", + topo: topoSingleSocketHT, + pod: makeMultiContainerPodWithOptions( + nil, + []*containerOptions{ + {request: "2000m", limit: "2000m", restartPolicy: v1.ContainerRestartPolicy("Never")}, // 0, 4 + {request: "2000m", limit: "2000m", restartPolicy: v1.ContainerRestartPolicy("Never")}}, // 1, 5 + ), + qosClass: v1.PodQOSGuaranteed, + podAllocated: "2000m", + resizeLimit: "2000m", + resizeRequest: "2000m", + containerName: "appContainer-0", + stPromised: state.ContainerCPUAssignments{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + containerName2: "appContainer-1", + expAllocErr: inconsistentCPUAllocationError{RequestedCPUs: "2", AllocatedCPUs: "2", Shared2Exclusive: false}, + expCSetAfterAlloc: cpuset.New(2, 3, 6, 7), + expCSetAfterResize: cpuset.New(2, 3, 6, 7), + expCSetAfterRemove: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + { + staticPolicyTest: staticPolicyTest{ + description: "SingleSocketHT, PodResize, Containers in exclusively allocated pool, Decrease appContainer-0 allocated CPUs", + topo: topoSingleSocketHT, + pod: makeMultiContainerPodWithOptions( + nil, + []*containerOptions{ + {request: "4000m", limit: "4000m", restartPolicy: v1.ContainerRestartPolicy("Never")}, // appContainer-0 CPUs 0, 4, 1, 5 + {request: "4000m", limit: "4000m", restartPolicy: v1.ContainerRestartPolicy("Never")}}, // appContainer-1 CPUS 2, 6, 3, 7 + ), + qosClass: v1.PodQOSGuaranteed, + podAllocated: "4000m", + resizeLimit: "2000m", + resizeRequest: "2000m", + containerName: "appContainer-0", + stPromised: state.ContainerCPUAssignments{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + containerName2: "appContainer-1", + expCSetAfterAlloc: cpuset.New(), + expCSetAfterResize: cpuset.New(), + expCSetAfterRemove: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + { + staticPolicyTest: staticPolicyTest{ + description: "SingleSocketHT, PodResize, Containers in shared pool with more than one core, Attempt to move to exclusively allocated pool", + topo: topoSingleSocketHT, + pod: makeMultiContainerPodWithOptions( + nil, + []*containerOptions{ + {request: "2100m", limit: "2100m", restartPolicy: v1.ContainerRestartPolicy("Never")}, // 0-7 + {request: "2100m", limit: "2100m", restartPolicy: v1.ContainerRestartPolicy("Never")}}, // 0-7 + ), + qosClass: v1.PodQOSGuaranteed, + podAllocated: "2100m", + resizeLimit: "2000m", + resizeRequest: "2000m", + containerName: "appContainer-0", + stPromised: state.ContainerCPUAssignments{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + containerName2: "appContainer-1", + expAllocErr: inconsistentCPUAllocationError{RequestedCPUs: "2", AllocatedCPUs: "2100m", Shared2Exclusive: true}, + expCSetAfterAlloc: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + expCSetAfterResize: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + expCSetAfterRemove: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + { + staticPolicyTest: staticPolicyTest{ + description: "SingleSocketHT, PodResize, appContainer-0 in shared pool, Increase CPU and keep appContainer-0 in shared pool", + topo: topoSingleSocketHT, + pod: makeMultiContainerPodWithOptions( + nil, + []*containerOptions{ + {request: "100m", limit: "100m", restartPolicy: v1.ContainerRestartPolicy("Never")}, // 2-3, 6-7 + {request: "4000m", limit: "4000m", restartPolicy: v1.ContainerRestartPolicy("Never")}}, // 0-1, 4-5 + ), + qosClass: v1.PodQOSGuaranteed, + podAllocated: "100m", + resizeLimit: "200m", + resizeRequest: "200m", + containerName: "appContainer-0", + stPromised: state.ContainerCPUAssignments{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + containerName2: "appContainer-1", + expCSetAfterAlloc: cpuset.New(2, 3, 6, 7), + expCSetAfterResize: cpuset.New(2, 3, 6, 7), + expCSetAfterRemove: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + { + staticPolicyTest: staticPolicyTest{ + description: "SingleSocketHT, PodResize, appContainer-0 in shared pool with more than one core, Increase CPU and keep appContainer-0 in shared pool", + topo: topoSingleSocketHT, + pod: makeMultiContainerPodWithOptions( + nil, + []*containerOptions{ + {request: "1100m", limit: "1100m", restartPolicy: v1.ContainerRestartPolicy("Never")}, // 0-7 + {request: "4000m", limit: "4000m", restartPolicy: v1.ContainerRestartPolicy("Never")}}, // 0-1, 4-5 + ), + qosClass: v1.PodQOSGuaranteed, + podAllocated: "1100m", + resizeLimit: "1200m", + resizeRequest: "1200m", + containerName: "appContainer-0", + stPromised: state.ContainerCPUAssignments{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + containerName2: "appContainer-1", + expCSetAfterAlloc: cpuset.New(2, 3, 6, 7), + expCSetAfterResize: cpuset.New(2, 3, 6, 7), + expCSetAfterRemove: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + { + staticPolicyTest: staticPolicyTest{ + description: "SingleSocketHT, PodResize, appContainer-0 in shared pool, appContainer-1 in exclusive pool, Decrease CPU and keep in shared pool", + topo: topoSingleSocketHT, + pod: makeMultiContainerPodWithOptions( + nil, + []*containerOptions{ + {request: "200m", limit: "200m", restartPolicy: v1.ContainerRestartPolicy("Never")}, // 0-7 + {request: "4000m", limit: "4000m", restartPolicy: v1.ContainerRestartPolicy("Never")}}, // 0-1, 4-5 + ), + qosClass: v1.PodQOSGuaranteed, + podAllocated: "200m", + resizeLimit: "100m", + resizeRequest: "100m", + containerName: "appContainer-0", + stPromised: state.ContainerCPUAssignments{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + containerName2: "appContainer-1", + expCSetAfterAlloc: cpuset.New(2, 3, 6, 7), + expCSetAfterResize: cpuset.New(2, 3, 6, 7), + expCSetAfterRemove: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + { + staticPolicyTest: staticPolicyTest{ + description: "SingleSocketHT, PodResize, appContainer-0 in exclusively allocated pool, Move to shared pool", + topo: topoSingleSocketHT, + pod: makeMultiContainerPodWithOptions( + nil, + []*containerOptions{ + {request: "2000m", limit: "2000m", restartPolicy: v1.ContainerRestartPolicy("Never")}, // 0-1, 4-5 + {request: "200m", limit: "200m", restartPolicy: v1.ContainerRestartPolicy("Never")}}, // 0-7 + ), + qosClass: v1.PodQOSGuaranteed, + podAllocated: "2000m", + resizeLimit: "1500m", + resizeRequest: "1500m", + containerName: "appContainer-0", + stPromised: state.ContainerCPUAssignments{}, + stAssignments: state.ContainerCPUAssignments{}, + stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + containerName2: "appContainer-1", + expAllocErr: inconsistentCPUAllocationError{RequestedCPUs: "1500m", AllocatedCPUs: "2", Shared2Exclusive: false}, + expCSetAfterAlloc: cpuset.New(1, 2, 3, 5, 6, 7), + expCSetAfterResize: cpuset.New(1, 2, 3, 5, 6, 7), + expCSetAfterRemove: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), + }, + } + + for _, testCase := range testCases { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.CPUManagerPolicyAlphaOptions, true) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.InPlacePodVerticalScaling, true) + t.Run(testCase.description, func(t *testing.T) { + + policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.New(), topologymanager.NewFakeManager(), nil) + + st := &mockState{ + promised: testCase.stPromised, + assignments: testCase.stAssignments, + defaultCPUSet: testCase.stDefaultCPUSet, + } + pod := testCase.pod + pod.Status.QOSClass = testCase.qosClass + + // allocate + for _, container := range append(pod.Spec.InitContainers, pod.Spec.Containers...) { + err := policy.Allocate(st, pod, &container) + if err != nil { + t.Errorf("StaticPolicy Allocate() error (%v). expected no error but got %v", + testCase.description, err) + } + } + if !reflect.DeepEqual(st.defaultCPUSet, testCase.expCSetAfterAlloc) { + t.Errorf("StaticPolicy Allocate() error (%v) before pod resize. expected default cpuset %v but got %v", + testCase.description, testCase.expCSetAfterAlloc, st.defaultCPUSet) + } + + // resize + pod.Status.ContainerStatuses = []v1.ContainerStatus{ + { + Name: testCase.containerName, + AllocatedResources: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(testCase.podAllocated), + }, + }, + } + pod.Spec.Containers[0].Resources = v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse(testCase.resizeLimit), + }, + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse(testCase.resizeRequest), + }, + } + podResized := pod + for _, container := range append(podResized.Spec.InitContainers, podResized.Spec.Containers...) { + err := policy.Allocate(st, podResized, &container) + if err != nil { + if !reflect.DeepEqual(err, testCase.expAllocErr) { + t.Errorf("StaticPolicy Allocate() error (%v), expected error: %v but got: %v", + testCase.description, testCase.expAllocErr, err) + } + } + } + + if testCase.expCSetAfterResizeSize > 0 { + // expCSetAfterResizeSize is used when testing scale down because allocated CPUs are not deterministic, + // since size of defaultCPUSet is deterministic and also interesection with expected allocation + // should not be nill. < ====== TODO esotsal + if !reflect.DeepEqual(st.defaultCPUSet.Size(), testCase.expCSetAfterResizeSize) { + t.Errorf("StaticPolicy Allocate() error (%v) after pod resize. expected default cpuset size equal to %v but got %v", + testCase.description, testCase.expCSetAfterResizeSize, st.defaultCPUSet.Size()) + } + } else { + if !reflect.DeepEqual(st.defaultCPUSet, testCase.expCSetAfterResize) { + t.Errorf("StaticPolicy Allocate() error (%v) after pod resize. expected default cpuset %v but got %v", + testCase.description, testCase.expCSetAfterResize, st.defaultCPUSet) + } + } + + // remove + err := policy.RemoveContainer(st, string(pod.UID), testCase.containerName) + if err != nil { + t.Errorf("StaticPolicy RemoveContainer() error (%v) after pod resize. expected no error but got %v", + testCase.description, err) + } + err = policy.RemoveContainer(st, string(pod.UID), testCase.containerName2) + if err != nil { + t.Errorf("StaticPolicy RemoveContainer() error (%v) after pod resize. expected no error but got %v", + testCase.description, err) + } + + if !reflect.DeepEqual(st.defaultCPUSet, testCase.expCSetAfterRemove) { + t.Errorf("StaticPolicy RemoveContainer() error (%v) after pod resize. expected default cpuset %v but got %v", + testCase.description, testCase.expCSetAfterRemove, st.defaultCPUSet) + } + if _, found := st.assignments[string(pod.UID)][testCase.containerName]; found { + t.Errorf("StaticPolicy RemoveContainer() error (%v) after pod resize. expected (pod %v, container %v) not be in assignments %v", + testCase.description, testCase.podUID, testCase.containerName, st.assignments) + } + }) + } +} func TestStaticPolicyRemove(t *testing.T) { testCases := []staticPolicyTest{ { @@ -805,6 +1677,11 @@ func TestStaticPolicyRemove(t *testing.T) { topo: topoSingleSocketHT, podUID: "fakePod", containerName: "fakeContainer1", + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer1": cpuset.New(1, 2, 3), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer1": cpuset.New(1, 2, 3), @@ -818,6 +1695,12 @@ func TestStaticPolicyRemove(t *testing.T) { topo: topoSingleSocketHT, podUID: "fakePod", containerName: "fakeContainer1", + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer1": cpuset.New(1, 2, 3), + "fakeContainer2": cpuset.New(4, 5, 6, 7), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer1": cpuset.New(1, 2, 3), @@ -832,6 +1715,12 @@ func TestStaticPolicyRemove(t *testing.T) { topo: topoSingleSocketHT, podUID: "fakePod", containerName: "fakeContainer1", + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer1": cpuset.New(1, 3, 5), + "fakeContainer2": cpuset.New(2, 4), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer1": cpuset.New(1, 3, 5), @@ -846,6 +1735,11 @@ func TestStaticPolicyRemove(t *testing.T) { topo: topoSingleSocketHT, podUID: "fakePod", containerName: "fakeContainer2", + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer1": cpuset.New(1, 3, 5), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer1": cpuset.New(1, 3, 5), @@ -863,6 +1757,7 @@ func TestStaticPolicyRemove(t *testing.T) { } st := &mockState{ + promised: testCase.stAssignments, assignments: testCase.stAssignments, defaultCPUSet: testCase.stDefaultCPUSet, } @@ -885,6 +1780,7 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { testCases := []struct { description string topo *topology.CPUTopology + stPromised state.ContainerCPUAssignments stAssignments state.ContainerCPUAssignments stDefaultCPUSet cpuset.CPUSet numRequested int @@ -894,6 +1790,7 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { { description: "Request 2 CPUs, No BitMask", topo: topoDualSocketHT, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11), numRequested: 2, @@ -903,6 +1800,7 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { { description: "Request 2 CPUs, BitMask on Socket 0", topo: topoDualSocketHT, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11), numRequested: 2, @@ -915,6 +1813,7 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { { description: "Request 2 CPUs, BitMask on Socket 1", topo: topoDualSocketHT, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11), numRequested: 2, @@ -927,6 +1826,7 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { { description: "Request 8 CPUs, BitMask on Socket 0", topo: topoDualSocketHT, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11), numRequested: 8, @@ -939,6 +1839,7 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { { description: "Request 8 CPUs, BitMask on Socket 1", topo: topoDualSocketHT, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11), numRequested: 8, @@ -956,6 +1857,7 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { } policy := p.(*staticPolicy) st := &mockState{ + promised: tc.stPromised, assignments: tc.stAssignments, defaultCPUSet: tc.stDefaultCPUSet, } @@ -965,7 +1867,7 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { continue } - cpuAlloc, err := policy.allocateCPUs(st, tc.numRequested, tc.socketMask, cpuset.New()) + cpuAlloc, err := policy.allocateCPUs(st, tc.numRequested, tc.socketMask, cpuset.New(), nil, nil) if err != nil { t.Errorf("StaticPolicy allocateCPUs() error (%v). expected CPUSet %v not error %v", tc.description, tc.expCSet, err) @@ -987,6 +1889,7 @@ type staticPolicyTestWithResvList struct { numReservedCPUs int reserved cpuset.CPUSet cpuPolicyOptions map[string]string + stPromised state.ContainerCPUAssignments stAssignments state.ContainerCPUAssignments stDefaultCPUSet cpuset.CPUSet pod *v1.Pod @@ -1004,6 +1907,7 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { topo: topoDualSocketHT, numReservedCPUs: 2, reserved: cpuset.New(0, 1), + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(), expCSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11), @@ -1014,6 +1918,7 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { numReservedCPUs: 2, reserved: cpuset.New(0, 1), cpuPolicyOptions: map[string]string{StrictCPUReservationOption: "true"}, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(), expCSet: cpuset.New(2, 3, 4, 5, 6, 7, 8, 9, 10, 11), @@ -1023,6 +1928,7 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { topo: topoDualSocketHT, numReservedCPUs: 2, reserved: cpuset.New(0, 1), + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(2, 3, 4, 5), expErr: fmt.Errorf("not all reserved cpus: \"0-1\" are present in defaultCpuSet: \"2-5\""), @@ -1033,6 +1939,7 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { numReservedCPUs: 2, reserved: cpuset.New(0, 1), cpuPolicyOptions: map[string]string{StrictCPUReservationOption: "true"}, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5), expErr: fmt.Errorf("some of strictly reserved cpus: \"0-1\" are present in defaultCpuSet: \"0-5\""), @@ -1042,6 +1949,7 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { topo: topoDualSocketHT, numReservedCPUs: 1, reserved: cpuset.New(0, 1), + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11), expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"), @@ -1059,6 +1967,7 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { } policy := p.(*staticPolicy) st := &mockState{ + promised: testCase.stAssignments, assignments: testCase.stAssignments, defaultCPUSet: testCase.stDefaultCPUSet, } @@ -1088,6 +1997,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 1, reserved: cpuset.New(0), + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"), @@ -1100,6 +2010,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 2, reserved: cpuset.New(0, 1), + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7), pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"), @@ -1112,6 +2023,11 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 2, reserved: cpuset.New(0, 1), + stPromised: state.ContainerCPUAssignments{ + "fakePod": map[string]cpuset.CPUSet{ + "fakeContainer100": cpuset.New(2, 3, 6, 7), + }, + }, stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.New(2, 3, 6, 7), @@ -1132,6 +2048,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { } st := &mockState{ + promised: testCase.stAssignments, assignments: testCase.stAssignments, defaultCPUSet: testCase.stDefaultCPUSet, } @@ -1190,6 +2107,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { FullPCPUsOnlyOption: "true", PreferAlignByUnCoreCacheOption: "true", }, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, // remove partially used uncores from the available CPUs to simulate fully clean slate stDefaultCPUSet: topoDualSocketSingleNumaPerSocketSMTUncore.CPUDetails.CPUs().Difference( @@ -1219,6 +2137,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { FullPCPUsOnlyOption: "true", PreferAlignByUnCoreCacheOption: "true", }, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, // remove partially used uncores from the available CPUs to simulate fully clean slate stDefaultCPUSet: topoDualSocketSingleNumaPerSocketSMTUncore.CPUDetails.CPUs().Difference( @@ -1249,6 +2168,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { FullPCPUsOnlyOption: "true", PreferAlignByUnCoreCacheOption: "true", }, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, // remove partially used uncores from the available CPUs to simulate fully clean slate stDefaultCPUSet: topoDualSocketSingleNumaPerSocketSMTUncore.CPUDetails.CPUs().Difference( @@ -1279,6 +2199,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { FullPCPUsOnlyOption: "true", PreferAlignByUnCoreCacheOption: "true", }, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, // remove partially used uncores from the available CPUs to simulate fully clean slate stDefaultCPUSet: topoDualSocketSingleNumaPerSocketSMTUncore.CPUDetails.CPUs().Difference( @@ -1311,6 +2232,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { FullPCPUsOnlyOption: "true", PreferAlignByUnCoreCacheOption: "true", }, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: topoDualSocketSingleNumaPerSocketSMTUncore.CPUDetails.CPUs(), pod: WithPodUID( @@ -1334,6 +2256,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { FullPCPUsOnlyOption: "true", PreferAlignByUnCoreCacheOption: "true", }, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: topoSingleSocketSingleNumaPerSocketSMTSmallUncore.CPUDetails.CPUs(), pod: WithPodUID( @@ -1357,6 +2280,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { FullPCPUsOnlyOption: "true", PreferAlignByUnCoreCacheOption: "true", }, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, // Uncore 1 fully allocated stDefaultCPUSet: topoSingleSocketSingleNumaPerSocketSMTSmallUncore.CPUDetails.CPUs().Difference( @@ -1383,6 +2307,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { FullPCPUsOnlyOption: "true", PreferAlignByUnCoreCacheOption: "true", }, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, // Uncore 2, 3, and 5 fully allocated stDefaultCPUSet: topoSingleSocketSingleNumaPerSocketNoSMTSmallUncore.CPUDetails.CPUs().Difference( @@ -1415,6 +2340,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { FullPCPUsOnlyOption: "true", PreferAlignByUnCoreCacheOption: "true", }, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, // uncore 1 fully allocated stDefaultCPUSet: topoSmallDualSocketSingleNumaPerSocketNoSMTUncore.CPUDetails.CPUs().Difference( @@ -1442,6 +2368,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { FullPCPUsOnlyOption: "true", PreferAlignByUnCoreCacheOption: "true", }, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, // uncore 1 fully allocated stDefaultCPUSet: topoSmallDualSocketSingleNumaPerSocketNoSMTUncore.CPUDetails.CPUs().Difference( @@ -1469,6 +2396,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { FullPCPUsOnlyOption: "true", PreferAlignByUnCoreCacheOption: "true", }, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, // 4 cpus allocated from uncore 1 stDefaultCPUSet: topoLargeSingleSocketSingleNumaPerSocketSMTUncore.CPUDetails.CPUs().Difference( @@ -1496,6 +2424,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { FullPCPUsOnlyOption: "true", PreferAlignByUnCoreCacheOption: "true", }, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, // 4 cpus allocated from uncore 1 stDefaultCPUSet: topoLargeSingleSocketSingleNumaPerSocketSMTUncore.CPUDetails.CPUs().Difference( @@ -1523,6 +2452,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { FullPCPUsOnlyOption: "true", PreferAlignByUnCoreCacheOption: "true", }, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: topoDualSocketMultiNumaPerSocketUncore.CPUDetails.CPUs(), pod: WithPodUID( @@ -1546,6 +2476,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { FullPCPUsOnlyOption: "true", PreferAlignByUnCoreCacheOption: "true", }, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: topoDualSocketSubNumaPerSocketHTMonolithicUncore.CPUDetails.CPUs(), pod: WithPodUID( @@ -1571,6 +2502,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { FullPCPUsOnlyOption: "true", PreferAlignByUnCoreCacheOption: "true", }, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, // CPUs 4-7 allocated stDefaultCPUSet: topoSingleSocketSingleNumaPerSocketPCoreHTMonolithicUncore.CPUDetails.CPUs().Difference( @@ -1599,6 +2531,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { FullPCPUsOnlyOption: "true", PreferAlignByUnCoreCacheOption: "true", }, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: topoLargeSingleSocketSingleNumaPerSocketUncore.CPUDetails.CPUs(), pod: WithPodUID( @@ -1622,6 +2555,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { FullPCPUsOnlyOption: "true", PreferAlignByUnCoreCacheOption: "true", }, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, // CPUs 6-9, 12-15, 18-19 allocated stDefaultCPUSet: topoSingleSocketSingleNumaPerSocketUncore.CPUDetails.CPUs().Difference( @@ -1657,6 +2591,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { FullPCPUsOnlyOption: "true", PreferAlignByUnCoreCacheOption: "true", }, + stPromised: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{}, // Every uncore has partially allocated 4 CPUs stDefaultCPUSet: topoSmallSingleSocketSingleNumaPerSocketNoSMTUncore.CPUDetails.CPUs().Difference( @@ -1689,6 +2624,7 @@ func TestStaticPolicyAddWithUncoreAlignment(t *testing.T) { } st := &mockState{ + promised: testCase.stAssignments, assignments: testCase.stAssignments, defaultCPUSet: testCase.stDefaultCPUSet.Difference(testCase.reserved), // ensure the cpumanager invariant } diff --git a/pkg/kubelet/cm/cpumanager/state/checkpoint.go b/pkg/kubelet/cm/cpumanager/state/checkpoint.go index 564c3482c0441..ea12e9be73e95 100644 --- a/pkg/kubelet/cm/cpumanager/state/checkpoint.go +++ b/pkg/kubelet/cm/cpumanager/state/checkpoint.go @@ -30,10 +30,20 @@ import ( var _ checkpointmanager.Checkpoint = &CPUManagerCheckpointV1{} var _ checkpointmanager.Checkpoint = &CPUManagerCheckpointV2{} +var _ checkpointmanager.Checkpoint = &CPUManagerCheckpointV3{} var _ checkpointmanager.Checkpoint = &CPUManagerCheckpoint{} -// CPUManagerCheckpoint struct is used to store cpu/pod assignments in a checkpoint in v2 format +// CPUManagerCheckpoint struct is used to store cpu/pod assignments in a checkpoint in v3 format type CPUManagerCheckpoint struct { + PolicyName string `json:"policyName"` + DefaultCPUSet string `json:"defaultCpuSet"` + Entries map[string]map[string]string `json:"entries,omitempty"` + Promised map[string]map[string]string `json:"promised,omitempty"` + Checksum checksum.Checksum `json:"checksum"` +} + +// CPUManagerCheckpoint struct is used to store cpu/pod assignments in a checkpoint in v2 format +type CPUManagerCheckpointV2 struct { PolicyName string `json:"policyName"` DefaultCPUSet string `json:"defaultCpuSet"` Entries map[string]map[string]string `json:"entries,omitempty"` @@ -48,13 +58,13 @@ type CPUManagerCheckpointV1 struct { Checksum checksum.Checksum `json:"checksum"` } -// CPUManagerCheckpointV2 struct is used to store cpu/pod assignments in a checkpoint in v2 format -type CPUManagerCheckpointV2 = CPUManagerCheckpoint +// CPUManagerCheckpointV3 struct is used to store cpu/pod assignments in a checkpoint in v3 format +type CPUManagerCheckpointV3 = CPUManagerCheckpoint // NewCPUManagerCheckpoint returns an instance of Checkpoint func NewCPUManagerCheckpoint() *CPUManagerCheckpoint { //nolint:staticcheck // unexported-type-in-api user-facing error message - return newCPUManagerCheckpointV2() + return newCPUManagerCheckpointV3() } func newCPUManagerCheckpointV1() *CPUManagerCheckpointV1 { @@ -69,6 +79,13 @@ func newCPUManagerCheckpointV2() *CPUManagerCheckpointV2 { } } +func newCPUManagerCheckpointV3() *CPUManagerCheckpointV3 { + return &CPUManagerCheckpointV3{ + Entries: make(map[string]map[string]string), + Promised: make(map[string]map[string]string), + } +} + // MarshalCheckpoint returns marshalled checkpoint in v1 format func (cp *CPUManagerCheckpointV1) MarshalCheckpoint() ([]byte, error) { // make sure checksum wasn't set before so it doesn't affect output checksum @@ -85,6 +102,14 @@ func (cp *CPUManagerCheckpointV2) MarshalCheckpoint() ([]byte, error) { return json.Marshal(*cp) } +// MarshalCheckpoint returns marshalled checkpoint in v3 format +func (cp *CPUManagerCheckpointV3) MarshalCheckpoint() ([]byte, error) { + // make sure checksum wasn't set before so it doesn't affect output checksum + cp.Checksum = 0 + cp.Checksum = checksum.New(cp) + return json.Marshal(*cp) +} + // UnmarshalCheckpoint tries to unmarshal passed bytes to checkpoint in v1 format func (cp *CPUManagerCheckpointV1) UnmarshalCheckpoint(blob []byte) error { return json.Unmarshal(blob, cp) @@ -95,6 +120,11 @@ func (cp *CPUManagerCheckpointV2) UnmarshalCheckpoint(blob []byte) error { return json.Unmarshal(blob, cp) } +// UnmarshalCheckpoint tries to unmarshal passed bytes to checkpoint in v3 format +func (cp *CPUManagerCheckpointV3) UnmarshalCheckpoint(blob []byte) error { + return json.Unmarshal(blob, cp) +} + // VerifyChecksum verifies that current checksum of checkpoint is valid in v1 format func (cp *CPUManagerCheckpointV1) VerifyChecksum() error { if cp.Checksum == 0 { @@ -109,7 +139,9 @@ func (cp *CPUManagerCheckpointV1) VerifyChecksum() error { cp.Checksum = ck hash := fnv.New32a() - fmt.Fprintf(hash, "%v", object) + if _, err := fmt.Fprintf(hash, "%v", object); err != nil { + return err + } actualCS := checksum.Checksum(hash.Sum32()) if cp.Checksum != actualCS { return &errors.CorruptCheckpointError{ @@ -123,6 +155,33 @@ func (cp *CPUManagerCheckpointV1) VerifyChecksum() error { // VerifyChecksum verifies that current checksum of checkpoint is valid in v2 format func (cp *CPUManagerCheckpointV2) VerifyChecksum() error { + if cp.Checksum == 0 { + // accept empty checksum for compatibility with old file backend + return nil + } + ck := cp.Checksum + cp.Checksum = 0 + object := dump.ForHash(cp) + object = strings.Replace(object, "CPUManagerCheckpointV2", "CPUManagerCheckpoint", 1) + cp.Checksum = ck + + hash := fnv.New32a() + if _, err := fmt.Fprintf(hash, "%v", object); err != nil { + return err + } + actualCS := checksum.Checksum(hash.Sum32()) + if cp.Checksum != actualCS { + return &errors.CorruptCheckpointError{ + ActualCS: uint64(actualCS), + ExpectedCS: uint64(cp.Checksum), + } + } + + return nil +} + +// VerifyChecksum verifies that current checksum of checkpoint is valid in v3 format +func (cp *CPUManagerCheckpointV3) VerifyChecksum() error { if cp.Checksum == 0 { // accept empty checksum for compatibility with old file backend return nil diff --git a/pkg/kubelet/cm/cpumanager/state/state.go b/pkg/kubelet/cm/cpumanager/state/state.go index 352fddfb9cdad..f1196601c1b7d 100644 --- a/pkg/kubelet/cm/cpumanager/state/state.go +++ b/pkg/kubelet/cm/cpumanager/state/state.go @@ -37,16 +37,20 @@ func (as ContainerCPUAssignments) Clone() ContainerCPUAssignments { // Reader interface used to read current cpu/pod assignment state type Reader interface { + GetPromisedCPUSet(podUID string, containerName string) (cpuset.CPUSet, bool) GetCPUSet(podUID string, containerName string) (cpuset.CPUSet, bool) GetDefaultCPUSet() cpuset.CPUSet GetCPUSetOrDefault(podUID string, containerName string) cpuset.CPUSet GetCPUAssignments() ContainerCPUAssignments + GetCPUPromised() ContainerCPUAssignments } type writer interface { + SetPromisedCPUSet(podUID string, containerName string, cpuset cpuset.CPUSet) SetCPUSet(podUID string, containerName string, cpuset cpuset.CPUSet) SetDefaultCPUSet(cpuset cpuset.CPUSet) SetCPUAssignments(ContainerCPUAssignments) + SetCPUPromised(ContainerCPUAssignments) Delete(podUID string, containerName string) ClearState() } diff --git a/pkg/kubelet/cm/cpumanager/state/state_checkpoint.go b/pkg/kubelet/cm/cpumanager/state/state_checkpoint.go index bda90ba1f4ca6..8312ac0e5084a 100644 --- a/pkg/kubelet/cm/cpumanager/state/state_checkpoint.go +++ b/pkg/kubelet/cm/cpumanager/state/state_checkpoint.go @@ -17,13 +17,14 @@ limitations under the License. package state import ( + "errors" "fmt" "path/filepath" "sync" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" - "k8s.io/kubernetes/pkg/kubelet/checkpointmanager/errors" + checkpointerrors "k8s.io/kubernetes/pkg/kubelet/checkpointmanager/errors" "k8s.io/kubernetes/pkg/kubelet/cm/containermap" "k8s.io/utils/cpuset" ) @@ -62,8 +63,8 @@ func NewCheckpointState(stateDir, checkpointName, policyName string, initialCont return stateCheckpoint, nil } -// migrateV1CheckpointToV2Checkpoint() converts checkpoints from the v1 format to the v2 format -func (sc *stateCheckpoint) migrateV1CheckpointToV2Checkpoint(src *CPUManagerCheckpointV1, dst *CPUManagerCheckpointV2) error { +// migrateV1CheckpointToV3Checkpoint() converts checkpoints from the v1 format to the v3 format +func (sc *stateCheckpoint) migrateV1CheckpointToV3Checkpoint(src *CPUManagerCheckpointV1, dst *CPUManagerCheckpointV3) error { if src.PolicyName != "" { dst.PolicyName = src.PolicyName } @@ -82,6 +83,42 @@ func (sc *stateCheckpoint) migrateV1CheckpointToV2Checkpoint(src *CPUManagerChec dst.Entries[podUID] = make(map[string]string) } dst.Entries[podUID][containerName] = cset + if dst.Promised == nil { + dst.Promised = make(map[string]map[string]string) + } + if _, exists := dst.Promised[podUID]; !exists { + dst.Promised[podUID] = make(map[string]string) + } + dst.Promised[podUID][containerName] = cset + } + return nil +} + +// migrateV2CheckpointToV3Checkpoint() converts checkpoints from the v2 format to the v3 format +func (sc *stateCheckpoint) migrateV2CheckpointToV3Checkpoint(src *CPUManagerCheckpointV2, dst *CPUManagerCheckpointV3) error { + if src.PolicyName != "" { + dst.PolicyName = src.PolicyName + } + if src.DefaultCPUSet != "" { + dst.DefaultCPUSet = src.DefaultCPUSet + } + for podUID := range src.Entries { + for containerName, cpuString := range src.Entries[podUID] { + if dst.Entries == nil { + dst.Entries = make(map[string]map[string]string) + } + if _, exists := dst.Entries[podUID]; !exists { + dst.Entries[podUID] = make(map[string]string) + } + dst.Entries[podUID][containerName] = cpuString + if dst.Promised == nil { + dst.Promised = make(map[string]map[string]string) + } + if _, exists := dst.Promised[podUID]; !exists { + dst.Promised[podUID] = make(map[string]string) + } + dst.Promised[podUID][containerName] = cpuString + } } return nil } @@ -94,44 +131,62 @@ func (sc *stateCheckpoint) restoreState() error { checkpointV1 := newCPUManagerCheckpointV1() checkpointV2 := newCPUManagerCheckpointV2() + checkpointV3 := newCPUManagerCheckpointV3() if err = sc.checkpointManager.GetCheckpoint(sc.checkpointName, checkpointV1); err != nil { - checkpointV1 = &CPUManagerCheckpointV1{} // reset it back to 0 if err = sc.checkpointManager.GetCheckpoint(sc.checkpointName, checkpointV2); err != nil { - if err == errors.ErrCheckpointNotFound { - return sc.storeState() + if err = sc.checkpointManager.GetCheckpoint(sc.checkpointName, checkpointV3); err != nil { + if errors.Is(err, checkpointerrors.ErrCheckpointNotFound) { + return sc.storeState() + } + return err + } + } else { + if err = sc.migrateV2CheckpointToV3Checkpoint(checkpointV2, checkpointV3); err != nil { + return fmt.Errorf("error migrating v2 checkpoint state to v3 checkpoint state: %w", err) } - return err + } + } else { + if err = sc.migrateV1CheckpointToV3Checkpoint(checkpointV1, checkpointV3); err != nil { + return fmt.Errorf("error migrating v1 checkpoint state to v2 checkpoint state: %w", err) } } - if err = sc.migrateV1CheckpointToV2Checkpoint(checkpointV1, checkpointV2); err != nil { - return fmt.Errorf("error migrating v1 checkpoint state to v2 checkpoint state: %s", err) - } - - if sc.policyName != checkpointV2.PolicyName { - return fmt.Errorf("configured policy %q differs from state checkpoint policy %q", sc.policyName, checkpointV2.PolicyName) + if sc.policyName != checkpointV3.PolicyName { + return fmt.Errorf("configured policy %q differs from state checkpoint policy %q", sc.policyName, checkpointV3.PolicyName) } var tmpDefaultCPUSet cpuset.CPUSet - if tmpDefaultCPUSet, err = cpuset.Parse(checkpointV2.DefaultCPUSet); err != nil { - return fmt.Errorf("could not parse default cpu set %q: %v", checkpointV2.DefaultCPUSet, err) + if tmpDefaultCPUSet, err = cpuset.Parse(checkpointV3.DefaultCPUSet); err != nil { + return fmt.Errorf("could not parse default cpu set %q: %w", checkpointV3.DefaultCPUSet, err) } var tmpContainerCPUSet cpuset.CPUSet tmpAssignments := ContainerCPUAssignments{} - for pod := range checkpointV2.Entries { - tmpAssignments[pod] = make(map[string]cpuset.CPUSet, len(checkpointV2.Entries[pod])) - for container, cpuString := range checkpointV2.Entries[pod] { + for pod := range checkpointV3.Entries { + tmpAssignments[pod] = make(map[string]cpuset.CPUSet, len(checkpointV3.Entries[pod])) + for container, cpuString := range checkpointV3.Entries[pod] { if tmpContainerCPUSet, err = cpuset.Parse(cpuString); err != nil { - return fmt.Errorf("could not parse cpuset %q for container %q in pod %q: %v", cpuString, container, pod, err) + return fmt.Errorf("could not parse cpuset %q for container %q in pod %q: %w", cpuString, container, pod, err) } tmpAssignments[pod][container] = tmpContainerCPUSet } } + tmpPromised := ContainerCPUAssignments{} + for pod := range checkpointV3.Promised { + tmpPromised[pod] = make(map[string]cpuset.CPUSet, len(checkpointV3.Promised[pod])) + for container, cpuString := range checkpointV3.Promised[pod] { + if tmpContainerCPUSet, err = cpuset.Parse(cpuString); err != nil { + return fmt.Errorf("could not parse cpuset %q for container %q in pod %q: %w", cpuString, container, pod, err) + } + tmpPromised[pod][container] = tmpContainerCPUSet + } + } + sc.cache.SetDefaultCPUSet(tmpDefaultCPUSet) sc.cache.SetCPUAssignments(tmpAssignments) + sc.cache.SetCPUPromised(tmpPromised) klog.V(2).InfoS("State checkpoint: restored state from checkpoint") klog.V(2).InfoS("State checkpoint: defaultCPUSet", "defaultCpuSet", tmpDefaultCPUSet.String()) @@ -153,6 +208,14 @@ func (sc *stateCheckpoint) storeState() error { } } + promised := sc.cache.GetCPUPromised() + for pod := range promised { + checkpoint.Promised[pod] = make(map[string]string, len(promised[pod])) + for container, cset := range promised[pod] { + checkpoint.Promised[pod][container] = cset.String() + } + } + err := sc.checkpointManager.CreateCheckpoint(sc.checkpointName, checkpoint) if err != nil { klog.ErrorS(err, "Failed to save checkpoint") @@ -161,6 +224,15 @@ func (sc *stateCheckpoint) storeState() error { return nil } +// GetPromisedCPUSet returns promised CPU set +func (sc *stateCheckpoint) GetPromisedCPUSet(podUID string, containerName string) (cpuset.CPUSet, bool) { + sc.mux.RLock() + defer sc.mux.RUnlock() + + res, ok := sc.cache.GetPromisedCPUSet(podUID, containerName) + return res, ok +} + // GetCPUSet returns current CPU set func (sc *stateCheckpoint) GetCPUSet(podUID string, containerName string) (cpuset.CPUSet, bool) { sc.mux.RLock() @@ -194,6 +266,25 @@ func (sc *stateCheckpoint) GetCPUAssignments() ContainerCPUAssignments { return sc.cache.GetCPUAssignments() } +// GetCPUPromised returns current CPU to pod promised +func (sc *stateCheckpoint) GetCPUPromised() ContainerCPUAssignments { + sc.mux.RLock() + defer sc.mux.RUnlock() + + return sc.cache.GetCPUPromised() +} + +// SetPromisedCPUSet sets CPU set +func (sc *stateCheckpoint) SetPromisedCPUSet(podUID string, containerName string, cset cpuset.CPUSet) { + sc.mux.Lock() + defer sc.mux.Unlock() + sc.cache.SetPromisedCPUSet(podUID, containerName, cset) + err := sc.storeState() + if err != nil { + klog.ErrorS(err, "Failed to store state to checkpoint", "podUID", podUID, "containerName", containerName) + } +} + // SetCPUSet sets CPU set func (sc *stateCheckpoint) SetCPUSet(podUID string, containerName string, cset cpuset.CPUSet) { sc.mux.Lock() @@ -227,6 +318,17 @@ func (sc *stateCheckpoint) SetCPUAssignments(a ContainerCPUAssignments) { } } +// SetCPUPromised sets CPU to pod promised +func (sc *stateCheckpoint) SetCPUPromised(a ContainerCPUAssignments) { + sc.mux.Lock() + defer sc.mux.Unlock() + sc.cache.SetCPUPromised(a) + err := sc.storeState() + if err != nil { + klog.ErrorS(err, "Failed to store state to checkpoint") + } +} + // Delete deletes assignment for specified pod func (sc *stateCheckpoint) Delete(podUID string, containerName string) { sc.mux.Lock() diff --git a/pkg/kubelet/cm/cpumanager/state/state_checkpoint_test.go b/pkg/kubelet/cm/cpumanager/state/state_checkpoint_test.go index 3cb0ab9eb0fb8..1f14bc6b669cd 100644 --- a/pkg/kubelet/cm/cpumanager/state/state_checkpoint_test.go +++ b/pkg/kubelet/cm/cpumanager/state/state_checkpoint_test.go @@ -365,6 +365,7 @@ func TestCheckpointStateHelpers(t *testing.T) { } } } + }) } } diff --git a/pkg/kubelet/cm/cpumanager/state/state_mem.go b/pkg/kubelet/cm/cpumanager/state/state_mem.go index cb01ea92609b0..c47ad1daaa84b 100644 --- a/pkg/kubelet/cm/cpumanager/state/state_mem.go +++ b/pkg/kubelet/cm/cpumanager/state/state_mem.go @@ -25,6 +25,7 @@ import ( type stateMemory struct { sync.RWMutex + promised ContainerCPUAssignments assignments ContainerCPUAssignments defaultCPUSet cpuset.CPUSet } @@ -35,11 +36,20 @@ var _ State = &stateMemory{} func NewMemoryState() State { klog.InfoS("Initialized new in-memory state store") return &stateMemory{ + promised: ContainerCPUAssignments{}, assignments: ContainerCPUAssignments{}, defaultCPUSet: cpuset.New(), } } +func (s *stateMemory) GetPromisedCPUSet(podUID string, containerName string) (cpuset.CPUSet, bool) { + s.RLock() + defer s.RUnlock() + + res, ok := s.promised[podUID][containerName] + return res.Clone(), ok +} + func (s *stateMemory) GetCPUSet(podUID string, containerName string) (cpuset.CPUSet, bool) { s.RLock() defer s.RUnlock() @@ -62,12 +72,30 @@ func (s *stateMemory) GetCPUSetOrDefault(podUID string, containerName string) cp return s.GetDefaultCPUSet() } +func (s *stateMemory) GetCPUPromised() ContainerCPUAssignments { + s.RLock() + defer s.RUnlock() + return s.promised.Clone() +} + func (s *stateMemory) GetCPUAssignments() ContainerCPUAssignments { s.RLock() defer s.RUnlock() return s.assignments.Clone() } +func (s *stateMemory) SetPromisedCPUSet(podUID string, containerName string, cset cpuset.CPUSet) { + s.Lock() + defer s.Unlock() + + if _, ok := s.promised[podUID]; !ok { + s.promised[podUID] = make(map[string]cpuset.CPUSet) + } + + s.promised[podUID][containerName] = cset + klog.InfoS("Updated promised CPUSet", "podUID", podUID, "containerName", containerName, "cpuSet", cset) +} + func (s *stateMemory) SetCPUSet(podUID string, containerName string, cset cpuset.CPUSet) { s.Lock() defer s.Unlock() @@ -88,6 +116,14 @@ func (s *stateMemory) SetDefaultCPUSet(cset cpuset.CPUSet) { klog.InfoS("Updated default CPUSet", "cpuSet", cset) } +func (s *stateMemory) SetCPUPromised(a ContainerCPUAssignments) { + s.Lock() + defer s.Unlock() + + s.promised = a.Clone() + klog.InfoS("Updated CPUSet promised", "promised", a) +} + func (s *stateMemory) SetCPUAssignments(a ContainerCPUAssignments) { s.Lock() defer s.Unlock() @@ -105,6 +141,13 @@ func (s *stateMemory) Delete(podUID string, containerName string) { delete(s.assignments, podUID) } klog.V(2).InfoS("Deleted CPUSet assignment", "podUID", podUID, "containerName", containerName) + + delete(s.promised[podUID], containerName) + if len(s.promised[podUID]) == 0 { + delete(s.promised, podUID) + } + klog.V(2).InfoS("Deleted CPUSet promised", "podUID", podUID, "containerName", containerName) + } func (s *stateMemory) ClearState() { @@ -113,5 +156,6 @@ func (s *stateMemory) ClearState() { s.defaultCPUSet = cpuset.CPUSet{} s.assignments = make(ContainerCPUAssignments) + s.promised = make(ContainerCPUAssignments) klog.V(2).InfoS("Cleared state") } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 71068f0e7ed3f..90e912262517e 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -2861,6 +2861,31 @@ func (kl *Kubelet) HandlePodSyncs(pods []*v1.Pod) { } } +func (kl *Kubelet) getMustKeepCPUs(pod *v1.Pod) error { + for i, container := range pod.Spec.Containers { + Command := strings.Split("cat /tmp/mustKeepCPUs", " ") + ctx := context.Background() + output, er := kl.RunInContainer( + ctx, + kubecontainer.GetPodFullName(pod), + pod.UID, + container.Name, + Command) + if er != nil { + klog.InfoS("Allocate: RunInContainer run error", "err", er) + continue + } + str := string(output) + str = strings.ReplaceAll(str, "<", "") + str = strings.ReplaceAll(str, ">", "") + str = strings.ReplaceAll(str, "\t", " ") + str = strings.ReplaceAll(str, "\n", "") + pod.Spec.Containers[i].Resources.MustKeepCPUs = str + klog.InfoS("RunInContainer", "kubecontainer.GetPodFullName(pod)", kubecontainer.GetPodFullName(pod), "container.Name", container.Name, "output:%s", output, "container.Resources.MustKeepCPUs", container.Resources.MustKeepCPUs) + } + return nil +} + // canResizePod determines if the requested resize is currently feasible. // pod should hold the desired (pre-allocated) spec. // Returns true if the resize can proceed; returns a reason and message @@ -2882,6 +2907,12 @@ func (kl *Kubelet) canResizePod(pod *v1.Pod) (bool, string, string) { } } + if v1qos.GetPodQOS(pod) == v1.PodQOSGuaranteed && utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScalingExclusiveCPUs) { + if kl.containerManager.GetNodeConfig().CPUManagerPolicy == "static" { + kl.getMustKeepCPUs(pod) + } + } + node, err := kl.getNodeAnyWay() if err != nil { klog.ErrorS(err, "getNodeAnyway function failed") diff --git a/pkg/kubelet/types/constants.go b/pkg/kubelet/types/constants.go index 791052dbbcece..afabc00c36ed5 100644 --- a/pkg/kubelet/types/constants.go +++ b/pkg/kubelet/types/constants.go @@ -38,3 +38,11 @@ const ( LimitedSwap SwapBehavior = "LimitedSwap" NoSwap SwapBehavior = "NoSwap" ) + +// InPlacePodVerticalScaling types +const ( + ErrorInconsistentCPUAllocation = "inconsistentCPUAllocationError" + ErrorProhibitedCPUAllocation = "prohibitedCPUAllocationError" + ErrorGetPromisedCPUSet = "getPromisedCPUSetError" + ErrorGetCPUSet = "getCPUSetError" +) diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index f7641e485aaf0..6d4374d164256 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -2707,6 +2707,8 @@ type ResourceRequirements struct { // +featureGate=DynamicResourceAllocation // +optional Claims []ResourceClaim `json:"claims,omitempty" protobuf:"bytes,3,opt,name=claims"` + // Add mustKeepCPUs + MustKeepCPUs string } // VolumeResourceRequirements describes the storage resource requirements for a volume. diff --git a/test/e2e/framework/pod/resize.go b/test/e2e/framework/pod/resize.go index af8d43ac34a07..b391f4b6b31e4 100644 --- a/test/e2e/framework/pod/resize.go +++ b/test/e2e/framework/pod/resize.go @@ -24,6 +24,9 @@ import ( "strconv" "strings" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,22 +37,19 @@ import ( kubeqos "k8s.io/kubernetes/pkg/kubelet/qos" "k8s.io/kubernetes/test/e2e/framework" imageutils "k8s.io/kubernetes/test/utils/image" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" + "k8s.io/utils/cpuset" ) const ( - CgroupCPUPeriod string = "/sys/fs/cgroup/cpu/cpu.cfs_period_us" - CgroupCPUShares string = "/sys/fs/cgroup/cpu/cpu.shares" - CgroupCPUQuota string = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us" - CgroupMemLimit string = "/sys/fs/cgroup/memory/memory.limit_in_bytes" - Cgroupv2MemLimit string = "/sys/fs/cgroup/memory.max" - Cgroupv2MemRequest string = "/sys/fs/cgroup/memory.min" - Cgroupv2CPULimit string = "/sys/fs/cgroup/cpu.max" - Cgroupv2CPURequest string = "/sys/fs/cgroup/cpu.weight" - CPUPeriod string = "100000" - MinContainerRuntimeVersion string = "1.6.9" + CgroupCPUPeriod string = "/sys/fs/cgroup/cpu/cpu.cfs_period_us" + CgroupCPUShares string = "/sys/fs/cgroup/cpu/cpu.shares" + CgroupCPUQuota string = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us" + CgroupMemLimit string = "/sys/fs/cgroup/memory/memory.limit_in_bytes" + Cgroupv2MemLimit string = "/sys/fs/cgroup/memory.max" + Cgroupv2MemRequest string = "/sys/fs/cgroup/memory.min" + Cgroupv2CPULimit string = "/sys/fs/cgroup/cpu.max" + Cgroupv2CPURequest string = "/sys/fs/cgroup/cpu.weight" + CPUPeriod string = "100000" ) var ( @@ -101,13 +101,15 @@ func (cr *ContainerResources) ResourceRequirements() *v1.ResourceRequirements { } type ResizableContainerInfo struct { - Name string - Resources *ContainerResources - CPUPolicy *v1.ResourceResizeRestartPolicy - MemPolicy *v1.ResourceResizeRestartPolicy - RestartCount int32 - RestartPolicy v1.ContainerRestartPolicy - InitCtr bool + Name string + Resources *ContainerResources + CPUPolicy *v1.ResourceResizeRestartPolicy + MemPolicy *v1.ResourceResizeRestartPolicy + RestartCount int32 + RestartPolicy v1.ContainerRestartPolicy + InitCtr bool + CPUsAllowedListValue string + CPUsAllowedList string } type containerPatch struct { @@ -535,3 +537,40 @@ func formatErrors(err error) error { } return fmt.Errorf("[\n%s\n]", strings.Join(errStrings, ",\n")) } + +func VerifyPodContainersCPUsAllowedListValue(f *framework.Framework, pod *v1.Pod, wantCtrs []ResizableContainerInfo) error { + ginkgo.GinkgoHelper() + verifyCPUsAllowedListValue := func(cName, expectedCPUsAllowedListValue string, expectedCPUsAllowedList string) error { + mycmd := "grep Cpus_allowed_list /proc/self/status | cut -f2" + calValue, _, err := ExecCommandInContainerWithFullOutput(f, pod.Name, cName, "/bin/sh", "-c", mycmd) + framework.Logf("Namespace %s Pod %s Container %s - looking for Cpus allowed list value %s in /proc/self/status", + pod.Namespace, pod.Name, cName, calValue) + if err != nil { + return fmt.Errorf("failed to find expected value '%s' in container '%s' Cpus allowed list '/proc/self/status'", cName, expectedCPUsAllowedListValue) + } + c, err := cpuset.Parse(calValue) + framework.ExpectNoError(err, "failed parsing Cpus allowed list for container %s in pod %s", cName, pod.Name) + cpuTotalValue := strconv.Itoa(c.Size()) + if cpuTotalValue != expectedCPUsAllowedListValue { + return fmt.Errorf("container '%s' cgroup value '%s' results to total CPUs '%s' not equal to expected '%s'", cName, calValue, cpuTotalValue, expectedCPUsAllowedListValue) + } + if expectedCPUsAllowedList != "" { + cExpected, err := cpuset.Parse(expectedCPUsAllowedList) + framework.ExpectNoError(err, "failed parsing Cpus allowed list for cexpectedCPUset") + if !c.Equals(cExpected) { + return fmt.Errorf("container '%s' cgroup value '%s' results to total CPUs '%v' not equal to expected '%v'", cName, calValue, c, cExpected) + } + } + return nil + } + for _, ci := range wantCtrs { + if ci.CPUsAllowedListValue == "" { + continue + } + err := verifyCPUsAllowedListValue(ci.Name, ci.CPUsAllowedListValue, ci.CPUsAllowedList) + if err != nil { + return err + } + } + return nil +} diff --git a/test/e2e_node/cpu_manager_metrics_test.go b/test/e2e_node/cpu_manager_metrics_test.go index 26e5a428d4387..68809ae9f6743 100644 --- a/test/e2e_node/cpu_manager_metrics_test.go +++ b/test/e2e_node/cpu_manager_metrics_test.go @@ -104,6 +104,8 @@ var _ = SIGDescribe("CPU Manager Metrics", framework.WithSerial(), feature.CPUMa enableCPUManagerOptions: true, options: cpuPolicyOptions, }, + false, + false, ) updateKubeletConfig(ctx, f, newCfg, true) }) @@ -403,6 +405,8 @@ var _ = SIGDescribe("CPU Manager Metrics", framework.WithSerial(), feature.CPUMa enableCPUManagerOptions: true, options: cpuPolicyOptions, }, + false, + false, ) updateKubeletConfig(ctx, f, newCfg, true) @@ -444,6 +448,8 @@ var _ = SIGDescribe("CPU Manager Metrics", framework.WithSerial(), feature.CPUMa enableCPUManagerOptions: true, options: cpuPolicyOptions, }, + false, + false, ) updateKubeletConfig(ctx, f, newCfg, true) diff --git a/test/e2e_node/cpu_manager_test.go b/test/e2e_node/cpu_manager_test.go index f8aa404ef0b7c..f468ba83a8692 100644 --- a/test/e2e_node/cpu_manager_test.go +++ b/test/e2e_node/cpu_manager_test.go @@ -297,7 +297,7 @@ type cpuManagerKubeletArguments struct { options map[string]string } -func configureCPUManagerInKubelet(oldCfg *kubeletconfig.KubeletConfiguration, kubeletArguments *cpuManagerKubeletArguments) *kubeletconfig.KubeletConfiguration { +func configureCPUManagerInKubelet(oldCfg *kubeletconfig.KubeletConfiguration, kubeletArguments *cpuManagerKubeletArguments, isInPlacePodVerticalScalingAllocatedStatusEnabled bool, isInPlacePodVerticalScalingExclusiveCPUsEnabled bool) *kubeletconfig.KubeletConfiguration { newCfg := oldCfg.DeepCopy() if newCfg.FeatureGates == nil { newCfg.FeatureGates = make(map[string]bool) @@ -306,6 +306,8 @@ func configureCPUManagerInKubelet(oldCfg *kubeletconfig.KubeletConfiguration, ku newCfg.FeatureGates["CPUManagerPolicyBetaOptions"] = kubeletArguments.enableCPUManagerOptions newCfg.FeatureGates["CPUManagerPolicyAlphaOptions"] = kubeletArguments.enableCPUManagerOptions newCfg.FeatureGates["DisableCPUQuotaWithExclusiveCPUs"] = kubeletArguments.disableCPUQuotaWithExclusiveCPUs + newCfg.FeatureGates["InPlacePodVerticalScalingExclusiveCPUs"] = isInPlacePodVerticalScalingExclusiveCPUsEnabled + newCfg.FeatureGates["InPlacePodVerticalScalingAllocatedStatus"] = isInPlacePodVerticalScalingAllocatedStatusEnabled newCfg.CPUManagerPolicy = kubeletArguments.policyName newCfg.CPUManagerReconcilePeriod = metav1.Duration{Duration: 1 * time.Second} @@ -893,7 +895,7 @@ func runCPUManagerTests(f *framework.Framework) { newCfg := configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ policyName: string(cpumanager.PolicyStatic), reservedSystemCPUs: cpuset.CPUSet{}, - }) + }, false, false) updateKubeletConfig(ctx, f, newCfg, true) ginkgo.By("running a non-Gu pod") @@ -932,10 +934,14 @@ func runCPUManagerTests(f *framework.Framework) { } reservedSystemCPUs := cpuset.New(0) - newCfg := configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ - policyName: string(cpumanager.PolicyStatic), - reservedSystemCPUs: reservedSystemCPUs, - }) + newCfg := configureCPUManagerInKubelet(oldCfg, + &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + reservedSystemCPUs: reservedSystemCPUs, + }, + false, + false, + ) updateKubeletConfig(ctx, f, newCfg, true) ginkgo.By("running a Gu pod - it shouldn't use reserved system CPUs") @@ -958,12 +964,16 @@ func runCPUManagerTests(f *framework.Framework) { cpuPolicyOptions := map[string]string{ cpumanager.StrictCPUReservationOption: "true", } - newCfg := configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ - policyName: string(cpumanager.PolicyStatic), - reservedSystemCPUs: reservedSystemCPUs, - enableCPUManagerOptions: true, - options: cpuPolicyOptions, - }) + newCfg := configureCPUManagerInKubelet(oldCfg, + &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + reservedSystemCPUs: reservedSystemCPUs, + enableCPUManagerOptions: true, + options: cpuPolicyOptions, + }, + false, + false, + ) updateKubeletConfig(ctx, f, newCfg, true) ginkgo.By("running a Gu pod - it shouldn't use reserved system CPUs") @@ -1002,7 +1012,7 @@ func runCPUManagerTests(f *framework.Framework) { reservedSystemCPUs: cpuset.New(0), enableCPUManagerOptions: true, options: cpuPolicyOptions, - }, + }, false, false, ) updateKubeletConfig(ctx, f, newCfg, true) @@ -1040,6 +1050,8 @@ func runCPUManagerTests(f *framework.Framework) { enableCPUManagerOptions: true, options: cpuPolicyOptions, }, + false, + false, ) updateKubeletConfig(ctx, f, newCfg, true) @@ -1062,6 +1074,8 @@ func runCPUManagerTests(f *framework.Framework) { reservedSystemCPUs: cpuset.New(0), disableCPUQuotaWithExclusiveCPUs: true, }, + false, + false, ) updateKubeletConfig(ctx, f, newCfg, true) @@ -1083,6 +1097,8 @@ func runCPUManagerTests(f *framework.Framework) { reservedSystemCPUs: cpuset.New(0), disableCPUQuotaWithExclusiveCPUs: false, }, + false, + false, ) updateKubeletConfig(ctx, f, newCfg, true) @@ -1100,10 +1116,14 @@ func runCPUManagerTests(f *framework.Framework) { } // Enable CPU Manager in the kubelet. - newCfg := configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ - policyName: string(cpumanager.PolicyStatic), - reservedSystemCPUs: cpuset.CPUSet{}, - }) + newCfg := configureCPUManagerInKubelet(oldCfg, + &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + reservedSystemCPUs: cpuset.CPUSet{}, + }, + false, + false, + ) updateKubeletConfig(ctx, f, newCfg, true) ginkgo.By("running a Gu pod with a regular init container and a restartable init container") @@ -1190,6 +1210,8 @@ func runCPUManagerTests(f *framework.Framework) { enableCPUManagerOptions: true, options: cpuPolicyOptions, }, + false, + false, ) updateKubeletConfig(ctx, f, newCfg, true) @@ -1264,6 +1286,8 @@ func runCPUManagerTests(f *framework.Framework) { enableCPUManagerOptions: true, options: cpuPolicyOptions, }, + false, + false, ) updateKubeletConfig(ctx, f, newCfg, true) // 'distribute-cpus-across-numa' policy option ensures that CPU allocations are evenly distributed diff --git a/test/e2e_node/pod_resize_test.go b/test/e2e_node/pod_resize_test.go new file mode 100644 index 0000000000000..f2458e873766f --- /dev/null +++ b/test/e2e_node/pod_resize_test.go @@ -0,0 +1,2504 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2enode + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + //"strings" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" + clientset "k8s.io/client-go/kubernetes" + kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager" + "k8s.io/kubernetes/test/e2e/framework" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + testutils "k8s.io/kubernetes/test/utils" + admissionapi "k8s.io/pod-security-admission/api" + "k8s.io/utils/cpuset" +) + +const ( + fakeExtendedResource = "dummy.com/dummy" +) + +func patchNode(ctx context.Context, client clientset.Interface, old *v1.Node, new *v1.Node) error { + oldData, err := json.Marshal(old) + if err != nil { + return err + } + + newData, err := json.Marshal(new) + if err != nil { + return err + } + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, &v1.Node{}) + if err != nil { + return fmt.Errorf("failed to create merge patch for node %q: %w", old.Name, err) + } + _, err = client.CoreV1().Nodes().Patch(ctx, old.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status") + return err +} + +func addExtendedResource(clientSet clientset.Interface, nodeName, extendedResourceName string, extendedResourceQuantity resource.Quantity) { + extendedResource := v1.ResourceName(extendedResourceName) + + ginkgo.By("Adding a custom resource") + OriginalNode, err := clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + + node := OriginalNode.DeepCopy() + node.Status.Capacity[extendedResource] = extendedResourceQuantity + node.Status.Allocatable[extendedResource] = extendedResourceQuantity + err = patchNode(context.Background(), clientSet, OriginalNode.DeepCopy(), node) + framework.ExpectNoError(err) + + gomega.Eventually(func() error { + node, err = clientSet.CoreV1().Nodes().Get(context.Background(), node.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + fakeResourceCapacity, exists := node.Status.Capacity[extendedResource] + if !exists { + return fmt.Errorf("node %s has no %s resource capacity", node.Name, extendedResourceName) + } + if expectedResource := resource.MustParse("123"); fakeResourceCapacity.Cmp(expectedResource) != 0 { + return fmt.Errorf("node %s has resource capacity %s, expected: %s", node.Name, fakeResourceCapacity.String(), expectedResource.String()) + } + + return nil + }).WithTimeout(30 * time.Second).WithPolling(time.Second).ShouldNot(gomega.HaveOccurred()) +} + +func removeExtendedResource(clientSet clientset.Interface, nodeName, extendedResourceName string) { + extendedResource := v1.ResourceName(extendedResourceName) + + ginkgo.By("Removing a custom resource") + originalNode, err := clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + + node := originalNode.DeepCopy() + delete(node.Status.Capacity, extendedResource) + delete(node.Status.Allocatable, extendedResource) + err = patchNode(context.Background(), clientSet, originalNode.DeepCopy(), node) + framework.ExpectNoError(err) + + gomega.Eventually(func() error { + node, err = clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + + if _, exists := node.Status.Capacity[extendedResource]; exists { + return fmt.Errorf("node %s has resource capacity %s which is expected to be removed", node.Name, extendedResourceName) + } + + return nil + }).WithTimeout(30 * time.Second).WithPolling(time.Second).ShouldNot(gomega.HaveOccurred()) +} + +func cpuManagerPolicyKubeletConfig(ctx context.Context, f *framework.Framework, oldCfg *kubeletconfig.KubeletConfiguration, cpuManagerPolicyName string, cpuManagerPolicyOptions map[string]string, isInPlacePodVerticalScalingAllocatedStatusEnabled bool, isInPlacePodVerticalScalingExclusiveCPUsEnabled bool) { + if cpuManagerPolicyName != "" { + if cpuManagerPolicyOptions != nil { + func() { + var cpuAlloc int64 + for policyOption, policyOptionValue := range cpuManagerPolicyOptions { + if policyOption == cpumanager.FullPCPUsOnlyOption && policyOptionValue == "true" { + _, cpuAlloc, _ = getLocalNodeCPUDetails(ctx, f) + smtLevel := getSMTLevel() + + // strict SMT alignment is trivially verified and granted on non-SMT systems + if smtLevel < 2 { + e2eskipper.Skipf("Skipping Pod Resize along side CPU Manager %s tests since SMT disabled", policyOption) + } + + // our tests want to allocate a full core, so we need at last 2*2=4 virtual cpus + if cpuAlloc < int64(smtLevel*2) { + e2eskipper.Skipf("Skipping Pod resize along side CPU Manager %s tests since the CPU capacity < 4", policyOption) + } + + framework.Logf("SMT level %d", smtLevel) + return + } + } + }() + + // TODO: we assume the first available CPUID is 0, which is pretty fair, but we should probably + // check what we do have in the node. + newCfg := configureCPUManagerInKubelet(oldCfg, + &cpuManagerKubeletArguments{ + policyName: cpuManagerPolicyName, + reservedSystemCPUs: cpuset.New(0), + enableCPUManagerOptions: true, + options: cpuManagerPolicyOptions, + }, + isInPlacePodVerticalScalingAllocatedStatusEnabled, + isInPlacePodVerticalScalingExclusiveCPUsEnabled, + ) + updateKubeletConfig(ctx, f, newCfg, true) + } else { + var cpuCap int64 + cpuCap, _, _ = getLocalNodeCPUDetails(ctx, f) + // Skip CPU Manager tests altogether if the CPU capacity < 2. + if cpuCap < 2 { + e2eskipper.Skipf("Skipping Pod Resize alongside CPU Manager tests since the CPU capacity < 2") + } + // Enable CPU Manager in the kubelet. + newCfg := configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: cpuManagerPolicyName, + reservedSystemCPUs: cpuset.CPUSet{}, + }, isInPlacePodVerticalScalingAllocatedStatusEnabled, isInPlacePodVerticalScalingExclusiveCPUsEnabled) + updateKubeletConfig(ctx, f, newCfg, true) + } + } +} + +type cpuManagerPolicyConfig struct { + name string + title string + options map[string]string +} + +func doPodResizeTests(policy cpuManagerPolicyConfig, isInPlacePodVerticalScalingAllocatedStatusEnabled bool, isInPlacePodVerticalScalingExclusiveCPUsEnabled bool) { + f := framework.NewDefaultFramework("pod-resize-test") + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + var podClient *e2epod.PodClient + var oldCfg *kubeletconfig.KubeletConfiguration + ginkgo.BeforeEach(func(ctx context.Context) { + var err error + node := getLocalNode(ctx, f) + if framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) { + e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping") + } + podClient = e2epod.NewPodClient(f) + if oldCfg == nil { + oldCfg, err = getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + } + }) + + type testCase struct { + name string + containers []e2epod.ResizableContainerInfo + patchString string + expected []e2epod.ResizableContainerInfo + addExtendedResource bool + } + + noRestart := v1.NotRequired + doRestart := v1.RestartContainer + tests := []testCase{ + { + name: "Guaranteed QoS pod, one container - increase CPU & memory", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"200m","memory":"400Mi"},"limits":{"cpu":"200m","memory":"400Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "400Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + }, + }, + { + name: "Guaranteed QoS pod, one container - decrease CPU & memory", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "300m", CPULim: "300m", MemReq: "500Mi", MemLim: "500Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"100m","memory":"250Mi"},"limits":{"cpu":"100m","memory":"250Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "250Mi", MemLim: "250Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + }, + }, + { + name: "Guaranteed QoS pod, one container - increase CPU & decrease memory", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"200m","memory":"100Mi"},"limits":{"cpu":"200m","memory":"100Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "100Mi", MemLim: "100Mi"}, + }, + }, + }, + { + name: "Guaranteed QoS pod, one container - decrease CPU & increase memory", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"50m","memory":"300Mi"},"limits":{"cpu":"50m","memory":"300Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "50m", CPULim: "50m", MemReq: "300Mi", MemLim: "300Mi"}, + }, + }, + }, + { + name: "Guaranteed QoS pod, three containers (c1, c2, c3) - increase: CPU (c1,c3), memory (c2) ; decrease: CPU (c2), memory (c1,c3)", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + { + Name: "c3", + Resources: &e2epod.ContainerResources{CPUReq: "300m", CPULim: "300m", MemReq: "300Mi", MemLim: "300Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"140m","memory":"50Mi"},"limits":{"cpu":"140m","memory":"50Mi"}}}, + {"name":"c2", "resources":{"requests":{"cpu":"150m","memory":"240Mi"},"limits":{"cpu":"150m","memory":"240Mi"}}}, + {"name":"c3", "resources":{"requests":{"cpu":"340m","memory":"250Mi"},"limits":{"cpu":"340m","memory":"250Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "140m", CPULim: "140m", MemReq: "50Mi", MemLim: "50Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "150m", CPULim: "150m", MemReq: "240Mi", MemLim: "240Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + { + Name: "c3", + Resources: &e2epod.ContainerResources{CPUReq: "340m", CPULim: "340m", MemReq: "250Mi", MemLim: "250Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease memory requests only", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"memory":"200Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "200Mi", MemLim: "500Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease memory limits only", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"limits":{"memory":"400Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "400Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase memory requests only", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"memory":"300Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "300Mi", MemLim: "500Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase memory limits only", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"limits":{"memory":"600Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "600Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease CPU requests only", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"100m"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease CPU limits only", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"limits":{"cpu":"300m"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase CPU requests only", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"150m"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "150m", CPULim: "200m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase CPU limits only", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"limits":{"cpu":"500m"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "500m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease CPU requests and limits", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"100m"},"limits":{"cpu":"200m"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase CPU requests and limits", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"200m"},"limits":{"cpu":"400m"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease CPU requests and increase CPU limits", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"100m"},"limits":{"cpu":"500m"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "500m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase CPU requests and decrease CPU limits", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"200m"},"limits":{"cpu":"300m"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "250Mi", MemLim: "500Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease memory requests and limits", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"memory":"100Mi"},"limits":{"memory":"300Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "300Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase memory requests and limits", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"memory":"300Mi"},"limits":{"memory":"500Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "300Mi", MemLim: "500Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease memory requests and increase memory limits", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"memory":"100Mi"},"limits":{"memory":"500Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "500Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase memory requests and decrease memory limits", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"memory":"300Mi"},"limits":{"memory":"300Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "300Mi", MemLim: "300Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease CPU requests and increase memory limits", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "200Mi", MemLim: "400Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"100m"},"limits":{"memory":"500Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "400m", MemReq: "200Mi", MemLim: "500Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase CPU requests and decrease memory limits", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "400m", MemReq: "200Mi", MemLim: "500Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"200m"},"limits":{"memory":"400Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "200Mi", MemLim: "400Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease memory requests and increase CPU limits", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"memory":"100Mi"},"limits":{"cpu":"300m"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "300m", MemReq: "100Mi", MemLim: "400Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase memory requests and decrease CPU limits", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "200Mi", MemLim: "400Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"memory":"300Mi"},"limits":{"cpu":"300m"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "300Mi", MemLim: "400Mi"}, + }, + }, + }, + { + name: "Burstable QoS pod, one container with cpu & memory requests - decrease memory request", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", MemReq: "500Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"memory":"400Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", MemReq: "400Mi"}, + }, + }, + }, + { + name: "Guaranteed QoS pod, one container - increase CPU (NotRequired) & memory (RestartContainer)", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &doRestart, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"200m","memory":"400Mi"},"limits":{"cpu":"200m","memory":"400Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "400Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &doRestart, + RestartCount: 1, + }, + }, + }, + { + name: "Burstable QoS pod, one container - decrease CPU (RestartContainer) & memory (NotRequired)", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, + CPUPolicy: &doRestart, + MemPolicy: &noRestart, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"50m","memory":"100Mi"},"limits":{"cpu":"100m","memory":"200Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "50m", CPULim: "100m", MemReq: "100Mi", MemLim: "200Mi"}, + CPUPolicy: &doRestart, + MemPolicy: &noRestart, + RestartCount: 1, + }, + }, + }, + { + name: "Burstable QoS pod, three containers - increase c1 resources, no change for c2, decrease c3 resources (no net change for pod)", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "200Mi", MemLim: "300Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &doRestart, + }, + { + Name: "c3", + Resources: &e2epod.ContainerResources{CPUReq: "300m", CPULim: "400m", MemReq: "300Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"150m","memory":"150Mi"},"limits":{"cpu":"250m","memory":"250Mi"}}}, + {"name":"c3", "resources":{"requests":{"cpu":"250m","memory":"250Mi"},"limits":{"cpu":"350m","memory":"350Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "150m", CPULim: "250m", MemReq: "150Mi", MemLim: "250Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "200Mi", MemLim: "300Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &doRestart, + }, + { + Name: "c3", + Resources: &e2epod.ContainerResources{CPUReq: "250m", CPULim: "350m", MemReq: "250Mi", MemLim: "350Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + }, + }, + { + name: "Burstable QoS pod, three containers - decrease c1 resources, increase c2 resources, no change for c3 (net increase for pod)", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "200Mi", MemLim: "300Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &doRestart, + }, + { + Name: "c3", + Resources: &e2epod.ContainerResources{CPUReq: "300m", CPULim: "400m", MemReq: "300Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"50m","memory":"50Mi"},"limits":{"cpu":"150m","memory":"150Mi"}}}, + {"name":"c2", "resources":{"requests":{"cpu":"350m","memory":"350Mi"},"limits":{"cpu":"450m","memory":"450Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "50m", CPULim: "150m", MemReq: "50Mi", MemLim: "150Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "350m", CPULim: "450m", MemReq: "350Mi", MemLim: "450Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &doRestart, + RestartCount: 1, + }, + { + Name: "c3", + Resources: &e2epod.ContainerResources{CPUReq: "300m", CPULim: "400m", MemReq: "300Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + }, + }, + { + name: "Burstable QoS pod, three containers - no change for c1, increase c2 resources, decrease c3 (net decrease for pod)", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, + CPUPolicy: &doRestart, + MemPolicy: &doRestart, + }, + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "200Mi", MemLim: "300Mi"}, + CPUPolicy: &doRestart, + MemPolicy: &noRestart, + }, + { + Name: "c3", + Resources: &e2epod.ContainerResources{CPUReq: "300m", CPULim: "400m", MemReq: "300Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &doRestart, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c2", "resources":{"requests":{"cpu":"250m","memory":"250Mi"},"limits":{"cpu":"350m","memory":"350Mi"}}}, + {"name":"c3", "resources":{"requests":{"cpu":"100m","memory":"100Mi"},"limits":{"cpu":"200m","memory":"200Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, + CPUPolicy: &doRestart, + MemPolicy: &doRestart, + }, + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "250m", CPULim: "350m", MemReq: "250Mi", MemLim: "350Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + RestartCount: 1, + }, + { + Name: "c3", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, + CPUPolicy: &doRestart, + MemPolicy: &doRestart, + RestartCount: 1, + }, + }, + }, + { + name: "Guaranteed QoS pod, one container - increase CPU & memory with an extended resource", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi", + ExtendedResourceReq: "1", ExtendedResourceLim: "1"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"200m","memory":"400Mi"},"limits":{"cpu":"200m","memory":"400Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "400Mi", MemLim: "400Mi", + ExtendedResourceReq: "1", ExtendedResourceLim: "1"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + }, + addExtendedResource: true, + }, + { + name: "Guaranteed QoS pod, one container - increase CPU & memory, with integer CPU requests", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "2", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"4","memory":"400Mi"},"limits":{"cpu":"4","memory":"400Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "4", CPULim: "4", MemReq: "400Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "4", + }, + }, + }, + { + name: "Burstable QoS pod, three containers - no change for c1, decrease c2 resources, decrease c3 (net decrease for pod)", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, + CPUPolicy: &doRestart, + MemPolicy: &doRestart, + }, + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "4", CPULim: "4", MemReq: "200Mi", MemLim: "300Mi"}, + CPUPolicy: &doRestart, + MemPolicy: &noRestart, + }, + { + Name: "c3", + Resources: &e2epod.ContainerResources{CPUReq: "300m", CPULim: "400m", MemReq: "300Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &doRestart, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c2", "resources":{"requests":{"cpu":"1","memory":"150Mi"},"limits":{"cpu":"1","memory":"250Mi"}}}, + {"name":"c3", "resources":{"requests":{"cpu":"100m","memory":"100Mi"},"limits":{"cpu":"200m","memory":"200Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, + CPUPolicy: &doRestart, + MemPolicy: &doRestart, + }, + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "1", CPULim: "1", MemReq: "150Mi", MemLim: "250Mi"}, + CPUPolicy: &doRestart, + MemPolicy: &noRestart, + RestartCount: 1, + }, + { + Name: "c3", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &doRestart, + RestartCount: 1, + }, + }, + }, + { + name: "Burstable QoS pod, three containers - no change for c1, increase c2 resources, decrease c3 (net increase for pod)", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, + CPUPolicy: &doRestart, + MemPolicy: &doRestart, + }, + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "200Mi", MemLim: "300Mi"}, + CPUPolicy: &doRestart, + MemPolicy: &noRestart, + }, + { + Name: "c3", + Resources: &e2epod.ContainerResources{CPUReq: "300m", CPULim: "400m", MemReq: "300Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &doRestart, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c2", "resources":{"requests":{"cpu":"4","memory":"250Mi"},"limits":{"cpu":"4","memory":"350Mi"}}}, + {"name":"c3", "resources":{"requests":{"cpu":"100m","memory":"100Mi"},"limits":{"cpu":"200m","memory":"200Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, + CPUPolicy: &doRestart, + MemPolicy: &doRestart, + }, + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "4", CPULim: "4", MemReq: "250Mi", MemLim: "350Mi"}, + CPUPolicy: &doRestart, + MemPolicy: &noRestart, + RestartCount: 1, + }, + { + Name: "c3", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &doRestart, + RestartCount: 1, + }, + }, + }, + { + name: "Guaranteed QoS pod, one container - decrease CPU & increase memory", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi"}, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"50m","memory":"300Mi"},"limits":{"cpu":"50m","memory":"300Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "50m", CPULim: "50m", MemReq: "300Mi", MemLim: "300Mi"}, + }, + }, + }, + { + name: "Guaranteed QoS pod, one container - decrease CPU & memory, with integer CPU requests", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "4", CPULim: "4", MemReq: "500Mi", MemLim: "500Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "4", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"2","memory":"250Mi"},"limits":{"cpu":"2","memory":"250Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "250Mi", MemLim: "250Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "2", + }, + }, + }, + { + name: "Guaranteed QoS pod, one container - decrease CPU & memory, with integer CPU requests", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "4", CPULim: "4", MemReq: "500Mi", MemLim: "500Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "4", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"2","memory":"250Mi"},"limits":{"cpu":"2","memory":"250Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "250Mi", MemLim: "250Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "2", + }, + }, + }, + { + name: "Guaranteed QoS pod, one container - increase CPU & decrease memory, with integer CPU requests", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "200Mi", MemLim: "200Mi"}, + CPUsAllowedListValue: "2", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"4","memory":"100Mi"},"limits":{"cpu":"4","memory":"100Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "4", CPULim: "4", MemReq: "100Mi", MemLim: "100Mi"}, + CPUsAllowedListValue: "4", + }, + }, + }, + { + name: "Guaranteed QoS pod, one container - increase CPU & decrease memory, with integer CPU requests", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "200Mi", MemLim: "200Mi"}, + CPUsAllowedListValue: "2", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"4","memory":"100Mi"},"limits":{"cpu":"4","memory":"100Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "4", CPULim: "4", MemReq: "100Mi", MemLim: "100Mi"}, + CPUsAllowedListValue: "4", + }, + }, + }, + { + name: "Guaranteed QoS pod, one container - increase CPU & memory, with integer CPU requests", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "2", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"4","memory":"400Mi"},"limits":{"cpu":"4","memory":"400Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "4", CPULim: "4", MemReq: "400Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "4", + }, + }, + }, + { + name: "Guaranteed QoS pod, one container - increase CPU (NotRequired) & memory (RestartContainer), with integer CPU requests", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &doRestart, + CPUsAllowedListValue: "2", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"4","memory":"400Mi"},"limits":{"cpu":"4","memory":"400Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "4", CPULim: "4", MemReq: "400Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &doRestart, + CPUsAllowedListValue: "4", + RestartCount: 1, + }, + }, + }, + { + name: "Guaranteed QoS pod, one container - increase CPU (NotRequired) & memory (RestartContainer), with integer CPU requests", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &doRestart, + CPUsAllowedListValue: "2", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"4","memory":"400Mi"},"limits":{"cpu":"4","memory":"400Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "4", CPULim: "4", MemReq: "400Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &doRestart, + CPUsAllowedListValue: "4", + RestartCount: 1, + }, + }, + }, + { + name: "Guaranteed QoS pod, three containers (c1, c2, c3) - increase CPU (c1,c3) and memory (c2) ; decrease CPU (c2) and memory (c1,c3)", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + { + Name: "c3", + Resources: &e2epod.ContainerResources{CPUReq: "300m", CPULim: "300m", MemReq: "300Mi", MemLim: "300Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"140m","memory":"50Mi"},"limits":{"cpu":"140m","memory":"50Mi"}}}, + {"name":"c2", "resources":{"requests":{"cpu":"150m","memory":"240Mi"},"limits":{"cpu":"150m","memory":"240Mi"}}}, + {"name":"c3", "resources":{"requests":{"cpu":"340m","memory":"250Mi"},"limits":{"cpu":"340m","memory":"250Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "140m", CPULim: "140m", MemReq: "50Mi", MemLim: "50Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "150m", CPULim: "150m", MemReq: "240Mi", MemLim: "240Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + { + Name: "c3", + Resources: &e2epod.ContainerResources{CPUReq: "340m", CPULim: "340m", MemReq: "250Mi", MemLim: "250Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + }, + }, + { + name: "Guaranteed QoS pod, three containers (c1, c2, c3) - increase CPU (c1,c3) and memory (c2) ; decrease CPU (c2) and memory (c1,c3), with integer CPU requests", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "100Mi", MemLim: "100Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "2", + }, + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "4", CPULim: "4", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "4", + }, + { + Name: "c3", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "300Mi", MemLim: "300Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "2", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"4","memory":"50Mi"},"limits":{"cpu":"4","memory":"50Mi"}}}, + {"name":"c2", "resources":{"requests":{"cpu":"2","memory":"240Mi"},"limits":{"cpu":"2","memory":"240Mi"}}}, + {"name":"c3", "resources":{"requests":{"cpu":"4","memory":"250Mi"},"limits":{"cpu":"4","memory":"250Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "4", CPULim: "4", MemReq: "50Mi", MemLim: "50Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "4", + }, + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "240Mi", MemLim: "240Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "2", + }, + { + Name: "c3", + Resources: &e2epod.ContainerResources{CPUReq: "4", CPULim: "4", MemReq: "250Mi", MemLim: "250Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "4", + }, + }, + }, + } + + timeouts := framework.NewTimeoutContext() + + for idx := range tests { + tc := tests[idx] + ginkgo.It(tc.name+policy.title+" (InPlacePodVerticalScalingAllocatedStatus="+strconv.FormatBool(isInPlacePodVerticalScalingAllocatedStatusEnabled)+", InPlacePodVerticalScalingExclusiveCPUs="+strconv.FormatBool(isInPlacePodVerticalScalingExclusiveCPUsEnabled)+")", func(ctx context.Context) { + cpuManagerPolicyKubeletConfig(ctx, f, oldCfg, policy.name, policy.options, isInPlacePodVerticalScalingAllocatedStatusEnabled, isInPlacePodVerticalScalingExclusiveCPUsEnabled) + + var testPod, patchedPod *v1.Pod + var pErr error + + tStamp := strconv.Itoa(time.Now().Nanosecond()) + testPod = e2epod.MakePodWithResizableContainers(f.Namespace.Name, "testpod", tStamp, tc.containers) + testPod.GenerateName = "resize-test-" + testPod = e2epod.MustMixinRestrictedPodSecurity(testPod) + + if tc.addExtendedResource { + nodes, err := e2enode.GetReadySchedulableNodes(context.Background(), f.ClientSet) + framework.ExpectNoError(err) + + for _, node := range nodes.Items { + addExtendedResource(f.ClientSet, node.Name, fakeExtendedResource, resource.MustParse("123")) + } + defer func() { + for _, node := range nodes.Items { + removeExtendedResource(f.ClientSet, node.Name, fakeExtendedResource) + } + }() + } + + ginkgo.By("creating pod") + newPod := podClient.CreateSync(ctx, testPod) + + ginkgo.By("verifying initial pod resources, allocations are as expected") + e2epod.VerifyPodResources(newPod, tc.containers) + ginkgo.By("verifying initial pod resize policy is as expected") + e2epod.VerifyPodResizePolicy(newPod, tc.containers) + + ginkgo.By("verifying initial pod status resources are as expected") + framework.ExpectNoError(e2epod.VerifyPodStatusResources(newPod, tc.containers)) + ginkgo.By("verifying initial cgroup config are as expected") + framework.ExpectNoError(e2epod.VerifyPodContainersCgroupValues(ctx, f, newPod, tc.containers)) + // TODO make this dynamic depending on Policy Name, Resources input and topology of target + // machine. + // For the moment skip below if CPU Manager Policy is set to none + if policy.name == string(cpumanager.PolicyStatic) { + ginkgo.By("verifying initial pod Cpus allowed list value") + gomega.Eventually(ctx, e2epod.VerifyPodContainersCPUsAllowedListValue, timeouts.PodStartShort, timeouts.Poll). + WithArguments(f, newPod, tc.containers). + Should(gomega.Succeed(), "failed to verify initial Pod CPUsAllowedListValue") + } + + patchAndVerify := func(patchString string, expectedContainers []e2epod.ResizableContainerInfo, initialContainers []e2epod.ResizableContainerInfo, opStr string) { + ginkgo.By(fmt.Sprintf("patching pod for %s", opStr)) + patchedPod, pErr = f.ClientSet.CoreV1().Pods(newPod.Namespace).Patch(ctx, newPod.Name, + types.StrategicMergePatchType, []byte(patchString), metav1.PatchOptions{}, "resize") + framework.ExpectNoError(pErr, fmt.Sprintf("failed to patch pod for %s", opStr)) + + ginkgo.By(fmt.Sprintf("verifying pod patched for %s", opStr)) + e2epod.VerifyPodResources(patchedPod, expectedContainers) + + ginkgo.By(fmt.Sprintf("waiting for %s to be actuated", opStr)) + resizedPod := e2epod.WaitForPodResizeActuation(ctx, f, podClient, newPod, expectedContainers) + e2epod.ExpectPodResized(ctx, f, resizedPod, expectedContainers) + + // Check cgroup values only for containerd versions before 1.6.9 + ginkgo.By(fmt.Sprintf("verifying pod container's cgroup values after %s", opStr)) + framework.ExpectNoError(e2epod.VerifyPodContainersCgroupValues(ctx, f, resizedPod, expectedContainers)) + + ginkgo.By(fmt.Sprintf("verifying pod resources after %s", opStr)) + e2epod.VerifyPodResources(resizedPod, expectedContainers) + + // TODO make this dynamic depending on Policy Name, Resources input and topology of target + // machine. + // For the moment skip below if CPU Manager Policy is set to none + if policy.name == string(cpumanager.PolicyStatic) { + ginkgo.By("verifying pod Cpus allowed list value after resize") + if isInPlacePodVerticalScalingExclusiveCPUsEnabled { + gomega.Eventually(ctx, e2epod.VerifyPodContainersCPUsAllowedListValue, timeouts.PodStartShort, timeouts.Poll). + WithArguments(f, resizedPod, tc.expected). + Should(gomega.Succeed(), "failed to verify Pod CPUsAllowedListValue for resizedPod with InPlacePodVerticalScalingExclusiveCPUs enabled") + } else { + gomega.Eventually(ctx, e2epod.VerifyPodContainersCPUsAllowedListValue, timeouts.PodStartShort, timeouts.Poll). + WithArguments(f, resizedPod, tc.containers). + Should(gomega.Succeed(), "failed to verify Pod CPUsAllowedListValue for resizedPod with InPlacePodVerticalScalingExclusiveCPUs disabled (default)") + } + } + } + + patchAndVerify(tc.patchString, tc.expected, tc.containers, "resize") + + rbPatchStr, err := e2epod.ResizeContainerPatch(tc.containers) + framework.ExpectNoError(err) + // Resize has been actuated, test rollback + patchAndVerify(rbPatchStr, tc.containers, tc.expected, "rollback") + + ginkgo.By("deleting pod") + deletePodSyncByName(ctx, f, newPod.Name) + // we need to wait for all containers to really be gone so cpumanager reconcile loop will not rewrite the cpu_manager_state. + // this is in turn needed because we will have an unavoidable (in the current framework) race with the + // reconcile loop which will make our attempt to delete the state file and to restore the old config go haywire + waitForAllContainerRemoval(ctx, newPod.Name, newPod.Namespace) + }) + } + + ginkgo.AfterEach(func(ctx context.Context) { + if oldCfg != nil { + updateKubeletConfig(ctx, f, oldCfg, true) + } + }) + +} + +func doPodResizeErrorTests(policy cpuManagerPolicyConfig, isInPlacePodVerticalScalingAllocatedStatusEnabled bool, isInPlacePodVerticalScalingExclusiveCPUsEnabled bool) { + f := framework.NewDefaultFramework("pod-resize-errors") + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + var podClient *e2epod.PodClient + var oldCfg *kubeletconfig.KubeletConfiguration + ginkgo.BeforeEach(func(ctx context.Context) { + var err error + node := getLocalNode(ctx, f) + if framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) { + e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping") + } + podClient = e2epod.NewPodClient(f) + if oldCfg == nil { + oldCfg, err = getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + } + }) + + type testCase struct { + name string + containers []e2epod.ResizableContainerInfo + patchString string + patchError string + expected []e2epod.ResizableContainerInfo + } + + tests := []testCase{ + { + name: "BestEffort QoS pod, one container - try requesting memory, expect error", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"memory":"400Mi"}}} + ]}}`, + patchError: "Pod QoS is immutable", + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + }, + }, + }, + { + name: "BestEffort QoS pod, three containers - try requesting memory for c1, expect error", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + }, + { + Name: "c2", + }, + { + Name: "c3", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"memory":"400Mi"}}} + ]}}`, + patchError: "Pod QoS is immutable", + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + }, + { + Name: "c2", + }, + { + Name: "c3", + }, + }, + }, + } + + timeouts := framework.NewTimeoutContext() + + for idx := range tests { + tc := tests[idx] + ginkgo.It(tc.name+policy.title+" (InPlacePodVerticalScalingAllocatedStatus="+strconv.FormatBool(isInPlacePodVerticalScalingAllocatedStatusEnabled)+", InPlacePodVerticalScalingExclusiveCPUs="+strconv.FormatBool(isInPlacePodVerticalScalingExclusiveCPUsEnabled)+")", func(ctx context.Context) { + var testPod, patchedPod *v1.Pod + var pErr error + + tStamp := strconv.Itoa(time.Now().Nanosecond()) + testPod = e2epod.MakePodWithResizableContainers(f.Namespace.Name, "testpod", tStamp, tc.containers) + testPod = e2epod.MustMixinRestrictedPodSecurity(testPod) + + ginkgo.By("creating pod") + newPod := podClient.CreateSync(ctx, testPod) + + perr := e2epod.WaitForPodCondition(ctx, f.ClientSet, newPod.Namespace, newPod.Name, "Ready", timeouts.PodStartSlow, testutils.PodRunningReady) + framework.ExpectNoError(perr, "pod %s/%s did not go running", newPod.Namespace, newPod.Name) + framework.Logf("pod %s/%s running", newPod.Namespace, newPod.Name) + + ginkgo.By("verifying initial pod resources, allocations, and policy are as expected") + e2epod.VerifyPodResources(newPod, tc.containers) + e2epod.VerifyPodResizePolicy(newPod, tc.containers) + + ginkgo.By("verifying initial pod status resources and cgroup config are as expected") + framework.ExpectNoError(e2epod.VerifyPodStatusResources(newPod, tc.containers)) + + ginkgo.By("patching pod for resize") + patchedPod, pErr = f.ClientSet.CoreV1().Pods(newPod.Namespace).Patch(ctx, newPod.Name, + types.StrategicMergePatchType, []byte(tc.patchString), metav1.PatchOptions{}) + if tc.patchError == "" { + framework.ExpectNoError(pErr, "failed to patch pod for resize") + } else { + gomega.Expect(pErr).To(gomega.HaveOccurred(), tc.patchError) + patchedPod = newPod + } + + ginkgo.By("verifying pod resources after patch") + e2epod.VerifyPodResources(patchedPod, tc.expected) + + deletePodSyncByName(ctx, f, newPod.Name) + // we need to wait for all containers to really be gone so cpumanager reconcile loop will not rewrite the cpu_manager_state. + // this is in turn needed because we will have an unavoidable (in the current framework) race with the + // reconcile loop which will make our attempt to delete the state file and to restore the old config go haywire + waitForAllContainerRemoval(ctx, newPod.Name, newPod.Namespace) + + }) + } + + ginkgo.AfterEach(func(ctx context.Context) { + if oldCfg != nil { + updateKubeletConfig(ctx, f, oldCfg, true) + } + }) + +} + +// NOTE: Pod resize scheduler resource quota tests are out of scope in e2e_node tests, +// because in e2e_node tests +// a) scheduler and controller manager is not running by the Node e2e +// b) api-server in services doesn't start with --enable-admission-plugins=ResourceQuota +// and is not possible to start it from TEST_ARGS +// Above tests are performed by doSheduletTests() and doPodResizeResourceQuotaTests() +// in test/e2e/node/pod_resize.go + +var _ = SIGDescribe("Pod InPlace Resize Container", framework.WithSerial(), func() { + + policiesGeneralAvailability := []cpuManagerPolicyConfig{ + { + name: string(cpumanager.PolicyNone), + title: "", + }, + { + name: string(cpumanager.PolicyStatic), + title: ", alongside CPU Manager Static Policy with no options", + options: map[string]string{ + cpumanager.FullPCPUsOnlyOption: "false", + cpumanager.DistributeCPUsAcrossNUMAOption: "false", + cpumanager.AlignBySocketOption: "false", + cpumanager.DistributeCPUsAcrossCoresOption: "false", + }, + }, + } + + policiesBeta := []cpuManagerPolicyConfig{ + { + name: string(cpumanager.PolicyStatic), + title: ", alongside CPU Manager Static Policy with FullPCPUsOnlyOption", + options: map[string]string{ + cpumanager.FullPCPUsOnlyOption: "true", + cpumanager.DistributeCPUsAcrossNUMAOption: "false", + cpumanager.AlignBySocketOption: "false", + cpumanager.DistributeCPUsAcrossCoresOption: "false", + }, + }, + } + + /*policiesAlpha := []cpuManagerPolicyConfig{ + { + name: string(cpumanager.PolicyStatic), + title: ", alongside CPU Manager Static Policy with DistributeCPUsAcrossNUMAOption", + options: map[string]string{ + cpumanager.FullPCPUsOnlyOption: "false", + cpumanager.DistributeCPUsAcrossNUMAOption: "true", + cpumanager.AlignBySocketOption: "false", + cpumanager.DistributeCPUsAcrossCoresOption: "false", + }, + }, + { + name: string(cpumanager.PolicyStatic), + title: ", alongside CPU Manager Static Policy with FullPCPUsOnlyOption, DistributeCPUsAcrossNUMAOption", + options: map[string]string{ + cpumanager.FullPCPUsOnlyOption: "true", + cpumanager.DistributeCPUsAcrossNUMAOption: "true", + cpumanager.AlignBySocketOption: "false", + cpumanager.DistributeCPUsAcrossCoresOption: "false", + }, + }, + { + name: string(cpumanager.PolicyStatic), + title: ", alongside CPU Manager Static Policy with AlignBySocketOption", + options: map[string]string{ + cpumanager.FullPCPUsOnlyOption: "false", + cpumanager.DistributeCPUsAcrossNUMAOption: "false", + cpumanager.AlignBySocketOption: "true", + cpumanager.DistributeCPUsAcrossCoresOption: "false", + }, + }, + { + name: string(cpumanager.PolicyStatic), + title: ", alongside CPU Manager Static Policy with FullPCPUsOnlyOption, AlignBySocketOption", + options: map[string]string{ + cpumanager.FullPCPUsOnlyOption: "true", + cpumanager.DistributeCPUsAcrossNUMAOption: "false", + cpumanager.AlignBySocketOption: "true", + cpumanager.DistributeCPUsAcrossCoresOption: "false", + }, + }, + { + name: string(cpumanager.PolicyStatic), + title: ", alongside CPU Manager Static Policy with DistributeCPUsAcrossNUMAOption, AlignBySocketOption", + options: map[string]string{ + cpumanager.FullPCPUsOnlyOption: "false", + cpumanager.DistributeCPUsAcrossNUMAOption: "true", + cpumanager.AlignBySocketOption: "true", + cpumanager.DistributeCPUsAcrossCoresOption: "false", + }, + }, + { + name: string(cpumanager.PolicyStatic), + title: ", alongside CPU Manager Static Policy with FullPCPUsOnlyOption, DistributeCPUsAcrossNUMAOption, AlignBySocketOption", + options: map[string]string{ + cpumanager.FullPCPUsOnlyOption: "true", + cpumanager.DistributeCPUsAcrossNUMAOption: "true", + cpumanager.AlignBySocketOption: "true", + cpumanager.DistributeCPUsAcrossCoresOption: "false", + }, + }, + { + name: string(cpumanager.PolicyStatic), + title: ", alongside CPU Manager Static Policy with DistributeCPUsAcrossCoresOption", + options: map[string]string{ + cpumanager.FullPCPUsOnlyOption: "false", + cpumanager.DistributeCPUsAcrossNUMAOption: "false", + cpumanager.AlignBySocketOption: "false", + cpumanager.DistributeCPUsAcrossCoresOption: "true", + }, + }, + { + name: string(cpumanager.PolicyStatic), + title: ", alongside CPU Manager Static Policy with DistributeCPUsAcrossCoresOption, AlignBySocketOption", + options: map[string]string{ + cpumanager.FullPCPUsOnlyOption: "false", + cpumanager.DistributeCPUsAcrossNUMAOption: "false", + cpumanager.AlignBySocketOption: "true", + cpumanager.DistributeCPUsAcrossCoresOption: "true", + }, + }, + }*/ + + for idp := range policiesGeneralAvailability { + doPodResizeTests(policiesGeneralAvailability[idp], false, false) + doPodResizeTests(policiesGeneralAvailability[idp], true, false) + doPodResizeTests(policiesGeneralAvailability[idp], false, true) + doPodResizeTests(policiesGeneralAvailability[idp], true, true) + doPodResizeErrorTests(policiesGeneralAvailability[idp], false, false) + doPodResizeErrorTests(policiesGeneralAvailability[idp], true, false) + doPodResizeErrorTests(policiesGeneralAvailability[idp], false, true) + doPodResizeErrorTests(policiesGeneralAvailability[idp], true, true) + } + + for idp := range policiesBeta { + doPodResizeTests(policiesBeta[idp], false, false) + doPodResizeTests(policiesBeta[idp], true, false) + doPodResizeTests(policiesBeta[idp], false, true) + doPodResizeTests(policiesBeta[idp], true, true) + doPodResizeErrorTests(policiesBeta[idp], false, false) + doPodResizeErrorTests(policiesBeta[idp], true, false) + doPodResizeErrorTests(policiesBeta[idp], false, true) + doPodResizeErrorTests(policiesBeta[idp], true, true) + } + + /*for idp := range policiesAlpha { + doPodResizeTests(policiesAlpha[idp], true, false) + doPodResizeTests(policiesAlpha[idp], true, true) + doPodResizeErrorTests(policiesAlpha[idp], true, false) + doPodResizeErrorTests(policiesAlpha[idp], true, true) + }*/ + +}) + +func doPodResizeExtendTests(policy cpuManagerPolicyConfig, isInPlacePodVerticalScalingAllocatedStatusEnabled bool, isInPlacePodVerticalScalingExclusiveCPUsEnabled bool) { + f := framework.NewDefaultFramework("pod-resize-test") + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + var podClient *e2epod.PodClient + var oldCfg *kubeletconfig.KubeletConfiguration + ginkgo.BeforeEach(func(ctx context.Context) { + var err error + node := getLocalNode(ctx, f) + if framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) { + e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping") + } + if isMultiNUMA() { + e2eskipper.Skipf("For simple test, only test one NUMA, multi NUMA -- skipping") + } + podClient = e2epod.NewPodClient(f) + if oldCfg == nil { + oldCfg, err = getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + } + }) + + type testCase struct { + name string + containers []e2epod.ResizableContainerInfo + patchString string + expected []e2epod.ResizableContainerInfo + addExtendedResource bool + skipFlag bool + } + + /*setCPUsForTestCase := func(ctx context.Context, tests *testCase, fullPCPUsOnly string) { + cpuCap, _, _ := getLocalNodeCPUDetails(ctx, f) + firstContainerCpuset := cpuset.New() + firstAdditionCpuset := cpuset.New() + firstExpectedCpuset := cpuset.New() + secondContainerCpuset := cpuset.New() + secondAdditionCpuset := cpuset.New() + secondExpectedCpuset := cpuset.New() + + if tests.name == "1 Guaranteed QoS pod, one container - increase CPU & memory, FullPCPUsOnlyOption = false" { + if cpuCap < 2 { + tests.skipFlag = true + } + firstContainerCpuset = cpuset.New(1) + if isHTEnabled() { + cpuList := mustParseCPUSet(getCPUSiblingList(0)).List() + firstContainerCpuset = cpuset.New(cpuList[1]) + } + tests.containers[0].CPUsAllowedList = firstContainerCpuset.String() + + firstAdditionCpuset = cpuset.New(2) + if isHTEnabled() { + cpuList := mustParseCPUSet(getCPUSiblingList(1)).List() + firstAdditionCpuset = cpuset.New(cpuList[0]) + } + firstExpectedCpuset = firstAdditionCpuset.Union(firstContainerCpuset) + tests.expected[0].CPUsAllowedList = firstExpectedCpuset.String() + } else if tests.name == "1 Guaranteed QoS pod, two containers - increase CPU & memory, FullPCPUsOnlyOption = false" { + if cpuCap < 4 { + tests.skipFlag = true + } + firstContainerCpuset = cpuset.New(1) + if isHTEnabled() { + cpuList := mustParseCPUSet(getCPUSiblingList(0)).List() + firstContainerCpuset = cpuset.New(cpuList[1]) + } + tests.containers[0].CPUsAllowedList = firstContainerCpuset.String() + + secondContainerCpuset = cpuset.New(1) + if isHTEnabled() { + cpuList := mustParseCPUSet(getCPUSiblingList(1)).List() + secondContainerCpuset = cpuset.New(cpuList[0]) + } + tests.containers[1].CPUsAllowedList = secondContainerCpuset.String() + + firstAdditionCpuset = cpuset.New(2) + if isHTEnabled() { + cpuList := mustParseCPUSet(getCPUSiblingList(1)).List() + firstAdditionCpuset = cpuset.New(cpuList[1]) + } + firstExpectedCpuset = firstAdditionCpuset.Union(firstContainerCpuset) + tests.expected[0].CPUsAllowedList = firstExpectedCpuset.String() + + secondAdditionCpuset = cpuset.New(2) + if isHTEnabled() { + cpuList := mustParseCPUSet(getCPUSiblingList(2)).List() + secondAdditionCpuset = cpuset.New(cpuList[0]) + } + secondExpectedCpuset = secondAdditionCpuset.Union(secondContainerCpuset) + tests.expected[1].CPUsAllowedList = secondExpectedCpuset.String() + } else if (tests.name == "1 Guaranteed QoS pod, one container - decrease CPU, FullPCPUsOnlyOption = false") || (tests.name == "1 Guaranteed QoS pod, one container - decrease CPU & memory with mustKeepCPUs, FullPCPUsOnlyOption = false") { + if cpuCap < 2 { + tests.skipFlag = true + } + firstContainerCpuset = cpuset.New(2, 3) + if isHTEnabled() { + cpuList := mustParseCPUSet(getCPUSiblingList(0)).List() + if cpuList[1] != 1 { + firstContainerCpuset = mustParseCPUSet(getCPUSiblingList(1)) + } + } + tests.containers[0].CPUsAllowedList = firstContainerCpuset.String() + + firstExpectedCpuset = cpuset.New(firstContainerCpuset.List()[1]) + tests.expected[0].CPUsAllowedList = firstExpectedCpuset.String() + if tests.name == "1 Guaranteed QoS pod, one container - decrease CPU & memory with mustKeepCPUs, FullPCPUsOnlyOption = false" { + startIndex := strings.Index(tests.patchString, `"mustKeepCPUs","value": "`) + len(`"mustKeepCPUs","value": "`) + endIndex := strings.Index(tests.patchString[startIndex:], `"`) + startIndex + tests.expected[0].CPUsAllowedList = tests.patchString[startIndex:endIndex] + ginkgo.By(fmt.Sprintf("startIndex:%d, endIndex:%d", startIndex, endIndex)) + } + } else if (tests.name == "1 Guaranteed QoS pod, one container - decrease CPU & memory, FullPCPUsOnlyOption = true") || (tests.name == "1 Guaranteed QoS pod, one container - decrease CPU with wrong mustKeepCPU, FullPCPUsOnlyOption = ture") || (tests.name == "1 Guaranteed QoS pod, one container - decrease CPU & memory with correct mustKeepCPU, FullPCPUsOnlyOption = true") { + if cpuCap < 4 { + tests.skipFlag = true + } + firstContainerCpuset = cpuset.New(2, 3, 4, 5) + if isHTEnabled() { + cpuList := mustParseCPUSet(getCPUSiblingList(0)).List() + if cpuList[1] != 1 { + firstContainerCpuset = mustParseCPUSet(getCPUSiblingList(1)) + firstContainerCpuset = firstContainerCpuset.Union(mustParseCPUSet(getCPUSiblingList(2))) + } + } + tests.containers[0].CPUsAllowedList = firstContainerCpuset.String() + + firstExpectedCpuset = mustParseCPUSet(getCPUSiblingList(1)) + tests.expected[0].CPUsAllowedList = firstExpectedCpuset.String() + if tests.name == "1 Guaranteed QoS pod, one container - decrease CPU & memory with correct mustKeepCPU, FullPCPUsOnlyOption = true" { + startIndex := strings.Index(tests.patchString, `"mustKeepCPUs","value": "`) + len(`"mustKeepCPUs","value": "`) + endIndex := strings.Index(tests.patchString[startIndex:], `"`) + startIndex + tests.expected[0].CPUsAllowedList = tests.patchString[startIndex:endIndex] + ginkgo.By(fmt.Sprintf("startIndex:%d, endIndex:%d", startIndex, endIndex)) + } + } + + ginkgo.By(fmt.Sprintf("firstContainerCpuset:%v, firstAdditionCpuset:%v, firstExpectedCpuset:%v", firstContainerCpuset, firstAdditionCpuset, firstExpectedCpuset)) + ginkgo.By(fmt.Sprintf("secondContainerCpuset:%v, secondAdditionCpuset:%v, secondExpectedCpuset:%v", secondContainerCpuset, secondAdditionCpuset, secondExpectedCpuset)) + }*/ + + noRestart := v1.NotRequired + testsWithFalseFullCPUs := []testCase{ + /*{ + name: "1 Guaranteed QoS pod, one container - increase CPU & memory, FullPCPUsOnlyOption = false", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "1", CPULim: "1", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "1", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"2","memory":"400Mi"},"limits":{"cpu":"2","memory":"400Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "400Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "2", + }, + }, + }, + { + name: "1 Guaranteed QoS pod, two containers - increase CPU & memory, FullPCPUsOnlyOption = false", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "1", CPULim: "1", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "1", + }, + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "1", CPULim: "1", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "1", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"2","memory":"400Mi"},"limits":{"cpu":"2","memory":"400Mi"}}}, + {"name":"c2", "resources":{"requests":{"cpu":"2","memory":"400Mi"},"limits":{"cpu":"2","memory":"400Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "400Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "2", + }, + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "400Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "2", + }, + }, + },*/ + { + name: "mustkeepCPU issue test", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "400Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "2", + CPUsAllowedList: cpuset.New(1, 11).String(), + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"6"},"limits":{"cpu":"6"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "6", CPULim: "6", MemReq: "400Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "6", + CPUsAllowedList: cpuset.New(1,2,3,11,12,13).String(), + }, + }, + }, + /*{ + name: "1 Guaranteed QoS pod, one container - decrease CPU & memory with mustKeepCPUs, FullPCPUsOnlyOption = false", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "2", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "env":[{"name":"mustKeepCPUs","value": "11"}], "resources":{"requests":{"cpu":"1","memory":"400Mi"},"limits":{"cpu":"1","memory":"400Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "1", CPULim: "1", MemReq: "400Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "1", + }, + }, + },*/ + } + + testsWithTrueFullCPUs := []testCase{ + { + name: "1 Guaranteed QoS pod, one container - decrease CPU & memory, FullPCPUsOnlyOption = true", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "4", CPULim: "4", MemReq: "400Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "4", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"2","memory":"200Mi"},"limits":{"cpu":"2","memory":"200Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "2", + }, + }, + }, + { + name: "1 Guaranteed QoS pod, one container - decrease CPU & memory with correct mustKeepCPU, FullPCPUsOnlyOption = true", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "4", CPULim: "4", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "4", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "env":[{"name":"mustKeepCPUs","value": "2,12"}], "resources":{"requests":{"cpu":"2"},"limits":{"cpu":"2"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "2", + }, + }, + }, + // Abnormal case, CPUs in mustKeepCPUs not full PCPUs, the mustKeepCPUs will be ignored + { + name: "1 Guaranteed QoS pod, one container - decrease CPU with wrong mustKeepCPU, FullPCPUsOnlyOption = ture", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "4", CPULim: "4", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "4", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "env":[{"name":"mustKeepCPUs","value": "1,2"}], "resources":{"requests":{"cpu":"2"},"limits":{"cpu":"2"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "2", + }, + }, + }, + } + + timeouts := framework.NewTimeoutContext() + + var tests []testCase + if policy.options[cpumanager.FullPCPUsOnlyOption] == "false" { + tests = testsWithFalseFullCPUs + } else if policy.options[cpumanager.FullPCPUsOnlyOption] == "true" { + tests = testsWithTrueFullCPUs + } + + for idx := range tests { + tc := tests[idx] + ginkgo.It(tc.name+policy.title+" (InPlacePodVerticalScalingAllocatedStatus="+strconv.FormatBool(isInPlacePodVerticalScalingAllocatedStatusEnabled)+", InPlacePodVerticalScalingExclusiveCPUs="+strconv.FormatBool(isInPlacePodVerticalScalingExclusiveCPUsEnabled)+")", func(ctx context.Context) { + cpuManagerPolicyKubeletConfig(ctx, f, oldCfg, policy.name, policy.options, isInPlacePodVerticalScalingAllocatedStatusEnabled, isInPlacePodVerticalScalingExclusiveCPUsEnabled) + + //setCPUsForTestCase(ctx, &tc, policy.options[cpumanager.FullPCPUsOnlyOption]) + if tc.skipFlag { + e2eskipper.Skipf("Skipping CPU Manager tests since the CPU not enough") + } + + var testPod, patchedPod *v1.Pod + var pErr error + + tStamp := strconv.Itoa(time.Now().Nanosecond()) + testPod = e2epod.MakePodWithResizableContainers(f.Namespace.Name, "testpod", tStamp, tc.containers) + testPod.GenerateName = "resize-test-" + testPod = e2epod.MustMixinRestrictedPodSecurity(testPod) + + if tc.addExtendedResource { + nodes, err := e2enode.GetReadySchedulableNodes(context.Background(), f.ClientSet) + framework.ExpectNoError(err) + + for _, node := range nodes.Items { + addExtendedResource(f.ClientSet, node.Name, fakeExtendedResource, resource.MustParse("123")) + } + defer func() { + for _, node := range nodes.Items { + removeExtendedResource(f.ClientSet, node.Name, fakeExtendedResource) + } + }() + } + + ginkgo.By("creating pod") + newPod := podClient.CreateSync(ctx, testPod) + + ginkgo.By("verifying initial pod resources, allocations are as expected") + e2epod.VerifyPodResources(newPod, tc.containers) + ginkgo.By("verifying initial pod resize policy is as expected") + e2epod.VerifyPodResizePolicy(newPod, tc.containers) + + ginkgo.By("verifying initial pod status resources are as expected") + framework.ExpectNoError(e2epod.VerifyPodStatusResources(newPod, tc.containers)) + ginkgo.By("verifying initial cgroup config are as expected") + framework.ExpectNoError(e2epod.VerifyPodContainersCgroupValues(ctx, f, newPod, tc.containers)) + // TODO make this dynamic depending on Policy Name, Resources input and topology of target + // machine. + // For the moment skip below if CPU Manager Policy is set to none + if policy.name == string(cpumanager.PolicyStatic) { + ginkgo.By("verifying initial pod Cpus allowed list value") + gomega.Eventually(ctx, e2epod.VerifyPodContainersCPUsAllowedListValue, timeouts.PodStartShort, timeouts.Poll). + WithArguments(f, newPod, tc.containers). + Should(gomega.Succeed(), "failed to verify initial Pod CPUsAllowedListValue") + } + + patchAndVerify := func(patchString string, expectedContainers []e2epod.ResizableContainerInfo, initialContainers []e2epod.ResizableContainerInfo, opStr string) { + ginkgo.By(fmt.Sprintf("patching pod for %s", opStr)) + patchedPod, pErr = f.ClientSet.CoreV1().Pods(newPod.Namespace).Patch(ctx, newPod.Name, + types.StrategicMergePatchType, []byte(patchString), metav1.PatchOptions{}, "resize") + framework.ExpectNoError(pErr, fmt.Sprintf("failed to patch pod for %s", opStr)) + + ginkgo.By(fmt.Sprintf("verifying pod patched for %s", opStr)) + e2epod.VerifyPodResources(patchedPod, expectedContainers) + + ginkgo.By(fmt.Sprintf("waiting for %s to be actuated", opStr)) + resizedPod := e2epod.WaitForPodResizeActuation(ctx, f, podClient, newPod, expectedContainers) + e2epod.ExpectPodResized(ctx, f, resizedPod, expectedContainers) + + // Check cgroup values only for containerd versions before 1.6.9 + ginkgo.By(fmt.Sprintf("verifying pod container's cgroup values after %s", opStr)) + framework.ExpectNoError(e2epod.VerifyPodContainersCgroupValues(ctx, f, resizedPod, expectedContainers)) + + ginkgo.By(fmt.Sprintf("verifying pod resources after %s", opStr)) + e2epod.VerifyPodResources(resizedPod, expectedContainers) + + // TODO make this dynamic depending on Policy Name, Resources input and topology of target + // machine. + // For the moment skip below if CPU Manager Policy is set to none + if policy.name == string(cpumanager.PolicyStatic) { + ginkgo.By(fmt.Sprintf("verifying pod Cpus allowed list value after %s", opStr)) + if isInPlacePodVerticalScalingExclusiveCPUsEnabled { + gomega.Eventually(ctx, e2epod.VerifyPodContainersCPUsAllowedListValue, timeouts.PodStartShort, timeouts.Poll). + WithArguments(f, resizedPod, expectedContainers). + Should(gomega.Succeed(), "failed to verify Pod CPUsAllowedListValue for resizedPod with InPlacePodVerticalScalingExclusiveCPUs enabled") + } else { + gomega.Eventually(ctx, e2epod.VerifyPodContainersCPUsAllowedListValue, timeouts.PodStartShort, timeouts.Poll). + WithArguments(f, resizedPod, tc.containers). + Should(gomega.Succeed(), "failed to verify Pod CPUsAllowedListValue for resizedPod with InPlacePodVerticalScalingExclusiveCPUs disabled (default)") + } + } + } + //time.Sleep(2 * time.Minute) + ginkgo.By("First patch") + patchAndVerify(tc.patchString, tc.expected, tc.containers, "resize") + time.Sleep(2 * time.Minute) + + secondScale := []testCase{ + { + name: "Add second scale", + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "6", CPULim: "6", MemReq: "400Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "6", + CPUsAllowedList: cpuset.New(1,2,3,11,12,13).String(), + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"4"},"limits":{"cpu":"4"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "4", CPULim: "4", MemReq: "400Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "4", + CPUsAllowedList: cpuset.New(1,3,11,13).String(), + }, + }, + }, + } + tc1 := secondScale[0] + patchAndVerify(tc1.patchString, tc1.expected, tc1.containers, "resize") + time.Sleep(2 * time.Minute) + + /*rbPatchStr, err := e2epod.ResizeContainerPatch(tc.containers) + framework.ExpectNoError(err) + + // Resize has been actuated, test rollback + ginkgo.By("Second patch for rollback") + patchAndVerify(rbPatchStr, tc.containers, tc.expected, "rollback")*/ + + ginkgo.By("deleting pod") + deletePodSyncByName(ctx, f, newPod.Name) + // we need to wait for all containers to really be gone so cpumanager reconcile loop will not rewrite the cpu_manager_state. + // this is in turn needed because we will have an unavoidable (in the current framework) race with the + // reconcile loop which will make our attempt to delete the state file and to restore the old config go haywire + waitForAllContainerRemoval(ctx, newPod.Name, newPod.Namespace) + }) + } + + ginkgo.AfterEach(func(ctx context.Context) { + if oldCfg != nil { + updateKubeletConfig(ctx, f, oldCfg, true) + } + }) + +} + +func doMultiPodResizeTests(policy cpuManagerPolicyConfig, isInPlacePodVerticalScalingAllocatedStatusEnabled bool, isInPlacePodVerticalScalingExclusiveCPUsEnabled bool) { + f := framework.NewDefaultFramework("pod-resize-test") + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + var podClient *e2epod.PodClient + var oldCfg *kubeletconfig.KubeletConfiguration + ginkgo.BeforeEach(func(ctx context.Context) { + var err error + node := getLocalNode(ctx, f) + if framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) { + e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping") + } + podClient = e2epod.NewPodClient(f) + if oldCfg == nil { + oldCfg, err = getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + } + }) + + type testPod struct { + containers []e2epod.ResizableContainerInfo + patchString string + expected []e2epod.ResizableContainerInfo + } + + type testCase struct { + name string + testPod1 testPod + testPod2 testPod + skipFlag bool + } + + setCPUsForTestCase := func(ctx context.Context, tests *testCase, fullPCPUsOnly string) { + cpuCap, _, _ := getLocalNodeCPUDetails(ctx, f) + firstContainerCpuset := cpuset.New() + firstAdditionCpuset := cpuset.New() + firstExpectedCpuset := cpuset.New() + secondContainerCpuset := cpuset.New() + secondAdditionCpuset := cpuset.New() + secondExpectedCpuset := cpuset.New() + + if tests.name == "1 Guaranteed QoS pod, two containers - increase CPU & memory, FullPCPUsOnlyOption = false" { + if cpuCap < 4 { + tests.skipFlag = true + } + firstContainerCpuset = cpuset.New(1) + if isHTEnabled() { + cpuList := mustParseCPUSet(getCPUSiblingList(0)).List() + firstContainerCpuset = cpuset.New(cpuList[1]) + } + tests.testPod1.containers[0].CPUsAllowedList = firstContainerCpuset.String() + + secondContainerCpuset = cpuset.New(1) + if isHTEnabled() { + cpuList := mustParseCPUSet(getCPUSiblingList(1)).List() + secondContainerCpuset = cpuset.New(cpuList[0]) + } + tests.testPod2.containers[1].CPUsAllowedList = secondContainerCpuset.String() + + firstAdditionCpuset = cpuset.New(2) + if isHTEnabled() { + cpuList := mustParseCPUSet(getCPUSiblingList(1)).List() + firstAdditionCpuset = cpuset.New(cpuList[1]) + } + firstExpectedCpuset = firstAdditionCpuset.Union(firstContainerCpuset) + tests.testPod1.expected[0].CPUsAllowedList = firstExpectedCpuset.String() + + secondAdditionCpuset = cpuset.New(2) + if isHTEnabled() { + cpuList := mustParseCPUSet(getCPUSiblingList(2)).List() + secondAdditionCpuset = cpuset.New(cpuList[0]) + } + secondExpectedCpuset = secondAdditionCpuset.Union(secondContainerCpuset) + tests.testPod2.expected[1].CPUsAllowedList = secondExpectedCpuset.String() + } + ginkgo.By(fmt.Sprintf("firstContainerCpuset:%v, firstAdditionCpuset:%v, firstExpectedCpuset:%v", firstContainerCpuset, firstAdditionCpuset, firstExpectedCpuset)) + ginkgo.By(fmt.Sprintf("secondContainerCpuset:%v, secondAdditionCpuset:%v, secondExpectedCpuset:%v", secondContainerCpuset, secondAdditionCpuset, secondExpectedCpuset)) + } + + noRestart := v1.NotRequired + tests := []testCase{ + { + name: "2 Guaranteed QoS pod, one container - increase CPU & memory, FullPCPUsOnlyOption = false", + testPod1: testPod{ + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "1", CPULim: "1", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "1", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"2","memory":"400Mi"},"limits":{"cpu":"2","memory":"400Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c1", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "400Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "2", + }, + }, + }, + testPod2: testPod{ + containers: []e2epod.ResizableContainerInfo{ + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "1", CPULim: "1", MemReq: "200Mi", MemLim: "200Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "1", + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c2", "resources":{"requests":{"cpu":"2","memory":"400Mi"},"limits":{"cpu":"2","memory":"400Mi"}}} + ]}}`, + expected: []e2epod.ResizableContainerInfo{ + { + Name: "c2", + Resources: &e2epod.ContainerResources{CPUReq: "2", CPULim: "2", MemReq: "400Mi", MemLim: "400Mi"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + CPUsAllowedListValue: "2", + }, + }, + }, + }, + } + + timeouts := framework.NewTimeoutContext() + + for idx := range tests { + tc := tests[idx] + ginkgo.It(tc.name+policy.title+" (InPlacePodVerticalScalingAllocatedStatus="+strconv.FormatBool(isInPlacePodVerticalScalingAllocatedStatusEnabled)+", InPlacePodVerticalScalingExclusiveCPUs="+strconv.FormatBool(isInPlacePodVerticalScalingExclusiveCPUsEnabled)+")", func(ctx context.Context) { + cpuManagerPolicyKubeletConfig(ctx, f, oldCfg, policy.name, policy.options, isInPlacePodVerticalScalingAllocatedStatusEnabled, isInPlacePodVerticalScalingExclusiveCPUsEnabled) + + setCPUsForTestCase(ctx, &tc, policy.options[cpumanager.FullPCPUsOnlyOption]) + if tc.skipFlag { + e2eskipper.Skipf("Skipping CPU Manager tests since the CPU not enough") + } + + var patchedPod *v1.Pod + var pErr error + + createAndVerify := func(podName string, podClient *e2epod.PodClient, testContainers []e2epod.ResizableContainerInfo) (newPod *v1.Pod) { + var testPod *v1.Pod + + tStamp := strconv.Itoa(time.Now().Nanosecond()) + testPod = e2epod.MakePodWithResizableContainers(f.Namespace.Name, fmt.Sprintf("resizepod-%s", podName), tStamp, testContainers) + testPod.GenerateName = "resize-test-" + testPod = e2epod.MustMixinRestrictedPodSecurity(testPod) + + ginkgo.By("creating pod") + newPod = podClient.CreateSync(ctx, testPod) + + ginkgo.By("verifying initial pod resources, allocations are as expected") + e2epod.VerifyPodResources(newPod, testContainers) + ginkgo.By("verifying initial pod resize policy is as expected") + e2epod.VerifyPodResizePolicy(newPod, testContainers) + + ginkgo.By("verifying initial pod status resources are as expected") + framework.ExpectNoError(e2epod.VerifyPodStatusResources(newPod, testContainers)) + ginkgo.By("verifying initial cgroup config are as expected") + framework.ExpectNoError(e2epod.VerifyPodContainersCgroupValues(ctx, f, newPod, testContainers)) + // TODO make this dynamic depending on Policy Name, Resources input and topology of target + // machine. + // For the moment skip below if CPU Manager Policy is set to none + if policy.name == string(cpumanager.PolicyStatic) { + ginkgo.By("verifying initial pod Cpus allowed list value") + gomega.Eventually(ctx, e2epod.VerifyPodContainersCPUsAllowedListValue, timeouts.PodStartShort, timeouts.Poll). + WithArguments(f, newPod, testContainers). + Should(gomega.Succeed(), "failed to verify initial Pod CPUsAllowedListValue") + } + return newPod + } + + newPod1 := createAndVerify("testpod1", podClient, tc.testPod1.containers) + newPod2 := createAndVerify("testpod2", podClient, tc.testPod2.containers) + + patchAndVerify := func(patchString string, expectedContainers []e2epod.ResizableContainerInfo, initialContainers []e2epod.ResizableContainerInfo, opStr string, newPod *v1.Pod) { + ginkgo.By(fmt.Sprintf("patching pod for %s", opStr)) + patchedPod, pErr = f.ClientSet.CoreV1().Pods(newPod.Namespace).Patch(ctx, newPod.Name, + types.StrategicMergePatchType, []byte(patchString), metav1.PatchOptions{}, "resize") + framework.ExpectNoError(pErr, fmt.Sprintf("failed to patch pod for %s", opStr)) + + ginkgo.By(fmt.Sprintf("verifying pod patched for %s", opStr)) + e2epod.VerifyPodResources(patchedPod, expectedContainers) + + ginkgo.By(fmt.Sprintf("waiting for %s to be actuated", opStr)) + resizedPod := e2epod.WaitForPodResizeActuation(ctx, f, podClient, newPod, expectedContainers) + e2epod.ExpectPodResized(ctx, f, resizedPod, expectedContainers) + + // Check cgroup values only for containerd versions before 1.6.9 + ginkgo.By(fmt.Sprintf("verifying pod container's cgroup values after %s", opStr)) + framework.ExpectNoError(e2epod.VerifyPodContainersCgroupValues(ctx, f, resizedPod, expectedContainers)) + + ginkgo.By(fmt.Sprintf("verifying pod resources after %s", opStr)) + e2epod.VerifyPodResources(resizedPod, expectedContainers) + + // TODO make this dynamic depending on Policy Name, Resources input and topology of target + // machine. + // For the moment skip below if CPU Manager Policy is set to none + if policy.name == string(cpumanager.PolicyStatic) { + ginkgo.By(fmt.Sprintf("verifying pod Cpus allowed list value after %s", opStr)) + if isInPlacePodVerticalScalingExclusiveCPUsEnabled { + gomega.Eventually(ctx, e2epod.VerifyPodContainersCPUsAllowedListValue, timeouts.PodStartShort, timeouts.Poll). + WithArguments(f, resizedPod, expectedContainers). + Should(gomega.Succeed(), "failed to verify Pod CPUsAllowedListValue for resizedPod with InPlacePodVerticalScalingExclusiveCPUs enabled") + } else { + gomega.Eventually(ctx, e2epod.VerifyPodContainersCPUsAllowedListValue, timeouts.PodStartShort, timeouts.Poll). + WithArguments(f, resizedPod, initialContainers). + Should(gomega.Succeed(), "failed to verify Pod CPUsAllowedListValue for resizedPod with InPlacePodVerticalScalingExclusiveCPUs disabled (default)") + } + } + } + + patchAndVerify(tc.testPod1.patchString, tc.testPod1.expected, tc.testPod1.containers, "resize", newPod1) + patchAndVerify(tc.testPod2.patchString, tc.testPod2.expected, tc.testPod2.containers, "resize", newPod2) + + rbPatchStr1, err1 := e2epod.ResizeContainerPatch(tc.testPod1.containers) + framework.ExpectNoError(err1) + rbPatchStr2, err2 := e2epod.ResizeContainerPatch(tc.testPod2.containers) + framework.ExpectNoError(err2) + // Resize has been actuated, test rollback + patchAndVerify(rbPatchStr1, tc.testPod1.containers, tc.testPod1.expected, "rollback", newPod1) + patchAndVerify(rbPatchStr2, tc.testPod2.containers, tc.testPod2.expected, "rollback", newPod2) + + ginkgo.By("deleting pod") + deletePodSyncByName(ctx, f, newPod1.Name) + deletePodSyncByName(ctx, f, newPod2.Name) + // we need to wait for all containers to really be gone so cpumanager reconcile loop will not rewrite the cpu_manager_state. + // this is in turn needed because we will have an unavoidable (in the current framework) race with the + // reconcile loop which will make our attempt to delete the state file and to restore the old config go haywire + waitForAllContainerRemoval(ctx, newPod1.Name, newPod1.Namespace) + waitForAllContainerRemoval(ctx, newPod2.Name, newPod2.Namespace) + }) + } + + ginkgo.AfterEach(func(ctx context.Context) { + if oldCfg != nil { + updateKubeletConfig(ctx, f, oldCfg, true) + } + }) +} + +var _ = SIGDescribe("Pod InPlace Resize Container Extended Cases", framework.WithSerial(), func() { + + policiesGeneralAvailability := []cpuManagerPolicyConfig{ + { + name: string(cpumanager.PolicyStatic), + title: ", alongside CPU Manager Static Policy with no options", + options: map[string]string{ + cpumanager.FullPCPUsOnlyOption: "false", + cpumanager.DistributeCPUsAcrossNUMAOption: "false", + cpumanager.AlignBySocketOption: "false", + cpumanager.DistributeCPUsAcrossCoresOption: "false", + }, + }, + { + name: string(cpumanager.PolicyStatic), + title: ", alongside CPU Manager Static Policy with FullPCPUsOnlyOption", + options: map[string]string{ + cpumanager.FullPCPUsOnlyOption: "true", + cpumanager.DistributeCPUsAcrossNUMAOption: "false", + cpumanager.AlignBySocketOption: "false", + cpumanager.DistributeCPUsAcrossCoresOption: "false", + }, + }, + } + + doPodResizeExtendTests(policiesGeneralAvailability[0], true, true) + //doPodResizeExtendTests(policiesGeneralAvailability[1], true, true) + //doMultiPodResizeTests(policiesGeneralAvailability[0], true, true) +}) \ No newline at end of file diff --git a/test/e2e_node/util.go b/test/e2e_node/util.go index d6d37fc650d14..600582b82b4ce 100644 --- a/test/e2e_node/util.go +++ b/test/e2e_node/util.go @@ -234,14 +234,14 @@ func waitForKubeletToStart(ctx context.Context, f *framework.Framework) { // wait until the kubelet health check will succeed gomega.Eventually(ctx, func() bool { return kubeletHealthCheck(kubeletHealthCheckURL) - }, 2*time.Minute, 5*time.Second).Should(gomega.BeTrueBecause("expected kubelet to be in healthy state")) + }, 5*time.Minute, 2*time.Second).Should(gomega.BeTrueBecause("expected kubelet to be in healthy state")) // Wait for the Kubelet to be ready. gomega.Eventually(ctx, func(ctx context.Context) bool { nodes, err := e2enode.TotalReady(ctx, f.ClientSet) framework.ExpectNoError(err) return nodes == 1 - }, time.Minute, time.Second).Should(gomega.BeTrueBecause("expected kubelet to be in ready state")) + }, 5*time.Minute, 2*time.Second).Should(gomega.BeTrueBecause("expected kubelet to be in ready state")) } func deleteStateFile(stateFileName string) { @@ -536,7 +536,7 @@ func waitForAllContainerRemoval(ctx context.Context, podName, podNS string) { return fmt.Errorf("expected all containers to be removed from CRI but %v containers still remain. Containers: %+v", len(containers), containers) } return nil - }, 2*time.Minute, 1*time.Second).Should(gomega.Succeed()) + }, 5*time.Minute, 2*time.Second).Should(gomega.Succeed()) } func getPidsForProcess(name, pidFile string) ([]int, error) {