From ea785e48ca1e67d0b1cc8575b394d72c4ef74d95 Mon Sep 17 00:00:00 2001 From: Mike Spreitzer Date: Wed, 15 Jan 2025 12:26:16 -0500 Subject: [PATCH 1/9] Increase leader-election nominal concurrency shares from 10 to 40 Because some testing shows leader election being starved. Signed-off-by: Mike Spreitzer --- .../k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go | 2 +- .../apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go index aca968de643f7..5e761ddaed989 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go @@ -208,7 +208,7 @@ var ( flowcontrol.PriorityLevelConfigurationSpec{ Type: flowcontrol.PriorityLevelEnablementLimited, Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ - NominalConcurrencyShares: ptr.To(int32(10)), + NominalConcurrencyShares: ptr.To(int32(40)), LendablePercent: ptr.To(int32(0)), LimitResponse: flowcontrol.LimitResponse{ Type: flowcontrol.LimitResponseTypeQueue, diff --git a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go index 09fffc4f88c1e..a661318bb51db 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go @@ -30,7 +30,7 @@ func TestBootstrapPriorityLevelConfigurationWithBorrowing(t *testing.T) { }{ { name: "leader-election", - nominalSharesExpected: 10, + nominalSharesExpected: 40, lendablePercentexpected: 0, }, { From 59d3e6027e1e4678766773b4fe712d489a9321d6 Mon Sep 17 00:00:00 2001 From: Mike Spreitzer Date: Wed, 26 Feb 2025 01:50:45 -0500 Subject: [PATCH 2/9] Increase leader-election's lendable percent .. so that it takes no larger bite from system capacity than before (modulo small difference due to total nominal shares). --- .../k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go | 2 +- .../apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go index 5e761ddaed989..15ddb09b4f88d 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go @@ -209,7 +209,7 @@ var ( Type: flowcontrol.PriorityLevelEnablementLimited, Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ NominalConcurrencyShares: ptr.To(int32(40)), - LendablePercent: ptr.To(int32(0)), + LendablePercent: ptr.To(int32(75)), LimitResponse: flowcontrol.LimitResponse{ Type: flowcontrol.LimitResponseTypeQueue, Queuing: &flowcontrol.QueuingConfiguration{ diff --git a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go index a661318bb51db..7d7a545007c9d 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go @@ -31,7 +31,7 @@ func TestBootstrapPriorityLevelConfigurationWithBorrowing(t *testing.T) { { name: "leader-election", nominalSharesExpected: 40, - lendablePercentexpected: 0, + lendablePercentexpected: 75, }, { name: "node-high", From a51944109f9032fcc6a48c18a6f591f382323599 Mon Sep 17 00:00:00 2001 From: Mike Spreitzer Date: Wed, 26 Feb 2025 11:23:15 -0500 Subject: [PATCH 3/9] Add FlowSchema and priority level for events --- .../pkg/apis/flowcontrol/bootstrap/default.go | 39 +++++++++++++++++++ .../flowcontrol/bootstrap/default_test.go | 22 +++++++++-- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go index 15ddb09b4f88d..73393d565b430 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go @@ -19,6 +19,7 @@ package bootstrap import ( coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" + eventsv1 "k8s.io/api/events/v1" flowcontrol "k8s.io/api/flowcontrol/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/authentication/serviceaccount" @@ -62,6 +63,8 @@ var ( // built-in workloads such as "deployments", "replicasets" and other low-level custom workload which // is important for the cluster. SuggestedPriorityLevelConfigurationWorkloadHigh, + // "events" is used for requests that create/update/delete Event objects. + SuggestedPriorityLevelConfigurationEvent, // "workload-low" is used by those workloads with lower priority which availability only has a // minor impact on the cluster. SuggestedPriorityLevelConfigurationWorkloadLow, @@ -70,6 +73,7 @@ var ( } SuggestedFlowSchemas = []*flowcontrol.FlowSchema{ SuggestedFlowSchemaSystemNodes, // references "system" priority-level + SuggestedFlowSchemaEvents, // references "events" priority-level SuggestedFlowSchemaSystemNodeHigh, // references "node-high" priority-level SuggestedFlowSchemaProbes, // references "exempt" priority-level SuggestedFlowSchemaSystemLeaderElection, // references "leader-election" priority-level @@ -238,6 +242,26 @@ var ( }, }, }) + // event priority-level. This is a relatively low priority configuration, loss of events + // is a relatively low harm thing. + SuggestedPriorityLevelConfigurationEvent = newPriorityLevelConfiguration( + "event", + flowcontrol.PriorityLevelConfigurationSpec{ + Type: flowcontrol.PriorityLevelEnablementLimited, + Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ + NominalConcurrencyShares: ptr.To(int32(5)), + LendablePercent: ptr.To(int32(0)), + BorrowingLimitPercent: ptr.To(int32(100)), + LimitResponse: flowcontrol.LimitResponse{ + Type: flowcontrol.LimitResponseTypeQueue, + Queuing: &flowcontrol.QueuingConfiguration{ + Queues: 31, + HandSize: 3, + QueueLengthLimit: 50, + }, + }, + }, + }) // workload-low priority-level SuggestedPriorityLevelConfigurationWorkloadLow = newPriorityLevelConfiguration( "workload-low", @@ -377,6 +401,21 @@ var ( }, }, ) + SuggestedFlowSchemaEvents = newFlowSchema( + "events", SuggestedPriorityLevelConfigurationEvent.Name, 450, + flowcontrol.FlowDistinguisherMethodByUserType, + flowcontrol.PolicyRulesWithSubjects{ + Subjects: groups(user.AllAuthenticated), // the nodes group + ResourceRules: []flowcontrol.ResourcePolicyRule{ + resourceRule( + []string{"create", "update", "patch", "delete"}, + []string{corev1.GroupName, eventsv1.GroupName}, + []string{"events"}, + []string{flowcontrol.NamespaceEvery}, + false), + }, + }, + ) SuggestedFlowSchemaSystemNodes = newFlowSchema( "system-nodes", "system", 500, flowcontrol.FlowDistinguisherMethodByUserType, diff --git a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go index 7d7a545007c9d..78c9276807f4b 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go @@ -17,9 +17,11 @@ limitations under the License. package bootstrap import ( + "fmt" "testing" flowcontrol "k8s.io/api/flowcontrol/v1" + "k8s.io/utils/ptr" ) func TestBootstrapPriorityLevelConfigurationWithBorrowing(t *testing.T) { @@ -27,6 +29,7 @@ func TestBootstrapPriorityLevelConfigurationWithBorrowing(t *testing.T) { name string nominalSharesExpected int32 lendablePercentexpected int32 + borrowingLimitPercent *int32 }{ { name: "leader-election", @@ -48,6 +51,12 @@ func TestBootstrapPriorityLevelConfigurationWithBorrowing(t *testing.T) { nominalSharesExpected: 40, lendablePercentexpected: 50, }, + { + name: "event", + nominalSharesExpected: 5, + lendablePercentexpected: 0, + borrowingLimitPercent: ptr.To(int32(100)), + }, { name: "workload-low", nominalSharesExpected: 100, @@ -93,10 +102,10 @@ func TestBootstrapPriorityLevelConfigurationWithBorrowing(t *testing.T) { t.Errorf("bootstrap PriorityLevelConfiguration %q: expected NominalConcurrencyShares: %d, but got: %d", test.name, test.nominalSharesExpected, bootstrapPL.Spec.Limited.NominalConcurrencyShares) } if test.lendablePercentexpected != *bootstrapPL.Spec.Limited.LendablePercent { - t.Errorf("bootstrap PriorityLevelConfiguration %q: expected NominalConcurrencyShares: %d, but got: %d", test.name, test.lendablePercentexpected, bootstrapPL.Spec.Limited.LendablePercent) + t.Errorf("bootstrap PriorityLevelConfiguration %q: expected LendablePercent: %d, but got: %d", test.name, test.lendablePercentexpected, bootstrapPL.Spec.Limited.LendablePercent) } - if bootstrapPL.Spec.Limited.BorrowingLimitPercent != nil { - t.Errorf("bootstrap PriorityLevelConfiguration %q: expected BorrowingLimitPercent to be nil, but got: %d", test.name, *bootstrapPL.Spec.Limited.BorrowingLimitPercent) + if !ptr.Equal(test.borrowingLimitPercent, bootstrapPL.Spec.Limited.BorrowingLimitPercent) { + t.Errorf("bootstrap PriorityLevelConfiguration %q: expected BorrowingLimitPercent to be %s, but got: %s", test.name, fmtPtr(test.borrowingLimitPercent), fmtPtr(bootstrapPL.Spec.Limited.BorrowingLimitPercent)) } } @@ -122,3 +131,10 @@ func TestBootstrapPriorityLevelConfigurationWithBorrowing(t *testing.T) { t.Errorf("Expected exempt priority level to have LendablePercent==0 but got %d instead", *exemptPL.Spec.Exempt.LendablePercent) } } + +func fmtPtr[Base any](ptr *Base) string { + if ptr == nil { + return "nil" + } + return fmt.Sprintf("&%v", *ptr) +} From b49eba01d72fd864729401a0e0aa05682da75984 Mon Sep 17 00:00:00 2001 From: Mike Spreitzer Date: Wed, 26 Feb 2025 11:48:47 -0500 Subject: [PATCH 4/9] Change APF config to stop working around lack of borrowing --- .../pkg/apis/flowcontrol/bootstrap/default.go | 18 +++++++++--------- .../apis/flowcontrol/bootstrap/default_test.go | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go index 73393d565b430..5f971bf4756c9 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go @@ -177,8 +177,8 @@ var ( flowcontrol.PriorityLevelConfigurationSpec{ Type: flowcontrol.PriorityLevelEnablementLimited, Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ - NominalConcurrencyShares: ptr.To(int32(30)), - LendablePercent: ptr.To(int32(33)), + NominalConcurrencyShares: ptr.To(int32(40)), + LendablePercent: ptr.To(int32(50)), LimitResponse: flowcontrol.LimitResponse{ Type: flowcontrol.LimitResponseTypeQueue, Queuing: &flowcontrol.QueuingConfiguration{ @@ -194,8 +194,8 @@ var ( flowcontrol.PriorityLevelConfigurationSpec{ Type: flowcontrol.PriorityLevelEnablementLimited, Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ - NominalConcurrencyShares: ptr.To(int32(40)), - LendablePercent: ptr.To(int32(25)), + NominalConcurrencyShares: ptr.To(int32(60)), + LendablePercent: ptr.To(int32(50)), LimitResponse: flowcontrol.LimitResponse{ Type: flowcontrol.LimitResponseTypeQueue, Queuing: &flowcontrol.QueuingConfiguration{ @@ -212,8 +212,8 @@ var ( flowcontrol.PriorityLevelConfigurationSpec{ Type: flowcontrol.PriorityLevelEnablementLimited, Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ - NominalConcurrencyShares: ptr.To(int32(40)), - LendablePercent: ptr.To(int32(75)), + NominalConcurrencyShares: ptr.To(int32(50)), + LendablePercent: ptr.To(int32(80)), LimitResponse: flowcontrol.LimitResponse{ Type: flowcontrol.LimitResponseTypeQueue, Queuing: &flowcontrol.QueuingConfiguration{ @@ -268,8 +268,8 @@ var ( flowcontrol.PriorityLevelConfigurationSpec{ Type: flowcontrol.PriorityLevelEnablementLimited, Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ - NominalConcurrencyShares: ptr.To(int32(100)), - LendablePercent: ptr.To(int32(90)), + NominalConcurrencyShares: ptr.To(int32(40)), + LendablePercent: ptr.To(int32(75)), LimitResponse: flowcontrol.LimitResponse{ Type: flowcontrol.LimitResponseTypeQueue, Queuing: &flowcontrol.QueuingConfiguration{ @@ -286,7 +286,7 @@ var ( flowcontrol.PriorityLevelConfigurationSpec{ Type: flowcontrol.PriorityLevelEnablementLimited, Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ - NominalConcurrencyShares: ptr.To(int32(20)), + NominalConcurrencyShares: ptr.To(int32(10)), LendablePercent: ptr.To(int32(50)), LimitResponse: flowcontrol.LimitResponse{ Type: flowcontrol.LimitResponseTypeQueue, diff --git a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go index 78c9276807f4b..b7fa01a4316c4 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go @@ -33,18 +33,18 @@ func TestBootstrapPriorityLevelConfigurationWithBorrowing(t *testing.T) { }{ { name: "leader-election", - nominalSharesExpected: 40, - lendablePercentexpected: 75, + nominalSharesExpected: 50, + lendablePercentexpected: 80, }, { name: "node-high", - nominalSharesExpected: 40, - lendablePercentexpected: 25, + nominalSharesExpected: 60, + lendablePercentexpected: 50, }, { name: "system", - nominalSharesExpected: 30, - lendablePercentexpected: 33, + nominalSharesExpected: 40, + lendablePercentexpected: 50, }, { name: "workload-high", @@ -59,12 +59,12 @@ func TestBootstrapPriorityLevelConfigurationWithBorrowing(t *testing.T) { }, { name: "workload-low", - nominalSharesExpected: 100, - lendablePercentexpected: 90, + nominalSharesExpected: 40, + lendablePercentexpected: 75, }, { name: "global-default", - nominalSharesExpected: 20, + nominalSharesExpected: 10, lendablePercentexpected: 50, }, { From 4a66e8113e73d77561882a8dacb913893515cb5e Mon Sep 17 00:00:00 2001 From: Mike Spreitzer Date: Wed, 26 Feb 2025 12:56:47 -0500 Subject: [PATCH 5/9] Introduce FeatureGate controlling which APF config to use --- .../internalbootstrap/default-internal.go | 65 +- .../internalbootstrap/defaults_test.go | 60 +- pkg/apis/flowcontrol/validation/validation.go | 21 +- .../flowcontrol/validation/validation_test.go | 10 +- pkg/controlplane/apiserver/apis.go | 2 +- pkg/controlplane/instance.go | 2 +- pkg/features/kube_features.go | 13 + .../flowcontrol/ensurer/flowschema_test.go | 2 +- .../prioritylevelconfiguration_test.go | 2 +- .../flowcontrol/rest/storage_flowcontrol.go | 47 +- .../pkg/apis/flowcontrol/base/types.go | 49 ++ .../apis/flowcontrol/bootstrap-old/default.go | 577 ++++++++++++++++++ .../flowcontrol/bootstrap-old/default_test.go | 124 ++++ .../{bootstrap => bootstrap-v134}/default.go | 118 ++-- .../default_test.go | 6 +- .../apis/flowcontrol/bootstrap/collection.go | 92 +++ .../apiserver/pkg/features/kube_features.go | 13 + .../filters/priority-and-fairness_test.go | 4 +- .../apiserver/pkg/server/options/feature.go | 3 + .../pkg/util/flowcontrol/apf_controller.go | 10 +- .../pkg/util/flowcontrol/apf_filter.go | 5 + .../pkg/util/flowcontrol/controller_test.go | 8 +- .../util/flowcontrol/exempt_borrowing_test.go | 2 +- .../pkg/util/flowcontrol/gen_test.go | 4 +- .../reference/versioned_feature_list.yaml | 6 + .../flowcontrol/fs_condition_test.go | 2 +- 26 files changed, 1087 insertions(+), 160 deletions(-) create mode 100644 staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/base/types.go create mode 100644 staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-old/default.go create mode 100644 staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-old/default_test.go rename staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/{bootstrap => bootstrap-v134}/default.go (83%) rename staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/{bootstrap => bootstrap-v134}/default_test.go (93%) create mode 100644 staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/collection.go diff --git a/pkg/apis/flowcontrol/internalbootstrap/default-internal.go b/pkg/apis/flowcontrol/internalbootstrap/default-internal.go index eb7a47e67d6c4..03da407e5b8db 100644 --- a/pkg/apis/flowcontrol/internalbootstrap/default-internal.go +++ b/pkg/apis/flowcontrol/internalbootstrap/default-internal.go @@ -24,30 +24,48 @@ import ( "k8s.io/kubernetes/pkg/apis/flowcontrol/install" ) -// MandatoryFlowSchemas holds the untyped renditions of the mandatory -// flow schemas. In this map the key is the schema's name and the +// FlowSchemasMap is a collection of unversioned (internal) FlowSchema objects indexed by name. +type FlowSchemasMap = map[string]*flowcontrol.FlowSchema + +// GetMandatoryFlowSchemasMap returns the unversioned (internal) mandatory FlowSchema objects, +// as a deeply immutable map. +// The arguments are the values of the features that control which config collection to use. +func GetMandatoryFlowSchemasMap(v134 bool) FlowSchemasMap { + return MandatoryFlowSchemasMap[bootstrap.CollectionID{V134: v134}] +} + +// MandatoryFlowSchemasMap holds the unversioned (internal) renditions of the mandatory +// flow schemas. In the outer map the key is CollectionId and +// in the inner map the key is the schema's name and the // value is the `*FlowSchema`. Nobody should mutate anything // reachable from this map. -var MandatoryFlowSchemas = internalizeFSes(bootstrap.MandatoryFlowSchemas) +var MandatoryFlowSchemasMap = map[bootstrap.CollectionID]FlowSchemasMap{ + {V134: false}: internalizeFSes(bootstrap.GetV1ConfigCollection(false).Mandatory.FlowSchemas), + {V134: true}: internalizeFSes(bootstrap.GetV1ConfigCollection(true).Mandatory.FlowSchemas), +} + +// PriorityLevelConfigurationsMap is a collection of unversioned (internal) PriorityLevelConfiguration objects, indexed by name. +type PriorityLevelConfigurationsMap = map[string]*flowcontrol.PriorityLevelConfiguration + +// GetMandatoryPriorityLevelConfigurationsMap returns the mandatory PriorityLevelConfiguration objects, +// as a deeply immutable map. +// The arguments are the values of the features that control which config collection to use. +func GetMandatoryPriorityLevelConfigurationsMap(v134 bool) PriorityLevelConfigurationsMap { + return MandatoryPriorityLevelConfigurationsMap[bootstrap.CollectionID{V134: v134}] +} -// MandatoryPriorityLevelConfigurations holds the untyped renditions of the -// mandatory priority level configuration objects. In this map the -// key is the object's name and the value is the +// MandatoryPriorityLevelConfigurationsMap holds the untyped renditions of the +// mandatory priority level configuration objects. In the outer map the key is `bootstrap.CollectionID` and +// in the inner map the key is the object's name and the value is the // `*PriorityLevelConfiguration`. Nobody should mutate anything // reachable from this map. -var MandatoryPriorityLevelConfigurations = internalizePLs(bootstrap.MandatoryPriorityLevelConfigurations) - -// NewAPFScheme constructs and returns a Scheme configured to handle -// the API object types that are used to configure API Priority and -// Fairness -func NewAPFScheme() *runtime.Scheme { - scheme := runtime.NewScheme() - install.Install(scheme) - return scheme +var MandatoryPriorityLevelConfigurationsMap = map[bootstrap.CollectionID]PriorityLevelConfigurationsMap{ + {V134: false}: internalizePLs(bootstrap.GetV1ConfigCollection(false).Mandatory.PriorityLevelConfigurations), + {V134: true}: internalizePLs(bootstrap.GetV1ConfigCollection(true).Mandatory.PriorityLevelConfigurations), } -func internalizeFSes(exts []*flowcontrolv1.FlowSchema) map[string]*flowcontrol.FlowSchema { - ans := make(map[string]*flowcontrol.FlowSchema, len(exts)) +func internalizeFSes(exts []*flowcontrolv1.FlowSchema) FlowSchemasMap { + ans := make(FlowSchemasMap, len(exts)) scheme := NewAPFScheme() for _, ext := range exts { var untyped flowcontrol.FlowSchema @@ -59,8 +77,8 @@ func internalizeFSes(exts []*flowcontrolv1.FlowSchema) map[string]*flowcontrol.F return ans } -func internalizePLs(exts []*flowcontrolv1.PriorityLevelConfiguration) map[string]*flowcontrol.PriorityLevelConfiguration { - ans := make(map[string]*flowcontrol.PriorityLevelConfiguration, len(exts)) +func internalizePLs(exts []*flowcontrolv1.PriorityLevelConfiguration) PriorityLevelConfigurationsMap { + ans := make(PriorityLevelConfigurationsMap, len(exts)) scheme := NewAPFScheme() for _, ext := range exts { var untyped flowcontrol.PriorityLevelConfiguration @@ -71,3 +89,12 @@ func internalizePLs(exts []*flowcontrolv1.PriorityLevelConfiguration) map[string } return ans } + +// NewAPFScheme constructs and returns a Scheme configured to handle +// the API object types that are used to configure API Priority and +// Fairness +func NewAPFScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + install.Install(scheme) + return scheme +} diff --git a/pkg/apis/flowcontrol/internalbootstrap/defaults_test.go b/pkg/apis/flowcontrol/internalbootstrap/defaults_test.go index 7c8b2dd103172..573d9aef2df0a 100644 --- a/pkg/apis/flowcontrol/internalbootstrap/defaults_test.go +++ b/pkg/apis/flowcontrol/internalbootstrap/defaults_test.go @@ -28,35 +28,37 @@ import ( ) func TestBootstrapConfigurationWithDefaulted(t *testing.T) { - scheme := NewAPFScheme() - - bootstrapFlowSchemas := make([]*flowcontrol.FlowSchema, 0) - bootstrapFlowSchemas = append(bootstrapFlowSchemas, bootstrap.MandatoryFlowSchemas...) - bootstrapFlowSchemas = append(bootstrapFlowSchemas, bootstrap.SuggestedFlowSchemas...) - for _, original := range bootstrapFlowSchemas { - t.Run(fmt.Sprintf("FlowSchema/%s", original.Name), func(t *testing.T) { - defaulted := original.DeepCopyObject().(*flowcontrol.FlowSchema) - scheme.Default(defaulted) - if apiequality.Semantic.DeepEqual(original, defaulted) { - t.Logf("Defaulting makes no change to FlowSchema: %q", original.Name) - return - } - t.Errorf("Expected defaulting to not change FlowSchema: %q, diff: %s", original.Name, cmp.Diff(original, defaulted)) - }) - } + t.Run("false", caseFn(false)) + t.Run("true", caseFn(true)) +} + +func caseFn(v134 bool) func(*testing.T) { + return func(t *testing.T) { + scheme := NewAPFScheme() + bootstrapFlowSchemas := bootstrap.GetFlowSchemas(v134) + for _, original := range bootstrapFlowSchemas { + t.Run(fmt.Sprintf("FlowSchema/%s", original.Name), func(t *testing.T) { + defaulted := original.DeepCopyObject().(*flowcontrol.FlowSchema) + scheme.Default(defaulted) + if apiequality.Semantic.DeepEqual(original, defaulted) { + t.Logf("Defaulting makes no change to FlowSchema: %q", original.Name) + return + } + t.Errorf("Expected defaulting to not change FlowSchema: %q, diff: %s", original.Name, cmp.Diff(original, defaulted)) + }) + } - bootstrapPriorityLevels := make([]*flowcontrol.PriorityLevelConfiguration, 0) - bootstrapPriorityLevels = append(bootstrapPriorityLevels, bootstrap.MandatoryPriorityLevelConfigurations...) - bootstrapPriorityLevels = append(bootstrapPriorityLevels, bootstrap.SuggestedPriorityLevelConfigurations...) - for _, original := range bootstrapPriorityLevels { - t.Run(fmt.Sprintf("PriorityLevelConfiguration/%s", original.Name), func(t *testing.T) { - defaulted := original.DeepCopyObject().(*flowcontrol.PriorityLevelConfiguration) - scheme.Default(defaulted) - if apiequality.Semantic.DeepEqual(original, defaulted) { - t.Logf("Defaulting makes no change to PriorityLevelConfiguration: %q", original.Name) - return - } - t.Errorf("Expected defaulting to not change PriorityLevelConfiguration: %q, diff: %s", original.Name, cmp.Diff(original, defaulted)) - }) + bootstrapPriorityLevels := bootstrap.GetPrioritylevelConfigurations(v134) + for _, original := range bootstrapPriorityLevels { + t.Run(fmt.Sprintf("PriorityLevelConfiguration/%s", original.Name), func(t *testing.T) { + defaulted := original.DeepCopyObject().(*flowcontrol.PriorityLevelConfiguration) + scheme.Default(defaulted) + if apiequality.Semantic.DeepEqual(original, defaulted) { + t.Logf("Defaulting makes no change to PriorityLevelConfiguration: %q", original.Name) + return + } + t.Errorf("Expected defaulting to not change PriorityLevelConfiguration: %q, diff: %s", original.Name, cmp.Diff(original, defaulted)) + }) + } } } diff --git a/pkg/apis/flowcontrol/validation/validation.go b/pkg/apis/flowcontrol/validation/validation.go index db3d8d699517f..ca9fc7416d724 100644 --- a/pkg/apis/flowcontrol/validation/validation.go +++ b/pkg/apis/flowcontrol/validation/validation.go @@ -83,13 +83,18 @@ func ValidateFlowSchema(fs *flowcontrol.FlowSchema) field.ErrorList { allErrs := apivalidation.ValidateObjectMeta(&fs.ObjectMeta, false, ValidateFlowSchemaName, field.NewPath("metadata")) specPath := field.NewPath("spec") allErrs = append(allErrs, ValidateFlowSchemaSpec(fs.Name, &fs.Spec, specPath)...) - if mand, ok := internalbootstrap.MandatoryFlowSchemas[fs.Name]; ok { + if mand, ok := internalbootstrap.GetMandatoryFlowSchemasMap(true)[fs.Name]; ok { + // The old config objects are a subset of the new config objects, + // as far as identify is concerned. The specs may differ. + mandOld := internalbootstrap.GetMandatoryFlowSchemasMap(false)[fs.Name] + // For now, accept either the old or the new spec. + // In a later release, change this to accept only the new spec. // Check for almost exact equality. This is a pretty // strict test, and it is OK in this context because both // sides of this comparison are intended to ultimately // come from the same code. - if !apiequality.Semantic.DeepEqual(fs.Spec, mand.Spec) { - allErrs = append(allErrs, field.Invalid(specPath, fs.Spec, fmt.Sprintf("spec of '%s' must equal the fixed value", fs.Name))) + if !(apiequality.Semantic.DeepEqual(fs.Spec, mand.Spec) || mandOld != nil && apiequality.Semantic.DeepEqual(fs.Spec, mandOld.Spec)) { + allErrs = append(allErrs, field.Invalid(specPath, fs.Spec, fmt.Sprintf("spec of '%s' must equal the old or new fixed value", fs.Name))) } } allErrs = append(allErrs, ValidateFlowSchemaStatus(&fs.Status, field.NewPath("status"))...) @@ -363,8 +368,9 @@ func ValidatePriorityLevelConfiguration(pl *flowcontrol.PriorityLevelConfigurati func ValidateIfMandatoryPriorityLevelConfigurationObject(pl *flowcontrol.PriorityLevelConfiguration, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList - mand, ok := internalbootstrap.MandatoryPriorityLevelConfigurations[pl.Name] + mand, ok := internalbootstrap.GetMandatoryPriorityLevelConfigurationsMap(true)[pl.Name] if !ok { + // The old config objects are a subset of the new, as far as identity is concerned. return allErrs } @@ -381,12 +387,15 @@ func ValidateIfMandatoryPriorityLevelConfigurationObject(pl *flowcontrol.Priorit return allErrs } + // For now, accept either the old or the new default config. + // In a later release, accept only the new. + mandOld := internalbootstrap.GetMandatoryPriorityLevelConfigurationsMap(false)[pl.Name] // Check for almost exact equality. This is a pretty // strict test, and it is OK in this context because both // sides of this comparison are intended to ultimately // come from the same code. - if !apiequality.Semantic.DeepEqual(pl.Spec, mand.Spec) { - allErrs = append(allErrs, field.Invalid(fldPath, pl.Spec, fmt.Sprintf("spec of '%s' must equal the fixed value", pl.Name))) + if !(apiequality.Semantic.DeepEqual(pl.Spec, mand.Spec) || mandOld != nil && apiequality.Semantic.DeepEqual(pl.Spec, mandOld.Spec)) { + allErrs = append(allErrs, field.Invalid(fldPath, pl.Spec, fmt.Sprintf("spec of '%s' must equal the old or new fixed value", pl.Name))) } return allErrs } diff --git a/pkg/apis/flowcontrol/validation/validation_test.go b/pkg/apis/flowcontrol/validation/validation_test.go index 65dcd0e5a718a..adcef472e7138 100644 --- a/pkg/apis/flowcontrol/validation/validation_test.go +++ b/pkg/apis/flowcontrol/validation/validation_test.go @@ -28,9 +28,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation/field" + fcboot "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/kubernetes/pkg/apis/flowcontrol" - "k8s.io/kubernetes/pkg/apis/flowcontrol/internalbootstrap" "k8s.io/utils/pointer" ) @@ -263,7 +263,7 @@ func TestFlowSchemaValidation(t *testing.T) { }, Spec: badExempt, }, - expectedErrors: field.ErrorList{field.Invalid(field.NewPath("spec"), badExempt, "spec of 'exempt' must equal the fixed value")}, + expectedErrors: field.ErrorList{field.Invalid(field.NewPath("spec"), badExempt, "spec of 'exempt' must equal the old or new fixed value")}, }, { name: "bad catch-all flow-schema should fail", flowSchema: &flowcontrol.FlowSchema{ @@ -272,7 +272,7 @@ func TestFlowSchemaValidation(t *testing.T) { }, Spec: badCatchAll, }, - expectedErrors: field.ErrorList{field.Invalid(field.NewPath("spec"), badCatchAll, "spec of 'catch-all' must equal the fixed value")}, + expectedErrors: field.ErrorList{field.Invalid(field.NewPath("spec"), badCatchAll, "spec of 'catch-all' must equal the old or new fixed value")}, }, { name: "catch-all flow-schema should work", flowSchema: &flowcontrol.FlowSchema{ @@ -769,7 +769,7 @@ func TestPriorityLevelConfigurationValidation(t *testing.T) { } validChangesInExemptFieldOfExemptPLFn := func() flowcontrol.PriorityLevelConfigurationSpec { - have, _ := internalbootstrap.MandatoryPriorityLevelConfigurations[flowcontrol.PriorityLevelConfigurationNameExempt] + have := fcboot.GetV1ConfigCollection(true).PriorityLevelConfigurationExempt return flowcontrol.PriorityLevelConfigurationSpec{ Type: flowcontrol.PriorityLevelEnablementExempt, Exempt: &flowcontrol.ExemptPriorityLevelConfiguration{ @@ -949,7 +949,7 @@ func TestPriorityLevelConfigurationValidation(t *testing.T) { }, Spec: badSpec, }, - expectedErrors: field.ErrorList{field.Invalid(field.NewPath("spec"), badSpec, "spec of 'catch-all' must equal the fixed value")}, + expectedErrors: field.ErrorList{field.Invalid(field.NewPath("spec"), badSpec, "spec of 'catch-all' must equal the old or new fixed value")}, }, { name: "backstop should work", priorityLevelConfiguration: &flowcontrol.PriorityLevelConfiguration{ diff --git a/pkg/controlplane/apiserver/apis.go b/pkg/controlplane/apiserver/apis.go index b164feffa5915..c609044f3da07 100644 --- a/pkg/controlplane/apiserver/apis.go +++ b/pkg/controlplane/apiserver/apis.go @@ -78,7 +78,7 @@ func (c *CompletedConfig) GenericStorageProviders(discovery discovery.DiscoveryI coordinationrest.RESTStorageProvider{}, rbacrest.RESTStorageProvider{Authorizer: c.Generic.Authorization.Authorizer}, svmrest.RESTStorageProvider{}, - flowcontrolrest.RESTStorageProvider{InformerFactory: c.Generic.SharedInformerFactory}, + flowcontrolrest.RESTStorageProvider{InformerFactory: c.Generic.SharedInformerFactory, FeatureGate: c.Generic.FeatureGate}, admissionregistrationrest.RESTStorageProvider{Authorizer: c.Generic.Authorization.Authorizer, DiscoveryClient: discovery}, eventsrest.RESTStorageProvider{TTL: c.EventTTL}, }, nil diff --git a/pkg/controlplane/instance.go b/pkg/controlplane/instance.go index dbfb35d733482..273700f5441ec 100644 --- a/pkg/controlplane/instance.go +++ b/pkg/controlplane/instance.go @@ -418,7 +418,7 @@ func (c CompletedConfig) StorageProviders(client *kubernetes.Clientset) ([]contr schedulingrest.RESTStorageProvider{}, storagerest.RESTStorageProvider{}, svmrest.RESTStorageProvider{}, - flowcontrolrest.RESTStorageProvider{InformerFactory: c.ControlPlane.Generic.SharedInformerFactory}, + flowcontrolrest.RESTStorageProvider{InformerFactory: c.ControlPlane.Generic.SharedInformerFactory, FeatureGate: c.ControlPlane.Generic.FeatureGate}, // keep apps after extensions so legacy clients resolve the extensions versions of shared resource names. // See https://github.com/kubernetes/kubernetes/issues/42392 appsrest.StorageProvider{}, diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index ec9a8d2fd681b..bc284d33f7783 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -62,6 +62,15 @@ const ( // Enables usage of any object for volume data source in PVCs AnyVolumeDataSource featuregate.Feature = "AnyVolumeDataSource" + // owner: @MikeSpreitzer, @tkashem, @linxiulei + // + // Make API Priority and Fairness use modern configuration, which + // differs from the old in these ways: + // - introduce priority level and flow schema for events; + // - generally reorganize to stop working around lack of borrowing; + // - increase the nominal concurrency shares for leader election. + APFv134Config featuregate.Feature = "APFv134Config" + // owner: @liggitt // kep: https://kep.k8s.io/4601 // @@ -1004,6 +1013,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.33"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // GA in 1.33 -> remove in 1.36 }, + APFv134Config: { + {Version: version.MustParse("1.34"), Default: true, PreRelease: featuregate.Beta}, + }, + AuthorizeNodeWithSelectors: { {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.Beta}, diff --git a/pkg/registry/flowcontrol/ensurer/flowschema_test.go b/pkg/registry/flowcontrol/ensurer/flowschema_test.go index 8272c7f0dedc8..821c5ad0f8fd2 100644 --- a/pkg/registry/flowcontrol/ensurer/flowschema_test.go +++ b/pkg/registry/flowcontrol/ensurer/flowschema_test.go @@ -24,7 +24,7 @@ import ( flowcontrolv1 "k8s.io/api/flowcontrol/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" + bootstrap "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-v134" "k8s.io/client-go/kubernetes/fake" flowcontrollisters "k8s.io/client-go/listers/flowcontrol/v1" toolscache "k8s.io/client-go/tools/cache" diff --git a/pkg/registry/flowcontrol/ensurer/prioritylevelconfiguration_test.go b/pkg/registry/flowcontrol/ensurer/prioritylevelconfiguration_test.go index 6aef244a849b0..80eafd97fc9d5 100644 --- a/pkg/registry/flowcontrol/ensurer/prioritylevelconfiguration_test.go +++ b/pkg/registry/flowcontrol/ensurer/prioritylevelconfiguration_test.go @@ -24,7 +24,7 @@ import ( flowcontrolv1 "k8s.io/api/flowcontrol/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" + bootstrap "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-v134" "k8s.io/client-go/kubernetes/fake" flowcontrollisters "k8s.io/client-go/listers/flowcontrol/v1" toolscache "k8s.io/client-go/tools/cache" diff --git a/pkg/registry/flowcontrol/rest/storage_flowcontrol.go b/pkg/registry/flowcontrol/rest/storage_flowcontrol.go index ca00bb71fcebc..c0edcf4d6cd55 100644 --- a/pkg/registry/flowcontrol/rest/storage_flowcontrol.go +++ b/pkg/registry/flowcontrol/rest/storage_flowcontrol.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" flowcontrolbootstrap "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" + "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" @@ -33,6 +34,7 @@ import ( flowcontrolclient "k8s.io/client-go/kubernetes/typed/flowcontrol/v1" flowcontrollisters "k8s.io/client-go/listers/flowcontrol/v1" "k8s.io/client-go/tools/cache" + "k8s.io/component-base/featuregate" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/flowcontrol" @@ -48,6 +50,7 @@ var _ genericapiserver.PostStartHookProvider = RESTStorageProvider{} // RESTStorageProvider is a provider of REST storage type RESTStorageProvider struct { InformerFactory informers.SharedInformerFactory + FeatureGate featuregate.FeatureGate } // PostStartHookName is the name of the post-start-hook provided by flow-control storage @@ -106,6 +109,7 @@ func (p RESTStorageProvider) GroupName() string { // PostStartHook returns the hook func that launches the config provider func (p RESTStorageProvider) PostStartHook() (string, genericapiserver.PostStartHookFunc, error) { bce := &bootstrapConfigurationEnsurer{ + v134Config: p.FeatureGate.Enabled(features.APFv134Config), informersSynced: []cache.InformerSynced{ p.InformerFactory.Flowcontrol().V1().PriorityLevelConfigurations().Informer().HasSynced, p.InformerFactory.Flowcontrol().V1().FlowSchemas().Informer().HasSynced, @@ -117,6 +121,7 @@ func (p RESTStorageProvider) PostStartHook() (string, genericapiserver.PostStart } type bootstrapConfigurationEnsurer struct { + v134Config bool informersSynced []cache.InformerSynced fsLister flowcontrollisters.FlowSchemaLister plcLister flowcontrollisters.PriorityLevelConfigurationLister @@ -142,7 +147,7 @@ func (bce *bootstrapConfigurationEnsurer) ensureAPFBootstrapConfiguration(hookCo ctx, time.Second, func(context.Context) (bool, error) { - if err := ensure(ctx, clientset, bce.fsLister, bce.plcLister); err != nil { + if err := ensure(ctx, bce.v134Config, clientset, bce.fsLister, bce.plcLister); err != nil { klog.ErrorS(err, "APF bootstrap ensurer ran into error, will retry later") return false, nil } @@ -163,7 +168,7 @@ func (bce *bootstrapConfigurationEnsurer) ensureAPFBootstrapConfiguration(hookCo wait.PollImmediateUntil( time.Minute, func() (bool, error) { - if err := ensure(hookContext, clientset, bce.fsLister, bce.plcLister); err != nil { + if err := ensure(hookContext, bce.v134Config, clientset, bce.fsLister, bce.plcLister); err != nil { klog.ErrorS(err, "APF bootstrap ensurer ran into error, will retry later") } // always auto update both suggested and mandatory configuration @@ -175,64 +180,64 @@ func (bce *bootstrapConfigurationEnsurer) ensureAPFBootstrapConfiguration(hookCo return nil } -func ensure(ctx context.Context, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { +func ensure(ctx context.Context, v134Config bool, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { - if err := ensureSuggestedConfiguration(ctx, clientset, fsLister, plcLister); err != nil { + if err := ensureSuggestedConfiguration(ctx, v134Config, clientset, fsLister, plcLister); err != nil { // We should not attempt creation of mandatory objects if ensuring the suggested // configuration resulted in an error. // This only happens when the stop channel is closed. return fmt.Errorf("failed ensuring suggested settings - %w", err) } - if err := ensureMandatoryConfiguration(ctx, clientset, fsLister, plcLister); err != nil { + if err := ensureMandatoryConfiguration(ctx, v134Config, clientset, fsLister, plcLister); err != nil { return fmt.Errorf("failed ensuring mandatory settings - %w", err) } - if err := removeDanglingBootstrapConfiguration(ctx, clientset, fsLister, plcLister); err != nil { + if err := removeDanglingBootstrapConfiguration(ctx, v134Config, clientset, fsLister, plcLister); err != nil { return fmt.Errorf("failed to delete removed settings - %w", err) } return nil } -func ensureSuggestedConfiguration(ctx context.Context, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { +func ensureSuggestedConfiguration(ctx context.Context, v134Config bool, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { plcOps := ensurer.NewPriorityLevelConfigurationOps(clientset.PriorityLevelConfigurations(), plcLister) - if err := ensurer.EnsureConfigurations(ctx, plcOps, flowcontrolbootstrap.SuggestedPriorityLevelConfigurations, ensurer.NewSuggestedEnsureStrategy[*flowcontrolv1.PriorityLevelConfiguration]()); err != nil { + config := flowcontrolbootstrap.GetV1ConfigCollection(v134Config).Suggested + if err := ensurer.EnsureConfigurations(ctx, plcOps, config.PriorityLevelConfigurations, ensurer.NewSuggestedEnsureStrategy[*flowcontrolv1.PriorityLevelConfiguration]()); err != nil { return err } fsOps := ensurer.NewFlowSchemaOps(clientset.FlowSchemas(), fsLister) - return ensurer.EnsureConfigurations(ctx, fsOps, flowcontrolbootstrap.SuggestedFlowSchemas, ensurer.NewSuggestedEnsureStrategy[*flowcontrolv1.FlowSchema]()) + return ensurer.EnsureConfigurations(ctx, fsOps, config.FlowSchemas, ensurer.NewSuggestedEnsureStrategy[*flowcontrolv1.FlowSchema]()) } -func ensureMandatoryConfiguration(ctx context.Context, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { +func ensureMandatoryConfiguration(ctx context.Context, v134Config bool, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { plcOps := ensurer.NewPriorityLevelConfigurationOps(clientset.PriorityLevelConfigurations(), plcLister) - if err := ensurer.EnsureConfigurations(ctx, plcOps, flowcontrolbootstrap.MandatoryPriorityLevelConfigurations, ensurer.NewMandatoryEnsureStrategy[*flowcontrolv1.PriorityLevelConfiguration]()); err != nil { + config := flowcontrolbootstrap.GetV1ConfigCollection(v134Config).Mandatory + if err := ensurer.EnsureConfigurations(ctx, plcOps, config.PriorityLevelConfigurations, ensurer.NewMandatoryEnsureStrategy[*flowcontrolv1.PriorityLevelConfiguration]()); err != nil { return err } fsOps := ensurer.NewFlowSchemaOps(clientset.FlowSchemas(), fsLister) - return ensurer.EnsureConfigurations(ctx, fsOps, flowcontrolbootstrap.MandatoryFlowSchemas, ensurer.NewMandatoryEnsureStrategy[*flowcontrolv1.FlowSchema]()) + return ensurer.EnsureConfigurations(ctx, fsOps, config.FlowSchemas, ensurer.NewMandatoryEnsureStrategy[*flowcontrolv1.FlowSchema]()) } -func removeDanglingBootstrapConfiguration(ctx context.Context, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { - if err := removeDanglingBootstrapFlowSchema(ctx, clientset, fsLister); err != nil { +func removeDanglingBootstrapConfiguration(ctx context.Context, v134Config bool, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { + if err := removeDanglingBootstrapFlowSchema(ctx, v134Config, clientset, fsLister); err != nil { return err } - return removeDanglingBootstrapPriorityLevel(ctx, clientset, plcLister) + return removeDanglingBootstrapPriorityLevel(ctx, v134Config, clientset, plcLister) } -func removeDanglingBootstrapFlowSchema(ctx context.Context, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister) error { - bootstrap := append(flowcontrolbootstrap.MandatoryFlowSchemas, flowcontrolbootstrap.SuggestedFlowSchemas...) +func removeDanglingBootstrapFlowSchema(ctx context.Context, v134Config bool, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister) error { fsOps := ensurer.NewFlowSchemaOps(clientset.FlowSchemas(), fsLister) - return ensurer.RemoveUnwantedObjects(ctx, fsOps, bootstrap) + return ensurer.RemoveUnwantedObjects(ctx, fsOps, flowcontrolbootstrap.GetFlowSchemas(v134Config)) } -func removeDanglingBootstrapPriorityLevel(ctx context.Context, clientset flowcontrolclient.FlowcontrolV1Interface, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { - bootstrap := append(flowcontrolbootstrap.MandatoryPriorityLevelConfigurations, flowcontrolbootstrap.SuggestedPriorityLevelConfigurations...) +func removeDanglingBootstrapPriorityLevel(ctx context.Context, v134Config bool, clientset flowcontrolclient.FlowcontrolV1Interface, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { plcOps := ensurer.NewPriorityLevelConfigurationOps(clientset.PriorityLevelConfigurations(), plcLister) - return ensurer.RemoveUnwantedObjects(ctx, plcOps, bootstrap) + return ensurer.RemoveUnwantedObjects(ctx, plcOps, flowcontrolbootstrap.GetPrioritylevelConfigurations(v134Config)) } // contextFromChannelAndMaxWaitDuration returns a Context that is bound to the diff --git a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/base/types.go b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/base/types.go new file mode 100644 index 0000000000000..bddfc5ac84c31 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/base/types.go @@ -0,0 +1,49 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package base + +import ( + flowcontrolv1 "k8s.io/api/flowcontrol/v1" +) + +// V1ConfigCollection is the collection of default V1 configuration objects to use. +// Deeply immutable. +type V1ConfigCollection struct { + // Mandatory holds the objects that define an apiserver's initial behavior. The + // registered defaulting procedures make no changes to these + // particular objects (this is verified in the unit tests of the + // internalbootstrap package; it can not be verified in this package + // because that would require importing k8s.io/kubernetes). + Mandatory V1ConfigSlices + // Suggested holds the objects that define the current suggested additional configuration. + Suggested V1ConfigSlices + // PriorityLevelConfigurationExempt also appears in `Mandatory.PriorityLevelConfigurations`. + PriorityLevelConfigurationExempt *flowcontrolv1.PriorityLevelConfiguration + // PriorityLevelConfigurationCatchAll also appears in `Mandatory.PriorityLevelConfigurations`. + PriorityLevelConfigurationCatchAll *flowcontrolv1.PriorityLevelConfiguration + // FlowSchemaExempt also appears in `Mandatory.FlowsSchemas`. + FlowSchemaExempt *flowcontrolv1.FlowSchema + // FlowSchemaCatchAll also appears in `Mandatory.FlowSchemas`. + FlowSchemaCatchAll *flowcontrolv1.FlowSchema +} + +// V1ConfigSlices is a collection of v1 APF configuration objects. +// Deeply immutable. +type V1ConfigSlices struct { + PriorityLevelConfigurations []*flowcontrolv1.PriorityLevelConfiguration + FlowSchemas []*flowcontrolv1.FlowSchema +} diff --git a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-old/default.go b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-old/default.go new file mode 100644 index 0000000000000..02bcd4cc4a96d --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-old/default.go @@ -0,0 +1,577 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + flowcontrol "k8s.io/api/flowcontrol/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/apis/flowcontrol/base" + "k8s.io/apiserver/pkg/authentication/serviceaccount" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/utils/ptr" +) + +var V1ConfigCollection = base.V1ConfigCollection{ + Mandatory: base.V1ConfigSlices{ + PriorityLevelConfigurations: []*flowcontrol.PriorityLevelConfiguration{ + MandatoryPriorityLevelConfigurationCatchAll, + MandatoryPriorityLevelConfigurationExempt, + }, + FlowSchemas: []*flowcontrol.FlowSchema{ + MandatoryFlowSchemaExempt, + MandatoryFlowSchemaCatchAll, + }, + }, + PriorityLevelConfigurationExempt: MandatoryPriorityLevelConfigurationExempt, + PriorityLevelConfigurationCatchAll: MandatoryPriorityLevelConfigurationCatchAll, + FlowSchemaExempt: MandatoryFlowSchemaExempt, + FlowSchemaCatchAll: MandatoryFlowSchemaCatchAll, + Suggested: base.V1ConfigSlices{ + PriorityLevelConfigurations: []*flowcontrol.PriorityLevelConfiguration{ + // "system" priority-level is for the system components that affects self-maintenance of the + // cluster and the availability of those running pods in the cluster, including kubelet and + // kube-proxy. + SuggestedPriorityLevelConfigurationSystem, + // "node-high" priority-level is for the node health reporting. It is separated from "system" + // to make sure that nodes are able to report their health even if kube-apiserver is not capable of + // handling load caused by pod startup (fetching secrets, events etc). + // NOTE: In large clusters 50% - 90% of all API calls use this priority-level. + SuggestedPriorityLevelConfigurationNodeHigh, + // "leader-election" is dedicated for controllers' leader-election, which majorly affects the + // availability of any controller runs in the cluster. + SuggestedPriorityLevelConfigurationLeaderElection, + // "workload-high" is used by those workloads with higher priority but their failure won't directly + // impact the existing running pods in the cluster, which includes kube-scheduler, and those well-known + // built-in workloads such as "deployments", "replicasets" and other low-level custom workload which + // is important for the cluster. + SuggestedPriorityLevelConfigurationWorkloadHigh, + // "workload-low" is used by those workloads with lower priority which availability only has a + // minor impact on the cluster. + SuggestedPriorityLevelConfigurationWorkloadLow, + // "global-default" serves the rest traffic not handled by the other suggested flow-schemas above. + SuggestedPriorityLevelConfigurationGlobalDefault, + }, + FlowSchemas: []*flowcontrol.FlowSchema{ + SuggestedFlowSchemaSystemNodes, // references "system" priority-level + SuggestedFlowSchemaSystemNodeHigh, // references "node-high" priority-level + SuggestedFlowSchemaProbes, // references "exempt" priority-level + SuggestedFlowSchemaSystemLeaderElection, // references "leader-election" priority-level + SuggestedFlowSchemaWorkloadLeaderElection, // references "leader-election" priority-level + SuggestedFlowSchemaEndpointsController, // references "workload-high" priority-level + SuggestedFlowSchemaKubeControllerManager, // references "workload-high" priority-level + SuggestedFlowSchemaKubeScheduler, // references "workload-high" priority-level + SuggestedFlowSchemaKubeSystemServiceAccounts, // references "workload-high" priority-level + SuggestedFlowSchemaServiceAccounts, // references "workload-low" priority-level + SuggestedFlowSchemaGlobalDefault, // references "global-default" priority-level + }, + }, +} + +// Mandatory PriorityLevelConfiguration objects +var ( + MandatoryPriorityLevelConfigurationExempt = newPriorityLevelConfiguration( + flowcontrol.PriorityLevelConfigurationNameExempt, + flowcontrol.PriorityLevelConfigurationSpec{ + Type: flowcontrol.PriorityLevelEnablementExempt, + Exempt: &flowcontrol.ExemptPriorityLevelConfiguration{ + NominalConcurrencyShares: ptr.To(int32(0)), + LendablePercent: ptr.To(int32(0)), + }, + }, + ) + MandatoryPriorityLevelConfigurationCatchAll = newPriorityLevelConfiguration( + flowcontrol.PriorityLevelConfigurationNameCatchAll, + flowcontrol.PriorityLevelConfigurationSpec{ + Type: flowcontrol.PriorityLevelEnablementLimited, + Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ + NominalConcurrencyShares: ptr.To(int32(5)), + LendablePercent: ptr.To(int32(0)), + LimitResponse: flowcontrol.LimitResponse{ + Type: flowcontrol.LimitResponseTypeReject, + }, + }, + }) +) + +// Mandatory FlowSchema objects +var ( + // "exempt" priority-level is used for preventing priority inversion and ensuring that sysadmin + // requests are always possible. + MandatoryFlowSchemaExempt = newFlowSchema( + "exempt", + flowcontrol.PriorityLevelConfigurationNameExempt, + 1, // matchingPrecedence + "", // distinguisherMethodType + flowcontrol.PolicyRulesWithSubjects{ + Subjects: groups(user.SystemPrivilegedGroup), + ResourceRules: []flowcontrol.ResourcePolicyRule{ + resourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.APIGroupAll}, + []string{flowcontrol.ResourceAll}, + []string{flowcontrol.NamespaceEvery}, + true, + ), + }, + NonResourceRules: []flowcontrol.NonResourcePolicyRule{ + nonResourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.NonResourceAll}, + ), + }, + }, + ) + // "catch-all" priority-level only gets a minimal positive share of concurrency and won't be reaching + // ideally unless you intentionally deleted the suggested "global-default". + MandatoryFlowSchemaCatchAll = newFlowSchema( + flowcontrol.FlowSchemaNameCatchAll, + flowcontrol.PriorityLevelConfigurationNameCatchAll, + 10000, // matchingPrecedence + flowcontrol.FlowDistinguisherMethodByUserType, // distinguisherMethodType + flowcontrol.PolicyRulesWithSubjects{ + Subjects: groups(user.AllUnauthenticated, user.AllAuthenticated), + ResourceRules: []flowcontrol.ResourcePolicyRule{ + resourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.APIGroupAll}, + []string{flowcontrol.ResourceAll}, + []string{flowcontrol.NamespaceEvery}, + true, + ), + }, + NonResourceRules: []flowcontrol.NonResourcePolicyRule{ + nonResourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.NonResourceAll}, + ), + }, + }, + ) +) + +// Suggested PriorityLevelConfiguration objects +var ( + // system priority-level + SuggestedPriorityLevelConfigurationSystem = newPriorityLevelConfiguration( + "system", + flowcontrol.PriorityLevelConfigurationSpec{ + Type: flowcontrol.PriorityLevelEnablementLimited, + Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ + NominalConcurrencyShares: ptr.To(int32(30)), + LendablePercent: ptr.To(int32(33)), + LimitResponse: flowcontrol.LimitResponse{ + Type: flowcontrol.LimitResponseTypeQueue, + Queuing: &flowcontrol.QueuingConfiguration{ + Queues: 64, + HandSize: 6, + QueueLengthLimit: 50, + }, + }, + }, + }) + SuggestedPriorityLevelConfigurationNodeHigh = newPriorityLevelConfiguration( + "node-high", + flowcontrol.PriorityLevelConfigurationSpec{ + Type: flowcontrol.PriorityLevelEnablementLimited, + Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ + NominalConcurrencyShares: ptr.To(int32(40)), + LendablePercent: ptr.To(int32(25)), + LimitResponse: flowcontrol.LimitResponse{ + Type: flowcontrol.LimitResponseTypeQueue, + Queuing: &flowcontrol.QueuingConfiguration{ + Queues: 64, + HandSize: 6, + QueueLengthLimit: 50, + }, + }, + }, + }) + // leader-election priority-level + SuggestedPriorityLevelConfigurationLeaderElection = newPriorityLevelConfiguration( + "leader-election", + flowcontrol.PriorityLevelConfigurationSpec{ + Type: flowcontrol.PriorityLevelEnablementLimited, + Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ + NominalConcurrencyShares: ptr.To(int32(10)), + LendablePercent: ptr.To(int32(0)), + LimitResponse: flowcontrol.LimitResponse{ + Type: flowcontrol.LimitResponseTypeQueue, + Queuing: &flowcontrol.QueuingConfiguration{ + Queues: 16, + HandSize: 4, + QueueLengthLimit: 50, + }, + }, + }, + }) + // workload-high priority-level + SuggestedPriorityLevelConfigurationWorkloadHigh = newPriorityLevelConfiguration( + "workload-high", + flowcontrol.PriorityLevelConfigurationSpec{ + Type: flowcontrol.PriorityLevelEnablementLimited, + Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ + NominalConcurrencyShares: ptr.To(int32(40)), + LendablePercent: ptr.To(int32(50)), + LimitResponse: flowcontrol.LimitResponse{ + Type: flowcontrol.LimitResponseTypeQueue, + Queuing: &flowcontrol.QueuingConfiguration{ + Queues: 128, + HandSize: 6, + QueueLengthLimit: 50, + }, + }, + }, + }) + // workload-low priority-level + SuggestedPriorityLevelConfigurationWorkloadLow = newPriorityLevelConfiguration( + "workload-low", + flowcontrol.PriorityLevelConfigurationSpec{ + Type: flowcontrol.PriorityLevelEnablementLimited, + Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ + NominalConcurrencyShares: ptr.To(int32(100)), + LendablePercent: ptr.To(int32(90)), + LimitResponse: flowcontrol.LimitResponse{ + Type: flowcontrol.LimitResponseTypeQueue, + Queuing: &flowcontrol.QueuingConfiguration{ + Queues: 128, + HandSize: 6, + QueueLengthLimit: 50, + }, + }, + }, + }) + // global-default priority-level + SuggestedPriorityLevelConfigurationGlobalDefault = newPriorityLevelConfiguration( + "global-default", + flowcontrol.PriorityLevelConfigurationSpec{ + Type: flowcontrol.PriorityLevelEnablementLimited, + Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ + NominalConcurrencyShares: ptr.To(int32(20)), + LendablePercent: ptr.To(int32(50)), + LimitResponse: flowcontrol.LimitResponse{ + Type: flowcontrol.LimitResponseTypeQueue, + Queuing: &flowcontrol.QueuingConfiguration{ + Queues: 128, + HandSize: 6, + QueueLengthLimit: 50, + }, + }, + }, + }) +) + +// Suggested FlowSchema objects. +// Ordered by matching precedence, so that their interactions are easier +// to follow while reading this source. +var ( + // the following flow schema exempts probes + SuggestedFlowSchemaProbes = newFlowSchema( + "probes", "exempt", 2, + "", // distinguisherMethodType + flowcontrol.PolicyRulesWithSubjects{ + Subjects: groups(user.AllUnauthenticated, user.AllAuthenticated), + NonResourceRules: []flowcontrol.NonResourcePolicyRule{ + nonResourceRule( + []string{"get"}, + []string{"/healthz", "/readyz", "/livez"}), + }, + }, + ) + SuggestedFlowSchemaSystemLeaderElection = newFlowSchema( + "system-leader-election", "leader-election", 100, + flowcontrol.FlowDistinguisherMethodByUserType, + flowcontrol.PolicyRulesWithSubjects{ + Subjects: append( + users(user.KubeControllerManager, user.KubeScheduler), + kubeSystemServiceAccount(flowcontrol.NameAll)...), + ResourceRules: []flowcontrol.ResourcePolicyRule{ + resourceRule( + []string{"get", "create", "update"}, + []string{coordinationv1.GroupName}, + []string{"leases"}, + []string{flowcontrol.NamespaceEvery}, + false), + }, + }, + ) + // We add an explicit rule for endpoint-controller with high precedence + // to ensure that those calls won't get caught by the following + // flow-schema. + // + // TODO(#80289): Get rid of this rule once we get rid of support for + // using endpoints and configmaps objects for leader election. + SuggestedFlowSchemaEndpointsController = newFlowSchema( + "endpoint-controller", "workload-high", 150, + flowcontrol.FlowDistinguisherMethodByUserType, + flowcontrol.PolicyRulesWithSubjects{ + Subjects: append( + users(user.KubeControllerManager), + kubeSystemServiceAccount("endpoint-controller", "endpointslicemirroring-controller")...), + ResourceRules: []flowcontrol.ResourcePolicyRule{ + resourceRule( + []string{"get", "create", "update"}, + []string{corev1.GroupName}, + []string{"endpoints"}, + []string{flowcontrol.NamespaceEvery}, + false), + }, + }, + ) + // TODO(#80289): Get rid of this rule once we get rid of support for + // using endpoints and configmaps objects for leader election. + SuggestedFlowSchemaWorkloadLeaderElection = newFlowSchema( + "workload-leader-election", "leader-election", 200, + flowcontrol.FlowDistinguisherMethodByUserType, + flowcontrol.PolicyRulesWithSubjects{ + Subjects: kubeSystemServiceAccount(flowcontrol.NameAll), + ResourceRules: []flowcontrol.ResourcePolicyRule{ + resourceRule( + []string{"get", "create", "update"}, + []string{corev1.GroupName}, + []string{"endpoints", "configmaps"}, + []string{flowcontrol.NamespaceEvery}, + false), + resourceRule( + []string{"get", "create", "update"}, + []string{coordinationv1.GroupName}, + []string{"leases"}, + []string{flowcontrol.NamespaceEvery}, + false), + }, + }, + ) + SuggestedFlowSchemaSystemNodeHigh = newFlowSchema( + "system-node-high", "node-high", 400, + flowcontrol.FlowDistinguisherMethodByUserType, + flowcontrol.PolicyRulesWithSubjects{ + Subjects: groups(user.NodesGroup), // the nodes group + ResourceRules: []flowcontrol.ResourcePolicyRule{ + resourceRule( + []string{flowcontrol.VerbAll}, + []string{corev1.GroupName}, + []string{"nodes", "nodes/status"}, + []string{flowcontrol.NamespaceEvery}, + true), + resourceRule( + []string{flowcontrol.VerbAll}, + []string{coordinationv1.GroupName}, + []string{"leases"}, + []string{flowcontrol.NamespaceEvery}, + false), + }, + }, + ) + SuggestedFlowSchemaSystemNodes = newFlowSchema( + "system-nodes", "system", 500, + flowcontrol.FlowDistinguisherMethodByUserType, + flowcontrol.PolicyRulesWithSubjects{ + Subjects: groups(user.NodesGroup), // the nodes group + ResourceRules: []flowcontrol.ResourcePolicyRule{resourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.APIGroupAll}, + []string{flowcontrol.ResourceAll}, + []string{flowcontrol.NamespaceEvery}, + true)}, + NonResourceRules: []flowcontrol.NonResourcePolicyRule{ + nonResourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.NonResourceAll}), + }, + }, + ) + SuggestedFlowSchemaKubeControllerManager = newFlowSchema( + "kube-controller-manager", "workload-high", 800, + flowcontrol.FlowDistinguisherMethodByNamespaceType, + flowcontrol.PolicyRulesWithSubjects{ + Subjects: users(user.KubeControllerManager), + ResourceRules: []flowcontrol.ResourcePolicyRule{resourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.APIGroupAll}, + []string{flowcontrol.ResourceAll}, + []string{flowcontrol.NamespaceEvery}, + true)}, + NonResourceRules: []flowcontrol.NonResourcePolicyRule{ + nonResourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.NonResourceAll}), + }, + }, + ) + SuggestedFlowSchemaKubeScheduler = newFlowSchema( + "kube-scheduler", "workload-high", 800, + flowcontrol.FlowDistinguisherMethodByNamespaceType, + flowcontrol.PolicyRulesWithSubjects{ + Subjects: users(user.KubeScheduler), + ResourceRules: []flowcontrol.ResourcePolicyRule{resourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.APIGroupAll}, + []string{flowcontrol.ResourceAll}, + []string{flowcontrol.NamespaceEvery}, + true)}, + NonResourceRules: []flowcontrol.NonResourcePolicyRule{ + nonResourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.NonResourceAll}), + }, + }, + ) + SuggestedFlowSchemaKubeSystemServiceAccounts = newFlowSchema( + "kube-system-service-accounts", "workload-high", 900, + flowcontrol.FlowDistinguisherMethodByNamespaceType, + flowcontrol.PolicyRulesWithSubjects{ + Subjects: kubeSystemServiceAccount(flowcontrol.NameAll), + ResourceRules: []flowcontrol.ResourcePolicyRule{resourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.APIGroupAll}, + []string{flowcontrol.ResourceAll}, + []string{flowcontrol.NamespaceEvery}, + true)}, + NonResourceRules: []flowcontrol.NonResourcePolicyRule{ + nonResourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.NonResourceAll}), + }, + }, + ) + SuggestedFlowSchemaServiceAccounts = newFlowSchema( + "service-accounts", "workload-low", 9000, + flowcontrol.FlowDistinguisherMethodByUserType, + flowcontrol.PolicyRulesWithSubjects{ + Subjects: groups(serviceaccount.AllServiceAccountsGroup), + ResourceRules: []flowcontrol.ResourcePolicyRule{resourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.APIGroupAll}, + []string{flowcontrol.ResourceAll}, + []string{flowcontrol.NamespaceEvery}, + true)}, + NonResourceRules: []flowcontrol.NonResourcePolicyRule{ + nonResourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.NonResourceAll}), + }, + }, + ) + SuggestedFlowSchemaGlobalDefault = newFlowSchema( + "global-default", "global-default", 9900, + flowcontrol.FlowDistinguisherMethodByUserType, + flowcontrol.PolicyRulesWithSubjects{ + Subjects: groups(user.AllUnauthenticated, user.AllAuthenticated), + ResourceRules: []flowcontrol.ResourcePolicyRule{resourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.APIGroupAll}, + []string{flowcontrol.ResourceAll}, + []string{flowcontrol.NamespaceEvery}, + true)}, + NonResourceRules: []flowcontrol.NonResourcePolicyRule{ + nonResourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.NonResourceAll}), + }, + }, + ) +) + +func newPriorityLevelConfiguration(name string, spec flowcontrol.PriorityLevelConfigurationSpec) *flowcontrol.PriorityLevelConfiguration { + return &flowcontrol.PriorityLevelConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: map[string]string{ + flowcontrol.AutoUpdateAnnotationKey: "true", + }, + }, + Spec: spec, + } +} + +func newFlowSchema(name, plName string, matchingPrecedence int32, dmType flowcontrol.FlowDistinguisherMethodType, rules ...flowcontrol.PolicyRulesWithSubjects) *flowcontrol.FlowSchema { + var dm *flowcontrol.FlowDistinguisherMethod + if dmType != "" { + dm = &flowcontrol.FlowDistinguisherMethod{Type: dmType} + } + return &flowcontrol.FlowSchema{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: map[string]string{ + flowcontrol.AutoUpdateAnnotationKey: "true", + }, + }, + Spec: flowcontrol.FlowSchemaSpec{ + PriorityLevelConfiguration: flowcontrol.PriorityLevelConfigurationReference{ + Name: plName, + }, + MatchingPrecedence: matchingPrecedence, + DistinguisherMethod: dm, + Rules: rules}, + } + +} + +func groups(names ...string) []flowcontrol.Subject { + ans := make([]flowcontrol.Subject, len(names)) + for idx, name := range names { + ans[idx] = flowcontrol.Subject{ + Kind: flowcontrol.SubjectKindGroup, + Group: &flowcontrol.GroupSubject{ + Name: name, + }, + } + } + return ans +} + +func users(names ...string) []flowcontrol.Subject { + ans := make([]flowcontrol.Subject, len(names)) + for idx, name := range names { + ans[idx] = flowcontrol.Subject{ + Kind: flowcontrol.SubjectKindUser, + User: &flowcontrol.UserSubject{ + Name: name, + }, + } + } + return ans +} + +func kubeSystemServiceAccount(names ...string) []flowcontrol.Subject { + subjects := []flowcontrol.Subject{} + for _, name := range names { + subjects = append(subjects, flowcontrol.Subject{ + Kind: flowcontrol.SubjectKindServiceAccount, + ServiceAccount: &flowcontrol.ServiceAccountSubject{ + Name: name, + Namespace: metav1.NamespaceSystem, + }, + }) + } + return subjects +} + +func resourceRule(verbs []string, groups []string, resources []string, namespaces []string, clusterScoped bool) flowcontrol.ResourcePolicyRule { + return flowcontrol.ResourcePolicyRule{ + Verbs: verbs, + APIGroups: groups, + Resources: resources, + Namespaces: namespaces, + ClusterScope: clusterScoped, + } +} + +func nonResourceRule(verbs []string, nonResourceURLs []string) flowcontrol.NonResourcePolicyRule { + return flowcontrol.NonResourcePolicyRule{Verbs: verbs, NonResourceURLs: nonResourceURLs} +} diff --git a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-old/default_test.go b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-old/default_test.go new file mode 100644 index 0000000000000..81aafaf9386cb --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-old/default_test.go @@ -0,0 +1,124 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "testing" + + flowcontrol "k8s.io/api/flowcontrol/v1" +) + +func TestBootstrapPriorityLevelConfigurationWithBorrowing(t *testing.T) { + tests := []struct { + name string + nominalSharesExpected int32 + lendablePercentexpected int32 + }{ + { + name: "leader-election", + nominalSharesExpected: 10, + lendablePercentexpected: 0, + }, + { + name: "node-high", + nominalSharesExpected: 40, + lendablePercentexpected: 25, + }, + { + name: "system", + nominalSharesExpected: 30, + lendablePercentexpected: 33, + }, + { + name: "workload-high", + nominalSharesExpected: 40, + lendablePercentexpected: 50, + }, + { + name: "workload-low", + nominalSharesExpected: 100, + lendablePercentexpected: 90, + }, + { + name: "global-default", + nominalSharesExpected: 20, + lendablePercentexpected: 50, + }, + { + name: "catch-all", + nominalSharesExpected: 5, + lendablePercentexpected: 0, + }, + } + + bootstrapPLs := func() map[string]*flowcontrol.PriorityLevelConfiguration { + list := make([]*flowcontrol.PriorityLevelConfiguration, 0) + list = append(list, V1ConfigCollection.Mandatory.PriorityLevelConfigurations...) + list = append(list, V1ConfigCollection.Suggested.PriorityLevelConfigurations...) + + m := map[string]*flowcontrol.PriorityLevelConfiguration{} + for i := range list { + m[list[i].Name] = list[i] + } + return m + }() + + for _, test := range tests { + bootstrapPL := bootstrapPLs[test.name] + if bootstrapPL == nil { + t.Errorf("Expected bootstrap PriorityLevelConfiguration %q, but not found in bootstrap configuration", test.name) + continue + } + delete(bootstrapPLs, test.name) + + if bootstrapPL.Spec.Type != flowcontrol.PriorityLevelEnablementLimited { + t.Errorf("bootstrap PriorityLevelConfiguration %q is not %q", test.name, flowcontrol.PriorityLevelEnablementLimited) + continue + } + if test.nominalSharesExpected != *bootstrapPL.Spec.Limited.NominalConcurrencyShares { + t.Errorf("bootstrap PriorityLevelConfiguration %q: expected NominalConcurrencyShares: %d, but got: %d", test.name, test.nominalSharesExpected, bootstrapPL.Spec.Limited.NominalConcurrencyShares) + } + if test.lendablePercentexpected != *bootstrapPL.Spec.Limited.LendablePercent { + t.Errorf("bootstrap PriorityLevelConfiguration %q: expected NominalConcurrencyShares: %d, but got: %d", test.name, test.lendablePercentexpected, bootstrapPL.Spec.Limited.LendablePercent) + } + if bootstrapPL.Spec.Limited.BorrowingLimitPercent != nil { + t.Errorf("bootstrap PriorityLevelConfiguration %q: expected BorrowingLimitPercent to be nil, but got: %d", test.name, *bootstrapPL.Spec.Limited.BorrowingLimitPercent) + } + } + + if len(bootstrapPLs) != 0 { + names := make([]string, 0) + for name, bpl := range bootstrapPLs { + if bpl.Spec.Type == flowcontrol.PriorityLevelEnablementExempt { + t.Logf("bootstrap PriorityLevelConfiguration %q is of %q type, skipped", name, flowcontrol.PriorityLevelConfigurationNameExempt) + continue + } + names = append(names, name) + } + + if len(names) != 0 { + t.Errorf("bootstrap PriorityLevelConfiguration objects not accounted by this test: %v", names) + } + } + exemptPL := MandatoryPriorityLevelConfigurationExempt + if exemptPL.Spec.Exempt.NominalConcurrencyShares != nil && *exemptPL.Spec.Exempt.NominalConcurrencyShares != 0 { + t.Errorf("Expected exempt priority level to have NominalConcurrencyShares==0 but got %d instead", *exemptPL.Spec.Exempt.NominalConcurrencyShares) + } + if exemptPL.Spec.Exempt.LendablePercent != nil && *exemptPL.Spec.Exempt.LendablePercent != 0 { + t.Errorf("Expected exempt priority level to have LendablePercent==0 but got %d instead", *exemptPL.Spec.Exempt.LendablePercent) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-v134/default.go similarity index 83% rename from staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go rename to staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-v134/default.go index 5f971bf4756c9..b32a399a48ea7 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-v134/default.go @@ -22,70 +22,70 @@ import ( eventsv1 "k8s.io/api/events/v1" flowcontrol "k8s.io/api/flowcontrol/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/apis/flowcontrol/base" "k8s.io/apiserver/pkg/authentication/serviceaccount" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/utils/ptr" ) -// The objects that define an apiserver's initial behavior. The -// registered defaulting procedures make no changes to these -// particular objects (this is verified in the unit tests of the -// internalbootstrap package; it can not be verified in this package -// because that would require importing k8s.io/kubernetes). -var ( - MandatoryPriorityLevelConfigurations = []*flowcontrol.PriorityLevelConfiguration{ - MandatoryPriorityLevelConfigurationCatchAll, - MandatoryPriorityLevelConfigurationExempt, - } - MandatoryFlowSchemas = []*flowcontrol.FlowSchema{ - MandatoryFlowSchemaExempt, - MandatoryFlowSchemaCatchAll, - } -) - -// The objects that define the current suggested additional configuration -var ( - SuggestedPriorityLevelConfigurations = []*flowcontrol.PriorityLevelConfiguration{ - // "system" priority-level is for the system components that affects self-maintenance of the - // cluster and the availability of those running pods in the cluster, including kubelet and - // kube-proxy. - SuggestedPriorityLevelConfigurationSystem, - // "node-high" priority-level is for the node health reporting. It is separated from "system" - // to make sure that nodes are able to report their health even if kube-apiserver is not capable of - // handling load caused by pod startup (fetching secrets, events etc). - // NOTE: In large clusters 50% - 90% of all API calls use this priority-level. - SuggestedPriorityLevelConfigurationNodeHigh, - // "leader-election" is dedicated for controllers' leader-election, which majorly affects the - // availability of any controller runs in the cluster. - SuggestedPriorityLevelConfigurationLeaderElection, - // "workload-high" is used by those workloads with higher priority but their failure won't directly - // impact the existing running pods in the cluster, which includes kube-scheduler, and those well-known - // built-in workloads such as "deployments", "replicasets" and other low-level custom workload which - // is important for the cluster. - SuggestedPriorityLevelConfigurationWorkloadHigh, - // "events" is used for requests that create/update/delete Event objects. - SuggestedPriorityLevelConfigurationEvent, - // "workload-low" is used by those workloads with lower priority which availability only has a - // minor impact on the cluster. - SuggestedPriorityLevelConfigurationWorkloadLow, - // "global-default" serves the rest traffic not handled by the other suggested flow-schemas above. - SuggestedPriorityLevelConfigurationGlobalDefault, - } - SuggestedFlowSchemas = []*flowcontrol.FlowSchema{ - SuggestedFlowSchemaSystemNodes, // references "system" priority-level - SuggestedFlowSchemaEvents, // references "events" priority-level - SuggestedFlowSchemaSystemNodeHigh, // references "node-high" priority-level - SuggestedFlowSchemaProbes, // references "exempt" priority-level - SuggestedFlowSchemaSystemLeaderElection, // references "leader-election" priority-level - SuggestedFlowSchemaWorkloadLeaderElection, // references "leader-election" priority-level - SuggestedFlowSchemaEndpointsController, // references "workload-high" priority-level - SuggestedFlowSchemaKubeControllerManager, // references "workload-high" priority-level - SuggestedFlowSchemaKubeScheduler, // references "workload-high" priority-level - SuggestedFlowSchemaKubeSystemServiceAccounts, // references "workload-high" priority-level - SuggestedFlowSchemaServiceAccounts, // references "workload-low" priority-level - SuggestedFlowSchemaGlobalDefault, // references "global-default" priority-level - } -) +var V1ConfigCollection = base.V1ConfigCollection{ + Mandatory: base.V1ConfigSlices{ + PriorityLevelConfigurations: []*flowcontrol.PriorityLevelConfiguration{ + MandatoryPriorityLevelConfigurationCatchAll, + MandatoryPriorityLevelConfigurationExempt, + }, + FlowSchemas: []*flowcontrol.FlowSchema{ + MandatoryFlowSchemaExempt, + MandatoryFlowSchemaCatchAll, + }, + }, + PriorityLevelConfigurationExempt: MandatoryPriorityLevelConfigurationExempt, + PriorityLevelConfigurationCatchAll: MandatoryPriorityLevelConfigurationCatchAll, + FlowSchemaExempt: MandatoryFlowSchemaExempt, + FlowSchemaCatchAll: MandatoryFlowSchemaCatchAll, + Suggested: base.V1ConfigSlices{ + PriorityLevelConfigurations: []*flowcontrol.PriorityLevelConfiguration{ + // "system" priority-level is for the system components that affects self-maintenance of the + // cluster and the availability of those running pods in the cluster, including kubelet and + // kube-proxy. + SuggestedPriorityLevelConfigurationSystem, + // "node-high" priority-level is for the node health reporting. It is separated from "system" + // to make sure that nodes are able to report their health even if kube-apiserver is not capable of + // handling load caused by pod startup (fetching secrets, events etc). + // NOTE: In large clusters 50% - 90% of all API calls use this priority-level. + SuggestedPriorityLevelConfigurationNodeHigh, + // "leader-election" is dedicated for controllers' leader-election, which majorly affects the + // availability of any controller runs in the cluster. + SuggestedPriorityLevelConfigurationLeaderElection, + // "workload-high" is used by those workloads with higher priority but their failure won't directly + // impact the existing running pods in the cluster, which includes kube-scheduler, and those well-known + // built-in workloads such as "deployments", "replicasets" and other low-level custom workload which + // is important for the cluster. + SuggestedPriorityLevelConfigurationWorkloadHigh, + // "events" is used for requests that create/update/delete Event objects. + SuggestedPriorityLevelConfigurationEvent, + // "workload-low" is used by those workloads with lower priority which availability only has a + // minor impact on the cluster. + SuggestedPriorityLevelConfigurationWorkloadLow, + // "global-default" serves the rest traffic not handled by the other suggested flow-schemas above. + SuggestedPriorityLevelConfigurationGlobalDefault, + }, + FlowSchemas: []*flowcontrol.FlowSchema{ + SuggestedFlowSchemaSystemNodes, // references "system" priority-level + SuggestedFlowSchemaEvents, // references "events" priority-level + SuggestedFlowSchemaSystemNodeHigh, // references "node-high" priority-level + SuggestedFlowSchemaProbes, // references "exempt" priority-level + SuggestedFlowSchemaSystemLeaderElection, // references "leader-election" priority-level + SuggestedFlowSchemaWorkloadLeaderElection, // references "leader-election" priority-level + SuggestedFlowSchemaEndpointsController, // references "workload-high" priority-level + SuggestedFlowSchemaKubeControllerManager, // references "workload-high" priority-level + SuggestedFlowSchemaKubeScheduler, // references "workload-high" priority-level + SuggestedFlowSchemaKubeSystemServiceAccounts, // references "workload-high" priority-level + SuggestedFlowSchemaServiceAccounts, // references "workload-low" priority-level + SuggestedFlowSchemaGlobalDefault, // references "global-default" priority-level + }, + }, +} // Mandatory PriorityLevelConfiguration objects var ( diff --git a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-v134/default_test.go similarity index 93% rename from staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go rename to staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-v134/default_test.go index b7fa01a4316c4..2f2b90366e143 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default_test.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-v134/default_test.go @@ -76,8 +76,8 @@ func TestBootstrapPriorityLevelConfigurationWithBorrowing(t *testing.T) { bootstrapPLs := func() map[string]*flowcontrol.PriorityLevelConfiguration { list := make([]*flowcontrol.PriorityLevelConfiguration, 0) - list = append(list, MandatoryPriorityLevelConfigurations...) - list = append(list, SuggestedPriorityLevelConfigurations...) + list = append(list, V1ConfigCollection.Mandatory.PriorityLevelConfigurations...) + list = append(list, V1ConfigCollection.Suggested.PriorityLevelConfigurations...) m := map[string]*flowcontrol.PriorityLevelConfiguration{} for i := range list { @@ -102,7 +102,7 @@ func TestBootstrapPriorityLevelConfigurationWithBorrowing(t *testing.T) { t.Errorf("bootstrap PriorityLevelConfiguration %q: expected NominalConcurrencyShares: %d, but got: %d", test.name, test.nominalSharesExpected, bootstrapPL.Spec.Limited.NominalConcurrencyShares) } if test.lendablePercentexpected != *bootstrapPL.Spec.Limited.LendablePercent { - t.Errorf("bootstrap PriorityLevelConfiguration %q: expected LendablePercent: %d, but got: %d", test.name, test.lendablePercentexpected, bootstrapPL.Spec.Limited.LendablePercent) + t.Errorf("bootstrap PriorityLevelConfiguration %q: expected LendablePercent: %d, but got: %s", test.name, test.lendablePercentexpected, fmtPtr(bootstrapPL.Spec.Limited.LendablePercent)) } if !ptr.Equal(test.borrowingLimitPercent, bootstrapPL.Spec.Limited.BorrowingLimitPercent) { t.Errorf("bootstrap PriorityLevelConfiguration %q: expected BorrowingLimitPercent to be %s, but got: %s", test.name, fmtPtr(test.borrowingLimitPercent), fmtPtr(bootstrapPL.Spec.Limited.BorrowingLimitPercent)) diff --git a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/collection.go b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/collection.go new file mode 100644 index 0000000000000..b418755e84829 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/collection.go @@ -0,0 +1,92 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + flowcontrolv1 "k8s.io/api/flowcontrol/v1" + "k8s.io/apiserver/pkg/apis/flowcontrol/base" + bootstrapold "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-old" + bootstrapv134 "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-v134" +) + +// This package provides access to the "default" configuration objects of +// API Priority and Fairness according to the settings of the Feature(s) +// that control which collection of defaults to use. +// Currently there is exactly one such Feature, APFv134Config. + +// GetV1ConfigCollection is given the values of the features that define +// the default APF configuration objects to use, and returns them (in V1 form). +// The returned value is deeply immutable. +// v134 is the value of the APFv134Config feature. +func GetV1ConfigCollection(v134 bool) *base.V1ConfigCollection { + if v134 { + return &bootstrapv134.V1ConfigCollection + } else { + return &bootstrapold.V1ConfigCollection + } +} + +// GetPrioritylevelConfigurations returns the default PriorityLevelConfiguration objects, +// as a deeply immutable slice. +// The types are the latest external type (version). +// The arguments are the values of the features that control which config collection to use. +func GetPrioritylevelConfigurations(v134 bool) []*flowcontrolv1.PriorityLevelConfiguration { + return PrioritylevelConfigurations[CollectionID{V134: v134}] +} + +// PrioritylevelConfigurations maps CollectionId to a slice of all the default objects. +// Deeply immutable. +var PrioritylevelConfigurations = map[CollectionID][]*flowcontrolv1.PriorityLevelConfiguration{ + {false}: Concat(bootstrapold.V1ConfigCollection.Mandatory.PriorityLevelConfigurations, + bootstrapold.V1ConfigCollection.Suggested.PriorityLevelConfigurations), + {true}: Concat(bootstrapv134.V1ConfigCollection.Mandatory.PriorityLevelConfigurations, + bootstrapv134.V1ConfigCollection.Suggested.PriorityLevelConfigurations), +} + +// GetFlowSchemas returns the default FlowSchema objects, +// as a deeply immutable slice. +// The types are the latest external type (version). +// The arguments are the values of the features that control which config collection to use. +func GetFlowSchemas(v134 bool) []*flowcontrolv1.FlowSchema { + return FlowSchemas[CollectionID{V134: v134}] +} + +// FlowSchemas maps CollectionId to a slice of all the default objects. +// Deeply immutable. +var FlowSchemas = map[CollectionID][]*flowcontrolv1.FlowSchema{ + {false}: Concat(bootstrapold.V1ConfigCollection.Mandatory.FlowSchemas, + bootstrapold.V1ConfigCollection.Suggested.FlowSchemas), + {true}: Concat(bootstrapv134.V1ConfigCollection.Mandatory.FlowSchemas, + bootstrapv134.V1ConfigCollection.Suggested.FlowSchemas), +} + +// CollectionID identifies the collection of API Priority and Fairness config objects +// to use as the defaults. +type CollectionID struct { + // V134=true means use the collection introduced for release 1.34. + // V134=false means use the older collection. + V134 bool +} + +// Concat concatenates the given slices, without reusing any given backing array. +func Concat[T any](slices ...[]T) []T { + var ans []T + for _, slice := range slices { + ans = append(ans, slice...) + } + return ans +} diff --git a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go index ec5df43795169..4d1f2c1452ee4 100644 --- a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go +++ b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go @@ -61,6 +61,15 @@ const ( // resources using the Kubernetes API only. AllowUnsafeMalformedObjectDeletion featuregate.Feature = "AllowUnsafeMalformedObjectDeletion" + // owner: @MikeSpreitzer, @tkashem, @linxiulei + // + // Make API Priority and Fairness use modern configuration, which + // differs from the old in these ways: + // - introduce priority level and flow schema for events; + // - generally reorganize to stop working around lack of borrowing; + // - increase the nominal concurrency shares for leader election. + APFv134Config featuregate.Feature = "APFv134Config" + // owner: @ilackams // // Enables compression of REST responses (GET and LIST only) @@ -269,6 +278,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha}, }, + APFv134Config: { + {Version: version.MustParse("1.34"), Default: true, PreRelease: featuregate.Beta}, + }, + AnonymousAuthConfigurableEndpoints: { {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.Beta}, diff --git a/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness_test.go b/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness_test.go index 18d99292ccd89..8bfa87e6f01bf 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness_test.go @@ -35,7 +35,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" + bootstrap "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-v134" "k8s.io/apiserver/pkg/authentication/user" apifilters "k8s.io/apiserver/pkg/endpoints/filters" epmetrics "k8s.io/apiserver/pkg/endpoints/metrics" @@ -1138,7 +1138,7 @@ func startAPFController(t *testing.T, stopCh <-chan struct{}, apfConfiguration [ clientset := newClientset(t, apfConfiguration...) // this test does not rely on resync, so resync period is set to zero factory := informers.NewSharedInformerFactory(clientset, 0) - controller := utilflowcontrol.New(factory, clientset.FlowcontrolV1(), serverConcurrency) + controller := utilflowcontrol.New(factory, clientset.FlowcontrolV1(), serverConcurrency, true) factory.Start(stopCh) diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/feature.go b/staging/src/k8s.io/apiserver/pkg/server/options/feature.go index a5b1372da4705..92e97fcdfd726 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/feature.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/feature.go @@ -22,6 +22,7 @@ import ( "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/server" utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol" "k8s.io/client-go/informers" @@ -78,10 +79,12 @@ func (o *FeatureOptions) ApplyTo(c *server.Config, clientset kubernetes.Interfac return fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight) } + v134Config := c.FeatureGate.Enabled(features.APFv134Config) c.FlowControl = utilflowcontrol.New( informers, clientset.FlowcontrolV1(), c.MaxRequestsInFlight+c.MaxMutatingRequestsInFlight, + v134Config, ) } diff --git a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go index 652ef57abb890..1e43f9f120a53 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go +++ b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go @@ -124,6 +124,7 @@ type configController struct { queueSetFactory fq.QueueSetFactory reqsGaugeVec metrics.RatioedGaugeVec execSeatsGaugeVec metrics.RatioedGaugeVec + newConfig bool // How this controller appears in an ObjectMeta ManagedFieldsEntry.Manager asFieldManager string @@ -276,6 +277,7 @@ func (stats *seatDemandStats) update(obs fq.IntegratorResults) { // NewTestableController is extra flexible to facilitate testing func newTestableController(config TestableConfig) *configController { cfgCtlr := &configController{ + newConfig: config.V134Config, name: config.Name, clock: config.Clock, queueSetFactory: config.QueueSetFactory, @@ -690,10 +692,10 @@ func (cfgCtlr *configController) lockAndDigestConfigObjects(newPLs []*flowcontro // Supply missing mandatory PriorityLevelConfiguration objects if !meal.haveExemptPL { - meal.imaginePL(fcboot.MandatoryPriorityLevelConfigurationExempt) + meal.imaginePL(fcboot.GetV1ConfigCollection(cfgCtlr.newConfig).PriorityLevelConfigurationExempt) } if !meal.haveCatchAllPL { - meal.imaginePL(fcboot.MandatoryPriorityLevelConfigurationCatchAll) + meal.imaginePL(fcboot.GetV1ConfigCollection(cfgCtlr.newConfig).PriorityLevelConfigurationCatchAll) } meal.finishQueueSetReconfigsLocked() @@ -785,10 +787,10 @@ func (meal *cfgMeal) digestFlowSchemasLocked(newFSs []*flowcontrol.FlowSchema) { // Supply missing mandatory FlowSchemas, in correct position if !haveExemptFS { - fsSeq = append(apihelpers.FlowSchemaSequence{fcboot.MandatoryFlowSchemaExempt}, fsSeq...) + fsSeq = append(apihelpers.FlowSchemaSequence{fcboot.GetV1ConfigCollection(meal.cfgCtlr.newConfig).FlowSchemaExempt}, fsSeq...) } if !haveCatchAllFS { - fsSeq = append(fsSeq, fcboot.MandatoryFlowSchemaCatchAll) + fsSeq = append(fsSeq, fcboot.GetV1ConfigCollection(meal.cfgCtlr.newConfig).FlowSchemaCatchAll) } meal.cfgCtlr.flowSchemas = fsSeq diff --git a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter.go b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter.go index 2a4bf10f7bb51..8387e86156db7 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter.go +++ b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter.go @@ -90,6 +90,7 @@ func New( informerFactory kubeinformers.SharedInformerFactory, flowcontrolClient flowcontrolclient.FlowcontrolV1Interface, serverConcurrencyLimit int, + v134Config bool, ) Interface { clk := eventclock.Real{} return NewTestable(TestableConfig{ @@ -103,6 +104,7 @@ func New( ReqsGaugeVec: metrics.PriorityLevelConcurrencyGaugeVec, ExecSeatsGaugeVec: metrics.PriorityLevelExecutionSeatsGaugeVec, QueueSetFactory: fqs.NewQueueSetFactory(clk), + V134Config: v134Config, }) } @@ -145,6 +147,9 @@ type TestableConfig struct { // QueueSetFactory for the queuing implementation QueueSetFactory fq.QueueSetFactory + + // Whether to use modern default config objects + V134Config bool } // NewTestable is extra flexible to facilitate testing diff --git a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/controller_test.go b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/controller_test.go index 2a80462888412..52ad58cd97231 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/controller_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/controller_test.go @@ -55,7 +55,7 @@ func TestMain(m *testing.M) { var mandPLs = func() map[string]*flowcontrol.PriorityLevelConfiguration { ans := make(map[string]*flowcontrol.PriorityLevelConfiguration) - for _, mand := range fcboot.MandatoryPriorityLevelConfigurations { + for _, mand := range fcboot.GetV1ConfigCollection(true).Mandatory.PriorityLevelConfigurations { ans[mand.Name] = mand } return ans @@ -224,7 +224,7 @@ func (cts *ctlrTestState) popHeldRequest() (plName string, hr *heldRequest, nCou var mandQueueSetNames = func() sets.String { mandQueueSetNames := sets.NewString() - for _, mpl := range fcboot.MandatoryPriorityLevelConfigurations { + for _, mpl := range fcboot.GetV1ConfigCollection(true).Mandatory.PriorityLevelConfigurations { mandQueueSetNames.Insert(mpl.Name) } return mandQueueSetNames @@ -523,8 +523,8 @@ func genPLs(rng *rand.Rand, trial string, oldPLNames sets.String, n int) (pls [] func genFSs(t *testing.T, rng *rand.Rand, trial string, goodPLNames, badPLNames sets.String, n int) (newFSs []*flowcontrol.FlowSchema, newFSMap map[string]*flowcontrol.FlowSchema, newFTRs map[string]*fsTestingRecord, catchAlls map[bool]*flowcontrol.FlowSchema) { newFTRs = map[string]*fsTestingRecord{} catchAlls = map[bool]*flowcontrol.FlowSchema{ - false: fcboot.MandatoryFlowSchemaCatchAll, - true: fcboot.MandatoryFlowSchemaCatchAll} + false: fcboot.GetV1ConfigCollection(true).FlowSchemaCatchAll, + true: fcboot.GetV1ConfigCollection(true).FlowSchemaCatchAll} newFSMap = map[string]*flowcontrol.FlowSchema{} add := func(ftr *fsTestingRecord) { newFSs = append(newFSs, ftr.fs) diff --git a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/exempt_borrowing_test.go b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/exempt_borrowing_test.go index 308c972f7cbfa..d2bbeba82c9b1 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/exempt_borrowing_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/exempt_borrowing_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - fcboot "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" + fcboot "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-v134" fqs "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset" testeventclock "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/testing/eventclock" "k8s.io/apiserver/pkg/util/flowcontrol/metrics" diff --git a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/gen_test.go b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/gen_test.go index 21b88367fbe96..f60011d0aaae4 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/gen_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/gen_test.go @@ -94,7 +94,7 @@ var flowDistinguisherMethodTypes = sets.NewString( ) var mandFTRExempt = &fsTestingRecord{ - fs: fcboot.MandatoryFlowSchemaExempt, + fs: fcboot.GetV1ConfigCollection(true).FlowSchemaExempt, wellFormed: true, digests: map[bool]map[bool][]RequestDigest{ false: { @@ -151,7 +151,7 @@ var mandFTRExempt = &fsTestingRecord{ } var mandFTRCatchAll = &fsTestingRecord{ - fs: fcboot.MandatoryFlowSchemaCatchAll, + fs: fcboot.GetV1ConfigCollection(true).FlowSchemaCatchAll, wellFormed: true, digests: map[bool]map[bool][]RequestDigest{ false: {}, diff --git a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml index dd0ad06e1fc37..3463a54b7c553 100644 --- a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml +++ b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml @@ -91,6 +91,12 @@ lockToDefault: true preRelease: GA version: "1.33" +- name: APFv134Config + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Beta + version: "1.34" - name: APIResponseCompression versionedSpecs: - default: false diff --git a/test/integration/apiserver/flowcontrol/fs_condition_test.go b/test/integration/apiserver/flowcontrol/fs_condition_test.go index a37b66a4fb3ed..8a8ca21991d52 100644 --- a/test/integration/apiserver/flowcontrol/fs_condition_test.go +++ b/test/integration/apiserver/flowcontrol/fs_condition_test.go @@ -38,7 +38,7 @@ func TestConditionIsolation(t *testing.T) { loopbackClient := clientset.NewForConfigOrDie(kubeConfig) - fsOrig := fcboot.SuggestedFlowSchemas[0] + fsOrig := fcboot.GetV1ConfigCollection(true).Suggested.FlowSchemas[0] t.Logf("Testing Status Condition isolation in FlowSchema %q", fsOrig.Name) fsClient := loopbackClient.FlowcontrolV1().FlowSchemas() var dangleOrig *flowcontrol.FlowSchemaCondition From e29bd1fa5f43c84efa25a15079ecf84ed82f50f4 Mon Sep 17 00:00:00 2001 From: Mike Spreitzer Date: Mon, 17 Mar 2025 04:17:23 -0400 Subject: [PATCH 6/9] Add integration test of APF config flapping Make APFv134Config Alpha at 1.33 --- pkg/features/kube_features.go | 1 + .../flowcontrol/rest/storage_flowcontrol.go | 15 +- .../apiserver/pkg/features/kube_features.go | 1 + .../pkg/util/flowcontrol/apf_controller.go | 12 +- .../reference/versioned_feature_list.yaml | 4 + .../apiserver/flowcontrol/concurrency_test.go | 40 +++- .../apiserver/flowcontrol/config_flap_test.go | 175 ++++++++++++++++++ .../apiserver/flowcontrol/fight_test.go | 2 +- .../flowcontrol/fs_condition_test.go | 5 +- 9 files changed, 226 insertions(+), 29 deletions(-) create mode 100644 test/integration/apiserver/flowcontrol/config_flap_test.go diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index bc284d33f7783..b70b91ce170d6 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -1014,6 +1014,7 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate }, APFv134Config: { + {Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("1.34"), Default: true, PreRelease: featuregate.Beta}, }, diff --git a/pkg/registry/flowcontrol/rest/storage_flowcontrol.go b/pkg/registry/flowcontrol/rest/storage_flowcontrol.go index c0edcf4d6cd55..04dcda184bc71 100644 --- a/pkg/registry/flowcontrol/rest/storage_flowcontrol.go +++ b/pkg/registry/flowcontrol/rest/storage_flowcontrol.go @@ -136,16 +136,14 @@ func (bce *bootstrapConfigurationEnsurer) ensureAPFBootstrapConfiguration(hookCo err = func() error { // get a derived context that gets cancelled after 5m or // when the StopCh gets closed, whichever happens first. - ctx, cancel := contextFromChannelAndMaxWaitDuration(hookContext.Done(), 5*time.Minute) + ctx, cancel := context.WithDeadline(hookContext, time.Now().Add(5*time.Minute)) defer cancel() if !cache.WaitForCacheSync(ctx.Done(), bce.informersSynced...) { return fmt.Errorf("APF bootstrap ensurer timed out waiting for cache sync") } - err = wait.PollImmediateUntilWithContext( - ctx, - time.Second, + err = wait.PollUntilContextCancel(ctx, time.Second, true, func(context.Context) (bool, error) { if err := ensure(ctx, bce.v134Config, clientset, bce.fsLister, bce.plcLister); err != nil { klog.ErrorS(err, "APF bootstrap ensurer ran into error, will retry later") @@ -165,15 +163,14 @@ func (bce *bootstrapConfigurationEnsurer) ensureAPFBootstrapConfiguration(hookCo // we have successfully initialized the bootstrap configuration, now we // spin up a goroutine which reconciles the bootstrap configuration periodically. go func() { - wait.PollImmediateUntil( - time.Minute, - func() (bool, error) { - if err := ensure(hookContext, bce.v134Config, clientset, bce.fsLister, bce.plcLister); err != nil { + _ = wait.PollUntilContextCancel(hookContext, time.Minute, true, + func(ctx context.Context) (bool, error) { + if err := ensure(ctx, bce.v134Config, clientset, bce.fsLister, bce.plcLister); err != nil { klog.ErrorS(err, "APF bootstrap ensurer ran into error, will retry later") } // always auto update both suggested and mandatory configuration return false, nil - }, hookContext.Done()) + }) klog.Info("APF bootstrap ensurer is exiting") }() diff --git a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go index 4d1f2c1452ee4..3ae05e3e342be 100644 --- a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go +++ b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go @@ -279,6 +279,7 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate }, APFv134Config: { + {Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("1.34"), Default: true, PreRelease: featuregate.Beta}, }, diff --git a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go index 1e43f9f120a53..c66f87aeb45d1 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go +++ b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go @@ -124,7 +124,7 @@ type configController struct { queueSetFactory fq.QueueSetFactory reqsGaugeVec metrics.RatioedGaugeVec execSeatsGaugeVec metrics.RatioedGaugeVec - newConfig bool + v134Config bool // How this controller appears in an ObjectMeta ManagedFieldsEntry.Manager asFieldManager string @@ -277,7 +277,7 @@ func (stats *seatDemandStats) update(obs fq.IntegratorResults) { // NewTestableController is extra flexible to facilitate testing func newTestableController(config TestableConfig) *configController { cfgCtlr := &configController{ - newConfig: config.V134Config, + v134Config: config.V134Config, name: config.Name, clock: config.Clock, queueSetFactory: config.QueueSetFactory, @@ -692,10 +692,10 @@ func (cfgCtlr *configController) lockAndDigestConfigObjects(newPLs []*flowcontro // Supply missing mandatory PriorityLevelConfiguration objects if !meal.haveExemptPL { - meal.imaginePL(fcboot.GetV1ConfigCollection(cfgCtlr.newConfig).PriorityLevelConfigurationExempt) + meal.imaginePL(fcboot.GetV1ConfigCollection(cfgCtlr.v134Config).PriorityLevelConfigurationExempt) } if !meal.haveCatchAllPL { - meal.imaginePL(fcboot.GetV1ConfigCollection(cfgCtlr.newConfig).PriorityLevelConfigurationCatchAll) + meal.imaginePL(fcboot.GetV1ConfigCollection(cfgCtlr.v134Config).PriorityLevelConfigurationCatchAll) } meal.finishQueueSetReconfigsLocked() @@ -787,10 +787,10 @@ func (meal *cfgMeal) digestFlowSchemasLocked(newFSs []*flowcontrol.FlowSchema) { // Supply missing mandatory FlowSchemas, in correct position if !haveExemptFS { - fsSeq = append(apihelpers.FlowSchemaSequence{fcboot.GetV1ConfigCollection(meal.cfgCtlr.newConfig).FlowSchemaExempt}, fsSeq...) + fsSeq = append(apihelpers.FlowSchemaSequence{fcboot.GetV1ConfigCollection(meal.cfgCtlr.v134Config).FlowSchemaExempt}, fsSeq...) } if !haveCatchAllFS { - fsSeq = append(fsSeq, fcboot.GetV1ConfigCollection(meal.cfgCtlr.newConfig).FlowSchemaCatchAll) + fsSeq = append(fsSeq, fcboot.GetV1ConfigCollection(meal.cfgCtlr.v134Config).FlowSchemaCatchAll) } meal.cfgCtlr.flowSchemas = fsSeq diff --git a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml index 3463a54b7c553..724fd6e9c7237 100644 --- a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml +++ b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml @@ -93,6 +93,10 @@ version: "1.33" - name: APFv134Config versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.33" - default: true lockToDefault: false preRelease: Beta diff --git a/test/integration/apiserver/flowcontrol/concurrency_test.go b/test/integration/apiserver/flowcontrol/concurrency_test.go index 34e44a5da7de8..26f184b7073e2 100644 --- a/test/integration/apiserver/flowcontrol/concurrency_test.go +++ b/test/integration/apiserver/flowcontrol/concurrency_test.go @@ -31,9 +31,12 @@ import ( flowcontrol "k8s.io/api/flowcontrol/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/features" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/component-base/featuregate" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + "k8s.io/kubernetes/pkg/controlplane" "k8s.io/kubernetes/test/integration/framework" "k8s.io/kubernetes/test/utils/ktesting" "k8s.io/utils/ptr" @@ -47,30 +50,49 @@ const ( timeout = time.Second * 10 ) -func setup(t testing.TB, maxReadonlyRequestsInFlight, maxMutatingRequestsInFlight int) (context.Context, *rest.Config, framework.TearDownFunc) { +func setup(t testing.TB, maxReadonlyRequestsInFlight, maxMutatingRequestsInFlight int, v134Config bool) (ktesting.TContext, clientset.Interface, *rest.Config, framework.TearDownFunc) { tCtx := ktesting.Init(t) + client, kubeConfig, tearDownFn := setupAnother(t, tCtx, maxReadonlyRequestsInFlight, maxMutatingRequestsInFlight, v134Config) + newTeardown := func() { + tCtx.Cancel("tearing down apiserver") + tearDownFn() + } + return tCtx, client, kubeConfig, newTeardown +} - _, kubeConfig, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ +func setupAnother(t testing.TB, ctx context.Context, maxReadonlyRequestsInFlight, maxMutatingRequestsInFlight int, v134Config bool) (clientset.Interface, *rest.Config, framework.TearDownFunc) { + client, kubeConfig, tearDownFn := framework.StartTestServer(ctx, t, framework.TestServerSetup{ ModifyServerRunOptions: func(opts *options.ServerRunOptions) { // Ensure all clients are allowed to send requests. opts.Authorization.Modes = []string{"AlwaysAllow"} opts.GenericServerRunOptions.MaxRequestsInFlight = maxReadonlyRequestsInFlight opts.GenericServerRunOptions.MaxMutatingRequestsInFlight = maxMutatingRequestsInFlight }, + ModifyServerConfig: func(c *controlplane.Config) { + fg := c.ControlPlane.Generic.FeatureGate + if fg.Enabled(features.APFv134Config) == v134Config { + return + } + desired := fmt.Sprintf("APFv134Config=%v", v134Config) + switch typed := fg.(type) { + case featuregate.MutableFeatureGate: + err := typed.Set(desired) + if err != nil { + t.Errorf("Failed to set %s: %s", desired, err.Error()) + } + default: + t.Error("Unable to turn on APFv134Config Feature due to immutable FeatureGate") + } + }, }) - newTeardown := func() { - tCtx.Cancel("tearing down apiserver") - tearDownFn() - } - return tCtx, kubeConfig, newTeardown + return client, kubeConfig, tearDownFn } func TestPriorityLevelIsolation(t *testing.T) { - ctx, kubeConfig, closeFn := setup(t, 1, 1) + ctx, loopbackClient, kubeConfig, closeFn := setup(t, 1, 1, true) defer closeFn() - loopbackClient := clientset.NewForConfigOrDie(kubeConfig) noxu1Client := getClientFor(kubeConfig, "noxu1") noxu2Client := getClientFor(kubeConfig, "noxu2") diff --git a/test/integration/apiserver/flowcontrol/config_flap_test.go b/test/integration/apiserver/flowcontrol/config_flap_test.go new file mode 100644 index 0000000000000..6dfeb4853a963 --- /dev/null +++ b/test/integration/apiserver/flowcontrol/config_flap_test.go @@ -0,0 +1,175 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flowcontrol + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "testing" + "time" + + flowcontrolv1 "k8s.io/api/flowcontrol/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + flowcontrolbootstrap "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + "k8s.io/kubernetes/test/utils/ktesting" +) + +// TestConfigFlap tests the ability of the APF config-producer to correctly switch +// between old and new config in the cluster. +// This test runs in three phases. +// First, it runs an apiserver configured with APFv134Config=true +// and checks that that config is established within 2.5 minutes; +// then the server is shut down. +// Second, it runs an apiserver configured with APFv134Config=false +// and checks that that config is established within 2.5 minutes; +// then the server is shut down. +// Finally, it runs an apiserver configured with APFv134Config=true +// and checks that that config is established within 2.5 minutes; +// then the server is shut down. +// Remember that the etcd server is bound in test_main, so all +// three phases share the same etcd server. +func TestConfigFlap(t *testing.T) { + tCtx := ktesting.Init(t) + if !flapTo(t, tCtx, "first", true) { + return + } + if !flapTo(t, tCtx, "first", false) { + return + } + if !flapTo(t, tCtx, "second", true) { + return + } + tCtx.Cancel("test function done") +} + +var errTestPhaseDone = errors.New("test phase done") + +func flapTo(t *testing.T, ctx context.Context, trial string, v134 bool) bool { + flapName := fmt.Sprintf("%s v134=%v", trial, v134) + t.Log("Preparing for " + flapName) + ctx, cancel := context.WithCancelCause(ctx) + client, _, tearDown := setupAnother(t, ctx, 100, 100, v134) + defer func() { + cancel(errTestPhaseDone) + tearDown() + }() + t.Log("Waiting for " + flapName) + if !waitForConfig(t, ctx, client, flapName, v134) { + t.Fatal("Failed to establish " + flapName) + return false + } + t.Log("Established " + flapName) + return true +} + +var errSuccess = errors.New("gotit") + +func waitForConfig(t *testing.T, ctx context.Context, client clientset.Interface, phaseName string, v134 bool) bool { + expectedSlices := flowcontrolbootstrap.GetV1ConfigCollection(v134) + expectedPLCs := slicesToMap(widenToObject, expectedSlices.Mandatory.PriorityLevelConfigurations, expectedSlices.Suggested.PriorityLevelConfigurations) + expectedFSes := slicesToMap(widenToObject, expectedSlices.Mandatory.FlowSchemas, expectedSlices.Suggested.FlowSchemas) + var iteration int + err := wait.PollUntilContextTimeout(ctx, 20*time.Second, 150*time.Second, false, func(ctx context.Context) (done bool, err error) { + defer func() { iteration++ }() + logger := klog.FromContext(ctx) + step := fmt.Sprintf("%s iteration %d", phaseName, iteration) + t.Log("Starting " + step) + actualPLCList, err := client.FlowcontrolV1().PriorityLevelConfigurations().List(ctx, metav1.ListOptions{}) + if err != nil { + logger.Error(err, "Failed to list PriorityLevelConfiguration objects") + return false, nil + } + actualPLCs := slicesToMap(plcToObject, actualPLCList.Items) + plcsGood := compareMaps(t, step, "PriorityLevelConfiguration", plcToSpec, expectedPLCs, actualPLCs) + actualFSList, err := client.FlowcontrolV1().FlowSchemas().List(ctx, metav1.ListOptions{}) + if err != nil { + logger.Error(err, "Failed to list FlowSchema objects") + return false, nil + } + actualFSes := slicesToMap(fsToObject, actualFSList.Items) + fsesGood := compareMaps(t, step, "FlowSchema", fsToSpec, expectedFSes, actualFSes) + t.Logf("In %s, plcsGood=%v, fsesGood=%v", step, plcsGood, fsesGood) + if plcsGood && fsesGood { + return false, errSuccess + } + return false, nil + }) + return errors.Is(err, errSuccess) +} + +func slicesToMap[Elt any](widen func(Elt) metav1.Object, slices ...[]Elt) map[string]Elt { + ans := make(map[string]Elt) + for _, aslice := range slices { + for _, elt := range aslice { + obj := widen(elt) + ans[obj.GetName()] = elt + } + } + return ans +} + +func widenToObject[Specific metav1.Object](obj Specific) metav1.Object { return obj } +func plcToObject(plc flowcontrolv1.PriorityLevelConfiguration) metav1.Object { return &plc } +func fsToObject(fs flowcontrolv1.FlowSchema) metav1.Object { return &fs } +func plcToSpec(plc *flowcontrolv1.PriorityLevelConfiguration) flowcontrolv1.PriorityLevelConfigurationSpec { + return plc.Spec +} +func fsToSpec(fs *flowcontrolv1.FlowSchema) flowcontrolv1.FlowSchemaSpec { + return fs.Spec +} + +func compareMaps[Elt, Spec any](t *testing.T, step, kind string, getSpec func(*Elt) Spec, expected map[string]*Elt, actual map[string]Elt) bool { + allGood := true + for name, expectedVal := range expected { + actualVal, ok := actual[name] + if !ok { + allGood = false + t.Logf("At %s: did not find expected %s %s", step, kind, name) + continue + } + expectedSpec := getSpec(expectedVal) + actualSpec := getSpec(&actualVal) + if !apiequality.Semantic.DeepEqual(expectedSpec, actualSpec) { + allGood = false + expectedBytes, err := json.Marshal(expectedSpec) + if err != nil { + t.Errorf("Failed to marshal expectedSpec %#v: %s", expectedSpec, err.Error()) + continue + } + actualBytes, err := json.Marshal(actualSpec) + if err != nil { + t.Errorf("Failed to marshal actualSpec %#v: %s", actualSpec, err.Error()) + continue + } + t.Logf("At %s: for %s %s, expectedSpec %s != actualSpec %s", step, kind, name, string(expectedBytes), string(actualBytes)) + } + } + for name := range actual { + _, ok := expected[name] + if !ok { + allGood = false + t.Logf("At %s: did not expect actual %s %s", step, kind, name) + } + } + return allGood +} diff --git a/test/integration/apiserver/flowcontrol/fight_test.go b/test/integration/apiserver/flowcontrol/fight_test.go index c0961795692af..f3d36155d63ba 100644 --- a/test/integration/apiserver/flowcontrol/fight_test.go +++ b/test/integration/apiserver/flowcontrol/fight_test.go @@ -167,7 +167,7 @@ func (ft *fightTest) evaluate(tBeforeCreate, tAfterCreate time.Time) { } } func TestConfigConsumerFight(t *testing.T) { - _, kubeConfig, closeFn := setup(t, 100, 100) + _, _, kubeConfig, closeFn := setup(t, 100, 100, true) defer closeFn() const teamSize = 3 ft := newFightTest(t, kubeConfig, teamSize) diff --git a/test/integration/apiserver/flowcontrol/fs_condition_test.go b/test/integration/apiserver/flowcontrol/fs_condition_test.go index 8a8ca21991d52..5367a72e7a813 100644 --- a/test/integration/apiserver/flowcontrol/fs_condition_test.go +++ b/test/integration/apiserver/flowcontrol/fs_condition_test.go @@ -28,16 +28,13 @@ import ( "k8s.io/apimachinery/pkg/util/wait" fcboot "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" flowcontrolapply "k8s.io/client-go/applyconfigurations/flowcontrol/v1" - clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" ) func TestConditionIsolation(t *testing.T) { - ctx, kubeConfig, closeFn := setup(t, 10, 10) + ctx, loopbackClient, _, closeFn := setup(t, 10, 10, true) defer closeFn() - loopbackClient := clientset.NewForConfigOrDie(kubeConfig) - fsOrig := fcboot.GetV1ConfigCollection(true).Suggested.FlowSchemas[0] t.Logf("Testing Status Condition isolation in FlowSchema %q", fsOrig.Name) fsClient := loopbackClient.FlowcontrolV1().FlowSchemas() From 8c8e38e7f285ef7ab4981865dfa15119b2a1a330 Mon Sep 17 00:00:00 2001 From: Mike Spreitzer Date: Tue, 18 Mar 2025 15:00:52 -0400 Subject: [PATCH 7/9] Use featuregatetesting.SetFeatureGateDuringTest .. and use it multiple times on the same Feature in the same test. The comment on this func says it will fail in that case, but it appears to actually succeed. Examining the body of the func, I do not see why it should fail. --- .../apiserver/flowcontrol/concurrency_test.go | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/test/integration/apiserver/flowcontrol/concurrency_test.go b/test/integration/apiserver/flowcontrol/concurrency_test.go index 26f184b7073e2..b8b8a0f86c7af 100644 --- a/test/integration/apiserver/flowcontrol/concurrency_test.go +++ b/test/integration/apiserver/flowcontrol/concurrency_test.go @@ -32,9 +32,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/component-base/featuregate" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/pkg/controlplane" "k8s.io/kubernetes/test/integration/framework" @@ -61,6 +62,9 @@ func setup(t testing.TB, maxReadonlyRequestsInFlight, maxMutatingRequestsInFligh } func setupAnother(t testing.TB, ctx context.Context, maxReadonlyRequestsInFlight, maxMutatingRequestsInFlight int, v134Config bool) (clientset.Interface, *rest.Config, framework.TearDownFunc) { + if utilfeature.DefaultFeatureGate.Enabled(features.APFv134Config) != v134Config { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APFv134Config, v134Config) + } client, kubeConfig, tearDownFn := framework.StartTestServer(ctx, t, framework.TestServerSetup{ ModifyServerRunOptions: func(opts *options.ServerRunOptions) { // Ensure all clients are allowed to send requests. @@ -70,18 +74,8 @@ func setupAnother(t testing.TB, ctx context.Context, maxReadonlyRequestsInFlight }, ModifyServerConfig: func(c *controlplane.Config) { fg := c.ControlPlane.Generic.FeatureGate - if fg.Enabled(features.APFv134Config) == v134Config { - return - } - desired := fmt.Sprintf("APFv134Config=%v", v134Config) - switch typed := fg.(type) { - case featuregate.MutableFeatureGate: - err := typed.Set(desired) - if err != nil { - t.Errorf("Failed to set %s: %s", desired, err.Error()) - } - default: - t.Error("Unable to turn on APFv134Config Feature due to immutable FeatureGate") + if enabled := fg.Enabled(features.APFv134Config); enabled != v134Config { + t.Fatalf("Wrong value for APFv134Config: %v", enabled) } }, }) From ea7fcdabbd400705d03c5c2e82a4e897dd6d5ae5 Mon Sep 17 00:00:00 2001 From: Mike Spreitzer Date: Wed, 19 Mar 2025 13:02:47 -0400 Subject: [PATCH 8/9] Use more canonical structure for multi-phase test --- .../apiserver/flowcontrol/config_flap_test.go | 33 +++++++------------ 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/test/integration/apiserver/flowcontrol/config_flap_test.go b/test/integration/apiserver/flowcontrol/config_flap_test.go index 6dfeb4853a963..89f2ddc40dc5a 100644 --- a/test/integration/apiserver/flowcontrol/config_flap_test.go +++ b/test/integration/apiserver/flowcontrol/config_flap_test.go @@ -31,7 +31,6 @@ import ( flowcontrolbootstrap "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" - "k8s.io/kubernetes/test/utils/ktesting" ) // TestConfigFlap tests the ability of the APF config-producer to correctly switch @@ -49,30 +48,22 @@ import ( // Remember that the etcd server is bound in test_main, so all // three phases share the same etcd server. func TestConfigFlap(t *testing.T) { - tCtx := ktesting.Init(t) - if !flapTo(t, tCtx, "first", true) { - return - } - if !flapTo(t, tCtx, "first", false) { - return - } - if !flapTo(t, tCtx, "second", true) { - return - } - tCtx.Cancel("test function done") + t.Run("Phase1", func(t *testing.T) { + flapTo(t, "first", true) + }) + t.Run("Phase2", func(t *testing.T) { + flapTo(t, "first", false) + }) + t.Run("Phase3", func(t *testing.T) { + flapTo(t, "second", true) + }) } -var errTestPhaseDone = errors.New("test phase done") - -func flapTo(t *testing.T, ctx context.Context, trial string, v134 bool) bool { +func flapTo(t *testing.T, trial string, v134 bool) bool { flapName := fmt.Sprintf("%s v134=%v", trial, v134) t.Log("Preparing for " + flapName) - ctx, cancel := context.WithCancelCause(ctx) - client, _, tearDown := setupAnother(t, ctx, 100, 100, v134) - defer func() { - cancel(errTestPhaseDone) - tearDown() - }() + ctx, client, _, tearDown := setup(t, 100, 100, v134) + defer tearDown() t.Log("Waiting for " + flapName) if !waitForConfig(t, ctx, client, flapName, v134) { t.Fatal("Failed to establish " + flapName) From 58b961e08a0074dc90ec48176cf0d0b59c1d9f5d Mon Sep 17 00:00:00 2001 From: Mike Spreitzer Date: Thu, 20 Mar 2025 16:32:10 -0400 Subject: [PATCH 9/9] Select APF config by FeatureGate .. instead of feature values. --- .../internalbootstrap/default-internal.go | 20 +- .../internalbootstrap/defaults_test.go | 11 +- pkg/apis/flowcontrol/validation/validation.go | 12 +- .../flowcontrol/validation/validation_test.go | 2 +- .../flowcontrol/rest/storage_flowcontrol.go | 39 +- .../apis/flowcontrol/bootstrap/collection.go | 34 +- .../apiserver/pkg/features/kube_features.go | 336 +++++++++--------- .../filters/priority-and-fairness_test.go | 11 +- .../apiserver/pkg/server/options/feature.go | 4 +- .../pkg/util/flowcontrol/apf_controller.go | 13 +- .../pkg/util/flowcontrol/apf_filter.go | 9 +- .../pkg/util/flowcontrol/apf_filter_test.go | 2 + .../pkg/util/flowcontrol/borrowing_test.go | 2 + .../pkg/util/flowcontrol/controller_test.go | 10 +- .../util/flowcontrol/exempt_borrowing_test.go | 12 +- .../pkg/util/flowcontrol/gen_test.go | 4 +- .../pkg/util/flowcontrol/max_seats_test.go | 2 + .../apiserver/flowcontrol/config_flap_test.go | 11 +- .../apiserver/flowcontrol/fight_test.go | 2 + .../flowcontrol/fs_condition_test.go | 2 +- 20 files changed, 295 insertions(+), 243 deletions(-) diff --git a/pkg/apis/flowcontrol/internalbootstrap/default-internal.go b/pkg/apis/flowcontrol/internalbootstrap/default-internal.go index 03da407e5b8db..894426a447b8d 100644 --- a/pkg/apis/flowcontrol/internalbootstrap/default-internal.go +++ b/pkg/apis/flowcontrol/internalbootstrap/default-internal.go @@ -20,6 +20,8 @@ import ( flowcontrolv1 "k8s.io/api/flowcontrol/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" + "k8s.io/apiserver/pkg/features" + "k8s.io/component-base/featuregate" "k8s.io/kubernetes/pkg/apis/flowcontrol" "k8s.io/kubernetes/pkg/apis/flowcontrol/install" ) @@ -30,18 +32,20 @@ type FlowSchemasMap = map[string]*flowcontrol.FlowSchema // GetMandatoryFlowSchemasMap returns the unversioned (internal) mandatory FlowSchema objects, // as a deeply immutable map. // The arguments are the values of the features that control which config collection to use. -func GetMandatoryFlowSchemasMap(v134 bool) FlowSchemasMap { - return MandatoryFlowSchemasMap[bootstrap.CollectionID{V134: v134}] +func GetMandatoryFlowSchemasMap(featureGate featuregate.FeatureGate) FlowSchemasMap { + return MandatoryFlowSchemasMap[bootstrap.CollectionID{V134: featureGate.Enabled(features.APFv134Config)}] } +var oldConfig = bootstrap.GetV1ConfigCollection(bootstrap.MakeGate(false)) + // MandatoryFlowSchemasMap holds the unversioned (internal) renditions of the mandatory // flow schemas. In the outer map the key is CollectionId and // in the inner map the key is the schema's name and the // value is the `*FlowSchema`. Nobody should mutate anything // reachable from this map. var MandatoryFlowSchemasMap = map[bootstrap.CollectionID]FlowSchemasMap{ - {V134: false}: internalizeFSes(bootstrap.GetV1ConfigCollection(false).Mandatory.FlowSchemas), - {V134: true}: internalizeFSes(bootstrap.GetV1ConfigCollection(true).Mandatory.FlowSchemas), + {V134: false}: internalizeFSes(oldConfig.Mandatory.FlowSchemas), + {V134: true}: internalizeFSes(bootstrap.Latest.Mandatory.FlowSchemas), } // PriorityLevelConfigurationsMap is a collection of unversioned (internal) PriorityLevelConfiguration objects, indexed by name. @@ -50,8 +54,8 @@ type PriorityLevelConfigurationsMap = map[string]*flowcontrol.PriorityLevelConfi // GetMandatoryPriorityLevelConfigurationsMap returns the mandatory PriorityLevelConfiguration objects, // as a deeply immutable map. // The arguments are the values of the features that control which config collection to use. -func GetMandatoryPriorityLevelConfigurationsMap(v134 bool) PriorityLevelConfigurationsMap { - return MandatoryPriorityLevelConfigurationsMap[bootstrap.CollectionID{V134: v134}] +func GetMandatoryPriorityLevelConfigurationsMap(featureGate featuregate.FeatureGate) PriorityLevelConfigurationsMap { + return MandatoryPriorityLevelConfigurationsMap[bootstrap.CollectionID{V134: featureGate.Enabled(features.APFv134Config)}] } // MandatoryPriorityLevelConfigurationsMap holds the untyped renditions of the @@ -60,8 +64,8 @@ func GetMandatoryPriorityLevelConfigurationsMap(v134 bool) PriorityLevelConfigur // `*PriorityLevelConfiguration`. Nobody should mutate anything // reachable from this map. var MandatoryPriorityLevelConfigurationsMap = map[bootstrap.CollectionID]PriorityLevelConfigurationsMap{ - {V134: false}: internalizePLs(bootstrap.GetV1ConfigCollection(false).Mandatory.PriorityLevelConfigurations), - {V134: true}: internalizePLs(bootstrap.GetV1ConfigCollection(true).Mandatory.PriorityLevelConfigurations), + {V134: false}: internalizePLs(oldConfig.Mandatory.PriorityLevelConfigurations), + {V134: true}: internalizePLs(bootstrap.Latest.Mandatory.PriorityLevelConfigurations), } func internalizeFSes(exts []*flowcontrolv1.FlowSchema) FlowSchemasMap { diff --git a/pkg/apis/flowcontrol/internalbootstrap/defaults_test.go b/pkg/apis/flowcontrol/internalbootstrap/defaults_test.go index 573d9aef2df0a..eb59ac4bcbd20 100644 --- a/pkg/apis/flowcontrol/internalbootstrap/defaults_test.go +++ b/pkg/apis/flowcontrol/internalbootstrap/defaults_test.go @@ -25,17 +25,18 @@ import ( flowcontrol "k8s.io/api/flowcontrol/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" + "k8s.io/component-base/featuregate" ) func TestBootstrapConfigurationWithDefaulted(t *testing.T) { - t.Run("false", caseFn(false)) - t.Run("true", caseFn(true)) + t.Run("v134Config=false", caseFn(bootstrap.MakeGate(false))) + t.Run("v134Config=true", caseFn(bootstrap.LatestFeatureGate)) } -func caseFn(v134 bool) func(*testing.T) { +func caseFn(featureGate featuregate.FeatureGate) func(*testing.T) { return func(t *testing.T) { scheme := NewAPFScheme() - bootstrapFlowSchemas := bootstrap.GetFlowSchemas(v134) + bootstrapFlowSchemas := bootstrap.GetFlowSchemas(featureGate) for _, original := range bootstrapFlowSchemas { t.Run(fmt.Sprintf("FlowSchema/%s", original.Name), func(t *testing.T) { defaulted := original.DeepCopyObject().(*flowcontrol.FlowSchema) @@ -48,7 +49,7 @@ func caseFn(v134 bool) func(*testing.T) { }) } - bootstrapPriorityLevels := bootstrap.GetPrioritylevelConfigurations(v134) + bootstrapPriorityLevels := bootstrap.GetPrioritylevelConfigurations(featureGate) for _, original := range bootstrapPriorityLevels { t.Run(fmt.Sprintf("PriorityLevelConfiguration/%s", original.Name), func(t *testing.T) { defaulted := original.DeepCopyObject().(*flowcontrol.PriorityLevelConfiguration) diff --git a/pkg/apis/flowcontrol/validation/validation.go b/pkg/apis/flowcontrol/validation/validation.go index ca9fc7416d724..b35a2f41d1e46 100644 --- a/pkg/apis/flowcontrol/validation/validation.go +++ b/pkg/apis/flowcontrol/validation/validation.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" + bootstrapv1 "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" "k8s.io/apiserver/pkg/util/shufflesharding" apivalidation "k8s.io/kubernetes/pkg/apis/core/validation" "k8s.io/kubernetes/pkg/apis/flowcontrol" @@ -78,15 +79,18 @@ var supportedLimitResponseType = sets.NewString( // PriorityLevelValidationOptions holds the validation options for a priority level object type PriorityLevelValidationOptions struct{} +var earliestGate = bootstrapv1.MakeGate(false) +var latestGate = bootstrapv1.LatestFeatureGate + // ValidateFlowSchema validates the content of flow-schema func ValidateFlowSchema(fs *flowcontrol.FlowSchema) field.ErrorList { allErrs := apivalidation.ValidateObjectMeta(&fs.ObjectMeta, false, ValidateFlowSchemaName, field.NewPath("metadata")) specPath := field.NewPath("spec") allErrs = append(allErrs, ValidateFlowSchemaSpec(fs.Name, &fs.Spec, specPath)...) - if mand, ok := internalbootstrap.GetMandatoryFlowSchemasMap(true)[fs.Name]; ok { + if mand, ok := internalbootstrap.GetMandatoryFlowSchemasMap(latestGate)[fs.Name]; ok { // The old config objects are a subset of the new config objects, // as far as identify is concerned. The specs may differ. - mandOld := internalbootstrap.GetMandatoryFlowSchemasMap(false)[fs.Name] + mandOld := internalbootstrap.GetMandatoryFlowSchemasMap(earliestGate)[fs.Name] // For now, accept either the old or the new spec. // In a later release, change this to accept only the new spec. // Check for almost exact equality. This is a pretty @@ -368,7 +372,7 @@ func ValidatePriorityLevelConfiguration(pl *flowcontrol.PriorityLevelConfigurati func ValidateIfMandatoryPriorityLevelConfigurationObject(pl *flowcontrol.PriorityLevelConfiguration, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList - mand, ok := internalbootstrap.GetMandatoryPriorityLevelConfigurationsMap(true)[pl.Name] + mand, ok := internalbootstrap.GetMandatoryPriorityLevelConfigurationsMap(latestGate)[pl.Name] if !ok { // The old config objects are a subset of the new, as far as identity is concerned. return allErrs @@ -389,7 +393,7 @@ func ValidateIfMandatoryPriorityLevelConfigurationObject(pl *flowcontrol.Priorit // For now, accept either the old or the new default config. // In a later release, accept only the new. - mandOld := internalbootstrap.GetMandatoryPriorityLevelConfigurationsMap(false)[pl.Name] + mandOld := internalbootstrap.GetMandatoryPriorityLevelConfigurationsMap(earliestGate)[pl.Name] // Check for almost exact equality. This is a pretty // strict test, and it is OK in this context because both // sides of this comparison are intended to ultimately diff --git a/pkg/apis/flowcontrol/validation/validation_test.go b/pkg/apis/flowcontrol/validation/validation_test.go index adcef472e7138..89b9fc9dd9160 100644 --- a/pkg/apis/flowcontrol/validation/validation_test.go +++ b/pkg/apis/flowcontrol/validation/validation_test.go @@ -769,7 +769,7 @@ func TestPriorityLevelConfigurationValidation(t *testing.T) { } validChangesInExemptFieldOfExemptPLFn := func() flowcontrol.PriorityLevelConfigurationSpec { - have := fcboot.GetV1ConfigCollection(true).PriorityLevelConfigurationExempt + have := fcboot.Latest.PriorityLevelConfigurationExempt return flowcontrol.PriorityLevelConfigurationSpec{ Type: flowcontrol.PriorityLevelEnablementExempt, Exempt: &flowcontrol.ExemptPriorityLevelConfiguration{ diff --git a/pkg/registry/flowcontrol/rest/storage_flowcontrol.go b/pkg/registry/flowcontrol/rest/storage_flowcontrol.go index 04dcda184bc71..c3e023a02ef8e 100644 --- a/pkg/registry/flowcontrol/rest/storage_flowcontrol.go +++ b/pkg/registry/flowcontrol/rest/storage_flowcontrol.go @@ -25,7 +25,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" flowcontrolbootstrap "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" - "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" @@ -109,7 +108,7 @@ func (p RESTStorageProvider) GroupName() string { // PostStartHook returns the hook func that launches the config provider func (p RESTStorageProvider) PostStartHook() (string, genericapiserver.PostStartHookFunc, error) { bce := &bootstrapConfigurationEnsurer{ - v134Config: p.FeatureGate.Enabled(features.APFv134Config), + featureGate: p.FeatureGate, informersSynced: []cache.InformerSynced{ p.InformerFactory.Flowcontrol().V1().PriorityLevelConfigurations().Informer().HasSynced, p.InformerFactory.Flowcontrol().V1().FlowSchemas().Informer().HasSynced, @@ -121,7 +120,7 @@ func (p RESTStorageProvider) PostStartHook() (string, genericapiserver.PostStart } type bootstrapConfigurationEnsurer struct { - v134Config bool + featureGate featuregate.FeatureGate informersSynced []cache.InformerSynced fsLister flowcontrollisters.FlowSchemaLister plcLister flowcontrollisters.PriorityLevelConfigurationLister @@ -145,7 +144,7 @@ func (bce *bootstrapConfigurationEnsurer) ensureAPFBootstrapConfiguration(hookCo err = wait.PollUntilContextCancel(ctx, time.Second, true, func(context.Context) (bool, error) { - if err := ensure(ctx, bce.v134Config, clientset, bce.fsLister, bce.plcLister); err != nil { + if err := ensure(ctx, bce.featureGate, clientset, bce.fsLister, bce.plcLister); err != nil { klog.ErrorS(err, "APF bootstrap ensurer ran into error, will retry later") return false, nil } @@ -165,7 +164,7 @@ func (bce *bootstrapConfigurationEnsurer) ensureAPFBootstrapConfiguration(hookCo go func() { _ = wait.PollUntilContextCancel(hookContext, time.Minute, true, func(ctx context.Context) (bool, error) { - if err := ensure(ctx, bce.v134Config, clientset, bce.fsLister, bce.plcLister); err != nil { + if err := ensure(ctx, bce.featureGate, clientset, bce.fsLister, bce.plcLister); err != nil { klog.ErrorS(err, "APF bootstrap ensurer ran into error, will retry later") } // always auto update both suggested and mandatory configuration @@ -177,29 +176,29 @@ func (bce *bootstrapConfigurationEnsurer) ensureAPFBootstrapConfiguration(hookCo return nil } -func ensure(ctx context.Context, v134Config bool, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { +func ensure(ctx context.Context, featureGate featuregate.FeatureGate, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { - if err := ensureSuggestedConfiguration(ctx, v134Config, clientset, fsLister, plcLister); err != nil { + if err := ensureSuggestedConfiguration(ctx, featureGate, clientset, fsLister, plcLister); err != nil { // We should not attempt creation of mandatory objects if ensuring the suggested // configuration resulted in an error. // This only happens when the stop channel is closed. return fmt.Errorf("failed ensuring suggested settings - %w", err) } - if err := ensureMandatoryConfiguration(ctx, v134Config, clientset, fsLister, plcLister); err != nil { + if err := ensureMandatoryConfiguration(ctx, featureGate, clientset, fsLister, plcLister); err != nil { return fmt.Errorf("failed ensuring mandatory settings - %w", err) } - if err := removeDanglingBootstrapConfiguration(ctx, v134Config, clientset, fsLister, plcLister); err != nil { + if err := removeDanglingBootstrapConfiguration(ctx, featureGate, clientset, fsLister, plcLister); err != nil { return fmt.Errorf("failed to delete removed settings - %w", err) } return nil } -func ensureSuggestedConfiguration(ctx context.Context, v134Config bool, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { +func ensureSuggestedConfiguration(ctx context.Context, featureGate featuregate.FeatureGate, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { plcOps := ensurer.NewPriorityLevelConfigurationOps(clientset.PriorityLevelConfigurations(), plcLister) - config := flowcontrolbootstrap.GetV1ConfigCollection(v134Config).Suggested + config := flowcontrolbootstrap.GetV1ConfigCollection(featureGate).Suggested if err := ensurer.EnsureConfigurations(ctx, plcOps, config.PriorityLevelConfigurations, ensurer.NewSuggestedEnsureStrategy[*flowcontrolv1.PriorityLevelConfiguration]()); err != nil { return err } @@ -208,9 +207,9 @@ func ensureSuggestedConfiguration(ctx context.Context, v134Config bool, clientse return ensurer.EnsureConfigurations(ctx, fsOps, config.FlowSchemas, ensurer.NewSuggestedEnsureStrategy[*flowcontrolv1.FlowSchema]()) } -func ensureMandatoryConfiguration(ctx context.Context, v134Config bool, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { +func ensureMandatoryConfiguration(ctx context.Context, featureGate featuregate.FeatureGate, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { plcOps := ensurer.NewPriorityLevelConfigurationOps(clientset.PriorityLevelConfigurations(), plcLister) - config := flowcontrolbootstrap.GetV1ConfigCollection(v134Config).Mandatory + config := flowcontrolbootstrap.GetV1ConfigCollection(featureGate).Mandatory if err := ensurer.EnsureConfigurations(ctx, plcOps, config.PriorityLevelConfigurations, ensurer.NewMandatoryEnsureStrategy[*flowcontrolv1.PriorityLevelConfiguration]()); err != nil { return err } @@ -219,22 +218,22 @@ func ensureMandatoryConfiguration(ctx context.Context, v134Config bool, clientse return ensurer.EnsureConfigurations(ctx, fsOps, config.FlowSchemas, ensurer.NewMandatoryEnsureStrategy[*flowcontrolv1.FlowSchema]()) } -func removeDanglingBootstrapConfiguration(ctx context.Context, v134Config bool, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { - if err := removeDanglingBootstrapFlowSchema(ctx, v134Config, clientset, fsLister); err != nil { +func removeDanglingBootstrapConfiguration(ctx context.Context, featureGate featuregate.FeatureGate, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { + if err := removeDanglingBootstrapFlowSchema(ctx, featureGate, clientset, fsLister); err != nil { return err } - return removeDanglingBootstrapPriorityLevel(ctx, v134Config, clientset, plcLister) + return removeDanglingBootstrapPriorityLevel(ctx, featureGate, clientset, plcLister) } -func removeDanglingBootstrapFlowSchema(ctx context.Context, v134Config bool, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister) error { +func removeDanglingBootstrapFlowSchema(ctx context.Context, featureGate featuregate.FeatureGate, clientset flowcontrolclient.FlowcontrolV1Interface, fsLister flowcontrollisters.FlowSchemaLister) error { fsOps := ensurer.NewFlowSchemaOps(clientset.FlowSchemas(), fsLister) - return ensurer.RemoveUnwantedObjects(ctx, fsOps, flowcontrolbootstrap.GetFlowSchemas(v134Config)) + return ensurer.RemoveUnwantedObjects(ctx, fsOps, flowcontrolbootstrap.GetFlowSchemas(featureGate)) } -func removeDanglingBootstrapPriorityLevel(ctx context.Context, v134Config bool, clientset flowcontrolclient.FlowcontrolV1Interface, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { +func removeDanglingBootstrapPriorityLevel(ctx context.Context, featureGate featuregate.FeatureGate, clientset flowcontrolclient.FlowcontrolV1Interface, plcLister flowcontrollisters.PriorityLevelConfigurationLister) error { plcOps := ensurer.NewPriorityLevelConfigurationOps(clientset.PriorityLevelConfigurations(), plcLister) - return ensurer.RemoveUnwantedObjects(ctx, plcOps, flowcontrolbootstrap.GetPrioritylevelConfigurations(v134Config)) + return ensurer.RemoveUnwantedObjects(ctx, plcOps, flowcontrolbootstrap.GetPrioritylevelConfigurations(featureGate)) } // contextFromChannelAndMaxWaitDuration returns a Context that is bound to the diff --git a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/collection.go b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/collection.go index b418755e84829..42cf5f1e81a9b 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/collection.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/collection.go @@ -17,10 +17,15 @@ limitations under the License. package bootstrap import ( + "fmt" + flowcontrolv1 "k8s.io/api/flowcontrol/v1" + "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/apis/flowcontrol/base" bootstrapold "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-old" bootstrapv134 "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-v134" + "k8s.io/apiserver/pkg/features" + "k8s.io/component-base/featuregate" ) // This package provides access to the "default" configuration objects of @@ -32,7 +37,8 @@ import ( // the default APF configuration objects to use, and returns them (in V1 form). // The returned value is deeply immutable. // v134 is the value of the APFv134Config feature. -func GetV1ConfigCollection(v134 bool) *base.V1ConfigCollection { +func GetV1ConfigCollection(featureGate featuregate.FeatureGate) *base.V1ConfigCollection { + v134 := featureGate.Enabled(features.APFv134Config) if v134 { return &bootstrapv134.V1ConfigCollection } else { @@ -40,12 +46,30 @@ func GetV1ConfigCollection(v134 bool) *base.V1ConfigCollection { } } +// LatestFeatureGate is a FeatureGate that calls for the latest edition of APF configuration. +var LatestFeatureGate = MakeGate(true) + +// Latest is shorthand for GetV1ConfigCollection(LatestFeatureGate). +var Latest = &bootstrapv134.V1ConfigCollection + +var defaultFeatures = features.DefaultVersionedKubernetesFeatureGates() + +// MakeGate makes a FeatureGate with the given settings for APF. +// The parameters are the values of the relevant Features; +// the parameter list will change as relevant Features are introduced and retired. +func MakeGate(v134Config bool) featuregate.FeatureGate { + fg := featuregate.NewFeatureGate() + runtime.Must(fg.AddVersioned(defaultFeatures)) + runtime.Must(fg.Set(fmt.Sprintf("%s=%v", features.APFv134Config, v134Config))) + return fg +} + // GetPrioritylevelConfigurations returns the default PriorityLevelConfiguration objects, // as a deeply immutable slice. // The types are the latest external type (version). // The arguments are the values of the features that control which config collection to use. -func GetPrioritylevelConfigurations(v134 bool) []*flowcontrolv1.PriorityLevelConfiguration { - return PrioritylevelConfigurations[CollectionID{V134: v134}] +func GetPrioritylevelConfigurations(featureGate featuregate.FeatureGate) []*flowcontrolv1.PriorityLevelConfiguration { + return PrioritylevelConfigurations[CollectionID{V134: featureGate.Enabled(features.APFv134Config)}] } // PrioritylevelConfigurations maps CollectionId to a slice of all the default objects. @@ -61,8 +85,8 @@ var PrioritylevelConfigurations = map[CollectionID][]*flowcontrolv1.PriorityLeve // as a deeply immutable slice. // The types are the latest external type (version). // The arguments are the values of the features that control which config collection to use. -func GetFlowSchemas(v134 bool) []*flowcontrolv1.FlowSchema { - return FlowSchemas[CollectionID{V134: v134}] +func GetFlowSchemas(featureGate featuregate.FeatureGate) []*flowcontrolv1.FlowSchema { + return FlowSchemas[CollectionID{V134: featureGate.Enabled(features.APFv134Config)}] } // FlowSchemas maps CollectionId to a slice of all the default objects. diff --git a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go index 3ae05e3e342be..001b4fc3840e7 100644 --- a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go +++ b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go @@ -255,7 +255,7 @@ const ( ) func init() { - runtime.Must(utilfeature.DefaultMutableFeatureGate.AddVersioned(defaultVersionedKubernetesFeatureGates)) + runtime.Must(utilfeature.DefaultMutableFeatureGate.AddVersioned(DefaultVersionedKubernetesFeatureGates())) } // defaultVersionedKubernetesFeatureGates consists of all known Kubernetes-specific feature keys with VersionedSpecs. @@ -264,170 +264,172 @@ func init() { // // Entries are alphabetized and separated from each other with blank lines to avoid sweeping gofmt changes // when adding or removing one entry. -var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{ - AggregatedDiscoveryRemoveBetaType: { - {Version: version.MustParse("1.0"), Default: false, PreRelease: featuregate.GA}, - {Version: version.MustParse("1.33"), Default: true, PreRelease: featuregate.Deprecated}, - }, - - AllowParsingUserUIDFromCertAuth: { - {Version: version.MustParse("1.33"), Default: true, PreRelease: featuregate.Beta}, - }, - - AllowUnsafeMalformedObjectDeletion: { - {Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha}, - }, - - APFv134Config: { - {Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("1.34"), Default: true, PreRelease: featuregate.Beta}, - }, - - AnonymousAuthConfigurableEndpoints: { - {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.Beta}, - }, - - APIResponseCompression: { - {Version: version.MustParse("1.8"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("1.16"), Default: true, PreRelease: featuregate.Beta}, - }, - - APIServerIdentity: { - {Version: version.MustParse("1.20"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("1.26"), Default: true, PreRelease: featuregate.Beta}, - }, - - APIServerTracing: { - {Version: version.MustParse("1.22"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("1.27"), Default: true, PreRelease: featuregate.Beta}, - }, - - APIServingWithRoutine: { - {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha}, - }, - - BtreeWatchCache: { - {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.Beta}, - {Version: version.MustParse("1.33"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, - }, - - AuthorizeWithSelectors: { - {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.Beta}, - }, - - CBORServingAndStorage: { - {Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha}, - }, - - ConcurrentWatchObjectDecode: { - {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Beta}, - }, - - ConsistentListFromCache: { - {Version: version.MustParse("1.28"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta}, - }, - - CoordinatedLeaderElection: { - {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Beta}, - }, - - KMSv1: { - {Version: version.MustParse("1.0"), Default: true, PreRelease: featuregate.GA}, - {Version: version.MustParse("1.28"), Default: true, PreRelease: featuregate.Deprecated}, - {Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Deprecated}, - }, - - ListFromCacheSnapshot: { - {Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Alpha}, - }, - - MutatingAdmissionPolicy: { - {Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha}, - }, - - OpenAPIEnums: { - {Version: version.MustParse("1.23"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("1.24"), Default: true, PreRelease: featuregate.Beta}, - }, - - RemoteRequestHeaderUID: { - {Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("1.33"), Default: true, PreRelease: featuregate.Beta}, - }, - - ResilientWatchCacheInitialization: { - {Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta}, - }, - - RetryGenerateName: { - {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta}, - {Version: version.MustParse("1.32"), Default: true, LockToDefault: true, PreRelease: featuregate.GA}, - }, - - SeparateCacheWatchRPC: { - {Version: version.MustParse("1.28"), Default: true, PreRelease: featuregate.Beta}, - {Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Deprecated}, - }, - - StorageVersionAPI: { - {Version: version.MustParse("1.20"), Default: false, PreRelease: featuregate.Alpha}, - }, - - StorageVersionHash: { - {Version: version.MustParse("1.14"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("1.15"), Default: true, PreRelease: featuregate.Beta}, - }, - - StreamingCollectionEncodingToJSON: { - {Version: version.MustParse("1.33"), Default: true, PreRelease: featuregate.Beta}, - }, - - StreamingCollectionEncodingToProtobuf: { - {Version: version.MustParse("1.33"), Default: true, PreRelease: featuregate.Beta}, - }, - - StrictCostEnforcementForVAP: { - {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Beta}, - {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, - }, - - StrictCostEnforcementForWebhooks: { - {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Beta}, - {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, - }, - - StructuredAuthenticationConfiguration: { - {Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("1.30"), Default: true, PreRelease: featuregate.Beta}, - }, - - StructuredAuthorizationConfiguration: { - {Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("1.30"), Default: true, PreRelease: featuregate.Beta}, - {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, - }, - - UnauthenticatedHTTP2DOSMitigation: { - {Version: version.MustParse("1.25"), Default: false, PreRelease: featuregate.Beta}, - {Version: version.MustParse("1.29"), Default: true, PreRelease: featuregate.Beta}, - }, - - WatchCacheInitializationPostStartHook: { - {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Beta}, - }, - - WatchFromStorageWithoutResourceVersion: { - {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Beta}, - {Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Deprecated, LockToDefault: true}, - }, - - WatchList: { - {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.Beta}, - }, +func DefaultVersionedKubernetesFeatureGates() map[featuregate.Feature]featuregate.VersionedSpecs { + return map[featuregate.Feature]featuregate.VersionedSpecs{ + AggregatedDiscoveryRemoveBetaType: { + {Version: version.MustParse("1.0"), Default: false, PreRelease: featuregate.GA}, + {Version: version.MustParse("1.33"), Default: true, PreRelease: featuregate.Deprecated}, + }, + + AllowParsingUserUIDFromCertAuth: { + {Version: version.MustParse("1.33"), Default: true, PreRelease: featuregate.Beta}, + }, + + AllowUnsafeMalformedObjectDeletion: { + {Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha}, + }, + + APFv134Config: { + {Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.34"), Default: true, PreRelease: featuregate.Beta}, + }, + + AnonymousAuthConfigurableEndpoints: { + {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.Beta}, + }, + + APIResponseCompression: { + {Version: version.MustParse("1.8"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.16"), Default: true, PreRelease: featuregate.Beta}, + }, + + APIServerIdentity: { + {Version: version.MustParse("1.20"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.26"), Default: true, PreRelease: featuregate.Beta}, + }, + + APIServerTracing: { + {Version: version.MustParse("1.22"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.27"), Default: true, PreRelease: featuregate.Beta}, + }, + + APIServingWithRoutine: { + {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha}, + }, + + BtreeWatchCache: { + {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.33"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, + }, + + AuthorizeWithSelectors: { + {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.Beta}, + }, + + CBORServingAndStorage: { + {Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha}, + }, + + ConcurrentWatchObjectDecode: { + {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Beta}, + }, + + ConsistentListFromCache: { + {Version: version.MustParse("1.28"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta}, + }, + + CoordinatedLeaderElection: { + {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Beta}, + }, + + KMSv1: { + {Version: version.MustParse("1.0"), Default: true, PreRelease: featuregate.GA}, + {Version: version.MustParse("1.28"), Default: true, PreRelease: featuregate.Deprecated}, + {Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Deprecated}, + }, + + ListFromCacheSnapshot: { + {Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Alpha}, + }, + + MutatingAdmissionPolicy: { + {Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha}, + }, + + OpenAPIEnums: { + {Version: version.MustParse("1.23"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.24"), Default: true, PreRelease: featuregate.Beta}, + }, + + RemoteRequestHeaderUID: { + {Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.33"), Default: true, PreRelease: featuregate.Beta}, + }, + + ResilientWatchCacheInitialization: { + {Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta}, + }, + + RetryGenerateName: { + {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.32"), Default: true, LockToDefault: true, PreRelease: featuregate.GA}, + }, + + SeparateCacheWatchRPC: { + {Version: version.MustParse("1.28"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Deprecated}, + }, + + StorageVersionAPI: { + {Version: version.MustParse("1.20"), Default: false, PreRelease: featuregate.Alpha}, + }, + + StorageVersionHash: { + {Version: version.MustParse("1.14"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.15"), Default: true, PreRelease: featuregate.Beta}, + }, + + StreamingCollectionEncodingToJSON: { + {Version: version.MustParse("1.33"), Default: true, PreRelease: featuregate.Beta}, + }, + + StreamingCollectionEncodingToProtobuf: { + {Version: version.MustParse("1.33"), Default: true, PreRelease: featuregate.Beta}, + }, + + StrictCostEnforcementForVAP: { + {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, + }, + + StrictCostEnforcementForWebhooks: { + {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, + }, + + StructuredAuthenticationConfiguration: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.30"), Default: true, PreRelease: featuregate.Beta}, + }, + + StructuredAuthorizationConfiguration: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.30"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, + }, + + UnauthenticatedHTTP2DOSMitigation: { + {Version: version.MustParse("1.25"), Default: false, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: featuregate.Beta}, + }, + + WatchCacheInitializationPostStartHook: { + {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Beta}, + }, + + WatchFromStorageWithoutResourceVersion: { + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Deprecated, LockToDefault: true}, + }, + + WatchList: { + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.Beta}, + }, + } } diff --git a/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness_test.go b/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness_test.go index 8bfa87e6f01bf..cbfaf74f5a30b 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness_test.go @@ -35,7 +35,8 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" - bootstrap "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-v134" + "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" + bootstraplatest "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-v134" "k8s.io/apiserver/pkg/authentication/user" apifilters "k8s.io/apiserver/pkg/endpoints/filters" epmetrics "k8s.io/apiserver/pkg/endpoints/metrics" @@ -95,7 +96,7 @@ func (t fakeApfFilter) Handle(ctx context.Context, if t.mockDecision == decisionSkipFilter { panic("Handle should not be invoked") } - noteFn(bootstrap.SuggestedFlowSchemaGlobalDefault, bootstrap.SuggestedPriorityLevelConfigurationGlobalDefault, requestDigest.User.GetName()) + noteFn(bootstraplatest.SuggestedFlowSchemaGlobalDefault, bootstraplatest.SuggestedPriorityLevelConfigurationGlobalDefault, requestDigest.User.GetName()) switch t.mockDecision { case decisionNoQueuingExecute: execFn() @@ -389,7 +390,7 @@ func (f *fakeWatchApfFilter) Handle(ctx context.Context, _ fq.QueueNoteFn, execFn func(), ) { - noteFn(bootstrap.SuggestedFlowSchemaGlobalDefault, bootstrap.SuggestedPriorityLevelConfigurationGlobalDefault, requestDigest.User.GetName()) + noteFn(bootstraplatest.SuggestedFlowSchemaGlobalDefault, bootstraplatest.SuggestedPriorityLevelConfigurationGlobalDefault, requestDigest.User.GetName()) canExecute := false func() { f.lock.Lock() @@ -645,7 +646,7 @@ func (f *fakeFilterRequestDigest) Handle(ctx context.Context, _ fq.QueueNoteFn, _ func(), ) { f.requestDigestGot = &requestDigest - noteFn(bootstrap.MandatoryFlowSchemaCatchAll, bootstrap.MandatoryPriorityLevelConfigurationCatchAll, "") + noteFn(bootstrap.Latest.FlowSchemaCatchAll, bootstrap.Latest.PriorityLevelConfigurationCatchAll, "") f.workEstimateGot = workEstimator() } @@ -1138,7 +1139,7 @@ func startAPFController(t *testing.T, stopCh <-chan struct{}, apfConfiguration [ clientset := newClientset(t, apfConfiguration...) // this test does not rely on resync, so resync period is set to zero factory := informers.NewSharedInformerFactory(clientset, 0) - controller := utilflowcontrol.New(factory, clientset.FlowcontrolV1(), serverConcurrency, true) + controller := utilflowcontrol.New(factory, clientset.FlowcontrolV1(), serverConcurrency, bootstrap.LatestFeatureGate) factory.Start(stopCh) diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/feature.go b/staging/src/k8s.io/apiserver/pkg/server/options/feature.go index 92e97fcdfd726..ebe92c754f816 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/feature.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/feature.go @@ -22,7 +22,6 @@ import ( "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/server" utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol" "k8s.io/client-go/informers" @@ -79,12 +78,11 @@ func (o *FeatureOptions) ApplyTo(c *server.Config, clientset kubernetes.Interfac return fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight) } - v134Config := c.FeatureGate.Enabled(features.APFv134Config) c.FlowControl = utilflowcontrol.New( informers, clientset.FlowcontrolV1(), c.MaxRequestsInFlight+c.MaxMutatingRequestsInFlight, - v134Config, + c.FeatureGate, ) } diff --git a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go index c66f87aeb45d1..4dc8b1525d02d 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go +++ b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go @@ -47,6 +47,7 @@ import ( fcrequest "k8s.io/apiserver/pkg/util/flowcontrol/request" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "k8s.io/component-base/featuregate" "k8s.io/klog/v2" "k8s.io/utils/clock" @@ -124,7 +125,7 @@ type configController struct { queueSetFactory fq.QueueSetFactory reqsGaugeVec metrics.RatioedGaugeVec execSeatsGaugeVec metrics.RatioedGaugeVec - v134Config bool + featureGate featuregate.FeatureGate // How this controller appears in an ObjectMeta ManagedFieldsEntry.Manager asFieldManager string @@ -277,7 +278,7 @@ func (stats *seatDemandStats) update(obs fq.IntegratorResults) { // NewTestableController is extra flexible to facilitate testing func newTestableController(config TestableConfig) *configController { cfgCtlr := &configController{ - v134Config: config.V134Config, + featureGate: config.FeatureGate, name: config.Name, clock: config.Clock, queueSetFactory: config.QueueSetFactory, @@ -692,10 +693,10 @@ func (cfgCtlr *configController) lockAndDigestConfigObjects(newPLs []*flowcontro // Supply missing mandatory PriorityLevelConfiguration objects if !meal.haveExemptPL { - meal.imaginePL(fcboot.GetV1ConfigCollection(cfgCtlr.v134Config).PriorityLevelConfigurationExempt) + meal.imaginePL(fcboot.GetV1ConfigCollection(cfgCtlr.featureGate).PriorityLevelConfigurationExempt) } if !meal.haveCatchAllPL { - meal.imaginePL(fcboot.GetV1ConfigCollection(cfgCtlr.v134Config).PriorityLevelConfigurationCatchAll) + meal.imaginePL(fcboot.GetV1ConfigCollection(cfgCtlr.featureGate).PriorityLevelConfigurationCatchAll) } meal.finishQueueSetReconfigsLocked() @@ -787,10 +788,10 @@ func (meal *cfgMeal) digestFlowSchemasLocked(newFSs []*flowcontrol.FlowSchema) { // Supply missing mandatory FlowSchemas, in correct position if !haveExemptFS { - fsSeq = append(apihelpers.FlowSchemaSequence{fcboot.GetV1ConfigCollection(meal.cfgCtlr.v134Config).FlowSchemaExempt}, fsSeq...) + fsSeq = append(apihelpers.FlowSchemaSequence{fcboot.GetV1ConfigCollection(meal.cfgCtlr.featureGate).FlowSchemaExempt}, fsSeq...) } if !haveCatchAllFS { - fsSeq = append(fsSeq, fcboot.GetV1ConfigCollection(meal.cfgCtlr.v134Config).FlowSchemaCatchAll) + fsSeq = append(fsSeq, fcboot.GetV1ConfigCollection(meal.cfgCtlr.featureGate).FlowSchemaCatchAll) } meal.cfgCtlr.flowSchemas = fsSeq diff --git a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter.go b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter.go index 8387e86156db7..a3c6566d4a3bd 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter.go +++ b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter.go @@ -30,6 +30,7 @@ import ( "k8s.io/apiserver/pkg/util/flowcontrol/metrics" fcrequest "k8s.io/apiserver/pkg/util/flowcontrol/request" kubeinformers "k8s.io/client-go/informers" + "k8s.io/component-base/featuregate" "k8s.io/klog/v2" "k8s.io/utils/clock" @@ -90,7 +91,7 @@ func New( informerFactory kubeinformers.SharedInformerFactory, flowcontrolClient flowcontrolclient.FlowcontrolV1Interface, serverConcurrencyLimit int, - v134Config bool, + featureGate featuregate.FeatureGate, ) Interface { clk := eventclock.Real{} return NewTestable(TestableConfig{ @@ -104,7 +105,7 @@ func New( ReqsGaugeVec: metrics.PriorityLevelConcurrencyGaugeVec, ExecSeatsGaugeVec: metrics.PriorityLevelExecutionSeatsGaugeVec, QueueSetFactory: fqs.NewQueueSetFactory(clk), - V134Config: v134Config, + FeatureGate: featureGate, }) } @@ -148,8 +149,8 @@ type TestableConfig struct { // QueueSetFactory for the queuing implementation QueueSetFactory fq.QueueSetFactory - // Whether to use modern default config objects - V134Config bool + // FeatureGate is here so that the correct configuration can be used + FeatureGate featuregate.FeatureGate } // NewTestable is extra flexible to facilitate testing diff --git a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter_test.go b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter_test.go index 910b133938577..c4df0bd3a132d 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter_test.go @@ -25,6 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" + fcboot "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" "k8s.io/apiserver/pkg/endpoints/request" fq "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing" fqs "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset" @@ -103,6 +104,7 @@ func TestQueueWaitTimeLatencyTracker(t *testing.T) { startTime := time.Now() clk, _ := eventclock.NewFake(startTime, 0, nil) controller := newTestableController(TestableConfig{ + FeatureGate: fcboot.LatestFeatureGate, Name: "Controller", Clock: clk, AsFieldManager: ConfigConsumerAsFieldManager, diff --git a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/borrowing_test.go b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/borrowing_test.go index a3db94fa39c7e..efb94ebee3d18 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/borrowing_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/borrowing_test.go @@ -27,6 +27,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" + fcboot "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/request" fq "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing" @@ -137,6 +138,7 @@ func TestBorrowing(t *testing.T) { flowcontrolClient := clientset.FlowcontrolV1() clk := eventclock.Real{} controller := newTestableController(TestableConfig{ + FeatureGate: fcboot.LatestFeatureGate, Name: "Controller", Clock: clk, AsFieldManager: ConfigConsumerAsFieldManager, diff --git a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/controller_test.go b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/controller_test.go index 52ad58cd97231..2089dbee3f6c3 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/controller_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/controller_test.go @@ -55,7 +55,7 @@ func TestMain(m *testing.M) { var mandPLs = func() map[string]*flowcontrol.PriorityLevelConfiguration { ans := make(map[string]*flowcontrol.PriorityLevelConfiguration) - for _, mand := range fcboot.GetV1ConfigCollection(true).Mandatory.PriorityLevelConfigurations { + for _, mand := range fcboot.Latest.Mandatory.PriorityLevelConfigurations { ans[mand.Name] = mand } return ans @@ -224,7 +224,7 @@ func (cts *ctlrTestState) popHeldRequest() (plName string, hr *heldRequest, nCou var mandQueueSetNames = func() sets.String { mandQueueSetNames := sets.NewString() - for _, mpl := range fcboot.GetV1ConfigCollection(true).Mandatory.PriorityLevelConfigurations { + for _, mpl := range fcboot.Latest.Mandatory.PriorityLevelConfigurations { mandQueueSetNames.Insert(mpl.Name) } return mandQueueSetNames @@ -246,6 +246,7 @@ func TestConfigConsumer(t *testing.T) { queues: map[string]*ctlrTestQueueSet{}, } ctlr := newTestableController(TestableConfig{ + FeatureGate: fcboot.LatestFeatureGate, Name: "Controller", Clock: clock.RealClock{}, AsFieldManager: ConfigConsumerAsFieldManager, @@ -377,6 +378,7 @@ func TestAPFControllerWithGracefulShutdown(t *testing.T) { queues: map[string]*ctlrTestQueueSet{}, } controller := newTestableController(TestableConfig{ + FeatureGate: fcboot.LatestFeatureGate, Name: "Controller", Clock: clock.RealClock{}, AsFieldManager: ConfigConsumerAsFieldManager, @@ -523,8 +525,8 @@ func genPLs(rng *rand.Rand, trial string, oldPLNames sets.String, n int) (pls [] func genFSs(t *testing.T, rng *rand.Rand, trial string, goodPLNames, badPLNames sets.String, n int) (newFSs []*flowcontrol.FlowSchema, newFSMap map[string]*flowcontrol.FlowSchema, newFTRs map[string]*fsTestingRecord, catchAlls map[bool]*flowcontrol.FlowSchema) { newFTRs = map[string]*fsTestingRecord{} catchAlls = map[bool]*flowcontrol.FlowSchema{ - false: fcboot.GetV1ConfigCollection(true).FlowSchemaCatchAll, - true: fcboot.GetV1ConfigCollection(true).FlowSchemaCatchAll} + false: fcboot.Latest.FlowSchemaCatchAll, + true: fcboot.Latest.FlowSchemaCatchAll} newFSMap = map[string]*flowcontrol.FlowSchema{} add := func(ftr *fsTestingRecord) { newFSs = append(newFSs, ftr.fs) diff --git a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/exempt_borrowing_test.go b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/exempt_borrowing_test.go index d2bbeba82c9b1..f2f693da01194 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/exempt_borrowing_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/exempt_borrowing_test.go @@ -20,7 +20,8 @@ import ( "testing" "time" - fcboot "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-v134" + fcboot "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" + fcbootlatest "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-v134" fqs "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset" testeventclock "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/testing/eventclock" "k8s.io/apiserver/pkg/util/flowcontrol/metrics" @@ -33,10 +34,10 @@ import ( func TestUpdateBorrowing(t *testing.T) { startTime := time.Now() clk, _ := testeventclock.NewFake(startTime, 0, nil) - plcExempt := fcboot.MandatoryPriorityLevelConfigurationExempt - plcHigh := fcboot.SuggestedPriorityLevelConfigurationWorkloadHigh - plcMid := fcboot.SuggestedPriorityLevelConfigurationWorkloadLow - plcLow := fcboot.MandatoryPriorityLevelConfigurationCatchAll + plcExempt := fcbootlatest.MandatoryPriorityLevelConfigurationExempt + plcHigh := fcbootlatest.SuggestedPriorityLevelConfigurationWorkloadHigh + plcMid := fcbootlatest.SuggestedPriorityLevelConfigurationWorkloadLow + plcLow := fcbootlatest.MandatoryPriorityLevelConfigurationCatchAll plcs := []*flowcontrol.PriorityLevelConfiguration{plcHigh, plcExempt, plcMid, plcLow} fses := []*flowcontrol.FlowSchema{} k8sClient := clientsetfake.NewSimpleClientset(plcLow, plcExempt, plcHigh, plcMid) @@ -46,6 +47,7 @@ func TestUpdateBorrowing(t *testing.T) { *plcMid.Spec.Limited.NominalConcurrencyShares+ *plcLow.Spec.Limited.NominalConcurrencyShares) * 6 config := TestableConfig{ + FeatureGate: fcboot.LatestFeatureGate, Name: "test", Clock: clk, AsFieldManager: "testfm", diff --git a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/gen_test.go b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/gen_test.go index f60011d0aaae4..5c5c67676b5a8 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/gen_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/gen_test.go @@ -94,7 +94,7 @@ var flowDistinguisherMethodTypes = sets.NewString( ) var mandFTRExempt = &fsTestingRecord{ - fs: fcboot.GetV1ConfigCollection(true).FlowSchemaExempt, + fs: fcboot.Latest.FlowSchemaExempt, wellFormed: true, digests: map[bool]map[bool][]RequestDigest{ false: { @@ -151,7 +151,7 @@ var mandFTRExempt = &fsTestingRecord{ } var mandFTRCatchAll = &fsTestingRecord{ - fs: fcboot.GetV1ConfigCollection(true).FlowSchemaCatchAll, + fs: fcboot.Latest.FlowSchemaCatchAll, wellFormed: true, digests: map[bool]map[bool][]RequestDigest{ false: {}, diff --git a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/max_seats_test.go b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/max_seats_test.go index 7c385625edbfc..5d4db169d9520 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/max_seats_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/max_seats_test.go @@ -22,6 +22,7 @@ import ( flowcontrolv1 "k8s.io/api/flowcontrol/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + fcboot "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" fqs "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset" "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/testing/eventclock" "k8s.io/apiserver/pkg/util/flowcontrol/metrics" @@ -102,6 +103,7 @@ func Test_GetMaxSeats(t *testing.T) { startTime := time.Now() clk, _ := eventclock.NewFake(startTime, 0, nil) c := newTestableController(TestableConfig{ + FeatureGate: fcboot.LatestFeatureGate, Name: "Controller", Clock: clk, InformerFactory: informerFactory, diff --git a/test/integration/apiserver/flowcontrol/config_flap_test.go b/test/integration/apiserver/flowcontrol/config_flap_test.go index 89f2ddc40dc5a..bc3f5cd04161d 100644 --- a/test/integration/apiserver/flowcontrol/config_flap_test.go +++ b/test/integration/apiserver/flowcontrol/config_flap_test.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" flowcontrolbootstrap "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" clientset "k8s.io/client-go/kubernetes" + "k8s.io/component-base/featuregate" "k8s.io/klog/v2" ) @@ -59,13 +60,17 @@ func TestConfigFlap(t *testing.T) { }) } +var featureGates = map[bool]featuregate.FeatureGate{ + false: flowcontrolbootstrap.MakeGate(false), + true: flowcontrolbootstrap.LatestFeatureGate} + func flapTo(t *testing.T, trial string, v134 bool) bool { flapName := fmt.Sprintf("%s v134=%v", trial, v134) t.Log("Preparing for " + flapName) ctx, client, _, tearDown := setup(t, 100, 100, v134) defer tearDown() t.Log("Waiting for " + flapName) - if !waitForConfig(t, ctx, client, flapName, v134) { + if !waitForConfig(t, ctx, client, flapName, featureGates[v134]) { t.Fatal("Failed to establish " + flapName) return false } @@ -75,8 +80,8 @@ func flapTo(t *testing.T, trial string, v134 bool) bool { var errSuccess = errors.New("gotit") -func waitForConfig(t *testing.T, ctx context.Context, client clientset.Interface, phaseName string, v134 bool) bool { - expectedSlices := flowcontrolbootstrap.GetV1ConfigCollection(v134) +func waitForConfig(t *testing.T, ctx context.Context, client clientset.Interface, phaseName string, featureGate featuregate.FeatureGate) bool { + expectedSlices := flowcontrolbootstrap.GetV1ConfigCollection(featureGate) expectedPLCs := slicesToMap(widenToObject, expectedSlices.Mandatory.PriorityLevelConfigurations, expectedSlices.Suggested.PriorityLevelConfigurations) expectedFSes := slicesToMap(widenToObject, expectedSlices.Mandatory.FlowSchemas, expectedSlices.Suggested.FlowSchemas) var iteration int diff --git a/test/integration/apiserver/flowcontrol/fight_test.go b/test/integration/apiserver/flowcontrol/fight_test.go index f3d36155d63ba..d098d592bcddb 100644 --- a/test/integration/apiserver/flowcontrol/fight_test.go +++ b/test/integration/apiserver/flowcontrol/fight_test.go @@ -26,6 +26,7 @@ import ( "time" flowcontrol "k8s.io/api/flowcontrol/v1" + fcboot "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" utilfc "k8s.io/apiserver/pkg/util/flowcontrol" fqtesting "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/testing" "k8s.io/apiserver/pkg/util/flowcontrol/metrics" @@ -130,6 +131,7 @@ func (ft *fightTest) createController(invert bool, i int) { foundToDangling = func(found bool) bool { return found } } ctlr := utilfc.NewTestable(utilfc.TestableConfig{ + FeatureGate: fcboot.LatestFeatureGate, Name: fieldMgr, FoundToDangling: foundToDangling, Clock: clock.RealClock{}, diff --git a/test/integration/apiserver/flowcontrol/fs_condition_test.go b/test/integration/apiserver/flowcontrol/fs_condition_test.go index 5367a72e7a813..42bb9f09f4460 100644 --- a/test/integration/apiserver/flowcontrol/fs_condition_test.go +++ b/test/integration/apiserver/flowcontrol/fs_condition_test.go @@ -35,7 +35,7 @@ func TestConditionIsolation(t *testing.T) { ctx, loopbackClient, _, closeFn := setup(t, 10, 10, true) defer closeFn() - fsOrig := fcboot.GetV1ConfigCollection(true).Suggested.FlowSchemas[0] + fsOrig := fcboot.Latest.Suggested.FlowSchemas[0] t.Logf("Testing Status Condition isolation in FlowSchema %q", fsOrig.Name) fsClient := loopbackClient.FlowcontrolV1().FlowSchemas() var dangleOrig *flowcontrol.FlowSchemaCondition