kubernetes · MikeSpreitzer · Jan 15, 2025 · Feb 26, 2025 · Feb 26, 2025 · Feb 26, 2025
diff --git a/pkg/apis/flowcontrol/internalbootstrap/default-internal.go b/pkg/apis/flowcontrol/internalbootstrap/default-internal.go
@@ -20,34 +20,56 @@ import (
 	flowcontrolv1 "k8s.io/api/flowcontrol/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap"
+	"k8s.io/apiserver/pkg/features"
+	"k8s.io/component-base/featuregate"
 	"k8s.io/kubernetes/pkg/apis/flowcontrol"
 	"k8s.io/kubernetes/pkg/apis/flowcontrol/install"
 )
 
-// MandatoryFlowSchemas holds the untyped renditions of the mandatory
-// flow schemas.  In this map the key is the schema's name and the
+// FlowSchemasMap is a collection of unversioned (internal) FlowSchema objects indexed by name.
+type FlowSchemasMap = map[string]*flowcontrol.FlowSchema
+
+// GetMandatoryFlowSchemasMap returns the unversioned (internal) mandatory FlowSchema objects,
+// as a deeply immutable map.
+// The arguments are the values of the features that control which config collection to use.
+func GetMandatoryFlowSchemasMap(featureGate featuregate.FeatureGate) FlowSchemasMap {
+	return MandatoryFlowSchemasMap[bootstrap.CollectionID{V134: featureGate.Enabled(features.APFv134Config)}]
+}
+
+var oldConfig = bootstrap.GetV1ConfigCollection(bootstrap.MakeGate(false))
+
+// MandatoryFlowSchemasMap holds the unversioned (internal) renditions of the mandatory
+// flow schemas.  In the outer map the key is CollectionId and
+// in the inner map the key is the schema's name and the
 // value is the `*FlowSchema`.  Nobody should mutate anything
 // reachable from this map.
-var MandatoryFlowSchemas = internalizeFSes(bootstrap.MandatoryFlowSchemas)
+var MandatoryFlowSchemasMap = map[bootstrap.CollectionID]FlowSchemasMap{
+	{V134: false}: internalizeFSes(oldConfig.Mandatory.FlowSchemas),
+	{V134: true}:  internalizeFSes(bootstrap.Latest.Mandatory.FlowSchemas),
+}
+
+// PriorityLevelConfigurationsMap is a collection of unversioned (internal) PriorityLevelConfiguration objects, indexed by name.
+type PriorityLevelConfigurationsMap = map[string]*flowcontrol.PriorityLevelConfiguration
 
-// MandatoryPriorityLevelConfigurations holds the untyped renditions of the
-// mandatory priority level configuration objects.  In this map the
-// key is the object's name and the value is the
+// GetMandatoryPriorityLevelConfigurationsMap returns the mandatory PriorityLevelConfiguration objects,
+// as a deeply immutable map.
+// The arguments are the values of the features that control which config collection to use.
+func GetMandatoryPriorityLevelConfigurationsMap(featureGate featuregate.FeatureGate) PriorityLevelConfigurationsMap {
+	return MandatoryPriorityLevelConfigurationsMap[bootstrap.CollectionID{V134: featureGate.Enabled(features.APFv134Config)}]
+}
+
+// MandatoryPriorityLevelConfigurationsMap holds the untyped renditions of the
+// mandatory priority level configuration objects.  In the outer map the key is `bootstrap.CollectionID` and
+// in the inner map the key is the object's name and the value is the
 // `*PriorityLevelConfiguration`.  Nobody should mutate anything
 // reachable from this map.
-var MandatoryPriorityLevelConfigurations = internalizePLs(bootstrap.MandatoryPriorityLevelConfigurations)
-
-// NewAPFScheme constructs and returns a Scheme configured to handle
-// the API object types that are used to configure API Priority and
-// Fairness
-func NewAPFScheme() *runtime.Scheme {
-	scheme := runtime.NewScheme()
-	install.Install(scheme)
-	return scheme
+var MandatoryPriorityLevelConfigurationsMap = map[bootstrap.CollectionID]PriorityLevelConfigurationsMap{
+	{V134: false}: internalizePLs(oldConfig.Mandatory.PriorityLevelConfigurations),
+	{V134: true}:  internalizePLs(bootstrap.Latest.Mandatory.PriorityLevelConfigurations),
 }
 
-func internalizeFSes(exts []*flowcontrolv1.FlowSchema) map[string]*flowcontrol.FlowSchema {
-	ans := make(map[string]*flowcontrol.FlowSchema, len(exts))
+func internalizeFSes(exts []*flowcontrolv1.FlowSchema) FlowSchemasMap {
+	ans := make(FlowSchemasMap, len(exts))
 	scheme := NewAPFScheme()
 	for _, ext := range exts {
 		var untyped flowcontrol.FlowSchema
@@ -59,8 +81,8 @@ func internalizeFSes(exts []*flowcontrolv1.FlowSchema) map[string]*flowcontrol.F
 	return ans
 }
 
-func internalizePLs(exts []*flowcontrolv1.PriorityLevelConfiguration) map[string]*flowcontrol.PriorityLevelConfiguration {
-	ans := make(map[string]*flowcontrol.PriorityLevelConfiguration, len(exts))
+func internalizePLs(exts []*flowcontrolv1.PriorityLevelConfiguration) PriorityLevelConfigurationsMap {
+	ans := make(PriorityLevelConfigurationsMap, len(exts))
 	scheme := NewAPFScheme()
 	for _, ext := range exts {
 		var untyped flowcontrol.PriorityLevelConfiguration
@@ -71,3 +93,12 @@ func internalizePLs(exts []*flowcontrolv1.PriorityLevelConfiguration) map[string
 	}
 	return ans
 }
+
+// NewAPFScheme constructs and returns a Scheme configured to handle
+// the API object types that are used to configure API Priority and
+// Fairness
+func NewAPFScheme() *runtime.Scheme {
+	scheme := runtime.NewScheme()
+	install.Install(scheme)
+	return scheme
+}
diff --git a/pkg/apis/flowcontrol/internalbootstrap/defaults_test.go b/pkg/apis/flowcontrol/internalbootstrap/defaults_test.go
@@ -25,38 +25,41 @@ import (
 	flowcontrol "k8s.io/api/flowcontrol/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	"k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap"
+	"k8s.io/component-base/featuregate"
 )
 
 func TestBootstrapConfigurationWithDefaulted(t *testing.T) {
-	scheme := NewAPFScheme()
-
-	bootstrapFlowSchemas := make([]*flowcontrol.FlowSchema, 0)
-	bootstrapFlowSchemas = append(bootstrapFlowSchemas, bootstrap.MandatoryFlowSchemas...)
-	bootstrapFlowSchemas = append(bootstrapFlowSchemas, bootstrap.SuggestedFlowSchemas...)
-	for _, original := range bootstrapFlowSchemas {
-		t.Run(fmt.Sprintf("FlowSchema/%s", original.Name), func(t *testing.T) {
-			defaulted := original.DeepCopyObject().(*flowcontrol.FlowSchema)
-			scheme.Default(defaulted)
-			if apiequality.Semantic.DeepEqual(original, defaulted) {
-				t.Logf("Defaulting makes no change to FlowSchema: %q", original.Name)
-				return
-			}
-			t.Errorf("Expected defaulting to not change FlowSchema: %q, diff: %s", original.Name, cmp.Diff(original, defaulted))
-		})
-	}
+	t.Run("v134Config=false", caseFn(bootstrap.MakeGate(false)))
+	t.Run("v134Config=true", caseFn(bootstrap.LatestFeatureGate))
+}
+
+func caseFn(featureGate featuregate.FeatureGate) func(*testing.T) {
+	return func(t *testing.T) {
+		scheme := NewAPFScheme()
+		bootstrapFlowSchemas := bootstrap.GetFlowSchemas(featureGate)
+		for _, original := range bootstrapFlowSchemas {
+			t.Run(fmt.Sprintf("FlowSchema/%s", original.Name), func(t *testing.T) {
+				defaulted := original.DeepCopyObject().(*flowcontrol.FlowSchema)
+				scheme.Default(defaulted)
+				if apiequality.Semantic.DeepEqual(original, defaulted) {
+					t.Logf("Defaulting makes no change to FlowSchema: %q", original.Name)
+					return
+				}
+				t.Errorf("Expected defaulting to not change FlowSchema: %q, diff: %s", original.Name, cmp.Diff(original, defaulted))
+			})
+		}
 
-	bootstrapPriorityLevels := make([]*flowcontrol.PriorityLevelConfiguration, 0)
-	bootstrapPriorityLevels = append(bootstrapPriorityLevels, bootstrap.MandatoryPriorityLevelConfigurations...)
-	bootstrapPriorityLevels = append(bootstrapPriorityLevels, bootstrap.SuggestedPriorityLevelConfigurations...)
-	for _, original := range bootstrapPriorityLevels {
-		t.Run(fmt.Sprintf("PriorityLevelConfiguration/%s", original.Name), func(t *testing.T) {
-			defaulted := original.DeepCopyObject().(*flowcontrol.PriorityLevelConfiguration)
-			scheme.Default(defaulted)
-			if apiequality.Semantic.DeepEqual(original, defaulted) {
-				t.Logf("Defaulting makes no change to PriorityLevelConfiguration: %q", original.Name)
-				return
-			}
-			t.Errorf("Expected defaulting to not change PriorityLevelConfiguration: %q, diff: %s", original.Name, cmp.Diff(original, defaulted))
-		})
+		bootstrapPriorityLevels := bootstrap.GetPrioritylevelConfigurations(featureGate)
+		for _, original := range bootstrapPriorityLevels {
+			t.Run(fmt.Sprintf("PriorityLevelConfiguration/%s", original.Name), func(t *testing.T) {
+				defaulted := original.DeepCopyObject().(*flowcontrol.PriorityLevelConfiguration)
+				scheme.Default(defaulted)
+				if apiequality.Semantic.DeepEqual(original, defaulted) {
+					t.Logf("Defaulting makes no change to PriorityLevelConfiguration: %q", original.Name)
+					return
+				}
+				t.Errorf("Expected defaulting to not change PriorityLevelConfiguration: %q, diff: %s", original.Name, cmp.Diff(original, defaulted))
+			})
+		}
 	}
 }
diff --git a/pkg/apis/flowcontrol/validation/validation.go b/pkg/apis/flowcontrol/validation/validation.go
@@ -28,6 +28,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/validation/field"
+	bootstrapv1 "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap"
 	"k8s.io/apiserver/pkg/util/shufflesharding"
 	apivalidation "k8s.io/kubernetes/pkg/apis/core/validation"
 	"k8s.io/kubernetes/pkg/apis/flowcontrol"
@@ -78,18 +79,26 @@ var supportedLimitResponseType = sets.NewString(
 // PriorityLevelValidationOptions holds the validation options for a priority level object
 type PriorityLevelValidationOptions struct{}
 
+var earliestGate = bootstrapv1.MakeGate(false)
+var latestGate = bootstrapv1.LatestFeatureGate
+
 // ValidateFlowSchema validates the content of flow-schema
 func ValidateFlowSchema(fs *flowcontrol.FlowSchema) field.ErrorList {
 	allErrs := apivalidation.ValidateObjectMeta(&fs.ObjectMeta, false, ValidateFlowSchemaName, field.NewPath("metadata"))
 	specPath := field.NewPath("spec")
 	allErrs = append(allErrs, ValidateFlowSchemaSpec(fs.Name, &fs.Spec, specPath)...)
-	if mand, ok := internalbootstrap.MandatoryFlowSchemas[fs.Name]; ok {
+	if mand, ok := internalbootstrap.GetMandatoryFlowSchemasMap(latestGate)[fs.Name]; ok {
+		// The old config objects are a subset of the new config objects,
+		// as far as identify is concerned. The specs may differ.
+		mandOld := internalbootstrap.GetMandatoryFlowSchemasMap(earliestGate)[fs.Name]
+		// For now, accept either the old or the new spec.
+		// In a later release, change this to accept only the new spec.
 		// Check for almost exact equality.  This is a pretty
 		// strict test, and it is OK in this context because both
 		// sides of this comparison are intended to ultimately
 		// come from the same code.
-		if !apiequality.Semantic.DeepEqual(fs.Spec, mand.Spec) {
-			allErrs = append(allErrs, field.Invalid(specPath, fs.Spec, fmt.Sprintf("spec of '%s' must equal the fixed value", fs.Name)))
+		if !(apiequality.Semantic.DeepEqual(fs.Spec, mand.Spec) || mandOld != nil && apiequality.Semantic.DeepEqual(fs.Spec, mandOld.Spec)) {
+			allErrs = append(allErrs, field.Invalid(specPath, fs.Spec, fmt.Sprintf("spec of '%s' must equal the old or new fixed value", fs.Name)))
 		}
 	}
 	allErrs = append(allErrs, ValidateFlowSchemaStatus(&fs.Status, field.NewPath("status"))...)
@@ -363,8 +372,9 @@ func ValidatePriorityLevelConfiguration(pl *flowcontrol.PriorityLevelConfigurati
 
 func ValidateIfMandatoryPriorityLevelConfigurationObject(pl *flowcontrol.PriorityLevelConfiguration, fldPath *field.Path) field.ErrorList {
 	var allErrs field.ErrorList
-	mand, ok := internalbootstrap.MandatoryPriorityLevelConfigurations[pl.Name]
+	mand, ok := internalbootstrap.GetMandatoryPriorityLevelConfigurationsMap(latestGate)[pl.Name]
 	if !ok {
+		// The old config objects are a subset of the new, as far as identity is concerned.
 		return allErrs
 	}
 
@@ -381,12 +391,15 @@ func ValidateIfMandatoryPriorityLevelConfigurationObject(pl *flowcontrol.Priorit
 		return allErrs
 	}
 
+	// For now, accept either the old or the new default config.
+	// In a later release, accept only the new.
+	mandOld := internalbootstrap.GetMandatoryPriorityLevelConfigurationsMap(earliestGate)[pl.Name]
 	// Check for almost exact equality.  This is a pretty
 	// strict test, and it is OK in this context because both
 	// sides of this comparison are intended to ultimately
 	// come from the same code.
-	if !apiequality.Semantic.DeepEqual(pl.Spec, mand.Spec) {
-		allErrs = append(allErrs, field.Invalid(fldPath, pl.Spec, fmt.Sprintf("spec of '%s' must equal the fixed value", pl.Name)))
+	if !(apiequality.Semantic.DeepEqual(pl.Spec, mand.Spec) || mandOld != nil && apiequality.Semantic.DeepEqual(pl.Spec, mandOld.Spec)) {
+		allErrs = append(allErrs, field.Invalid(fldPath, pl.Spec, fmt.Sprintf("spec of '%s' must equal the old or new fixed value", pl.Name)))
 	}
 	return allErrs
 }

diff --git a/pkg/apis/flowcontrol/validation/validation_test.go b/pkg/apis/flowcontrol/validation/validation_test.go
@@ -28,9 +28,9 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/validation/field"
+	fcboot "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/kubernetes/pkg/apis/flowcontrol"
-	"k8s.io/kubernetes/pkg/apis/flowcontrol/internalbootstrap"
 	"k8s.io/utils/pointer"
 )
 
@@ -263,7 +263,7 @@ func TestFlowSchemaValidation(t *testing.T) {
 			},
 			Spec: badExempt,
 		},
-		expectedErrors: field.ErrorList{field.Invalid(field.NewPath("spec"), badExempt, "spec of 'exempt' must equal the fixed value")},
+		expectedErrors: field.ErrorList{field.Invalid(field.NewPath("spec"), badExempt, "spec of 'exempt' must equal the old or new fixed value")},
 	}, {
 		name: "bad catch-all flow-schema should fail",
 		flowSchema: &flowcontrol.FlowSchema{
@@ -272,7 +272,7 @@ func TestFlowSchemaValidation(t *testing.T) {
 			},
 			Spec: badCatchAll,
 		},
-		expectedErrors: field.ErrorList{field.Invalid(field.NewPath("spec"), badCatchAll, "spec of 'catch-all' must equal the fixed value")},
+		expectedErrors: field.ErrorList{field.Invalid(field.NewPath("spec"), badCatchAll, "spec of 'catch-all' must equal the old or new fixed value")},
 	}, {
 		name: "catch-all flow-schema should work",
 		flowSchema: &flowcontrol.FlowSchema{
@@ -769,7 +769,7 @@ func TestPriorityLevelConfigurationValidation(t *testing.T) {
 	}
 
 	validChangesInExemptFieldOfExemptPLFn := func() flowcontrol.PriorityLevelConfigurationSpec {
-		have, _ := internalbootstrap.MandatoryPriorityLevelConfigurations[flowcontrol.PriorityLevelConfigurationNameExempt]
+		have := fcboot.Latest.PriorityLevelConfigurationExempt
 		return flowcontrol.PriorityLevelConfigurationSpec{
 			Type: flowcontrol.PriorityLevelEnablementExempt,
 			Exempt: &flowcontrol.ExemptPriorityLevelConfiguration{
@@ -949,7 +949,7 @@ func TestPriorityLevelConfigurationValidation(t *testing.T) {
 			},
 			Spec: badSpec,
 		},
-		expectedErrors: field.ErrorList{field.Invalid(field.NewPath("spec"), badSpec, "spec of 'catch-all' must equal the fixed value")},
+		expectedErrors: field.ErrorList{field.Invalid(field.NewPath("spec"), badSpec, "spec of 'catch-all' must equal the old or new fixed value")},
 	}, {
 		name: "backstop should work",
 		priorityLevelConfiguration: &flowcontrol.PriorityLevelConfiguration{

diff --git a/pkg/controlplane/apiserver/apis.go b/pkg/controlplane/apiserver/apis.go
@@ -78,7 +78,7 @@ func (c *CompletedConfig) GenericStorageProviders(discovery discovery.DiscoveryI
 		coordinationrest.RESTStorageProvider{},
 		rbacrest.RESTStorageProvider{Authorizer: c.Generic.Authorization.Authorizer},
 		svmrest.RESTStorageProvider{},
-		flowcontrolrest.RESTStorageProvider{InformerFactory: c.Generic.SharedInformerFactory},
+		flowcontrolrest.RESTStorageProvider{InformerFactory: c.Generic.SharedInformerFactory, FeatureGate: c.Generic.FeatureGate},
 		admissionregistrationrest.RESTStorageProvider{Authorizer: c.Generic.Authorization.Authorizer, DiscoveryClient: discovery},
 		eventsrest.RESTStorageProvider{TTL: c.EventTTL},
 	}, nil

diff --git a/pkg/controlplane/instance.go b/pkg/controlplane/instance.go
@@ -418,7 +418,7 @@ func (c CompletedConfig) StorageProviders(client *kubernetes.Clientset) ([]contr
 		schedulingrest.RESTStorageProvider{},
 		storagerest.RESTStorageProvider{},
 		svmrest.RESTStorageProvider{},
-		flowcontrolrest.RESTStorageProvider{InformerFactory: c.ControlPlane.Generic.SharedInformerFactory},
+		flowcontrolrest.RESTStorageProvider{InformerFactory: c.ControlPlane.Generic.SharedInformerFactory, FeatureGate: c.ControlPlane.Generic.FeatureGate},
 		// keep apps after extensions so legacy clients resolve the extensions versions of shared resource names.
 		// See https://github.com/kubernetes/kubernetes/issues/42392
 		appsrest.StorageProvider{},

diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go
@@ -62,6 +62,15 @@ const (
 	// Enables usage of any object for volume data source in PVCs
 	AnyVolumeDataSource featuregate.Feature = "AnyVolumeDataSource"
 
+	// owner: @MikeSpreitzer, @tkashem, @linxiulei
+	//
+	// Make API Priority and Fairness use modern configuration, which
+	// differs from the old in these ways:
+	// - introduce priority level and flow schema for events;
+	// - generally reorganize to stop working around lack of borrowing;
+	// - increase the nominal concurrency shares for leader election.
+	APFv134Config featuregate.Feature = "APFv134Config"
+
 	// owner: @liggitt
 	// kep: https://kep.k8s.io/4601
 	//
@@ -1004,6 +1013,11 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
 		{Version: version.MustParse("1.33"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // GA in 1.33 -> remove in 1.36
 	},
 
+	APFv134Config: {
+		{Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Alpha},
+		{Version: version.MustParse("1.34"), Default: true, PreRelease: featuregate.Beta},
+	},
+
 	AuthorizeNodeWithSelectors: {
 		{Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha},
 		{Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.Beta},

diff --git a/pkg/registry/flowcontrol/ensurer/flowschema_test.go b/pkg/registry/flowcontrol/ensurer/flowschema_test.go
@@ -24,7 +24,7 @@ import (
 	flowcontrolv1 "k8s.io/api/flowcontrol/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap"
+	bootstrap "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-v134"
 	"k8s.io/client-go/kubernetes/fake"
 	flowcontrollisters "k8s.io/client-go/listers/flowcontrol/v1"
 	toolscache "k8s.io/client-go/tools/cache"

diff --git a/pkg/registry/flowcontrol/ensurer/prioritylevelconfiguration_test.go b/pkg/registry/flowcontrol/ensurer/prioritylevelconfiguration_test.go
@@ -24,7 +24,7 @@ import (
 	flowcontrolv1 "k8s.io/api/flowcontrol/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap"
+	bootstrap "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap-v134"
 	"k8s.io/client-go/kubernetes/fake"
 	flowcontrollisters "k8s.io/client-go/listers/flowcontrol/v1"
 	toolscache "k8s.io/client-go/tools/cache"