Skip to content

[WIP] OCPSTRAT-2876: Allow inclusion/exclusion of manifests based on the OCP major version #1282

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
JoelSpeed wants to merge 8 commits into
base: main
Choose a base branch
from
Draft

[WIP] OCPSTRAT-2876: Allow inclusion/exclusion of manifests based on the OCP major version #1282

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
44606be
Update library-go to include feature gating manifest-inclusion
JoelSpeed Dec 17, 2025
40928f0
Pass feature gating information through to sync workers
JoelSpeed Dec 17, 2025
ded94a9
Filter all manifests at render based on featureset, featuregates and …
JoelSpeed Dec 19, 2025
73277b1
Ensure bootstrap pod is included in render
JoelSpeed Dec 19, 2025
1e704d1
Rename enabledFeatureGates to enabledCVOFeatureGates
JoelSpeed Jan 21, 2026
d0b4061
Adjust syncPayload handling to explicit switch on various changes
JoelSpeed Jan 23, 2026
96ce762
test: Add CVO scenario for feature gate based inclusion
DavidHurta Jan 26, 2026
ef0e8c6
Add major version manifest filtering
JoelSpeed Dec 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions bootstrap/bootstrap-pod.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ metadata:
namespace: openshift-cluster-version
labels:
k8s-app: cluster-version-operator
annotations:
include.release.openshift.io/{{ .ClusterProfile }}: "true"
spec:
containers:
- name: cluster-version-operator
Expand Down
6 changes: 4 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ require (
github.com/onsi/ginkgo/v2 v2.21.0
github.com/onsi/gomega v1.35.1
github.com/openshift-eng/openshift-tests-extension v0.0.0-20250220212757-b9c4d98a0c45
github.com/openshift/api v0.0.0-20260116192047-6fb7fdae95fd
github.com/openshift/api v0.0.0-20260126183958-606bd613f9f7
github.com/openshift/client-go v0.0.0-20260108185524-48f4ccfc4e13
github.com/openshift/library-go v0.0.0-20260108135436-db8dbd64c462
github.com/openshift/library-go v0.0.0-20260121132910-dc3a1c884c04
github.com/operator-framework/api v0.17.1
github.com/operator-framework/operator-lifecycle-manager v0.22.0
github.com/pkg/errors v0.9.1
Expand All @@ -34,6 +34,8 @@ require (
k8s.io/utils v0.0.0-20250604170112-4c0f3b243397
)

replace github.com/openshift/library-go => github.com/JoelSpeed/library-go v0.0.0-20260130121715-6e5ccfd0da42

require (
cel.dev/expr v0.24.0 // indirect
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
Expand Down
8 changes: 4 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
github.com/JoelSpeed/library-go v0.0.0-20260130121715-6e5ccfd0da42 h1:uIrR1qWklQm8U7WgtSZnyxj8bxCRtzvQ9JDKWvihOCg=
github.com/JoelSpeed/library-go v0.0.0-20260130121715-6e5ccfd0da42/go.mod h1:DCRz1EgdayEmr9b6KXKDL+DWBN0rGHu/VYADeHzPoOk=
github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
Expand Down Expand Up @@ -86,12 +88,10 @@ github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4=
github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
github.com/openshift-eng/openshift-tests-extension v0.0.0-20250220212757-b9c4d98a0c45 h1:hXpbYtP3iTh8oy/RKwKkcMziwchY3fIk95ciczf7cOA=
github.com/openshift-eng/openshift-tests-extension v0.0.0-20250220212757-b9c4d98a0c45/go.mod h1:6gkP5f2HL0meusT0Aim8icAspcD1cG055xxBZ9yC68M=
github.com/openshift/api v0.0.0-20260116192047-6fb7fdae95fd h1:Hpjq/55Qb0Gy65RewTSWWlyxpSovaRF86EPjYQkdlRU=
github.com/openshift/api v0.0.0-20260116192047-6fb7fdae95fd/go.mod h1:d5uzF0YN2nQQFA0jIEWzzOZ+edmo6wzlGLvx5Fhz4uY=
github.com/openshift/api v0.0.0-20260126183958-606bd613f9f7 h1:96rhgJpWlWzKEslMd6aYFMixV9vQVY32M71JcO4Gzn0=
github.com/openshift/api v0.0.0-20260126183958-606bd613f9f7/go.mod h1:d5uzF0YN2nQQFA0jIEWzzOZ+edmo6wzlGLvx5Fhz4uY=
github.com/openshift/client-go v0.0.0-20260108185524-48f4ccfc4e13 h1:6rd4zSo2UaWQcAPZfHK9yzKVqH0BnMv1hqMzqXZyTds=
github.com/openshift/client-go v0.0.0-20260108185524-48f4ccfc4e13/go.mod h1:YvOmPmV7wcJxpfhTDuFqqs2Xpb3M3ovsM6Qs/i2ptq4=
github.com/openshift/library-go v0.0.0-20260108135436-db8dbd64c462 h1:zX9Od4Jg8sVmwQLwk6Vd+BX7tcyC/462FVvDdzHEPPk=
github.com/openshift/library-go v0.0.0-20260108135436-db8dbd64c462/go.mod h1:nIzWQQE49XbiKizVnVOip9CEB7HJ0hoJwNi3g3YKnKc=
github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12 h1:AKx/w1qpS8We43bsRgf8Nll3CGlDHpr/WAXvuedTNZI=
github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
github.com/operator-framework/api v0.17.1 h1:J/6+Xj4IEV8C7hcirqUFwOiZAU3PbnJhWvB0/bB51c4=
Expand Down
3 changes: 2 additions & 1 deletion hack/cluster-version-util/task_graph.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"time"

"github.com/spf13/cobra"
"k8s.io/apimachinery/pkg/util/sets"

"github.com/openshift/cluster-version-operator/pkg/payload"
)
Expand All @@ -30,7 +31,7 @@ func newTaskGraphCmd() *cobra.Command {

func runTaskGraphCmd(cmd *cobra.Command, args []string) error {
manifestDir := args[0]
release, err := payload.LoadUpdate(manifestDir, "", "", "", payload.DefaultClusterProfile, nil)
release, err := payload.LoadUpdate(manifestDir, "", "", "", payload.DefaultClusterProfile, nil, sets.Set[string]{})
if err != nil {
return err
}
Expand Down
10 changes: 10 additions & 0 deletions lib/manifest/manifest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@ type InclusionConfiguration struct {

// Platform, if non-nil, excludes CredentialsRequests manifests unless they match the infrastructure platform.
Platform *string

// EnabledFeatureGates excludes manifests with a feature gate requirement when the condition is not met.
EnabledFeatureGates sets.Set[string]

// MajorVersion, if non-nil, excludes manifests unless they match the major version.
MajorVersion *uint64
}

// GetImplicitlyEnabledCapabilities returns a set of capabilities that are implicitly enabled after a cluster update.
Expand All @@ -57,6 +63,8 @@ func GetImplicitlyEnabledCapabilities(
manifestInclusionConfiguration.Profile,
manifestInclusionConfiguration.Capabilities,
manifestInclusionConfiguration.Overrides,
manifestInclusionConfiguration.EnabledFeatureGates,
manifestInclusionConfiguration.MajorVersion,
true,
)
// update manifest is enabled, no need to check
Expand All @@ -74,6 +82,8 @@ func GetImplicitlyEnabledCapabilities(
manifestInclusionConfiguration.Profile,
manifestInclusionConfiguration.Capabilities,
manifestInclusionConfiguration.Overrides,
manifestInclusionConfiguration.EnabledFeatureGates,
manifestInclusionConfiguration.MajorVersion,
true,
); err != nil {
continue
Expand Down
110 changes: 97 additions & 13 deletions pkg/cvo/cvo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
informerscorev1 "k8s.io/client-go/informers/core/v1"
"k8s.io/client-go/kubernetes"
Expand Down Expand Up @@ -120,6 +121,7 @@ type Operator struct {
cmConfigLister listerscorev1.ConfigMapNamespaceLister
cmConfigManagedLister listerscorev1.ConfigMapNamespaceLister
proxyLister configlistersv1.ProxyLister
featureGateLister configlistersv1.FeatureGateLister
cacheSynced []cache.InformerSynced

// queue tracks applying updates to a cluster.
Expand Down Expand Up @@ -178,16 +180,23 @@ type Operator struct {
// to select the manifests that will be applied in the cluster. The starting value cannot be changed in the executing
// CVO but the featurechangestopper controller will detect a feature set change in the cluster and shutdown the CVO.
// Enforcing featuresets is a standard GA CVO behavior that supports the feature gating functionality across the whole
// cluster, as opposed to the enabledFeatureGates which controls what gated behaviors of CVO itself are enabled by
// cluster, as opposed to the enabledCVOFeatureGates which controls what gated behaviors of CVO itself are enabled by
// the cluster feature gates.
// See: https://github.com/openshift/enhancements/blob/master/enhancements/update/cvo-techpreview-manifests.md
requiredFeatureSet configv1.FeatureSet

// enabledFeatureGates is the checker for what gated CVO behaviors are enabled or disabled by specific cluster-level
// enabledCVOFeatureGates is the checker for what gated CVO behaviors are enabled or disabled by specific cluster-level
// feature gates. It allows multiplexing the cluster-level feature gates to more granular CVO-level gates. Similarly
// to the requiredFeatureSet, the enabledFeatureGates cannot be changed in the executing CVO but the
// to the requiredFeatureSet, the enabledCVOFeatureGates cannot be changed in the executing CVO but the
// featurechangestopper controller will detect when cluster feature gate config changes and shutdown the CVO.
enabledFeatureGates featuregates.CvoGateChecker
enabledCVOFeatureGates featuregates.CvoGateChecker

// featureGatesMutex protects access to enabledManifestFeatureGates
featureGatesMutex sync.RWMutex
// enabledManifestFeatureGates is the set of feature gates that are currently enabled for the manifests that are applied to the cluster.
// This is the full set of enabled feature gates extracted from the FeatureGate object.
// We use this set as a filter to determine which of the manifests from the payload should or should not be applied to the cluster.
enabledManifestFeatureGates sets.Set[string]

clusterProfile string
uid types.UID
Expand All @@ -213,6 +222,7 @@ func New(
cmConfigManagedInformer informerscorev1.ConfigMapInformer,
proxyInformer configinformersv1.ProxyInformer,
operatorInformerFactory operatorexternalversions.SharedInformerFactory,
featureGateInformer configinformersv1.FeatureGateInformer,
client clientset.Interface,
kubeClient kubernetes.Interface,
operatorClient operatorclientset.Interface,
Expand All @@ -225,6 +235,7 @@ func New(
alwaysEnableCapabilities []configv1.ClusterVersionCapability,
featureSet configv1.FeatureSet,
cvoGates featuregates.CvoGateChecker,
startingEnabledManifestFeatureGates sets.Set[string],
) (*Operator, error) {
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartLogging(klog.Infof)
Expand All @@ -248,18 +259,19 @@ func New(
kubeClient: kubeClient,
operatorClient: operatorClient,
eventRecorder: eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: namespace}),
queue: workqueue.NewTypedRateLimitingQueueWithConfig[any](workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "clusterversion"}),
availableUpdatesQueue: workqueue.NewTypedRateLimitingQueueWithConfig[any](workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "availableupdates"}),
upgradeableQueue: workqueue.NewTypedRateLimitingQueueWithConfig[any](workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "upgradeable"}),
queue: workqueue.NewTypedRateLimitingQueueWithConfig(workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "clusterversion"}),
availableUpdatesQueue: workqueue.NewTypedRateLimitingQueueWithConfig(workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "availableupdates"}),
upgradeableQueue: workqueue.NewTypedRateLimitingQueueWithConfig(workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "upgradeable"}),

hypershift: hypershift,
exclude: exclude,
clusterProfile: clusterProfile,
conditionRegistry: standard.NewConditionRegistry(promqlTarget),
injectClusterIdIntoPromQL: injectClusterIdIntoPromQL,

requiredFeatureSet: featureSet,
enabledFeatureGates: cvoGates,
requiredFeatureSet: featureSet,
enabledCVOFeatureGates: cvoGates,
enabledManifestFeatureGates: startingEnabledManifestFeatureGates,

alwaysEnableCapabilities: alwaysEnableCapabilities,
}
Expand All @@ -276,6 +288,9 @@ func New(
if _, err := coInformer.Informer().AddEventHandler(optr.clusterOperatorEventHandler()); err != nil {
return nil, err
}
if _, err := featureGateInformer.Informer().AddEventHandler(optr.featureGateEventHandler()); err != nil {
return nil, err
}

optr.coLister = coInformer.Lister()
optr.cacheSynced = append(optr.cacheSynced, coInformer.Informer().HasSynced)
Expand All @@ -287,6 +302,9 @@ func New(
optr.cmConfigLister = cmConfigInformer.Lister().ConfigMaps(internal.ConfigNamespace)
optr.cmConfigManagedLister = cmConfigManagedInformer.Lister().ConfigMaps(internal.ConfigManagedNamespace)

optr.featureGateLister = featureGateInformer.Lister()
optr.cacheSynced = append(optr.cacheSynced, featureGateInformer.Informer().HasSynced)

// make sure this is initialized after all the listers are initialized
optr.upgradeableChecks = optr.defaultUpgradeableChecks()

Expand Down Expand Up @@ -318,7 +336,7 @@ func (optr *Operator) LoadInitialPayload(ctx context.Context, restConfig *rest.C
}

update, err := payload.LoadUpdate(optr.defaultPayloadDir(), optr.release.Image, optr.exclude, string(optr.requiredFeatureSet),
optr.clusterProfile, configv1.KnownClusterVersionCapabilities)
optr.clusterProfile, configv1.KnownClusterVersionCapabilities, optr.getEnabledFeatureGates())

if err != nil {
return nil, fmt.Errorf("the local release contents are invalid - no current version can be determined from disk: %v", err)
Expand Down Expand Up @@ -779,7 +797,7 @@ func (optr *Operator) sync(ctx context.Context, key string) error {
}

// inform the config sync loop about our desired state
status := optr.configSync.Update(ctx, config.Generation, desired, config, state)
status := optr.configSync.Update(ctx, config.Generation, desired, config, state, optr.getEnabledFeatureGates())

// write cluster version status
return optr.syncStatus(ctx, original, config, status, errs)
Expand Down Expand Up @@ -1084,10 +1102,76 @@ func (optr *Operator) HTTPClient() (*http.Client, error) {
}, nil
}

// featureGateEventHandler handles changes to FeatureGate objects and updates the cluster feature gates
func (optr *Operator) featureGateEventHandler() cache.ResourceEventHandler {
workQueueKey := optr.queueKey()
return cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
if optr.updateEnabledFeatureGates(obj) {
optr.queue.Add(workQueueKey)
}
},
UpdateFunc: func(old, new interface{}) {
if optr.updateEnabledFeatureGates(new) {
optr.queue.Add(workQueueKey)
}
},
}
}

// updateEnabledFeatureGates updates the cluster feature gates based on a FeatureGate object.
// Returns true or false based on whether or not the gates were actually updated.
// This allows us to avoid unnecessary work if the gates have not changed.
func (optr *Operator) updateEnabledFeatureGates(obj interface{}) bool {
featureGate, ok := obj.(*configv1.FeatureGate)
if !ok {
klog.Warningf("Expected FeatureGate object but got %T", obj)
return false
}

newGates := optr.extractEnabledGates(featureGate)

optr.featureGatesMutex.Lock()
defer optr.featureGatesMutex.Unlock()

// Check if gates actually changed to avoid unnecessary work
if !optr.enabledManifestFeatureGates.Equal(newGates) {

klog.V(2).Infof("Cluster feature gates changed from %v to %v",
sets.List(optr.enabledManifestFeatureGates), sets.List(newGates))

optr.enabledManifestFeatureGates = newGates
return true
}

return false
}

// getEnabledFeatureGates returns a copy of the current cluster feature gates for safe consumption
func (optr *Operator) getEnabledFeatureGates() sets.Set[string] {
optr.featureGatesMutex.RLock()
defer optr.featureGatesMutex.RUnlock()

// Return a copy to prevent external modification
return optr.enabledManifestFeatureGates.Clone()
}

// extractEnabledGates extracts the list of enabled feature gates for the current cluster version
func (optr *Operator) extractEnabledGates(featureGate *configv1.FeatureGate) sets.Set[string] {
// Find the feature gate details for the current loaded payload version.
currentVersion := optr.currentVersion().Version
if currentVersion == "" {
klog.Warningf("Payload has not been initialized yet, using the operator version %s", optr.enabledCVOFeatureGates.DesiredVersion())
currentVersion = optr.enabledCVOFeatureGates.DesiredVersion()
}

return featuregates.ExtractEnabledGates(featureGate, currentVersion)
}

// shouldReconcileCVOConfiguration returns whether the CVO should reconcile its configuration using the API server.
//
// enabledFeatureGates must be initialized before the function is called.
// enabledCVOFeatureGates must be initialized before the function is called.
func (optr *Operator) shouldReconcileCVOConfiguration() bool {
// The relevant CRD and CR are not applied in HyperShift, which configures the CVO via a configuration file
return optr.enabledFeatureGates.CVOConfiguration() && !optr.hypershift
return optr.enabledCVOFeatureGates.CVOConfiguration() && !optr.hypershift
}
Loading