From b3e23211221c5179903001185ad0dbcd93309e44 Mon Sep 17 00:00:00 2001 From: Wei Weng Date: Thu, 5 Mar 2026 11:08:52 -0500 Subject: [PATCH 1/4] feat: new selection mode so that CRP can select some namespace resources (#452) * add selection mode that places namespace and namespace scoped resources Signed-off-by: Wei Weng * remove namespace selector must be 1st restriction Signed-off-by: Wei Weng * use CEL validation Signed-off-by: Wei Weng * try double quote Signed-off-by: Wei Weng * simplify if else Signed-off-by: Wei Weng --------- Signed-off-by: Wei Weng Co-authored-by: Wei Weng --- .../v1beta1/clusterresourceplacement_types.go | 97 +++- ...tes-fleet.io_clusterresourceoverrides.yaml | 66 ++- ...t.io_clusterresourceoverridesnapshots.yaml | 66 ++- ...es-fleet.io_clusterresourceplacements.yaml | 52 +- ...ubernetes-fleet.io_resourceplacements.yaml | 52 +- .../controller/resource_selector_resolver.go | 95 +++- .../resource_selector_resolver_test.go | 518 ++++++++++++++++++ pkg/utils/test_util.go | 14 + pkg/utils/validator/placement.go | 18 +- pkg/utils/validator/placement_test.go | 180 ++++++ .../api_validation_integration_test.go | 312 +++++++++++ ...t_namespace_with_resourceselectors_test.go | 278 ++++++++++ test/e2e/utils_test.go | 49 ++ 13 files changed, 1770 insertions(+), 27 deletions(-) create mode 100644 test/e2e/placement_namespace_with_resourceselectors_test.go diff --git a/apis/placement/v1beta1/clusterresourceplacement_types.go b/apis/placement/v1beta1/clusterresourceplacement_types.go index 3fa18bfe5..5e3cfbd43 100644 --- a/apis/placement/v1beta1/clusterresourceplacement_types.go +++ b/apis/placement/v1beta1/clusterresourceplacement_types.go @@ -124,6 +124,9 @@ type ClusterResourcePlacement struct { } // PlacementSpec defines the desired state of ClusterResourcePlacement and ResourcePlacement. +// +kubebuilder:validation:XValidation:rule="size(self.resourceSelectors.filter(x, x.kind == 'Namespace' && x.group == \"\" && x.version == 'v1' && has(x.selectionScope) && x.selectionScope == 'NamespaceWithResourceSelectors')) <= 1",message="only one namespace selector with NamespaceWithResourceSelectors mode is allowed" +// +kubebuilder:validation:XValidation:rule="size(self.resourceSelectors.filter(x, x.kind == 'Namespace' && x.group == \"\" && x.version == 'v1' && has(x.selectionScope) && x.selectionScope == 'NamespaceWithResourceSelectors')) == 0 || (size(self.resourceSelectors.filter(x, x.kind == 'Namespace' && x.group == \"\" && x.version == 'v1' && has(x.selectionScope) && x.selectionScope == 'NamespaceWithResourceSelectors' && has(x.name) && size(x.name) > 0 && !has(x.labelSelector))) == 1)",message="namespace selector with NamespaceWithResourceSelectors mode must select by name (not by label)" +// +kubebuilder:validation:XValidation:rule="size(self.resourceSelectors.filter(x, x.kind == 'Namespace' && x.group == \"\" && x.version == 'v1' && has(x.selectionScope) && x.selectionScope == 'NamespaceWithResourceSelectors')) == 0 || size(self.resourceSelectors.filter(x, x.kind == 'Namespace' && x.group == \"\" && x.version == 'v1')) == 1",message="when using NamespaceWithResourceSelectors mode, only one namespace selector is allowed (cannot mix with other namespace selectors)" type PlacementSpec struct { // ResourceSelectors is an array of selectors used to select cluster scoped resources. The selectors are `ORed`. // You can have 1-100 selectors. @@ -185,7 +188,32 @@ type ResourceSelectorTerm struct { Version string `json:"version"` // Kind of the to be selected resource. - // Note: When `Kind` is `namespace`, by default ALL the resources under the selected namespaces are selected. + // + // Special behavior when Kind is `namespace` (ClusterResourcePlacement only): + // Note: ResourcePlacement cannot select namespaces since it is namespace-scoped and selects resources within a namespace. + // + // For ClusterResourcePlacement, you can use SelectionScope to control what gets selected: + // - NamespaceOnly: Only the namespace object itself + // - NamespaceWithResources: The namespace AND all resources within it (default) + // - NamespaceWithResourceSelectors: The namespace AND resources specified by additional selectors + // + // When SelectionScope is NamespaceWithResourceSelectors, you can define additional ResourceSelectorTerms + // (after the namespace selector) to specify which resources to include. These additional selectors can + // target both namespace-scoped resources (within the selected namespace) and cluster-scoped resources. + // + // Important requirements for NamespaceWithResourceSelectors mode: + // - Exactly one namespace selector with this mode is allowed + // - The namespace selector must select by name (not by label) + // - Only one namespace selector is allowed when using this mode (cannot mix with other namespace selectors) + // - All requirements are validated via CEL at API validation time + // - If the selected namespace is deleted after CRP creation, the controller will report an error condition + // + // Example using NamespaceWithResourceSelectors: + // - Namespace selector: {Group: "", Version: "v1", Kind: "Namespace", Name: "prod", SelectionScope: "NamespaceWithResourceSelectors"} + // - Additional selector: {Group: "apps", Version: "v1", Kind: "Deployment", LabelSelector: {app: "frontend"}} + // - Third selector: {Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole", Name: "admin"} + // This selects: the "prod" namespace, all Deployments with label app=frontend in "prod", and the "admin" ClusterRole. + // // +kubebuilder:validation:Required Kind string `json:"kind"` @@ -202,21 +230,80 @@ type ResourceSelectorTerm struct { LabelSelector *metav1.LabelSelector `json:"labelSelector,omitempty"` // SelectionScope defines the scope of resource selections when the Kind is `namespace`. - // +kubebuilder:validation:Enum=NamespaceOnly;NamespaceWithResources + // This field is only applicable when Kind is "Namespace" and is ignored for other resource kinds. + // See the Kind field documentation for detailed examples and usage patterns. + // +kubebuilder:validation:Enum=NamespaceOnly;NamespaceWithResources;NamespaceWithResourceSelectors // +kubebuilder:default=NamespaceWithResources // +kubebuilder:validation:Optional SelectionScope SelectionScope `json:"selectionScope,omitempty"` } -// SelectionScope defines the scope of resource selections. +// SelectionScope defines the scope of resource selections when selecting namespaces. +// This only applies when a ResourceSelectorTerm has Kind="Namespace". type SelectionScope string const ( - // NamespaceOnly means only the namespace itself is selected. + // NamespaceOnly means only the namespace object itself is selected. + // + // Use case: When you want to create/manage only the namespace without any resources inside it. + // Example: Creating a namespace with specific labels/annotations on member clusters. NamespaceOnly SelectionScope = "NamespaceOnly" - // NamespaceWithResources means all the resources under the namespace including namespace itself are selected. + // NamespaceWithResources means the namespace and ALL resources within it are selected. + // This is the default behavior for backward compatibility. + // + // Use case: When you want to replicate an entire namespace with all its contents to member clusters. + // Example: Copying a complete application stack (deployments, services, configmaps, etc.) across clusters. + // + // Note: This is the default value. When you select a namespace without specifying SelectionScope, + // this mode is used automatically. NamespaceWithResources SelectionScope = "NamespaceWithResources" + + // NamespaceWithResourceSelectors allows fine-grained selection of specific resources within a namespace. + // The namespace itself is always selected, and you can optionally specify which resources to include + // by adding additional ResourceSelectorTerm entries after the namespace selector. + // + // Use cases: + // 1. Select only specific resource types from a namespace (e.g., only Deployments and Services) + // 2. Select resources matching certain labels within a namespace + // 3. Include specific cluster-scoped resources along with namespace-scoped resources + // + // How "additional selectors" work: + // - Exactly one namespace selector with NamespaceWithResourceSelectors mode is required + // - This selector must select a namespace by name (label selectors not allowed) + // - ADDITIONAL selectors specify which resources to include: + // - Namespace-scoped resources are filtered to only those within the selected namespace + // - Cluster-scoped resources are included as specified (not limited to the namespace) + // - If no additional selectors are provided, only the namespace object itself is selected + // + // Example 1 - Select specific deployments from a namespace: + // Selector 1: {Group: "", Version: "v1", Kind: "Namespace", Name: "production", SelectionScope: "NamespaceWithResourceSelectors"} + // Selector 2: {Group: "apps", Version: "v1", Kind: "Deployment", LabelSelector: {tier: "frontend"}} + // Result: The "production" namespace + all Deployments labeled tier=frontend within "production" + // + // Example 2 - Select namespace with multiple resource types: + // Selector 1: {Group: "", Version: "v1", Kind: "Namespace", Name: "app", SelectionScope: "NamespaceWithResourceSelectors"} + // Selector 2: {Group: "apps", Version: "v1", Kind: "Deployment"} + // Selector 3: {Group: "", Version: "v1", Kind: "Service"} + // Result: The "app" namespace + ALL Deployments and Services within "app" + // + // Example 3 - Include cluster-scoped resources: + // Selector 1: {Group: "", Version: "v1", Kind: "Namespace", Name: "app", SelectionScope: "NamespaceWithResourceSelectors"} + // Selector 2: {Group: "apps", Version: "v1", Kind: "Deployment"} + // Selector 3: {Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole", Name: "app-admin"} + // Result: The "app" namespace + ALL Deployments in "app" + the "app-admin" ClusterRole + // + // Important constraints: + // - Exactly ONE namespace selector with NamespaceWithResourceSelectors mode is allowed + // - The namespace selector must select by name (label selectors not allowed) + // - Only ONE namespace selector total is allowed when using this mode (cannot mix with other namespace selectors) + // - All constraints are enforced via CEL at API validation time + // + // Runtime behavior: + // - If the selected namespace is deleted after the CRP is created, the controller will detect this during + // the next reconciliation and report an error condition in the CRP status + // - The CRP will transition to a failed state until the namespace is recreated or the CRP is updated + NamespaceWithResourceSelectors SelectionScope = "NamespaceWithResourceSelectors" ) // PlacementPolicy contains the rules to select target member clusters to place the selected resources. diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceoverrides.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceoverrides.yaml index 5c92a300d..71474ff9b 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceoverrides.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceoverrides.yaml @@ -432,7 +432,31 @@ spec: kind: description: |- Kind of the to be selected resource. - Note: When `Kind` is `namespace`, by default ALL the resources under the selected namespaces are selected. + + Special behavior when Kind is `namespace` (ClusterResourcePlacement only): + Note: ResourcePlacement cannot select namespaces since it is namespace-scoped and selects resources within a namespace. + + For ClusterResourcePlacement, you can use SelectionScope to control what gets selected: + - NamespaceOnly: Only the namespace object itself + - NamespaceWithResources: The namespace AND all resources within it (default) + - NamespaceWithResourceSelectors: The namespace AND resources specified by additional selectors + + When SelectionScope is NamespaceWithResourceSelectors, you can define additional ResourceSelectorTerms + (after the namespace selector) to specify which resources to include. These additional selectors can + target both namespace-scoped resources (within the selected namespace) and cluster-scoped resources. + + Important requirements for NamespaceWithResourceSelectors mode: + - Exactly one namespace selector with this mode is allowed + - The namespace selector must select by name (not by label) + - Only one namespace selector is allowed when using this mode (cannot mix with other namespace selectors) + - All requirements are validated via CEL at API validation time + - If the selected namespace is deleted after CRP creation, the controller will report an error condition + + Example using NamespaceWithResourceSelectors: + - Namespace selector: {Group: "", Version: "v1", Kind: "Namespace", Name: "prod", SelectionScope: "NamespaceWithResourceSelectors"} + - Additional selector: {Group: "apps", Version: "v1", Kind: "Deployment", LabelSelector: {app: "frontend"}} + - Third selector: {Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole", Name: "admin"} + This selects: the "prod" namespace, all Deployments with label app=frontend in "prod", and the "admin" ClusterRole. type: string labelSelector: description: |- @@ -487,11 +511,14 @@ spec: type: string selectionScope: default: NamespaceWithResources - description: SelectionScope defines the scope of resource selections - when the Kind is `namespace`. + description: |- + SelectionScope defines the scope of resource selections when the Kind is `namespace`. + This field is only applicable when Kind is "Namespace" and is ignored for other resource kinds. + See the Kind field documentation for detailed examples and usage patterns. enum: - NamespaceOnly - NamespaceWithResources + - NamespaceWithResourceSelectors type: string version: description: Version of the to be selected resource. @@ -805,7 +832,31 @@ spec: kind: description: |- Kind of the to be selected resource. - Note: When `Kind` is `namespace`, by default ALL the resources under the selected namespaces are selected. + + Special behavior when Kind is `namespace` (ClusterResourcePlacement only): + Note: ResourcePlacement cannot select namespaces since it is namespace-scoped and selects resources within a namespace. + + For ClusterResourcePlacement, you can use SelectionScope to control what gets selected: + - NamespaceOnly: Only the namespace object itself + - NamespaceWithResources: The namespace AND all resources within it (default) + - NamespaceWithResourceSelectors: The namespace AND resources specified by additional selectors + + When SelectionScope is NamespaceWithResourceSelectors, you can define additional ResourceSelectorTerms + (after the namespace selector) to specify which resources to include. These additional selectors can + target both namespace-scoped resources (within the selected namespace) and cluster-scoped resources. + + Important requirements for NamespaceWithResourceSelectors mode: + - Exactly one namespace selector with this mode is allowed + - The namespace selector must select by name (not by label) + - Only one namespace selector is allowed when using this mode (cannot mix with other namespace selectors) + - All requirements are validated via CEL at API validation time + - If the selected namespace is deleted after CRP creation, the controller will report an error condition + + Example using NamespaceWithResourceSelectors: + - Namespace selector: {Group: "", Version: "v1", Kind: "Namespace", Name: "prod", SelectionScope: "NamespaceWithResourceSelectors"} + - Additional selector: {Group: "apps", Version: "v1", Kind: "Deployment", LabelSelector: {app: "frontend"}} + - Third selector: {Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole", Name: "admin"} + This selects: the "prod" namespace, all Deployments with label app=frontend in "prod", and the "admin" ClusterRole. type: string labelSelector: description: |- @@ -860,11 +911,14 @@ spec: type: string selectionScope: default: NamespaceWithResources - description: SelectionScope defines the scope of resource selections - when the Kind is `namespace`. + description: |- + SelectionScope defines the scope of resource selections when the Kind is `namespace`. + This field is only applicable when Kind is "Namespace" and is ignored for other resource kinds. + See the Kind field documentation for detailed examples and usage patterns. enum: - NamespaceOnly - NamespaceWithResources + - NamespaceWithResourceSelectors type: string version: description: Version of the to be selected resource. diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceoverridesnapshots.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceoverridesnapshots.yaml index d8289eec7..336d3a861 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceoverridesnapshots.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceoverridesnapshots.yaml @@ -464,7 +464,31 @@ spec: kind: description: |- Kind of the to be selected resource. - Note: When `Kind` is `namespace`, by default ALL the resources under the selected namespaces are selected. + + Special behavior when Kind is `namespace` (ClusterResourcePlacement only): + Note: ResourcePlacement cannot select namespaces since it is namespace-scoped and selects resources within a namespace. + + For ClusterResourcePlacement, you can use SelectionScope to control what gets selected: + - NamespaceOnly: Only the namespace object itself + - NamespaceWithResources: The namespace AND all resources within it (default) + - NamespaceWithResourceSelectors: The namespace AND resources specified by additional selectors + + When SelectionScope is NamespaceWithResourceSelectors, you can define additional ResourceSelectorTerms + (after the namespace selector) to specify which resources to include. These additional selectors can + target both namespace-scoped resources (within the selected namespace) and cluster-scoped resources. + + Important requirements for NamespaceWithResourceSelectors mode: + - Exactly one namespace selector with this mode is allowed + - The namespace selector must select by name (not by label) + - Only one namespace selector is allowed when using this mode (cannot mix with other namespace selectors) + - All requirements are validated via CEL at API validation time + - If the selected namespace is deleted after CRP creation, the controller will report an error condition + + Example using NamespaceWithResourceSelectors: + - Namespace selector: {Group: "", Version: "v1", Kind: "Namespace", Name: "prod", SelectionScope: "NamespaceWithResourceSelectors"} + - Additional selector: {Group: "apps", Version: "v1", Kind: "Deployment", LabelSelector: {app: "frontend"}} + - Third selector: {Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole", Name: "admin"} + This selects: the "prod" namespace, all Deployments with label app=frontend in "prod", and the "admin" ClusterRole. type: string labelSelector: description: |- @@ -519,11 +543,14 @@ spec: type: string selectionScope: default: NamespaceWithResources - description: SelectionScope defines the scope of resource - selections when the Kind is `namespace`. + description: |- + SelectionScope defines the scope of resource selections when the Kind is `namespace`. + This field is only applicable when Kind is "Namespace" and is ignored for other resource kinds. + See the Kind field documentation for detailed examples and usage patterns. enum: - NamespaceOnly - NamespaceWithResources + - NamespaceWithResourceSelectors type: string version: description: Version of the to be selected resource. @@ -851,7 +878,31 @@ spec: kind: description: |- Kind of the to be selected resource. - Note: When `Kind` is `namespace`, by default ALL the resources under the selected namespaces are selected. + + Special behavior when Kind is `namespace` (ClusterResourcePlacement only): + Note: ResourcePlacement cannot select namespaces since it is namespace-scoped and selects resources within a namespace. + + For ClusterResourcePlacement, you can use SelectionScope to control what gets selected: + - NamespaceOnly: Only the namespace object itself + - NamespaceWithResources: The namespace AND all resources within it (default) + - NamespaceWithResourceSelectors: The namespace AND resources specified by additional selectors + + When SelectionScope is NamespaceWithResourceSelectors, you can define additional ResourceSelectorTerms + (after the namespace selector) to specify which resources to include. These additional selectors can + target both namespace-scoped resources (within the selected namespace) and cluster-scoped resources. + + Important requirements for NamespaceWithResourceSelectors mode: + - Exactly one namespace selector with this mode is allowed + - The namespace selector must select by name (not by label) + - Only one namespace selector is allowed when using this mode (cannot mix with other namespace selectors) + - All requirements are validated via CEL at API validation time + - If the selected namespace is deleted after CRP creation, the controller will report an error condition + + Example using NamespaceWithResourceSelectors: + - Namespace selector: {Group: "", Version: "v1", Kind: "Namespace", Name: "prod", SelectionScope: "NamespaceWithResourceSelectors"} + - Additional selector: {Group: "apps", Version: "v1", Kind: "Deployment", LabelSelector: {app: "frontend"}} + - Third selector: {Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole", Name: "admin"} + This selects: the "prod" namespace, all Deployments with label app=frontend in "prod", and the "admin" ClusterRole. type: string labelSelector: description: |- @@ -906,11 +957,14 @@ spec: type: string selectionScope: default: NamespaceWithResources - description: SelectionScope defines the scope of resource - selections when the Kind is `namespace`. + description: |- + SelectionScope defines the scope of resource selections when the Kind is `namespace`. + This field is only applicable when Kind is "Namespace" and is ignored for other resource kinds. + See the Kind field documentation for detailed examples and usage patterns. enum: - NamespaceOnly - NamespaceWithResources + - NamespaceWithResourceSelectors type: string version: description: Version of the to be selected resource. diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml index a4fb8185e..9299ee0af 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml @@ -2028,7 +2028,31 @@ spec: kind: description: |- Kind of the to be selected resource. - Note: When `Kind` is `namespace`, by default ALL the resources under the selected namespaces are selected. + + Special behavior when Kind is `namespace` (ClusterResourcePlacement only): + Note: ResourcePlacement cannot select namespaces since it is namespace-scoped and selects resources within a namespace. + + For ClusterResourcePlacement, you can use SelectionScope to control what gets selected: + - NamespaceOnly: Only the namespace object itself + - NamespaceWithResources: The namespace AND all resources within it (default) + - NamespaceWithResourceSelectors: The namespace AND resources specified by additional selectors + + When SelectionScope is NamespaceWithResourceSelectors, you can define additional ResourceSelectorTerms + (after the namespace selector) to specify which resources to include. These additional selectors can + target both namespace-scoped resources (within the selected namespace) and cluster-scoped resources. + + Important requirements for NamespaceWithResourceSelectors mode: + - Exactly one namespace selector with this mode is allowed + - The namespace selector must select by name (not by label) + - Only one namespace selector is allowed when using this mode (cannot mix with other namespace selectors) + - All requirements are validated via CEL at API validation time + - If the selected namespace is deleted after CRP creation, the controller will report an error condition + + Example using NamespaceWithResourceSelectors: + - Namespace selector: {Group: "", Version: "v1", Kind: "Namespace", Name: "prod", SelectionScope: "NamespaceWithResourceSelectors"} + - Additional selector: {Group: "apps", Version: "v1", Kind: "Deployment", LabelSelector: {app: "frontend"}} + - Third selector: {Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole", Name: "admin"} + This selects: the "prod" namespace, all Deployments with label app=frontend in "prod", and the "admin" ClusterRole. type: string labelSelector: description: |- @@ -2083,11 +2107,14 @@ spec: type: string selectionScope: default: NamespaceWithResources - description: SelectionScope defines the scope of resource selections - when the Kind is `namespace`. + description: |- + SelectionScope defines the scope of resource selections when the Kind is `namespace`. + This field is only applicable when Kind is "Namespace" and is ignored for other resource kinds. + See the Kind field documentation for detailed examples and usage patterns. enum: - NamespaceOnly - NamespaceWithResources + - NamespaceWithResourceSelectors type: string version: description: Version of the to be selected resource. @@ -2500,6 +2527,25 @@ spec: - message: statusReportingScope is immutable rule: '!has(oldSelf.statusReportingScope) || self.statusReportingScope == oldSelf.statusReportingScope' + - message: only one namespace selector with NamespaceWithResourceSelectors + mode is allowed + rule: size(self.resourceSelectors.filter(x, x.kind == 'Namespace' && + x.group == "" && x.version == 'v1' && has(x.selectionScope) && x.selectionScope + == 'NamespaceWithResourceSelectors')) <= 1 + - message: namespace selector with NamespaceWithResourceSelectors mode + must select by name (not by label) + rule: size(self.resourceSelectors.filter(x, x.kind == 'Namespace' && + x.group == "" && x.version == 'v1' && has(x.selectionScope) && x.selectionScope + == 'NamespaceWithResourceSelectors')) == 0 || (size(self.resourceSelectors.filter(x, + x.kind == 'Namespace' && x.group == "" && x.version == 'v1' && has(x.selectionScope) + && x.selectionScope == 'NamespaceWithResourceSelectors' && has(x.name) + && size(x.name) > 0 && !has(x.labelSelector))) == 1) + - message: when using NamespaceWithResourceSelectors mode, only one namespace + selector is allowed (cannot mix with other namespace selectors) + rule: size(self.resourceSelectors.filter(x, x.kind == 'Namespace' && + x.group == "" && x.version == 'v1' && has(x.selectionScope) && x.selectionScope + == 'NamespaceWithResourceSelectors')) == 0 || size(self.resourceSelectors.filter(x, + x.kind == 'Namespace' && x.group == "" && x.version == 'v1')) == 1 status: description: The observed status of ClusterResourcePlacement. properties: diff --git a/config/crd/bases/placement.kubernetes-fleet.io_resourceplacements.yaml b/config/crd/bases/placement.kubernetes-fleet.io_resourceplacements.yaml index 5df58390b..303f9e9b8 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_resourceplacements.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_resourceplacements.yaml @@ -538,7 +538,31 @@ spec: kind: description: |- Kind of the to be selected resource. - Note: When `Kind` is `namespace`, by default ALL the resources under the selected namespaces are selected. + + Special behavior when Kind is `namespace` (ClusterResourcePlacement only): + Note: ResourcePlacement cannot select namespaces since it is namespace-scoped and selects resources within a namespace. + + For ClusterResourcePlacement, you can use SelectionScope to control what gets selected: + - NamespaceOnly: Only the namespace object itself + - NamespaceWithResources: The namespace AND all resources within it (default) + - NamespaceWithResourceSelectors: The namespace AND resources specified by additional selectors + + When SelectionScope is NamespaceWithResourceSelectors, you can define additional ResourceSelectorTerms + (after the namespace selector) to specify which resources to include. These additional selectors can + target both namespace-scoped resources (within the selected namespace) and cluster-scoped resources. + + Important requirements for NamespaceWithResourceSelectors mode: + - Exactly one namespace selector with this mode is allowed + - The namespace selector must select by name (not by label) + - Only one namespace selector is allowed when using this mode (cannot mix with other namespace selectors) + - All requirements are validated via CEL at API validation time + - If the selected namespace is deleted after CRP creation, the controller will report an error condition + + Example using NamespaceWithResourceSelectors: + - Namespace selector: {Group: "", Version: "v1", Kind: "Namespace", Name: "prod", SelectionScope: "NamespaceWithResourceSelectors"} + - Additional selector: {Group: "apps", Version: "v1", Kind: "Deployment", LabelSelector: {app: "frontend"}} + - Third selector: {Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole", Name: "admin"} + This selects: the "prod" namespace, all Deployments with label app=frontend in "prod", and the "admin" ClusterRole. type: string labelSelector: description: |- @@ -593,11 +617,14 @@ spec: type: string selectionScope: default: NamespaceWithResources - description: SelectionScope defines the scope of resource selections - when the Kind is `namespace`. + description: |- + SelectionScope defines the scope of resource selections when the Kind is `namespace`. + This field is only applicable when Kind is "Namespace" and is ignored for other resource kinds. + See the Kind field documentation for detailed examples and usage patterns. enum: - NamespaceOnly - NamespaceWithResources + - NamespaceWithResourceSelectors type: string version: description: Version of the to be selected resource. @@ -1003,6 +1030,25 @@ spec: x-kubernetes-validations: - message: policy cannot be removed once set rule: '!(has(oldSelf.policy) && !has(self.policy))' + - message: only one namespace selector with NamespaceWithResourceSelectors + mode is allowed + rule: size(self.resourceSelectors.filter(x, x.kind == 'Namespace' && + x.group == "" && x.version == 'v1' && has(x.selectionScope) && x.selectionScope + == 'NamespaceWithResourceSelectors')) <= 1 + - message: namespace selector with NamespaceWithResourceSelectors mode + must select by name (not by label) + rule: size(self.resourceSelectors.filter(x, x.kind == 'Namespace' && + x.group == "" && x.version == 'v1' && has(x.selectionScope) && x.selectionScope + == 'NamespaceWithResourceSelectors')) == 0 || (size(self.resourceSelectors.filter(x, + x.kind == 'Namespace' && x.group == "" && x.version == 'v1' && has(x.selectionScope) + && x.selectionScope == 'NamespaceWithResourceSelectors' && has(x.name) + && size(x.name) > 0 && !has(x.labelSelector))) == 1) + - message: when using NamespaceWithResourceSelectors mode, only one namespace + selector is allowed (cannot mix with other namespace selectors) + rule: size(self.resourceSelectors.filter(x, x.kind == 'Namespace' && + x.group == "" && x.version == 'v1' && has(x.selectionScope) && x.selectionScope + == 'NamespaceWithResourceSelectors')) == 0 || size(self.resourceSelectors.filter(x, + x.kind == 'Namespace' && x.group == "" && x.version == 'v1')) == 1 status: description: The observed status of ResourcePlacement. properties: diff --git a/pkg/utils/controller/resource_selector_resolver.go b/pkg/utils/controller/resource_selector_resolver.go index c2c37c662..8fb3db6a7 100644 --- a/pkg/utils/controller/resource_selector_resolver.go +++ b/pkg/utils/controller/resource_selector_resolver.go @@ -237,6 +237,38 @@ func generateRawContent(object *unstructured.Unstructured) ([]byte, error) { func (rs *ResourceSelectorResolver) gatherSelectedResource(placementKey types.NamespacedName, selectors []placementv1beta1.ResourceSelectorTerm) ([]*unstructured.Unstructured, error) { var resources []*unstructured.Unstructured var resourceMap = make(map[placementv1beta1.ResourceIdentifier]bool) + + // First pass: check if there's a namespace selector with NamespaceWithResourceSelectors mode + // If found, collect the selected namespace (must be exactly one) for filtering resources in the second pass + var selectedNamespace string + for _, selector := range selectors { + gvk := schema.GroupVersionKind{ + Group: selector.Group, + Version: selector.Version, + Kind: selector.Kind, + } + if gvk == utils.NamespaceGVK && placementKey.Namespace == "" && selector.SelectionScope == placementv1beta1.NamespaceWithResourceSelectors { + namespaces, err := rs.fetchSelectedNamespaces(selector, placementKey.Name) + if err != nil { + return nil, err + } + // NamespaceWithResourceSelectors mode requires exactly one namespace + if len(namespaces) == 0 { + err := fmt.Errorf("invalid clusterResourcePlacement %s: NamespaceWithResourceSelectors mode requires exactly one namespace, but no namespaces were selected", placementKey.Name) + klog.ErrorS(err, "No namespaces selected with NamespaceWithResourceSelectors mode", "placement", placementKey.Name) + return nil, NewUserError(err) + } + if len(namespaces) > 1 { + err := fmt.Errorf("invalid clusterResourcePlacement %s: NamespaceWithResourceSelectors mode requires exactly one namespace, but %d namespaces were selected", placementKey.Name, len(namespaces)) + klog.ErrorS(err, "Multiple namespaces selected with NamespaceWithResourceSelectors mode", "placement", placementKey.Name, "namespaceCount", len(namespaces)) + return nil, NewUserError(err) + } + selectedNamespace = namespaces[0] + break + } + } + + // Second pass: fetch resources based on selectors for _, selector := range selectors { gvk := schema.GroupVersionKind{ Group: selector.Group, @@ -248,16 +280,45 @@ func (rs *ResourceSelectorResolver) gatherSelectedResource(placementKey types.Na klog.V(2).InfoS("Skip select resource", "group version kind", gvk.String()) continue } + var objs []runtime.Object var err error - if gvk == utils.NamespaceGVK && placementKey.Namespace == "" && selector.SelectionScope != placementv1beta1.NamespaceOnly { - objs, err = rs.fetchNamespaceResources(selector, placementKey.Name) + + // Case 1: Namespace selector + if gvk == utils.NamespaceGVK { + if placementKey.Namespace != "" { + // RP trying to select namespace - this is an error because Namespace is cluster-scoped + err := fmt.Errorf("invalid placement %s: cannot select cluster-scoped resource Namespace in a resourcePlacement", placementKey) + klog.ErrorS(err, "Invalid resource selector for RP", "selector", selector) + return nil, NewUserError(err) + } + // CRP namespace selector + switch selector.SelectionScope { + case placementv1beta1.NamespaceOnly, placementv1beta1.NamespaceWithResourceSelectors: + // Just the namespace objects + objs, err = rs.fetchResources(selector, placementKey) + default: + // NamespaceWithResources mode or empty (default behavior): namespace + all resources inside + objs, err = rs.fetchNamespaceResources(selector, placementKey.Name) + } } else { - objs, err = rs.fetchResources(selector, placementKey) + // Case 2: Non-namespace selector + isNamespaceScoped := !rs.InformerManager.IsClusterScopedResources(gvk) + isCRP := placementKey.Namespace == "" + if isCRP && isNamespaceScoped { + // Special case when there is a namespaced resource with NamespaceWithResourceSelectors selection mode: + // Use selective namespace from 1st pass + namespacedKey := types.NamespacedName{Name: placementKey.Name, Namespace: selectedNamespace} + objs, err = rs.fetchResources(selector, namespacedKey) + } else { + objs, err = rs.fetchResources(selector, placementKey) + } } + if err != nil { return nil, err } + for _, obj := range objs { uObj := obj.(*unstructured.Unstructured) ri := placementv1beta1.ResourceIdentifier{ @@ -530,6 +591,34 @@ func (rs *ResourceSelectorResolver) fetchResources(selector placementv1beta1.Res return selectedObjs, nil } +// fetchSelectedNamespaces retrieves the namespace names based on the namespace selector. +func (rs *ResourceSelectorResolver) fetchSelectedNamespaces(selector placementv1beta1.ResourceSelectorTerm, placementName string) ([]string, error) { + klog.V(2).InfoS("start to fetch selected namespaces", "selector", selector, "placement", placementName) + + // Use fetchResources to get namespace objects (handles label selector, name selector, deletion timestamps) + placementKey := types.NamespacedName{Name: placementName} + objs, err := rs.fetchResources(selector, placementKey) + if err != nil { + return nil, err + } + + // Filter by skipped namespaces and extract names + namespaceNames := make([]string, 0, len(objs)) + for _, obj := range objs { + ns, err := meta.Accessor(obj) + if err != nil { + return nil, NewUnexpectedBehaviorError(fmt.Errorf("failed to access the namespace object: %w", err)) + } + if !utils.ShouldPropagateNamespace(ns.GetName(), rs.SkippedNamespaces) { + klog.V(2).InfoS("skip namespace that is not allowed to propagate", "namespace", ns.GetName(), "placement", placementName) + continue + } + namespaceNames = append(namespaceNames, ns.GetName()) + } + + return namespaceNames, nil +} + func (rs *ResourceSelectorResolver) ShouldPropagateObj(namespace, placementName string, obj runtime.Object) (bool, error) { uObj := obj.DeepCopyObject().(*unstructured.Unstructured) uObjKObj := klog.KObj(uObj) diff --git a/pkg/utils/controller/resource_selector_resolver_test.go b/pkg/utils/controller/resource_selector_resolver_test.go index d0069412d..0167c7871 100644 --- a/pkg/utils/controller/resource_selector_resolver_test.go +++ b/pkg/utils/controller/resource_selector_resolver_test.go @@ -19,6 +19,7 @@ package controller import ( "errors" "math/rand" + "strings" "testing" "time" @@ -1336,6 +1337,243 @@ func TestGatherSelectedResource(t *testing.T) { }(), wantError: ErrUnexpectedBehavior, }, + { + name: "should select namespace and specific resources with NamespaceWithResourceSelectors mode", + placementName: types.NamespacedName{Name: "test-placement"}, + selectors: []fleetv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-ns", + SelectionScope: fleetv1beta1.NamespaceWithResourceSelectors, + }, + { + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: "test-deployment", + }, + { + Group: "", + Version: "v1", + Kind: "ConfigMap", + Name: "test-configmap", + }, + }, + resourceConfig: utils.NewResourceConfig(false), // default deny list + informerManager: func() *testinformer.FakeManager { + return &testinformer.FakeManager{ + Listers: map[schema.GroupVersionResource]*testinformer.FakeLister{ + utils.NamespaceGVR: {Objects: []runtime.Object{testNamespace, prodNamespace}}, + utils.DeploymentGVR: {Objects: []runtime.Object{testDeployment}}, + utils.ConfigMapGVR: {Objects: []runtime.Object{testConfigMap}}, + }, + NamespaceScopedResources: []schema.GroupVersionResource{utils.DeploymentGVR, utils.ConfigMapGVR}, + } + }(), + // Should select namespace, deployment, and configmap from test-ns + want: []*unstructured.Unstructured{testNamespace, testConfigMap, testDeployment}, + }, + { + name: "should select namespace and resources by label selector with NamespaceWithResourceSelectors mode", + placementName: types.NamespacedName{Name: "test-placement"}, + selectors: []fleetv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-ns", + SelectionScope: fleetv1beta1.NamespaceWithResourceSelectors, + }, + { + Group: "apps", + Version: "v1", + Kind: "Deployment", + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + }, + }, + resourceConfig: utils.NewResourceConfig(false), // default deny list + informerManager: func() *testinformer.FakeManager { + // Create a deployment with matching labels for this test + deploymentWithLabels := testDeployment.DeepCopy() + deploymentWithLabels.SetLabels(map[string]string{"app": "test"}) + + return &testinformer.FakeManager{ + IsClusterScopedResource: true, + APIResources: map[schema.GroupVersionKind]bool{ + utils.NamespaceGVK: true, // cluster-scoped + utils.DeploymentGVK: false, // namespace-scoped + }, + Listers: map[schema.GroupVersionResource]*testinformer.FakeLister{ + utils.NamespaceGVR: {Objects: []runtime.Object{testNamespace}}, + utils.DeploymentGVR: {Objects: []runtime.Object{deploymentWithLabels}}, + }, + NamespaceScopedResources: []schema.GroupVersionResource{utils.DeploymentGVR}, + } + }(), + // Should select namespace and deployment with matching labels + want: func() []*unstructured.Unstructured { + deploymentWithLabels := testDeployment.DeepCopy() + deploymentWithLabels.SetLabels(map[string]string{"app": "test"}) + return []*unstructured.Unstructured{testNamespace, deploymentWithLabels} + }(), + }, + { + name: "should select namespace and cluster-scoped resources with NamespaceWithResourceSelectors mode", + placementName: types.NamespacedName{Name: "test-placement"}, + selectors: []fleetv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-ns", + SelectionScope: fleetv1beta1.NamespaceWithResourceSelectors, + }, + { + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + Name: "test-cluster-role", + }, + { + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: "test-deployment", + }, + }, + resourceConfig: utils.NewResourceConfig(false), // default deny list + informerManager: func() *testinformer.FakeManager { + return &testinformer.FakeManager{ + IsClusterScopedResource: true, + APIResources: map[schema.GroupVersionKind]bool{ + utils.NamespaceGVK: true, // cluster-scoped + utils.ClusterRoleGVK: true, // cluster-scoped + utils.DeploymentGVK: false, // namespace-scoped + }, + Listers: map[schema.GroupVersionResource]*testinformer.FakeLister{ + utils.NamespaceGVR: {Objects: []runtime.Object{testNamespace}}, + utils.ClusterRoleGVR: {Objects: []runtime.Object{testClusterRole}}, + utils.DeploymentGVR: {Objects: []runtime.Object{testDeployment}}, + }, + NamespaceScopedResources: []schema.GroupVersionResource{utils.DeploymentGVR}, + } + }(), + // Should select namespace, cluster role, and deployment + want: []*unstructured.Unstructured{testNamespace, testClusterRole, testDeployment}, + }, + { + name: "should select only namespace with NamespaceWithResourceSelectors mode when no other selectors provided", + placementName: types.NamespacedName{Name: "test-placement"}, + selectors: []fleetv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-ns", + SelectionScope: fleetv1beta1.NamespaceWithResourceSelectors, + }, + }, + resourceConfig: utils.NewResourceConfig(false), // default deny list + informerManager: func() *testinformer.FakeManager { + return &testinformer.FakeManager{ + Listers: map[schema.GroupVersionResource]*testinformer.FakeLister{ + utils.NamespaceGVR: {Objects: []runtime.Object{testNamespace, prodNamespace}}, + utils.DeploymentGVR: {Objects: []runtime.Object{testDeployment}}, + utils.ConfigMapGVR: {Objects: []runtime.Object{testConfigMap}}, + }, + NamespaceScopedResources: []schema.GroupVersionResource{utils.DeploymentGVR, utils.ConfigMapGVR}, + } + }(), + // Should select only the namespace, not its child resources + want: []*unstructured.Unstructured{testNamespace}, + }, + { + name: "should error when selecting multiple namespaces with NamespaceWithResourceSelectors mode using label selector", + placementName: types.NamespacedName{Name: "test-placement"}, + selectors: []fleetv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "environment": "test", + }, + }, + SelectionScope: fleetv1beta1.NamespaceWithResourceSelectors, + }, + { + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: "test-deployment", + }, + }, + resourceConfig: utils.NewResourceConfig(false), // default deny list + informerManager: func() *testinformer.FakeManager { + // Create a second test namespace with the same label + testNamespace2 := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Namespace", + "metadata": map[string]interface{}{ + "name": "test-ns-2", + "labels": map[string]interface{}{ + "environment": "test", + }, + }, + }, + } + testNamespace2.SetGroupVersionKind(utils.NamespaceGVK) + + return &testinformer.FakeManager{ + Listers: map[schema.GroupVersionResource]*testinformer.FakeLister{ + utils.NamespaceGVR: {Objects: []runtime.Object{testNamespace, testNamespace2}}, + utils.DeploymentGVR: {Objects: []runtime.Object{testDeployment}}, + }, + NamespaceScopedResources: []schema.GroupVersionResource{utils.DeploymentGVR}, + } + }(), + // Should error because multiple namespaces match the label selector + wantError: ErrUserError, + }, + { + name: "should error when selecting zero namespaces with NamespaceWithResourceSelectors mode", + placementName: types.NamespacedName{Name: "test-placement"}, + selectors: []fleetv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "non-existent-namespace", + SelectionScope: fleetv1beta1.NamespaceWithResourceSelectors, + }, + { + Group: "", + Version: "v1", + Kind: "ConfigMap", + Name: "test-configmap", + }, + }, + resourceConfig: utils.NewResourceConfig(false), // default deny list + informerManager: func() *testinformer.FakeManager { + return &testinformer.FakeManager{ + Listers: map[schema.GroupVersionResource]*testinformer.FakeLister{ + utils.NamespaceGVR: {Objects: []runtime.Object{testNamespace}}, // testNamespace doesn't match "non-existent-namespace" + utils.ConfigMapGVR: {Objects: []runtime.Object{testConfigMap}}, + }, + NamespaceScopedResources: []schema.GroupVersionResource{utils.ConfigMapGVR}, + } + }(), + // Should error because no namespaces match the selector + wantError: ErrUserError, + }, } for _, tt := range tests { @@ -1963,6 +2201,286 @@ func TestSortResources(t *testing.T) { } } +func TestFetchSelectedNamespaces(t *testing.T) { + testNs1 := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Namespace", + "metadata": map[string]interface{}{ + "name": "test-ns-1", + "labels": map[string]interface{}{ + "env": "prod", + }, + }, + }, + } + testNs1.SetGroupVersionKind(utils.NamespaceGVK) + + testNs2 := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Namespace", + "metadata": map[string]interface{}{ + "name": "test-ns-2", + "labels": map[string]interface{}{ + "env": "dev", + }, + }, + }, + } + testNs2.SetGroupVersionKind(utils.NamespaceGVK) + + skippedNs := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Namespace", + "metadata": map[string]interface{}{ + "name": "kube-system", + }, + }, + } + skippedNs.SetGroupVersionKind(utils.NamespaceGVK) + + tests := []struct { + name string + selector fleetv1beta1.ResourceSelectorTerm + skippedNamespaces map[string]bool + informerManager *testinformer.FakeManager + want []string + wantErr bool + }{ + { + name: "select namespace by name", + selector: fleetv1beta1.ResourceSelectorTerm{ + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-ns-1", + }, + skippedNamespaces: nil, + informerManager: &testinformer.FakeManager{ + Listers: map[schema.GroupVersionResource]*testinformer.FakeLister{ + utils.NamespaceGVR: {Objects: []runtime.Object{testNs1, testNs2}}, + }, + }, + want: []string{"test-ns-1"}, + }, + { + name: "select namespaces by label selector", + selector: fleetv1beta1.ResourceSelectorTerm{ + Group: "", + Version: "v1", + Kind: "Namespace", + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"env": "prod"}, + }, + }, + skippedNamespaces: nil, + informerManager: &testinformer.FakeManager{ + Listers: map[schema.GroupVersionResource]*testinformer.FakeLister{ + utils.NamespaceGVR: {Objects: []runtime.Object{testNs1, testNs2}}, + }, + }, + want: []string{"test-ns-1"}, + }, + { + name: "filter out skipped namespaces", + selector: fleetv1beta1.ResourceSelectorTerm{ + Group: "", + Version: "v1", + Kind: "Namespace", + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{}, + }, + }, + skippedNamespaces: map[string]bool{"kube-system": true}, + informerManager: &testinformer.FakeManager{ + Listers: map[schema.GroupVersionResource]*testinformer.FakeLister{ + utils.NamespaceGVR: {Objects: []runtime.Object{testNs1, skippedNs}}, + }, + }, + want: []string{"test-ns-1"}, + }, + { + name: "no namespaces match selector", + selector: fleetv1beta1.ResourceSelectorTerm{ + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "non-existent", + }, + skippedNamespaces: nil, + informerManager: &testinformer.FakeManager{ + Listers: map[schema.GroupVersionResource]*testinformer.FakeLister{ + utils.NamespaceGVR: {Objects: []runtime.Object{testNs1, testNs2}}, + }, + }, + want: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rsr := &ResourceSelectorResolver{ + SkippedNamespaces: tt.skippedNamespaces, + ResourceConfig: utils.NewResourceConfig(false), + InformerManager: tt.informerManager, + RestMapper: newFakeRESTMapper(), + } + + got, err := rsr.fetchSelectedNamespaces(tt.selector, "test-placement") + if (err != nil) != tt.wantErr { + t.Errorf("fetchSelectedNamespaces() error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("fetchSelectedNamespaces() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestGatherSelectedResource_ErrorCases(t *testing.T) { + testNamespace := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Namespace", + "metadata": map[string]interface{}{ + "name": "test-ns", + }, + }, + } + testNamespace.SetGroupVersionKind(utils.NamespaceGVK) + + testDeployment := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]interface{}{ + "name": "test-deployment", + "namespace": "test-ns", + }, + }, + } + testDeployment.SetGroupVersionKind(utils.DeploymentGVK) + + tests := []struct { + name string + placementName types.NamespacedName + selectors []fleetv1beta1.ResourceSelectorTerm + resourceConfig *utils.ResourceConfig + informerManager *testinformer.FakeManager + wantError error + wantErrMsg string + }{ + { + name: "ResourcePlacement trying to select namespace should fail", + placementName: types.NamespacedName{Name: "test-placement", Namespace: "some-ns"}, + selectors: []fleetv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-ns", + }, + }, + resourceConfig: utils.NewResourceConfig(false), + informerManager: &testinformer.FakeManager{ + Listers: map[schema.GroupVersionResource]*testinformer.FakeLister{ + utils.NamespaceGVR: {Objects: []runtime.Object{testNamespace}}, + }, + }, + wantError: ErrUserError, + wantErrMsg: "cannot select cluster-scoped resource Namespace in a resourcePlacement", + }, + { + name: "CRP selecting namespace-scoped resource without NamespaceWithResourceSelectors mode", + placementName: types.NamespacedName{Name: "test-placement"}, + selectors: []fleetv1beta1.ResourceSelectorTerm{ + { + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: "test-deployment", + }, + }, + resourceConfig: utils.NewResourceConfig(false), + informerManager: &testinformer.FakeManager{ + IsClusterScopedResource: true, + APIResources: map[schema.GroupVersionKind]bool{ + utils.DeploymentGVK: false, // namespace-scoped + }, + Listers: map[schema.GroupVersionResource]*testinformer.FakeLister{ + utils.DeploymentGVR: {Objects: []runtime.Object{testDeployment}}, + }, + NamespaceScopedResources: []schema.GroupVersionResource{utils.DeploymentGVR}, + }, + wantError: ErrUserError, + wantErrMsg: "cannot select namespace-scoped resource", + }, + { + name: "duplicate resource should fail", + placementName: types.NamespacedName{Name: "test-placement"}, + selectors: []fleetv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-ns", + SelectionScope: fleetv1beta1.NamespaceWithResourceSelectors, + }, + { + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: "test-deployment", + }, + // Duplicate selector + { + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: "test-deployment", + }, + }, + resourceConfig: utils.NewResourceConfig(false), + informerManager: &testinformer.FakeManager{ + IsClusterScopedResource: true, + APIResources: map[schema.GroupVersionKind]bool{ + utils.NamespaceGVK: true, // cluster-scoped + utils.DeploymentGVK: false, // namespace-scoped + }, + Listers: map[schema.GroupVersionResource]*testinformer.FakeLister{ + utils.NamespaceGVR: {Objects: []runtime.Object{testNamespace}}, + utils.DeploymentGVR: {Objects: []runtime.Object{testDeployment}}, + }, + NamespaceScopedResources: []schema.GroupVersionResource{utils.DeploymentGVR}, + }, + wantError: ErrUserError, + wantErrMsg: "found duplicate resource", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rsr := &ResourceSelectorResolver{ + ResourceConfig: tt.resourceConfig, + InformerManager: tt.informerManager, + RestMapper: newFakeRESTMapper(), + } + + _, err := rsr.gatherSelectedResource(tt.placementName, tt.selectors) + if gotErr, wantErr := err != nil, tt.wantError != nil; gotErr != wantErr || !errors.Is(err, tt.wantError) { + t.Errorf("gatherSelectedResource() error = %v, wantError %v", err, tt.wantError) + return + } + if tt.wantError != nil && !strings.Contains(err.Error(), tt.wantErrMsg) { + t.Errorf("gatherSelectedResource() error = %v, wantErrMsg %v", err, tt.wantErrMsg) + } + }) + } +} + func TestShouldPropagateObj(t *testing.T) { tests := []struct { name string diff --git a/pkg/utils/test_util.go b/pkg/utils/test_util.go index cecd852c3..a665d0bb2 100644 --- a/pkg/utils/test_util.go +++ b/pkg/utils/test_util.go @@ -175,5 +175,19 @@ func (m TestMapper) RESTMapping(gk schema.GroupKind, _ ...string) (*meta.RESTMap Scope: nil, }, nil } + if gk.Kind == "Namespace" { + return &meta.RESTMapping{ + Resource: NamespaceGVR, + GroupVersionKind: NamespaceGVK, + Scope: nil, + }, nil + } + if gk.Kind == "ConfigMap" { + return &meta.RESTMapping{ + Resource: ConfigMapGVR, + GroupVersionKind: ConfigMapGVK, + Scope: nil, + }, nil + } return nil, errors.New("test error: mapping does not exist") } diff --git a/pkg/utils/validator/placement.go b/pkg/utils/validator/placement.go index 86da654b4..6c75c89cb 100644 --- a/pkg/utils/validator/placement.go +++ b/pkg/utils/validator/placement.go @@ -41,6 +41,7 @@ import ( placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" "github.com/kubefleet-dev/kubefleet/pkg/propertyprovider" + "github.com/kubefleet-dev/kubefleet/pkg/utils" "github.com/kubefleet-dev/kubefleet/pkg/utils/controller" "github.com/kubefleet-dev/kubefleet/pkg/utils/informer" ) @@ -65,6 +66,18 @@ var ( resourceCapacityTypes = supportedResourceCapacityTypes() ) +// hasNamespaceWithResourceSelectorsMode checks if any namespace selector has NamespaceWithResourceSelectors mode. +func hasNamespaceWithResourceSelectorsMode(resourceSelectors []placementv1beta1.ResourceSelectorTerm) bool { + for _, selector := range resourceSelectors { + if selector.Group == utils.NamespaceGVK.Group && selector.Version == utils.NamespaceGVK.Version && selector.Kind == utils.NamespaceGVK.Kind { + if selector.SelectionScope == placementv1beta1.NamespaceWithResourceSelectors { + return true + } + } + } + return false +} + // validatePlacement validates a placement object (either ClusterResourcePlacement or ResourcePlacement). func validatePlacement(name string, resourceSelectors []placementv1beta1.ResourceSelectorTerm, policy *placementv1beta1.PlacementPolicy, strategy placementv1beta1.RolloutStrategy, isClusterScoped bool) error { allErr := make([]error, 0) @@ -73,6 +86,8 @@ func validatePlacement(name string, resourceSelectors []placementv1beta1.Resourc allErr = append(allErr, fmt.Errorf("the name field cannot have length exceeding %d", validation.DNS1035LabelMaxLength)) } + hasNsWithResourceSelectorsMode := hasNamespaceWithResourceSelectorsMode(resourceSelectors) + for _, selector := range resourceSelectors { if selector.LabelSelector != nil { if len(selector.Name) != 0 { @@ -97,7 +112,8 @@ func validatePlacement(name string, resourceSelectors []placementv1beta1.Resourc Kind: selector.Kind, } // Only check cluster scope for ClusterResourcePlacement - if isClusterScoped && !ResourceInformer.IsClusterScopedResources(gvk) { + // Exception: NamespaceWithResourceSelectors mode allows namespace-scoped resources + if isClusterScoped && !ResourceInformer.IsClusterScopedResources(gvk) && !hasNsWithResourceSelectorsMode { allErr = append(allErr, fmt.Errorf("the resource is not found in schema (please retry) or it is not a cluster scoped resource: %v", gvk)) } diff --git a/pkg/utils/validator/placement_test.go b/pkg/utils/validator/placement_test.go index 6afd0ae7f..81a828123 100644 --- a/pkg/utils/validator/placement_test.go +++ b/pkg/utils/validator/placement_test.go @@ -42,6 +42,115 @@ var ( } ) +func TestHasNamespaceWithResourceSelectorsMode(t *testing.T) { + tests := map[string]struct { + resourceSelectors []placementv1beta1.ResourceSelectorTerm + wantHasNamespaceWithResourceSelectors bool + }{ + "no namespace selectors": { + resourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + Name: "test-cluster-role", + }, + }, + wantHasNamespaceWithResourceSelectors: false, + }, + "one namespace selector with NamespaceOnly mode": { + resourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-namespace", + }, + }, + wantHasNamespaceWithResourceSelectors: false, + }, + "one namespace selector with NamespaceWithResources mode": { + resourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-namespace", + SelectionScope: placementv1beta1.NamespaceWithResources, + }, + }, + wantHasNamespaceWithResourceSelectors: false, + }, + "one namespace selector with NamespaceWithResourceSelectors mode": { + resourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-namespace", + SelectionScope: placementv1beta1.NamespaceWithResourceSelectors, + }, + }, + wantHasNamespaceWithResourceSelectors: true, + }, + "multiple namespace selectors": { + resourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "namespace-1", + }, + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "namespace-2", + SelectionScope: placementv1beta1.NamespaceWithResourceSelectors, + }, + }, + wantHasNamespaceWithResourceSelectors: true, + }, + "mixed selectors with NamespaceWithResourceSelectors": { + resourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-namespace", + SelectionScope: placementv1beta1.NamespaceWithResourceSelectors, + }, + { + Group: "", + Version: "v1", + Kind: "ConfigMap", + Name: "test-configmap", + }, + { + Group: "", + Version: "v1", + Kind: "Secret", + Name: "test-secret", + }, + }, + wantHasNamespaceWithResourceSelectors: true, + }, + "empty resource selectors": { + resourceSelectors: []placementv1beta1.ResourceSelectorTerm{}, + wantHasNamespaceWithResourceSelectors: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + gotHasMode := hasNamespaceWithResourceSelectorsMode(tc.resourceSelectors) + if gotHasMode != tc.wantHasNamespaceWithResourceSelectors { + t.Errorf("hasNamespaceWithResourceSelectorsMode() = %v, want %v", gotHasMode, tc.wantHasNamespaceWithResourceSelectors) + } + }) + } +} + func TestValidateClusterResourcePlacement(t *testing.T) { tests := map[string]struct { crp *placementv1beta1.ClusterResourcePlacement @@ -190,6 +299,77 @@ func TestValidateClusterResourcePlacement(t *testing.T) { wantErr: true, wantErrMsg: "resource is not found in schema (please retry) or it is not a cluster scoped resource", }, + "valid CRP with NamespaceWithResourceSelectors and namespaced resources": { + crp: &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-crp", + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-namespace", + SelectionScope: placementv1beta1.NamespaceWithResourceSelectors, + }, + { + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: "test-deployment", + }, + { + Group: "", + Version: "v1", + Kind: "ConfigMap", + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + }, + }, + }, + }, + resourceInformer: &testinformer.FakeManager{ + APIResources: map[schema.GroupVersionKind]bool{ + utils.NamespaceGVK: true, + utils.DeploymentGVK: true, + utils.ConfigMapGVK: true, + }, + }, + wantErr: false, + }, + "valid CRP with NamespaceWithResourceSelectors and cluster-scoped resources": { + crp: &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-crp", + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-namespace", + SelectionScope: placementv1beta1.NamespaceWithResourceSelectors, + }, + { + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + Name: "test-cluster-role", + }, + }, + }, + }, + resourceInformer: &testinformer.FakeManager{ + APIResources: map[schema.GroupVersionKind]bool{ + utils.NamespaceGVK: true, + utils.ClusterRoleGVK: true, + }, + }, + wantErr: false, + }, } for testName, testCase := range tests { t.Run(testName, func(t *testing.T) { diff --git a/test/apis/placement/v1beta1/api_validation_integration_test.go b/test/apis/placement/v1beta1/api_validation_integration_test.go index 904a691e6..aae616ef7 100644 --- a/test/apis/placement/v1beta1/api_validation_integration_test.go +++ b/test/apis/placement/v1beta1/api_validation_integration_test.go @@ -624,6 +624,318 @@ var _ = Describe("Test placement v1beta1 API validation", func() { }) }) + Context("Test ClusterResourcePlacement SelectionScope enum validation - allow cases", func() { + var crp placementv1beta1.ClusterResourcePlacement + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + + AfterEach(func() { + Expect(hubClient.Delete(ctx, &crp)).Should(Succeed()) + }) + + It("should allow creation of ClusterResourcePlacement with SelectionScope=NamespaceOnly", func() { + crp = placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "prod", + SelectionScope: placementv1beta1.NamespaceOnly, + }, + }, + }, + } + Expect(hubClient.Create(ctx, &crp)).Should(Succeed()) + Expect(crp.Spec.ResourceSelectors[0].SelectionScope).Should(Equal(placementv1beta1.NamespaceOnly)) + }) + + It("should allow creation of ClusterResourcePlacement with SelectionScope=NamespaceWithResources", func() { + crp = placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "prod", + SelectionScope: placementv1beta1.NamespaceWithResources, + }, + }, + }, + } + Expect(hubClient.Create(ctx, &crp)).Should(Succeed()) + Expect(crp.Spec.ResourceSelectors[0].SelectionScope).Should(Equal(placementv1beta1.NamespaceWithResources)) + }) + + It("should allow creation of ClusterResourcePlacement with SelectionScope=NamespaceWithResourceSelectors", func() { + crp = placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "prod", + SelectionScope: placementv1beta1.NamespaceWithResourceSelectors, + }, + }, + }, + } + Expect(hubClient.Create(ctx, &crp)).Should(Succeed()) + Expect(crp.Spec.ResourceSelectors[0].SelectionScope).Should(Equal(placementv1beta1.NamespaceWithResourceSelectors)) + }) + + It("should allow creation of ClusterResourcePlacement with SelectionScope not specified (defaults to NamespaceWithResources)", func() { + crp = placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "prod", + // SelectionScope not specified - should default to NamespaceWithResources + }, + }, + }, + } + Expect(hubClient.Create(ctx, &crp)).Should(Succeed()) + Expect(crp.Spec.ResourceSelectors[0].SelectionScope).Should(Equal(placementv1beta1.NamespaceWithResources)) + }) + + It("should allow creation of ClusterResourcePlacement with empty SelectionScope (defaults to NamespaceWithResources)", func() { + crp = placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "prod", + SelectionScope: "", + }, + }, + }, + } + Expect(hubClient.Create(ctx, &crp)).Should(Succeed()) + Expect(crp.Spec.ResourceSelectors[0].SelectionScope).Should(Equal(placementv1beta1.NamespaceWithResources)) + }) + + It("should allow creation of CRP with NamespaceWithResourceSelectors selecting by name and additional resource selectors", func() { + crp = placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "prod", + SelectionScope: placementv1beta1.NamespaceWithResourceSelectors, + }, + { + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: "frontend", + }, + { + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + Name: "test-cluster-role", + }, + }, + }, + } + Expect(hubClient.Create(ctx, &crp)).Should(Succeed()) + }) + }) + + Context("Test ClusterResourcePlacement SelectionScope enum validation - deny cases", func() { + var crp placementv1beta1.ClusterResourcePlacement + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + + It("should deny creation of ClusterResourcePlacement with invalid SelectionScope value", func() { + crp = placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "prod", + SelectionScope: "InvalidScope", + }, + }, + }, + } + err := hubClient.Create(ctx, &crp) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create CRP call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("supported values: \"NamespaceOnly\", \"NamespaceWithResources\", \"NamespaceWithResourceSelectors\"")) + }) + + It("should deny update of ClusterResourcePlacement SelectionScope to invalid value", func() { + crp = placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "prod", + SelectionScope: placementv1beta1.NamespaceWithResources, + }, + }, + }, + } + Expect(hubClient.Create(ctx, &crp)).Should(Succeed()) + + // Try to update to invalid SelectionScope + crp.Spec.ResourceSelectors[0].SelectionScope = "InvalidScope" + err := hubClient.Update(ctx, &crp) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Update CRP call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("supported values: \"NamespaceOnly\", \"NamespaceWithResources\", \"NamespaceWithResourceSelectors\"")) + + // Cleanup + Expect(hubClient.Delete(ctx, &crp)).Should(Succeed()) + }) + + It("should deny creation of CRP with multiple namespace selectors with NamespaceWithResourceSelectors mode", func() { + crp = placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "prod-1", + SelectionScope: placementv1beta1.NamespaceWithResourceSelectors, + }, + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "prod-2", + SelectionScope: placementv1beta1.NamespaceWithResourceSelectors, + }, + }, + }, + } + err := hubClient.Create(ctx, &crp) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create CRP call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("only one namespace selector with NamespaceWithResourceSelectors mode is allowed")) + }) + + It("should deny creation of CRP with NamespaceWithResourceSelectors using label selector", func() { + crp = placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"env": "prod"}, + }, + SelectionScope: placementv1beta1.NamespaceWithResourceSelectors, + }, + }, + }, + } + err := hubClient.Create(ctx, &crp) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create CRP call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("namespace selector with NamespaceWithResourceSelectors mode must select by name")) + }) + + It("should deny creation of CRP with NamespaceWithResourceSelectors using empty name", func() { + crp = placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "", // Empty name + SelectionScope: placementv1beta1.NamespaceWithResourceSelectors, + }, + }, + }, + } + err := hubClient.Create(ctx, &crp) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create CRP call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("namespace selector with NamespaceWithResourceSelectors mode must select by name")) + }) + + It("should deny creation of CRP mixing NamespaceWithResourceSelectors with other namespace selectors", func() { + crp = placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "prod", + SelectionScope: placementv1beta1.NamespaceWithResourceSelectors, + }, + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "staging", + // Default SelectionScope is NamespaceWithResources + }, + }, + }, + } + err := hubClient.Create(ctx, &crp) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create CRP call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("when using NamespaceWithResourceSelectors mode, only one namespace selector is allowed")) + }) + }) + Context("Test ResourcePlacement API validation - invalid cases", func() { var rp placementv1beta1.ResourcePlacement rpName := fmt.Sprintf(rpNameTemplate, GinkgoParallelProcess()) diff --git a/test/e2e/placement_namespace_with_resourceselectors_test.go b/test/e2e/placement_namespace_with_resourceselectors_test.go new file mode 100644 index 000000000..8806a76d4 --- /dev/null +++ b/test/e2e/placement_namespace_with_resourceselectors_test.go @@ -0,0 +1,278 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package e2e + +import ( + "fmt" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" +) + +// Helper functions for NamespaceWithResourceSelectors tests + +// deleteCRP deletes a ClusterResourcePlacement. +func deleteCRP(crpName string) { + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + } + Expect(hubClient.Delete(ctx, crp)).To(Succeed(), "Failed to delete CRP %s", crpName) +} + +// checkCRPStatusWithResources verifies the CRP status matches expected resources. +func checkCRPStatusWithResources(crpName string, expectedResources []placementv1beta1.ResourceIdentifier) { + crpStatusUpdatedActual := crpStatusUpdatedActual(expectedResources, allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) +} + +// checkFinalizersRemoved verifies controller finalizers are removed from CRP. +func checkFinalizersRemoved(crpName string) { + finalizerRemovedActual := allFinalizersExceptForCustomDeletionBlockerRemovedFromPlacementActual(types.NamespacedName{Name: crpName}) + Eventually(finalizerRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove controller finalizers from CRP %s", crpName) +} + +// Tests for NamespaceWithResourceSelectors SelectionScope mode +var _ = Describe("CRP with NamespaceWithResourceSelectors selecting single namespace and specific resources", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + testNamespace := fmt.Sprintf(workNamespaceNameTemplate, GinkgoParallelProcess()) + configMapName := fmt.Sprintf(appConfigMapNameTemplate, GinkgoParallelProcess()) + + BeforeAll(func() { + By("creating test namespace and resources") + createWorkResources() + }) + + AfterAll(func() { + By(fmt.Sprintf("deleting CRP and related resources %s", crpName)) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should create CRP with NamespaceWithResourceSelectors mode", func() { + createCRPWithSelectors(crpName, []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Kind: "Namespace", + Version: "v1", + Name: testNamespace, + SelectionScope: placementv1beta1.NamespaceWithResourceSelectors, + }, + { + Group: "", + Kind: "ConfigMap", + Version: "v1", + Name: configMapName, + }, + }) + }) + + It("should update CRP status as expected", func() { + checkCRPStatusWithResources(crpName, []placementv1beta1.ResourceIdentifier{ + { + Group: "", + Kind: "Namespace", + Version: "v1", + Name: testNamespace, + }, + { + Group: "", + Kind: "ConfigMap", + Version: "v1", + Name: configMapName, + Namespace: testNamespace, + }, + }) + }) + + It("should place the namespace and configmap on member clusters", func() { + checkNamespacePlacedOnClusters(testNamespace) + checkConfigMapPlacedOnClusters(testNamespace, configMapName) + }) + + It("can delete the CRP", func() { + deleteCRP(crpName) + }) + + It("should remove placed resources from all member clusters", checkIfRemovedWorkResourcesFromAllMemberClusters) + + It("should remove controller finalizers from CRP", func() { + checkFinalizersRemoved(crpName) + }) +}) + +var _ = Describe("CRP with NamespaceWithResourceSelectors selecting only namespace", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + testNamespace := fmt.Sprintf(workNamespaceNameTemplate, GinkgoParallelProcess()) + + BeforeAll(func() { + By("creating test namespace and resources") + createWorkResources() + }) + + AfterAll(func() { + By(fmt.Sprintf("deleting CRP and related resources %s", crpName)) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should create CRP with NamespaceWithResourceSelectors selecting only namespace", func() { + createCRPWithSelectors(crpName, []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Kind: "Namespace", + Version: "v1", + Name: testNamespace, + SelectionScope: placementv1beta1.NamespaceWithResourceSelectors, + }, + }) + }) + + It("should update CRP status with only namespace", func() { + checkCRPStatusWithResources(crpName, []placementv1beta1.ResourceIdentifier{ + { + Group: "", + Kind: "Namespace", + Version: "v1", + Name: testNamespace, + }, + }) + }) + + It("should place only the namespace on member clusters", func() { + checkNamespacePlacedOnClusters(testNamespace) + }) + + It("can delete the CRP", func() { + deleteCRP(crpName) + }) + + It("should remove placed namespace from all member clusters", func() { + checkNamespaceRemovedFromClusters(testNamespace) + }) + + It("should remove controller finalizers from CRP", func() { + checkFinalizersRemoved(crpName) + }) +}) + +var _ = Describe("CRP with NamespaceWithResourceSelectors with cluster-scoped resources", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + testNamespace := fmt.Sprintf(workNamespaceNameTemplate, GinkgoParallelProcess()) + clusterRoleName := fmt.Sprintf("test-clusterrole-%d", GinkgoParallelProcess()) + + BeforeAll(func() { + By("creating test namespace") + createWorkResources() + }) + + AfterAll(func() { + By(fmt.Sprintf("deleting CRP and related resources %s", crpName)) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should create CRP with NamespaceWithResourceSelectors and cluster-scoped resource", func() { + createCRPWithSelectors(crpName, []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Kind: "Namespace", + Version: "v1", + Name: testNamespace, + SelectionScope: placementv1beta1.NamespaceWithResourceSelectors, + }, + { + Group: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Version: "v1", + Name: clusterRoleName, + }, + }) + }) + + It("should update CRP status with namespace only (ClusterRole doesn't exist)", func() { + checkCRPStatusWithResources(crpName, []placementv1beta1.ResourceIdentifier{ + { + Group: "", + Kind: "Namespace", + Version: "v1", + Name: testNamespace, + }, + }) + }) + + It("can delete the CRP", func() { + deleteCRP(crpName) + }) + + It("should remove placed resources from all member clusters", func() { + checkNamespaceRemovedFromClusters(testNamespace) + }) + + It("should remove controller finalizers from CRP", func() { + checkFinalizersRemoved(crpName) + }) +}) + +// Negative test case: selecting a non-existent namespace should show error in status +var _ = Describe("CRP with NamespaceWithResourceSelectors selecting non-existent namespace", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + nonExistentNamespace := fmt.Sprintf("non-existent-ns-%d", GinkgoParallelProcess()) + + AfterAll(func() { + By("cleaning up CRP") + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should create CRP that selects a non-existent namespace by name", func() { + createCRPWithSelectors(crpName, []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Kind: "Namespace", + Version: "v1", + Name: nonExistentNamespace, + SelectionScope: placementv1beta1.NamespaceWithResourceSelectors, + }, + }) + }) + + It("should show error in CRP status for non-existent namespace", func() { + Eventually(func() bool { + var crp placementv1beta1.ClusterResourcePlacement + if err := hubClient.Get(ctx, types.NamespacedName{Name: crpName}, &crp); err != nil { + return false + } + // Check if there's a condition indicating the error about namespace not being selected + for _, cond := range crp.Status.Conditions { + if strings.Contains(cond.Message, "NamespaceWithResourceSelectors mode requires exactly one namespace, but no namespaces were selected") { + return true + } + } + return false + }, eventuallyDuration, eventuallyInterval).Should(BeTrue(), "CRP %s should have error condition for non-existent namespace", crpName) + }) + + It("can delete the CRP", func() { + deleteCRP(crpName) + }) + + It("should remove controller finalizers from CRP", func() { + checkFinalizersRemoved(crpName) + }) +}) diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index b23f45644..66c6bca86 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -1057,6 +1057,55 @@ func checkIfRemovedConfigMapFromMemberClustersConsistently(clusters []*framework } } +// createCRPWithSelectors creates a ClusterResourcePlacement with the given selectors. +func createCRPWithSelectors(crpName string, selectors []placementv1beta1.ResourceSelectorTerm) { + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: selectors, + }, + } + By(fmt.Sprintf("creating CRP %s", crpName)) + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP %s", crpName) +} + +// checkNamespacePlacedOnClusters verifies a namespace is placed on all member clusters. +func checkNamespacePlacedOnClusters(namespaceName string) { + for idx := range allMemberClusters { + cluster := allMemberClusters[idx] + ns := &corev1.Namespace{} + Eventually(func() error { + return cluster.KubeClient.Get(ctx, types.NamespacedName{Name: namespaceName}, ns) + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to get namespace %s on cluster %s", namespaceName, cluster.ClusterName) + } +} + +// checkNamespaceRemovedFromClusters verifies a namespace is removed from all member clusters. +func checkNamespaceRemovedFromClusters(namespaceName string) { + for idx := range allMemberClusters { + cluster := allMemberClusters[idx] + Eventually(func() bool { + ns := &corev1.Namespace{} + err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: namespaceName}, ns) + return err != nil + }, eventuallyDuration, eventuallyInterval).Should(BeTrue(), "Namespace %s should be removed from cluster %s", namespaceName, cluster.ClusterName) + } +} + +// checkConfigMapPlacedOnClusters verifies a configmap is placed on all member clusters. +func checkConfigMapPlacedOnClusters(namespaceName, configMapName string) { + for idx := range allMemberClusters { + cluster := allMemberClusters[idx] + cm := &corev1.ConfigMap{} + Eventually(func() error { + return cluster.KubeClient.Get(ctx, types.NamespacedName{Name: configMapName, Namespace: namespaceName}, cm) + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to get configmap %s on cluster %s", configMapName, cluster.ClusterName) + } +} + func checkNamespaceExistsWithOwnerRefOnMemberCluster(nsName, crpName string) { Consistently(func() error { ns := &corev1.Namespace{} From 4d96d72e8ddd20bc6eb77e2497103507b104637d Mon Sep 17 00:00:00 2001 From: Simon Waight Date: Fri, 6 Mar 2026 08:18:25 +1100 Subject: [PATCH 2/4] Update PR template to add clear link requirement. (#476) Signed-off-by: Simon Waight --- .github/PULL_REQUEST_TEMPLATE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index fbf6bfb2a..a95df8430 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -13,6 +13,7 @@ Fixes # I have: +- [ ] Associated this change with a known KubeFleet Issue (Bug, Feature, etc). - [ ] Run `make reviewable` to ensure this PR is ready for review. ### How has this code been tested From b625d1936a035174925c1264b6aeab4ca261277b Mon Sep 17 00:00:00 2001 From: Yetkin Timocin Date: Thu, 5 Mar 2026 14:53:28 -0800 Subject: [PATCH 3/4] fix: correct pre-selected to preselected in scheduler framework test (#492) --- pkg/scheduler/framework/framework_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/scheduler/framework/framework_test.go b/pkg/scheduler/framework/framework_test.go index ec156568d..c33b019f7 100644 --- a/pkg/scheduler/framework/framework_test.go +++ b/pkg/scheduler/framework/framework_test.go @@ -6550,7 +6550,7 @@ func TestRunSchedulingCycleForPickAllPlacementType_StableStatusOutputInLargeFlee profile := NewProfile("TestOnly") dummyLabelBasedFilterPluginName := fmt.Sprintf(dummyAllPurposePluginNameFormat, 0) - wantLabelKey := "pre-selected" + wantLabelKey := "preselected" wantLabelValue := "true" wantLabels := map[string]string{ wantLabelKey: wantLabelValue, From c92d4c3e69fa96496afa2158ea89ab9a0dd25828 Mon Sep 17 00:00:00 2001 From: Wei Weng Date: Fri, 6 Mar 2026 15:40:24 +0000 Subject: [PATCH 4/4] Revert "Update PR template to add clear link requirement. (#476)" This reverts commit 4d96d72e8ddd20bc6eb77e2497103507b104637d. --- .github/PULL_REQUEST_TEMPLATE.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a95df8430..fbf6bfb2a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -13,7 +13,6 @@ Fixes # I have: -- [ ] Associated this change with a known KubeFleet Issue (Bug, Feature, etc). - [ ] Run `make reviewable` to ensure this PR is ready for review. ### How has this code been tested