diff --git a/api/v1beta1/spec.go b/api/v1beta1/spec.go index b3f3c4e9..9e4e3478 100644 --- a/api/v1beta1/spec.go +++ b/api/v1beta1/spec.go @@ -660,6 +660,14 @@ type KustomizationRef struct { // +kubebuilder:default:=false // +optional SkipNamespaceCreation bool `json:"skipNamespaceCreation,omitempty"` + + // Force indicates whether Sveltos should delete and recreate a resource defined in this + // KustomizationRef when an update is rejected with an error that only a delete+recreate + // can resolve (eg an invalid combination of fields, or a field enforced as immutable). + // By default, such errors are surfaced instead of recreating the resource. + // +kubebuilder:default:=false + // +optional + Force bool `json:"force,omitempty"` } // StopMatchingBehavior indicates what will happen when Cluster stops matching @@ -779,6 +787,14 @@ type PolicyRef struct { // +optional SkipNamespaceCreation bool `json:"skipNamespaceCreation,omitempty"` + // Force indicates whether Sveltos should delete and recreate a resource defined in this + // PolicyRef when an update is rejected with an error that only a delete+recreate can + // resolve (eg an invalid combination of fields, or a field enforced as immutable). + // By default, such errors are surfaced instead of recreating the resource. + // +kubebuilder:default:=false + // +optional + Force bool `json:"force,omitempty"` + // RemoteURL configures fetching content from an HTTP/HTTPS endpoint or an OCI registry. // When set, Kind/Name/Namespace must be omitted. // +optional diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 9e31aec6..dd08190d 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -21,12 +21,11 @@ limitations under the License. package v1beta1 import ( + apiv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" - - apiv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. diff --git a/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml b/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml index a2c1a359..f9e056e0 100644 --- a/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml +++ b/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml @@ -767,6 +767,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + KustomizationRef when an update is rejected with an error that only a delete+recreate + can resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: @@ -1064,6 +1072,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + PolicyRef when an update is rejected with an error that only a delete+recreate can + resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: diff --git a/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml b/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml index ca604389..5196a03a 100644 --- a/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml +++ b/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml @@ -669,6 +669,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + KustomizationRef when an update is rejected with an error that only a delete+recreate + can resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: @@ -966,6 +974,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + PolicyRef when an update is rejected with an error that only a delete+recreate can + resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: @@ -2183,6 +2199,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + PolicyRef when an update is rejected with an error that only a delete+recreate can + resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: @@ -2545,6 +2569,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + PolicyRef when an update is rejected with an error that only a delete+recreate can + resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: diff --git a/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml b/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml index 8cd4dead..3fa1dbc6 100644 --- a/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml +++ b/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml @@ -806,6 +806,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + KustomizationRef when an update is rejected with an error that only a delete+recreate + can resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: @@ -1103,6 +1111,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + PolicyRef when an update is rejected with an error that only a delete+recreate can + resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: diff --git a/config/crd/bases/config.projectsveltos.io_profiles.yaml b/config/crd/bases/config.projectsveltos.io_profiles.yaml index 3ef6d24d..84cb4643 100644 --- a/config/crd/bases/config.projectsveltos.io_profiles.yaml +++ b/config/crd/bases/config.projectsveltos.io_profiles.yaml @@ -767,6 +767,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + KustomizationRef when an update is rejected with an error that only a delete+recreate + can resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: @@ -1064,6 +1072,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + PolicyRef when an update is rejected with an error that only a delete+recreate can + resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: diff --git a/controllers/export_test.go b/controllers/export_test.go index af23095b..bfc4a0a2 100644 --- a/controllers/export_test.go +++ b/controllers/export_test.go @@ -107,16 +107,18 @@ var ( GenericDeploy = genericDeploy GenericUndeploy = genericUndeploy - GetEntryKey = getEntryKey - DeployContentOfConfigMap = deployContentOfConfigMap - DeployContentOfSecret = deployContentOfSecret - DeployContent = deployContent - GetClusterSummaryAdmin = getClusterSummaryAdmin - CollectContent = collectContent - UndeployStaleResources = undeployStaleResources - GetDeployedGroupVersionKinds = getDeployedGroupVersionKinds - GetSecret = getSecret - ReadFiles = readFiles + GetEntryKey = getEntryKey + DeployContentOfConfigMap = deployContentOfConfigMap + DeployContentOfSecret = deployContentOfSecret + DeployContent = deployContent + GetClusterSummaryAdmin = getClusterSummaryAdmin + CollectContent = collectContent + CollectReferencedObjects = collectReferencedObjects + PrepareBundleSettersWithResourceInfo = prepareBundleSettersWithResourceInfo + UndeployStaleResources = undeployStaleResources + GetDeployedGroupVersionKinds = getDeployedGroupVersionKinds + GetSecret = getSecret + ReadFiles = readFiles AddExtraLabels = addExtraLabels AddExtraAnnotations = addExtraAnnotations diff --git a/controllers/handlers_helm.go b/controllers/handlers_helm.go index 9b48ae66..7e246c30 100644 --- a/controllers/handlers_helm.go +++ b/controllers/handlers_helm.go @@ -4574,8 +4574,8 @@ func addExtraMetadata(ctx context.Context, requestedChart *configv1beta1.HelmCha isDriftDetection := clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1beta1.SyncModeContinuousWithDriftDetection isDryRun := clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1beta1.SyncModeDryRun - _, err = deployer.UpdateResource(ctx, dr, isDriftDetection, isDryRun, clusterSummary.Spec.ClusterProfileSpec.DriftExclusions, - r, []string{}, logger) + _, err = deployer.UpdateResource(ctx, dr, isDriftDetection, isDryRun, getForceValue(requestedChart.Options), + clusterSummary.Spec.ClusterProfileSpec.DriftExclusions, r, []string{}, logger) if err != nil { logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to update resource %s %s/%s: %v", r.GetKind(), r.GetNamespace(), r.GetName(), err)) diff --git a/controllers/handlers_kustomize.go b/controllers/handlers_kustomize.go index 2a363158..82344fa8 100644 --- a/controllers/handlers_kustomize.go +++ b/controllers/handlers_kustomize.go @@ -1011,8 +1011,8 @@ func deployKustomizeResources(ctx context.Context, c client.Client, remoteRestCo Name: kustomizationRef.Name, } localReports, err = deployUnstructured(ctx, true, localConfig, c, objectsToDeployLocally, - ref, kustomizationRef.Tier, kustomizationRef.SkipNamespaceCreation, libsveltosv1beta1.FeatureKustomize, - dCtx.clusterSummary, []string{}, logger) + ref, kustomizationRef.Tier, kustomizationRef.SkipNamespaceCreation, kustomizationRef.Force, + libsveltosv1beta1.FeatureKustomize, dCtx.clusterSummary, []string{}, logger) if err != nil { logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to deploy to management cluster %v", err)) return localReports, nil, err @@ -1025,7 +1025,7 @@ func deployKustomizeResources(ctx context.Context, c client.Client, remoteRestCo bundleResources[key] = convertPointerSliceToValueSlice(objectsToDeployRemotely) setters := prepareBundleSettersWithResourceInfo(ref.Kind, ref.Namespace, ref.Name, kustomizationRef.Tier, - kustomizationRef.SkipNamespaceCreation) + kustomizationRef.SkipNamespaceCreation, kustomizationRef.Force) return localReports, nil, pullmode.StageResourcesForDeployment(ctx, getManagementClusterClient(), dCtx.clusterSummary.Spec.ClusterNamespace, dCtx.clusterSummary.Spec.ClusterName, configv1beta1.ClusterSummaryKind, @@ -1038,8 +1038,8 @@ func deployKustomizeResources(ctx context.Context, c client.Client, remoteRestCo } remoteReports, err = deployUnstructured(ctx, false, remoteRestConfig, remoteClient, objectsToDeployRemotely, - ref, kustomizationRef.Tier, kustomizationRef.SkipNamespaceCreation, libsveltosv1beta1.FeatureKustomize, - dCtx.clusterSummary, []string{}, logger) + ref, kustomizationRef.Tier, kustomizationRef.SkipNamespaceCreation, kustomizationRef.Force, + libsveltosv1beta1.FeatureKustomize, dCtx.clusterSummary, []string{}, logger) if err != nil { logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to deploy to remote cluster %v", err)) return localReports, remoteReports, err diff --git a/controllers/handlers_utils.go b/controllers/handlers_utils.go index 928542a6..bd4e5d4d 100644 --- a/controllers/handlers_utils.go +++ b/controllers/handlers_utils.go @@ -73,6 +73,7 @@ type referencedObject struct { corev1.ObjectReference Tier int32 SkipNamespaceCreation bool + Force bool Optional bool Path string // URL and related fields are set only for URL-based PolicyRefs (Kind == urlSourceKind). @@ -97,7 +98,7 @@ func deployContentOfConfigMap(ctx context.Context, deployingToMgmtCluster bool, ) ([]libsveltosv1beta1.ResourceReport, error) { resourceReports, err := deployContent(ctx, deployingToMgmtCluster, destConfig, destClient, configMap, configMap.Data, - reference.Tier, reference.SkipNamespaceCreation, dCtx, logger) + reference.Tier, reference.SkipNamespaceCreation, reference.Force, dCtx, logger) if err != nil { return resourceReports, fmt.Errorf("processing ConfigMap %s/%s: %w", configMap.Namespace, configMap.Name, err) } @@ -120,7 +121,7 @@ func deployContentOfSecret(ctx context.Context, deployingToMgmtCluster bool, des } resourceReports, err := deployContent(ctx, deployingToMgmtCluster, destConfig, destClient, secret, data, - reference.Tier, reference.SkipNamespaceCreation, dCtx, logger) + reference.Tier, reference.SkipNamespaceCreation, reference.Force, dCtx, logger) if err != nil { return resourceReports, fmt.Errorf("processing Secret %s/%s: %w", secret.Namespace, secret.Name, err) } @@ -171,7 +172,7 @@ func deployContentOfSource(ctx context.Context, deployingToMgmtCluster bool, des } return deployContent(ctx, deployingToMgmtCluster, destConfig, destClient, source, content, - reference.Tier, reference.SkipNamespaceCreation, dCtx, logger) + reference.Tier, reference.SkipNamespaceCreation, reference.Force, dCtx, logger) } func readFiles(dir string) (map[string]string, error) { @@ -281,7 +282,7 @@ func prepareReports(resources []*unstructured.Unstructured) []libsveltosv1beta1. // the policies deployed in the form of kind.group:namespace:name for namespaced policies // and kind.group::name for cluster wide policies. func deployContent(ctx context.Context, deployingToMgmtCluster bool, destConfig *rest.Config, destClient client.Client, - referencedObject client.Object, data map[string]string, referenceTier int32, skipNamespaceCreation bool, + referencedObject client.Object, data map[string]string, referenceTier int32, skipNamespaceCreation, force bool, dCtx *deploymentContext, logger logr.Logger) (reports []libsveltosv1beta1.ResourceReport, err error) { subresources := getSubresources(referencedObject) @@ -314,7 +315,8 @@ func deployContent(ctx context.Context, deployingToMgmtCluster bool, destConfig // In pull mode we return reports with action Create. Those will only be used to update deployed GVK. // sveltos-applier will take care of sending proper reports - setters := prepareBundleSettersWithResourceInfo(ref.Kind, ref.Namespace, ref.Name, referenceTier, skipNamespaceCreation) + setters := prepareBundleSettersWithResourceInfo(ref.Kind, ref.Namespace, ref.Name, referenceTier, + skipNamespaceCreation, force) return prepareReports(resources), pullmode.StageResourcesForDeployment(ctx, getManagementClusterClient(), @@ -324,7 +326,7 @@ func deployContent(ctx context.Context, deployingToMgmtCluster bool, destConfig } return deployUnstructured(ctx, deployingToMgmtCluster, destConfig, destClient, resources, ref, referenceTier, - skipNamespaceCreation, libsveltosv1beta1.FeatureResources, dCtx.clusterSummary, subresources, logger) + skipNamespaceCreation, force, libsveltosv1beta1.FeatureResources, dCtx.clusterSummary, subresources, logger) } // adjustNamespace fixes namespace. @@ -396,8 +398,9 @@ func applyPatches(ctx context.Context, clusterSummary *configv1beta1.ClusterSumm //nolint:funlen // requires a lot of arguments because kustomize and plain resources are using this function func deployUnstructured(ctx context.Context, deployingToMgmtCluster bool, destConfig *rest.Config, destClient client.Client, referencedUnstructured []*unstructured.Unstructured, referencedObject *corev1.ObjectReference, - referenceTier int32, skipNamespaceCreation bool, featureID libsveltosv1beta1.FeatureID, clusterSummary *configv1beta1.ClusterSummary, - subresources []string, logger logr.Logger) (reports []libsveltosv1beta1.ResourceReport, err error) { + referenceTier int32, skipNamespaceCreation, forceRecreate bool, featureID libsveltosv1beta1.FeatureID, + clusterSummary *configv1beta1.ClusterSummary, subresources []string, logger logr.Logger, +) (reports []libsveltosv1beta1.ResourceReport, err error) { profile, profileTier, err := configv1beta1.GetProfileOwnerAndTier(ctx, getManagementClusterClient(), clusterSummary) if err != nil { @@ -507,6 +510,7 @@ func deployUnstructured(ctx context.Context, deployingToMgmtCluster bool, destCo dr, clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1beta1.SyncModeContinuousWithDriftDetection, clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1beta1.SyncModeDryRun, + forceRecreate, clusterSummary.Spec.ClusterProfileSpec.DriftExclusions, policy, subresources, @@ -728,6 +732,15 @@ func appendPathAnnotations(object client.Object, path string) { object.SetAnnotations(annotations) } +// setCommonReferencedObjectFields copies the options shared by every PolicyRef kind +// (Tier, Optional, SkipNamespaceCreation, Force) from reference onto object. +func setCommonReferencedObjectFields(object *referencedObject, reference *configv1beta1.PolicyRef) { + object.Tier = reference.Tier + object.Optional = reference.Optional + object.SkipNamespaceCreation = reference.SkipNamespaceCreation + object.Force = reference.Force +} + // collectReferencedObjects collects all referenced configMaps/secrets in control cluster // local contains all configMaps/Secrets whose content need to be deployed locally (in the management cluster) // remote contains all configMap/Secrets whose content need to be deployed remotely (in the managed cluster) @@ -746,9 +759,7 @@ func collectReferencedObjects(references []configv1beta1.PolicyRef) (local, remo object.URL = reference.RemoteURL.URL object.IsTemplate = reference.RemoteURL.Template object.SecretRef = reference.RemoteURL.SecretRef - object.Tier = reference.Tier - object.Optional = reference.Optional - object.SkipNamespaceCreation = reference.SkipNamespaceCreation + setCommonReferencedObjectFields(&object, reference) if reference.DeploymentType == configv1beta1.DeploymentTypeLocal { local = append(local, object) @@ -766,9 +777,7 @@ func collectReferencedObjects(references []configv1beta1.PolicyRef) (local, remo Namespace: reference.Namespace, Name: reference.Name, } - object.Tier = reference.Tier - object.Optional = reference.Optional - object.SkipNamespaceCreation = reference.SkipNamespaceCreation + setCommonReferencedObjectFields(&object, reference) case string(libsveltosv1beta1.SecretReferencedResourceKind): object.ObjectReference = corev1.ObjectReference{ APIVersion: coreAPIVersion, @@ -776,9 +785,7 @@ func collectReferencedObjects(references []configv1beta1.PolicyRef) (local, remo Namespace: reference.Namespace, Name: reference.Name, } - object.Tier = reference.Tier - object.Optional = reference.Optional - object.SkipNamespaceCreation = reference.SkipNamespaceCreation + setCommonReferencedObjectFields(&object, reference) case sourcev1.GitRepositoryKind: object.ObjectReference = corev1.ObjectReference{ APIVersion: sourcev1.GroupVersion.String(), @@ -786,9 +793,7 @@ func collectReferencedObjects(references []configv1beta1.PolicyRef) (local, remo Namespace: reference.Namespace, Name: reference.Name, } - object.Tier = reference.Tier - object.Optional = reference.Optional - object.SkipNamespaceCreation = reference.SkipNamespaceCreation + setCommonReferencedObjectFields(&object, reference) object.Path = reference.Path case sourcev1.OCIRepositoryKind: object.ObjectReference = corev1.ObjectReference{ @@ -797,9 +802,7 @@ func collectReferencedObjects(references []configv1beta1.PolicyRef) (local, remo Namespace: reference.Namespace, Name: reference.Name, } - object.Tier = reference.Tier - object.Optional = reference.Optional - object.SkipNamespaceCreation = reference.SkipNamespaceCreation + setCommonReferencedObjectFields(&object, reference) object.Path = reference.Path case sourcev1.BucketKind: object.ObjectReference = corev1.ObjectReference{ @@ -808,9 +811,7 @@ func collectReferencedObjects(references []configv1beta1.PolicyRef) (local, remo Namespace: reference.Namespace, Name: reference.Name, } - object.Tier = reference.Tier - object.Optional = reference.Optional - object.SkipNamespaceCreation = reference.SkipNamespaceCreation + setCommonReferencedObjectFields(&object, reference) object.Path = reference.Path } @@ -1753,12 +1754,12 @@ func getReloaderClient(ctx context.Context, clusterNamespace, clusterName string } func prepareBundleSettersWithResourceInfo(referenceKind, referenceNamespace, referenceName string, - tier int32, skipNamespaceCreation bool) []pullmode.BundleOption { + tier int32, skipNamespaceCreation, force bool) []pullmode.BundleOption { setters := make([]pullmode.BundleOption, 0) setters = append(setters, - pullmode.WithResourceInfo(referenceKind, referenceNamespace, referenceName, tier, skipNamespaceCreation)) + pullmode.WithResourceInfo(referenceKind, referenceNamespace, referenceName, tier, skipNamespaceCreation, force)) return setters } diff --git a/controllers/handlers_utils_test.go b/controllers/handlers_utils_test.go index 2bc301c1..55279b28 100644 --- a/controllers/handlers_utils_test.go +++ b/controllers/handlers_utils_test.go @@ -48,6 +48,7 @@ import ( libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" "github.com/projectsveltos/libsveltos/lib/deployer" "github.com/projectsveltos/libsveltos/lib/k8s_utils" + "github.com/projectsveltos/libsveltos/lib/pullmode" ) const ( @@ -292,7 +293,7 @@ var _ = Describe("HandlersUtils", func() { isDryRun := clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1beta1.SyncModeDryRun isDriftDetection := clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1beta1.SyncModeContinuousWithDriftDetection // following will successfully create deployment - _, err = deployer.UpdateResource(context.TODO(), dr, isDriftDetection, isDryRun, clusterSummary.Spec.ClusterProfileSpec.DriftExclusions, + _, err = deployer.UpdateResource(context.TODO(), dr, isDriftDetection, isDryRun, false, clusterSummary.Spec.ClusterProfileSpec.DriftExclusions, u, nil, textlogger.NewLogger(textlogger.NewConfig())) Expect(err).To(BeNil()) @@ -336,7 +337,7 @@ var _ = Describe("HandlersUtils", func() { isDryRun = clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1beta1.SyncModeDryRun isDriftDetection = clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1beta1.SyncModeContinuousWithDriftDetection // New deploy will not override replicas - _, err = deployer.UpdateResource(context.TODO(), dr, isDriftDetection, isDryRun, clusterSummary.Spec.ClusterProfileSpec.DriftExclusions, + _, err = deployer.UpdateResource(context.TODO(), dr, isDriftDetection, isDryRun, false, clusterSummary.Spec.ClusterProfileSpec.DriftExclusions, u, nil, textlogger.NewLogger(textlogger.NewConfig())) Expect(err).To(BeNil()) @@ -361,7 +362,7 @@ var _ = Describe("HandlersUtils", func() { isDryRun := false isDriftDetection := true - _, err = deployer.UpdateResource(context.TODO(), dr, isDriftDetection, isDryRun, clusterSummary.Spec.ClusterProfileSpec.DriftExclusions, + _, err = deployer.UpdateResource(context.TODO(), dr, isDriftDetection, isDryRun, false, clusterSummary.Spec.ClusterProfileSpec.DriftExclusions, u, nil, textlogger.NewLogger(textlogger.NewConfig())) Expect(err).To(BeNil()) @@ -405,7 +406,7 @@ var _ = Describe("HandlersUtils", func() { }, timeout, pollingInterval).Should(BeTrue()) // New deploy will override labels - _, err = deployer.UpdateResource(context.TODO(), dr, isDriftDetection, isDryRun, clusterSummary.Spec.ClusterProfileSpec.DriftExclusions, + _, err = deployer.UpdateResource(context.TODO(), dr, isDriftDetection, isDryRun, false, clusterSummary.Spec.ClusterProfileSpec.DriftExclusions, u, nil, textlogger.NewLogger(textlogger.NewConfig())) Expect(err).To(BeNil()) @@ -429,7 +430,7 @@ var _ = Describe("HandlersUtils", func() { isDryRun := clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1beta1.SyncModeDryRun isDriftDetection := clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1beta1.SyncModeContinuousWithDriftDetection // following will successfully create deployment - _, err = deployer.UpdateResource(context.TODO(), dr, isDriftDetection, isDryRun, clusterSummary.Spec.ClusterProfileSpec.DriftExclusions, + _, err = deployer.UpdateResource(context.TODO(), dr, isDriftDetection, isDryRun, false, clusterSummary.Spec.ClusterProfileSpec.DriftExclusions, u, nil, textlogger.NewLogger(textlogger.NewConfig())) Expect(err).To(BeNil()) @@ -481,7 +482,7 @@ var _ = Describe("HandlersUtils", func() { isDryRun = clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1beta1.SyncModeDryRun isDriftDetection = clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1beta1.SyncModeContinuousWithDriftDetection // New deploy will not override replicas - _, err = deployer.UpdateResource(context.TODO(), dr, isDriftDetection, isDryRun, clusterSummary.Spec.ClusterProfileSpec.DriftExclusions, + _, err = deployer.UpdateResource(context.TODO(), dr, isDriftDetection, isDryRun, false, clusterSummary.Spec.ClusterProfileSpec.DriftExclusions, u, []string{testStatusField}, textlogger.NewLogger(textlogger.NewConfig())) Expect(err).To(BeNil()) @@ -561,7 +562,7 @@ var _ = Describe("HandlersUtils", func() { // created) resourceReports, err := controllers.DeployContent(context.TODO(), false, testEnv.Config, testEnv.Client, secret, map[string]string{testServiceKey: services}, - defaultTier, false, controllers.NewDeploymentContext(clusterSummary, clusterObjects, nil), + defaultTier, false, false, controllers.NewDeploymentContext(clusterSummary, clusterObjects, nil), textlogger.NewLogger(textlogger.NewConfig())) Expect(err).To(BeNil()) By("Validating action for all resourceReports is Create") @@ -593,7 +594,7 @@ var _ = Describe("HandlersUtils", func() { // ( if the ClusterProfile were to be changed from DryRun, nothing would happen). resourceReports, err = controllers.DeployContent(context.TODO(), false, testEnv.Config, testEnv.Client, secret, map[string]string{testServiceKey: services}, - defaultTier, false, controllers.NewDeploymentContext(clusterSummary, clusterObjects, nil), + defaultTier, false, false, controllers.NewDeploymentContext(clusterSummary, clusterObjects, nil), textlogger.NewLogger(textlogger.NewConfig())) Expect(err).To(BeNil()) By("Validating action for all resourceReports is NoAction") @@ -631,7 +632,7 @@ var _ = Describe("HandlersUtils", func() { // (if the ClusterProfile were to be changed from DryRun, both service would be updated). resourceReports, err = controllers.DeployContent(context.TODO(), false, testEnv.Config, testEnv.Client, secret, map[string]string{testServiceKey: newContent}, - defaultTier, false, controllers.NewDeploymentContext(clusterSummary, clusterObjects, nil), + defaultTier, false, false, controllers.NewDeploymentContext(clusterSummary, clusterObjects, nil), textlogger.NewLogger(textlogger.NewConfig())) Expect(err).To(BeNil()) By("Validating action for all resourceReports is Update") @@ -642,7 +643,7 @@ var _ = Describe("HandlersUtils", func() { tmpSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: randomString(), Name: randomString()}} resourceReports, err = controllers.DeployContent(context.TODO(), false, testEnv.Config, testEnv.Client, tmpSecret, map[string]string{testServiceKey: services}, - defaultTier, false, controllers.NewDeploymentContext(clusterSummary, clusterObjects, nil), + defaultTier, false, false, controllers.NewDeploymentContext(clusterSummary, clusterObjects, nil), textlogger.NewLogger(textlogger.NewConfig())) Expect(err).To(BeNil()) By("Validating action for all resourceReports is Conflict") @@ -1510,6 +1511,68 @@ status: }) }) +var _ = Describe("collectReferencedObjects", func() { + It("copies Force from PolicyRef onto the referencedObject for every supported Kind", func() { + references := []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: randomString(), Name: randomString(), + DeploymentType: configv1beta1.DeploymentTypeRemote, + Force: true, + }, + { + Kind: string(libsveltosv1beta1.SecretReferencedResourceKind), + Namespace: randomString(), Name: randomString(), + DeploymentType: configv1beta1.DeploymentTypeRemote, + Force: false, + }, + } + + _, remote := controllers.CollectReferencedObjects(references) + Expect(len(remote)).To(Equal(2)) + Expect(remote[0].Force).To(BeTrue()) + Expect(remote[1].Force).To(BeFalse()) + }) + + It("copies Force from a RemoteURL PolicyRef onto the referencedObject", func() { + references := []configv1beta1.PolicyRef{ + { + RemoteURL: &configv1beta1.RemoteURL{ + URL: "https://example.com/manifest.yaml", + }, + DeploymentType: configv1beta1.DeploymentTypeRemote, + Force: true, + }, + } + + _, remote := controllers.CollectReferencedObjects(references) + Expect(len(remote)).To(Equal(1)) + Expect(remote[0].Force).To(BeTrue()) + }) +}) + +var _ = Describe("prepareBundleSettersWithResourceInfo", func() { + It("threads force onto the ConfigurationBundle options", func() { + kind := randomString() + namespace := randomString() + name := randomString() + var tier int32 = 100 + + setters := controllers.PrepareBundleSettersWithResourceInfo(kind, namespace, name, tier, true, true) + Expect(len(setters)).To(Equal(1)) + + bundleOptions := &pullmode.BundleOptions{} + setters[0](bundleOptions) + + Expect(bundleOptions.ReferencedObjectKind).To(Equal(kind)) + Expect(bundleOptions.ReferencedObjectNamespace).To(Equal(namespace)) + Expect(bundleOptions.ReferencedObjectName).To(Equal(name)) + Expect(bundleOptions.ReferencedTier).To(Equal(tier)) + Expect(bundleOptions.SkipNamespaceCreation).To(BeTrue()) + Expect(bundleOptions.Force).To(BeTrue()) + }) +}) + // validateResourceReports validates that number of resourceResources with certain actions // match the expected number per action func validateResourceReports(resourceReports []libsveltosv1beta1.ResourceReport, diff --git a/controllers/url_source.go b/controllers/url_source.go index 3a5f10b4..82e79775 100644 --- a/controllers/url_source.go +++ b/controllers/url_source.go @@ -332,7 +332,7 @@ func deployContentOfURL(ctx context.Context, deployingToMgmtCluster bool, destCo l.V(logs.LogDebug).Info("deploying URL content") return deployContent(ctx, deployingToMgmtCluster, destConfig, destClient, syntheticSource, - data, ref.Tier, ref.SkipNamespaceCreation, dCtx, l) + data, ref.Tier, ref.SkipNamespaceCreation, ref.Force, dCtx, l) } // minURLInterval returns the shortest polling interval across all URL-based PolicyRefs, diff --git a/go.mod b/go.mod index 98c69998..afa0d624 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/onsi/gomega v1.42.1 github.com/opencontainers/image-spec v1.1.1 github.com/pkg/errors v0.9.1 - github.com/projectsveltos/libsveltos v1.12.0 + github.com/projectsveltos/libsveltos v1.12.1-0.20260715200227-148c08381bed github.com/prometheus/client_golang v1.23.2 github.com/robfig/cron v1.2.0 github.com/sigstore/cosign/v3 v3.1.1 diff --git a/go.sum b/go.sum index c26a53aa..04c0d4ac 100644 --- a/go.sum +++ b/go.sum @@ -644,8 +644,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= -github.com/projectsveltos/libsveltos v1.12.0 h1:xQfo/AEh3vVRbfWVazpsgBoRgBG5vzm/sJmXp5YrUEg= -github.com/projectsveltos/libsveltos v1.12.0/go.mod h1:4/vcbYFCFE8uEGIHmltriAoHxdB8jgS2zI+9gruOfJ4= +github.com/projectsveltos/libsveltos v1.12.1-0.20260715200227-148c08381bed h1:k5t1S+iaj2IqDUbL4za4ywEKoWJx0/gdEaXDTij5ZhQ= +github.com/projectsveltos/libsveltos v1.12.1-0.20260715200227-148c08381bed/go.mod h1:4/vcbYFCFE8uEGIHmltriAoHxdB8jgS2zI+9gruOfJ4= github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5 h1:khnc+994UszxZYu69J+R5FKiLA/Nk1JQj0EYAkwTWz0= github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5/go.mod h1:yVL8KQFa9tmcxgwl9nwIMtKgtmIVC1zaFRSCfOwYvPY= github.com/projectsveltos/lua-utils/glua-runes v0.0.0-20251212200258-2b3cdcb7c0f5 h1:YbsebwRwTRhV8QacvEAdFqxcxHdeu7JTVtsBovbkgos= diff --git a/manifest/manifest.yaml b/manifest/manifest.yaml index e5dccb8b..d91782ae 100644 --- a/manifest/manifest.yaml +++ b/manifest/manifest.yaml @@ -1076,6 +1076,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + KustomizationRef when an update is rejected with an error that only a delete+recreate + can resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: @@ -1373,6 +1381,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + PolicyRef when an update is rejected with an error that only a delete+recreate can + resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: @@ -3271,6 +3287,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + KustomizationRef when an update is rejected with an error that only a delete+recreate + can resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: @@ -3568,6 +3592,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + PolicyRef when an update is rejected with an error that only a delete+recreate can + resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: @@ -4785,6 +4817,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + PolicyRef when an update is rejected with an error that only a delete+recreate can + resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: @@ -5147,6 +5187,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + PolicyRef when an update is rejected with an error that only a delete+recreate can + resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: @@ -6484,6 +6532,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + KustomizationRef when an update is rejected with an error that only a delete+recreate + can resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: @@ -6781,6 +6837,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + PolicyRef when an update is rejected with an error that only a delete+recreate can + resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: @@ -8710,6 +8774,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + KustomizationRef when an update is rejected with an error that only a delete+recreate + can resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: @@ -9007,6 +9079,14 @@ spec: - Local - Remote type: string + force: + default: false + description: |- + Force indicates whether Sveltos should delete and recreate a resource defined in this + PolicyRef when an update is rejected with an error that only a delete+recreate can + resolve (eg an invalid combination of fields, or a field enforced as immutable). + By default, such errors are surfaced instead of recreating the resource. + type: boolean kind: description: |- Kind of the resource. Supported kinds are: diff --git a/test/fv/policy_ref_force_test.go b/test/fv/policy_ref_force_test.go new file mode 100644 index 00000000..ff969f58 --- /dev/null +++ b/test/fv/policy_ref_force_test.go @@ -0,0 +1,207 @@ +/* +Copyright 2026. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fv_test + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" + "github.com/projectsveltos/addon-controller/lib/clusterops" + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" +) + +const ( + depNoStrategy = `apiVersion: apps/v1 +kind: Deployment +metadata: + name: %s + namespace: %s +spec: + replicas: 1 + selector: + matchLabels: + app: %s + template: + metadata: + labels: + app: %s + spec: + containers: + - name: main + image: nginx:latest` + + depRecreateStrategy = `apiVersion: apps/v1 +kind: Deployment +metadata: + name: %s + namespace: %s +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: %s + template: + metadata: + labels: + app: %s + spec: + containers: + - name: main + image: nginx:latest` +) + +// This test only runs in push mode: sveltos-applier does not yet act on +// ConfigurationBundle.Spec.Force in pull mode. +var _ = Describe("Feature", func() { + const ( + namePrefix = "policyref-force-" + ) + + It("Force lets Sveltos recreate a Deployment when a strategy change is rejected by the API server", + Label("FV"), func() { + configMapNs := randomString() + Byf("Create configMap's namespace %s", configMapNs) + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapNs, + }, + } + Expect(k8sClient.Create(context.TODO(), ns)).To(Succeed()) + + deploymentName := randomString() + Byf("Create a configMap with a Deployment (no strategy set)") + configMap := createConfigMapWithPolicy(configMapNs, namePrefix+randomString(), + fmt.Sprintf(depNoStrategy, deploymentName, configMapNs, deploymentName, deploymentName)) + Expect(k8sClient.Create(context.TODO(), configMap)).To(Succeed()) + + Byf("Create a ClusterProfile matching Cluster %s/%s", kindWorkloadCluster.GetNamespace(), kindWorkloadCluster.GetName()) + clusterProfile := getClusterProfile(namePrefix, map[string]string{key: value}) + clusterProfile.Spec.SyncMode = configv1beta1.SyncModeContinuous + Expect(k8sClient.Create(context.TODO(), clusterProfile)).To(Succeed()) + + verifyClusterProfileMatches(clusterProfile) + + verifyClusterSummary(clusterops.ClusterProfileLabelName, clusterProfile.Name, &clusterProfile.Spec, + kindWorkloadCluster.GetNamespace(), kindWorkloadCluster.GetName(), getClusterType()) + + Byf("Update ClusterProfile %s to reference ConfigMap %s/%s", clusterProfile.Name, configMap.Namespace, configMap.Name) + currentClusterProfile := &configv1beta1.ClusterProfile{} + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: configMap.Namespace, + Name: configMap.Name, + }, + } + Expect(k8sClient.Update(context.TODO(), currentClusterProfile)).To(Succeed()) + + clusterSummary := verifyClusterSummary(clusterops.ClusterProfileLabelName, + currentClusterProfile.Name, ¤tClusterProfile.Spec, + kindWorkloadCluster.GetNamespace(), kindWorkloadCluster.GetName(), getClusterType()) + + Byf("Verifying ClusterSummary %s status is set to Provisioned for Resources feature", clusterSummary.Name) + verifyFeatureStatusIsProvisioned(clusterSummary.Namespace, clusterSummary.Name, libsveltosv1beta1.FeatureResources) + + Byf("Getting client to access the workload cluster") + workloadClient, err := getKindWorkloadClusterKubeconfig() + Expect(err).To(BeNil()) + Expect(workloadClient).ToNot(BeNil()) + + Byf("Verifying Deployment %s/%s is created in the workload cluster with a defaulted rollingUpdate strategy", + configMapNs, deploymentName) + Eventually(func() bool { + depl := &appsv1.Deployment{} + err = workloadClient.Get(context.TODO(), + types.NamespacedName{Namespace: configMapNs, Name: deploymentName}, depl) + if err != nil { + return false + } + return depl.Spec.Strategy.RollingUpdate != nil + }, timeout, pollingInterval).Should(BeTrue()) + + By("Updating ConfigMap to set the Deployment's strategy to Recreate") + currentConfigMap := &corev1.ConfigMap{} + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Namespace: configMap.Namespace, Name: configMap.Name}, currentConfigMap)).To(Succeed()) + currentConfigMap = updateConfigMapWithPolicy(currentConfigMap, + fmt.Sprintf(depRecreateStrategy, deploymentName, configMapNs, deploymentName, deploymentName)) + Expect(k8sClient.Update(context.TODO(), currentConfigMap)).To(Succeed()) + + Byf("Verifying ClusterSummary %s reports a failure for Resources feature (rollingUpdate forbidden with Recreate)", + clusterSummary.Name) + Eventually(func() bool { + currentClusterSummary := &configv1beta1.ClusterSummary{} + err = k8sClient.Get(context.TODO(), + types.NamespacedName{Namespace: clusterSummary.Namespace, Name: clusterSummary.Name}, + currentClusterSummary) + if err != nil { + return false + } + for i := range currentClusterSummary.Status.FeatureSummaries { + fs := ¤tClusterSummary.Status.FeatureSummaries[i] + if fs.FeatureID == libsveltosv1beta1.FeatureResources { + return fs.FailureMessage != nil + } + } + return false + }, timeout, pollingInterval).Should(BeTrue()) + + By("Setting Force on the PolicyRef") + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: clusterProfile.Name}, + currentClusterProfile)).To(Succeed()) + currentClusterProfile.Spec.PolicyRefs[0].Force = true + return k8sClient.Update(context.TODO(), currentClusterProfile) + }) + Expect(err).To(BeNil()) + + Byf("Verifying ClusterSummary %s status is back to Provisioned for Resources feature", clusterSummary.Name) + verifyFeatureStatusIsProvisioned(clusterSummary.Namespace, clusterSummary.Name, libsveltosv1beta1.FeatureResources) + + Byf("Verifying Deployment %s/%s was recreated with strategy Recreate and no rollingUpdate", + configMapNs, deploymentName) + Eventually(func() bool { + depl := &appsv1.Deployment{} + err = workloadClient.Get(context.TODO(), + types.NamespacedName{Namespace: configMapNs, Name: deploymentName}, depl) + if err != nil { + return false + } + return depl.Spec.Strategy.Type == appsv1.RecreateDeploymentStrategyType && + depl.Spec.Strategy.RollingUpdate == nil + }, timeout, pollingInterval).Should(BeTrue()) + + deleteClusterProfile(clusterProfile) + + currentNs := &corev1.Namespace{} + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: configMapNs}, currentNs)).To(Succeed()) + Expect(k8sClient.Delete(context.TODO(), currentNs)).To(Succeed()) + }) +})