Feature: Allow workloadKind: Deployment

CHANGELOG.md

@@ -11,9 +11,15 @@ All notable changes to this project will be documented in this file.
   Also, a rego-rule library has been added to make it easier to call resource-info-fetcher from within OPA.
   The API (especially the response) might change in the future once more data catalogs are supported ([#863]).
 - Allow specifying the maximum number of cached entries in the user-info-fetcher ([#863]).
+- The `servers` role can now run as a `Deployment` instead of a `DaemonSet`, selected via
+  `spec.servers.roleConfig.workloadKind`. ([#873]).
+- A `PodDisruptionBudget` is now written out for the `servers` role when it runs as a `Deployment`,
+  with `maxUnavailable: 1`. Configurable via `spec.servers.roleConfig.podDisruptionBudget` ([#873]).
 
 ### Changed
 
+- OPA Pods now default to a soft anti-affinity that spreads them across nodes. This is a no-op for a
+  `DaemonSet`, which already runs one Pod per node, but keeps a `Deployment`'s replicas from being deployed together ([#873]).
 - Internal operator refactoring: introduce a build() step in the reconciler that
   assembles all relevant Kubernetes resources before anything is applied ([#852]).
 - Bump `stackable-operator` to 0.114.0 ([#867]).
@@ -27,6 +33,7 @@ All notable changes to this project will be documented in this file.
   which could cause problems with GitOps tools (e.g. ArgoCD) reporting a diff in the custom resources.
   See [our internal issue](https://github.com/stackabletech/hdfs-operator/issues/626) and [the fix](https://github.com/kube-rs/kube/pull/2042) for details ([#871]).
 
+[#873]: https://github.com/stackabletech/opa-operator/pull/873
 [#852]: https://github.com/stackabletech/opa-operator/pull/852
 [#861]: https://github.com/stackabletech/opa-operator/pull/861
 [#863]: https://github.com/stackabletech/opa-operator/pull/863

deploy/helm/opa-operator/templates/clusterrole-operator.yaml

@@ -63,12 +63,26 @@ rules:
       - bind
     resourceNames:
       - {{ include "operator.name" . }}-clusterrole
-  # DaemonSet created per role group. Applied via SSA, tracked for orphan cleanup, and
-  # owned by the controller.
+  # DaemonSet or Deployment created per role group, depending on the role's `workloadKind`.
+  # Applied via SSA, tracked for orphan cleanup, and owned by the controller. 
   - apiGroups:
       - apps
     resources:
       - daemonsets
+      - deployments
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - watch
+  # PodDisruptionBudget created per role, when the role has it enabled. Also needs `delete`, because
+  # disabling it (or switching to a DaemonSet) must clean the existing budget up.
+  - apiGroups:
+      - policy
+    resources:
+      - poddisruptionbudgets
     verbs:
       - create
       - delete

extra/crds.yaml

@@ -1490,10 +1490,54 @@ spec:
                     type: object
                     x-kubernetes-preserve-unknown-fields: true
                   roleConfig:
-                    default: {}
-                    description: |-
-                      This is a product-agnostic RoleConfig, with nothing in it. It is used e.g. by products that have
-                      nothing configurable at role level.
+                    default:
+                      podDisruptionBudget:
+                        enabled: null
+                        maxUnavailable: null
+                      workloadKind: DaemonSet
+                    description: Role-level configuration for the OPA servers.
+                    properties:
+                      podDisruptionBudget:
+                        default:
+                          enabled: null
+                          maxUnavailable: null
+                        description: |-
+                          This struct is used to configure:
+
+                          1. If PodDisruptionBudgets are created by the operator
+                          2. The allowed number of Pods to be unavailable (`maxUnavailable`)
+
+                          Documentation:
+                          [allowed Pod disruptions documentation](https://docs.stackable.tech/home/nightly/concepts/operations/pod_disruptions).
+                        properties:
+                          enabled:
+                            description: |-
+                              Whether a PodDisruptionBudget should be written out for this role.
+
+                              Defaults to `true` when `workloadKind` is `Deployment` and to `false` when it is
+                              `DaemonSet`, since a PodDisruptionBudget doesn't make sense for a DaemonSet.
+                            nullable: true
+                            type: boolean
+                          maxUnavailable:
+                            description: The number of Pods that are allowed to be down simultaneous.
+                            format: uint16
+                            maximum: 65535.0
+                            minimum: 0.0
+                            nullable: true
+                            type: integer
+                        type: object
+                      workloadKind:
+                        default: DaemonSet
+                        description: |-
+                          The Kubernetes workload the OPA servers run as.
+
+                          * `DaemonSet`: one Pod per node. `replicas` is ignored.
+
+                          * `Deployment`: fixed number of Pods, configured by `replicas`.
+                        enum:
+                        - DaemonSet
+                        - Deployment
+                        type: string
                     type: object
                   roleGroups:
                     additionalProperties:
@@ -3747,10 +3791,54 @@ spec:
                     type: object
                     x-kubernetes-preserve-unknown-fields: true
                   roleConfig:
-                    default: {}
-                    description: |-
-                      This is a product-agnostic RoleConfig, with nothing in it. It is used e.g. by products that have
-                      nothing configurable at role level.
+                    default:
+                      podDisruptionBudget:
+                        enabled: null
+                        maxUnavailable: null
+                      workloadKind: DaemonSet
+                    description: Role-level configuration for the OPA servers.
+                    properties:
+                      podDisruptionBudget:
+                        default:
+                          enabled: null
+                          maxUnavailable: null
+                        description: |-
+                          This struct is used to configure:
+
+                          1. If PodDisruptionBudgets are created by the operator
+                          2. The allowed number of Pods to be unavailable (`maxUnavailable`)
+
+                          Documentation:
+                          [allowed Pod disruptions documentation](https://docs.stackable.tech/home/nightly/concepts/operations/pod_disruptions).
+                        properties:
+                          enabled:
+                            description: |-
+                              Whether a PodDisruptionBudget should be written out for this role.
+
+                              Defaults to `true` when `workloadKind` is `Deployment` and to `false` when it is
+                              `DaemonSet`, since a PodDisruptionBudget doesn't make sense for a DaemonSet.
+                            nullable: true
+                            type: boolean
+                          maxUnavailable:
+                            description: The number of Pods that are allowed to be down simultaneous.
+                            format: uint16
+                            maximum: 65535.0
+                            minimum: 0.0
+                            nullable: true
+                            type: integer
+                        type: object
+                      workloadKind:
+                        default: DaemonSet
+                        description: |-
+                          The Kubernetes workload the OPA servers run as.
+
+                          * `DaemonSet`: one Pod per node. `replicas` is ignored.
+
+                          * `Deployment`: fixed number of Pods, configured by `replicas`.
+                        enum:
+                        - DaemonSet
+                        - Deployment
+                        type: string
                     type: object
                   roleGroups:
                     additionalProperties:

rust/operator-binary/src/controller/build.rs

@@ -4,6 +4,7 @@
 use std::str::FromStr;
 
 use snafu::{ResultExt, Snafu};
+use stackable_opa_operator::crd::v1alpha2;
 use stackable_operator::{
     builder::meta::ObjectMetaBuilder,
     utils::cluster_info::KubernetesClusterInfo,
@@ -14,13 +15,17 @@ use crate::controller::{
     KubernetesResources, RoleGroupName, ValidatedCluster,
     build::resource::{
         config_map::build_rolegroup_config_map,
-        daemonset::build_server_rolegroup_daemonset,
         discovery::build_discovery_config_map,
+        pdb::build_role_pod_disruption_budget,
         rbac::{build_role_binding, build_service_account},
         service::{
             build_rolegroup_headless_service, build_rolegroup_metrics_service,
             build_server_role_service,
         },
+        workload::{
+            daemonset::build_server_rolegroup_daemonset,
+            deployment::build_server_rolegroup_deployment,
+        },
     },
 };
 
@@ -37,7 +42,13 @@ pub enum Error {
 
     #[snafu(display("failed to build DaemonSet for role group {role_group}"))]
     DaemonSet {
-        source: resource::daemonset::Error,
+        source: resource::workload::Error,
+        role_group: RoleGroupName,
+    },
+
+    #[snafu(display("failed to build Deployment for role group {role_group}"))]
+    Deployment {
+        source: resource::workload::Error,
         role_group: RoleGroupName,
     },
 
@@ -62,13 +73,25 @@ pub fn build(
     cluster_info: &KubernetesClusterInfo,
 ) -> Result<KubernetesResources, Error> {
     let mut daemon_sets = vec![];
+    let mut deployments = vec![];
     let mut services = vec![];
     let mut config_maps = vec![];
+    let mut pod_disruption_budgets = vec![];
 
     // The role-level load-balanced Service, which is not bound to a single role group.
     services.push(build_server_role_service(cluster));
 
-    for role_group_configs in cluster.role_group_configs.values() {
+    // Iterating with the role key, because the workload kind is configured per role.
+    for (opa_role, role_group_configs) in &cluster.role_group_configs {
+        let role_config = cluster.role_config(opa_role);
+        let workload_kind = &role_config.workload_kind;
+
+        pod_disruption_budgets.extend(build_role_pod_disruption_budget(
+            cluster,
+            opa_role,
+            role_config,
+        ));
+
         for (role_group_name, role_group) in role_group_configs {
             config_maps.push(
                 build_rolegroup_config_map(cluster, role_group_name, role_group).context(
@@ -79,20 +102,37 @@ pub fn build(
             );
             services.push(build_rolegroup_headless_service(cluster, role_group_name));
             services.push(build_rolegroup_metrics_service(cluster, role_group_name));
-            daemon_sets.push(
-                build_server_rolegroup_daemonset(
-                    cluster,
-                    role_group_name,
-                    role_group,
-                    opa_bundle_builder_image,
-                    user_info_fetcher_image,
-                    resource_info_fetcher_image,
-                    cluster_info,
-                )
-                .context(DaemonSetSnafu {
-                    role_group: role_group_name.clone(),
-                })?,
-            );
+            // Exactly one workload object per role group, of the kind its role asks for.
+            match workload_kind {
+                v1alpha2::WorkloadKind::DaemonSet => daemon_sets.push(
+                    build_server_rolegroup_daemonset(
+                        cluster,
+                        role_group_name,
+                        role_group,
+                        opa_bundle_builder_image,
+                        user_info_fetcher_image,
+                        resource_info_fetcher_image,
+                        cluster_info,
+                    )
+                    .context(DaemonSetSnafu {
+                        role_group: role_group_name.clone(),
+                    })?,
+                ),
+                v1alpha2::WorkloadKind::Deployment => deployments.push(
+                    build_server_rolegroup_deployment(
+                        cluster,
+                        role_group_name,
+                        role_group,
+                        opa_bundle_builder_image,
+                        user_info_fetcher_image,
+                        resource_info_fetcher_image,
+                        cluster_info,
+                    )
+                    .context(DeploymentSnafu {
+                        role_group: role_group_name.clone(),
+                    })?,
+                ),
+            }
         }
     }
 
@@ -101,10 +141,12 @@ pub fn build(
 
     Ok(KubernetesResources {
         daemon_sets,
+        deployments,
         services,
         config_maps,
         service_accounts: vec![build_service_account(cluster)],
         role_bindings: vec![build_role_binding(cluster)],
+        pod_disruption_budgets,
     })
 }
 
@@ -185,11 +227,12 @@ mod tests {
         )
         .expect("build succeeds");
 
-        // One DaemonSet per role group.
+        // One DaemonSet per role group, as `workloadKind` defaults to `DaemonSet`.
         assert_eq!(
             sorted_names(&resources.daemon_sets),
             ["test-opa-server-default"]
         );
+        assert!(resources.deployments.is_empty());
         // The role-level Service plus a headless and a metrics Service per role group.
         assert_eq!(
             sorted_names(&resources.services),
@@ -213,5 +256,59 @@ mod tests {
             sorted_names(&resources.role_bindings),
             ["test-opa-rolebinding"]
         );
+        // The default `DaemonSet` gets no PodDisruptionBudget, so existing installations gain no
+        // new object on upgrade.
+        assert!(resources.pod_disruption_budgets.is_empty());
+    }
+
+    /// `workloadKind` decides which workload object a role group gets. Exactly one kind is built, so
+    /// the other list stays empty and `ClusterResources` sweeps the workload that is no longer
+    /// wanted when the administrator switches modes.
+    #[test]
+    fn build_dispatches_on_workload_kind() {
+        let build_with = |workload_kind| {
+            build(
+                &validated_cluster_from_spec(json!({
+                    "image": { "productVersion": "1.2.3" },
+                    "servers": {
+                        "roleConfig": { "workloadKind": workload_kind },
+                        "roleGroups": { "default": {} },
+                    },
+                })),
+                "bundle-builder-image",
+                "user-info-fetcher-image",
+                "resource-info-fetcher-image",
+                &cluster_info(),
+            )
+            .expect("build succeeds")
+        };
+
+        let daemon_set_mode = build_with("DaemonSet");
+        assert_eq!(
+            sorted_names(&daemon_set_mode.daemon_sets),
+            ["test-opa-server-default"]
+        );
+        assert!(daemon_set_mode.deployments.is_empty());
+
+        let deployment_mode = build_with("Deployment");
+        assert_eq!(
+            sorted_names(&deployment_mode.deployments),
+            ["test-opa-server-default"]
+        );
+        assert!(deployment_mode.daemon_sets.is_empty());
+
+        // One role-level PodDisruptionBudget for a Deployment, none for a DaemonSet whose Pods
+        // `kubectl drain` skips anyway.
+        assert!(daemon_set_mode.pod_disruption_budgets.is_empty());
+        assert_eq!(
+            sorted_names(&deployment_mode.pod_disruption_budgets),
+            ["test-opa-server"]
+        );
+
+        // Products consume the discovery ConfigMap, so it must not depend on the workload kind.
+        assert_eq!(
+            sorted_names(&daemon_set_mode.config_maps),
+            sorted_names(&deployment_mode.config_maps)
+        );
     }
 }

rust/operator-binary/src/controller/build/resource/mod.rs

@@ -2,7 +2,8 @@
 //! Kubernetes resources, one module per resource kind.
 
 pub mod config_map;
-pub mod daemonset;
 pub mod discovery;
+pub mod pdb;
 pub mod rbac;
 pub mod service;
+pub mod workload;