diff --git a/examples/49-control-plane-on-private-subnets.yaml b/examples/49-control-plane-on-private-subnets.yaml new file mode 100644 index 0000000000..09865c4d0a --- /dev/null +++ b/examples/49-control-plane-on-private-subnets.yaml @@ -0,0 +1,29 @@ +# An example config for restricting the EKS control plane's cross-account ENIs to private +# subnets at cluster creation time. +# To create the cluster, run `eksctl create cluster -f 49-control-plane-on-private-subnets.yaml` +# +# Public subnets are still created and used for the NAT gateways and for internet-facing load +# balancers; only the subnets passed to the EKS API are restricted. Requires at least two +# private subnets across at least two availability zones. + +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig +metadata: + name: cluster-49 + region: us-west-2 + +availabilityZones: + - us-west-2a + - us-west-2b + +vpc: + controlPlaneOnPrivateSubnets: true + nat: + gateway: HighlyAvailable + clusterEndpoints: + publicAccess: true + privateAccess: true + +managedNodeGroups: + - name: mng1 + privateNetworking: true diff --git a/pkg/apis/eksctl.io/v1alpha5/assets/schema.json b/pkg/apis/eksctl.io/v1alpha5/assets/schema.json index e3cd90d5ba..ec6d058020 100755 --- a/pkg/apis/eksctl.io/v1alpha5/assets/schema.json +++ b/pkg/apis/eksctl.io/v1alpha5/assets/schema.json @@ -1163,6 +1163,11 @@ "description": "controls how the control plane routes egress traffic. Valid values: \"AWS_MANAGED\" (default), \"CUSTOMER_ROUTED\"", "x-intellij-html-description": "controls how the control plane routes egress traffic. Valid values: "AWSMANAGED" (default), "CUSTOMERROUTED"" }, + "controlPlaneOnPrivateSubnets": { + "type": "boolean", + "description": "restricts the control plane (the cross-account ENIs that EKS places in the cluster subnets) to private subnets only, excluding public subnets. It applies both when eksctl creates the VPC and when a pre-existing VPC is used. Cannot be combined with ControlPlaneSubnetIDs. Requires at least two private subnets spanning at least two availability zones, which must have NAT or the relevant VPC endpoints for nodes to reach the API server.", + "x-intellij-html-description": "restricts the control plane (the cross-account ENIs that EKS places in the cluster subnets) to private subnets only, excluding public subnets. It applies both when eksctl creates the VPC and when a pre-existing VPC is used. Cannot be combined with ControlPlaneSubnetIDs. Requires at least two private subnets spanning at least two availability zones, which must have NAT or the relevant VPC endpoints for nodes to reach the API server." + }, "controlPlaneSecurityGroupIDs": { "items": { "type": "string" @@ -1259,6 +1264,7 @@ "clusterEndpoints", "publicAccessCIDRs", "controlPlaneSubnetIDs", + "controlPlaneOnPrivateSubnets", "controlPlaneSecurityGroupIDs", "controlPlaneEgressMode" ], diff --git a/pkg/apis/eksctl.io/v1alpha5/types.go b/pkg/apis/eksctl.io/v1alpha5/types.go index 83934de0b4..a5175b0d5e 100644 --- a/pkg/apis/eksctl.io/v1alpha5/types.go +++ b/pkg/apis/eksctl.io/v1alpha5/types.go @@ -927,6 +927,12 @@ func (c *ClusterConfig) IsControlPlaneOnOutposts() bool { return c.Outpost != nil && c.Outpost.ControlPlaneOutpostARN != "" } +// IsControlPlaneOnPrivateSubnets returns true if the control plane's cross-account ENIs +// should be restricted to private subnets only. +func (c *ClusterConfig) IsControlPlaneOnPrivateSubnets() bool { + return c.VPC != nil && IsEnabled(c.VPC.ControlPlaneOnPrivateSubnets) +} + // GetOutpost returns the Outpost info. func (c *ClusterConfig) GetOutpost() *Outpost { return c.Outpost diff --git a/pkg/apis/eksctl.io/v1alpha5/validation.go b/pkg/apis/eksctl.io/v1alpha5/validation.go index 070e7a2519..efb69ddf97 100644 --- a/pkg/apis/eksctl.io/v1alpha5/validation.go +++ b/pkg/apis/eksctl.io/v1alpha5/validation.go @@ -501,6 +501,10 @@ func (c *ClusterConfig) ValidateVPCConfig() error { return errors.New("only one of vpc.securityGroup and vpc.controlPlaneSecurityGroupIDs can be specified") } + if err := c.validateControlPlaneOnPrivateSubnets(); err != nil { + return err + } + if (c.VPC.IPv6Cidr != "" || c.VPC.IPv6Pool != "") && !c.IPv6Enabled() { return fmt.Errorf("Ipv6Cidr and Ipv6CidrPool are only supported when IPFamily is set to IPv6") } @@ -554,6 +558,67 @@ func (c *ClusterConfig) ValidateVPCConfig() error { return nil } +// validateControlPlaneOnPrivateSubnets validates vpc.controlPlaneOnPrivateSubnets against +// the rest of the VPC configuration. +func (c *ClusterConfig) validateControlPlaneOnPrivateSubnets() error { + if !IsEnabled(c.VPC.ControlPlaneOnPrivateSubnets) { + return nil + } + + if len(c.VPC.ControlPlaneSubnetIDs) > 0 { + return errors.New("only one of vpc.controlPlaneSubnetIDs and vpc.controlPlaneOnPrivateSubnets can be specified") + } + + // The control plane is already restricted to private subnets on Outposts, where a + // single subnet in a single zone is expected, so the checks below do not apply. + if c.IsControlPlaneOnOutposts() { + return nil + } + + // Subnets are nil when eksctl creates the VPC. Private subnets are then derived from + // availabilityZones by vpc.SetSubnets, which runs after validation and always covers + // every requested zone, so there is nothing to check yet. + if c.VPC.Subnets == nil { + return nil + } + + if numPrivate := len(c.VPC.Subnets.Private); numPrivate < MinRequiredSubnets { + return fmt.Errorf("vpc.controlPlaneOnPrivateSubnets requires at least %d private subnets, got %d", MinRequiredSubnets, numPrivate) + } + + if azs := distinctSubnetAZs(c.VPC.Subnets.Private); len(azs) < MinRequiredAvailabilityZones { + return fmt.Errorf("vpc.controlPlaneOnPrivateSubnets requires private subnets in at least %d availability zones, got %d (%v)", MinRequiredAvailabilityZones, len(azs), azs) + } + + return nil +} + +// distinctSubnetAZs returns the unique availability zones covered by the given subnets. +// A subnet's zone is taken from its AZ field, falling back to the mapping key, which is an +// AZ name in the common form. +// +// This is best-effort: subnets given only by ID have their zone resolved from EC2 later, so +// their real zone is unknown here and the mapping key is used instead. Such a configuration +// is allowed through and is rejected by the EKS API if the subnets turn out to share a zone. +// The check is deliberately permissive rather than risk rejecting a valid pre-existing VPC. +func distinctSubnetAZs(subnets AZSubnetMapping) []string { + seen := make(map[string]struct{}, len(subnets)) + azs := make([]string, 0, len(subnets)) + for key, spec := range subnets { + az := spec.AZ + if az == "" { + az = key + } + if _, ok := seen[az]; ok { + continue + } + seen[az] = struct{}{} + azs = append(azs, az) + } + slices.Sort(azs) + return azs +} + func (c *ClusterConfig) unsupportedVPCCNIAddonVersion() (bool, error) { for _, addon := range c.Addons { if addon.Name == VPCCNIAddon { diff --git a/pkg/apis/eksctl.io/v1alpha5/validation_test.go b/pkg/apis/eksctl.io/v1alpha5/validation_test.go index ce8a9501c7..5955636c37 100644 --- a/pkg/apis/eksctl.io/v1alpha5/validation_test.go +++ b/pkg/apis/eksctl.io/v1alpha5/validation_test.go @@ -1526,6 +1526,121 @@ var _ = Describe("ClusterConfig validation", func() { }) }) + Context("controlPlaneOnPrivateSubnets", func() { + privateSubnets := func(azs ...string) api.AZSubnetMapping { + m := api.NewAZSubnetMapping() + for i, az := range azs { + m.Set(fmt.Sprintf("subnet-alias-%d", i), api.AZSubnetSpec{ + ID: fmt.Sprintf("subnet-%d", i), + AZ: az, + }) + } + return m + } + + When("it is enabled and eksctl creates the VPC", func() { + It("does not reject the config, since subnets are derived from availabilityZones later", func() { + cfg.VPC.Subnets = nil + cfg.VPC.ControlPlaneOnPrivateSubnets = api.Enabled() + err = cfg.ValidateVPCConfig() + Expect(err).NotTo(HaveOccurred()) + }) + }) + + When("it is enabled with two private subnets across two AZs", func() { + It("does not return an error", func() { + cfg.VPC.Subnets = &api.ClusterSubnets{ + Private: privateSubnets("us-west-2a", "us-west-2b"), + } + cfg.VPC.ControlPlaneOnPrivateSubnets = api.Enabled() + err = cfg.ValidateVPCConfig() + Expect(err).NotTo(HaveOccurred()) + }) + }) + + When("it is enabled together with controlPlaneSubnetIDs", func() { + It("returns an error", func() { + cfg.VPC.ControlPlaneOnPrivateSubnets = api.Enabled() + cfg.VPC.ControlPlaneSubnetIDs = []string{"subnet-1234", "subnet-5678"} + err = cfg.ValidateVPCConfig() + Expect(err).To(MatchError("only one of vpc.controlPlaneSubnetIDs and vpc.controlPlaneOnPrivateSubnets can be specified")) + }) + }) + + When("it is enabled but the VPC has no private subnets", func() { + It("returns an error instead of silently using public subnets", func() { + cfg.VPC.Subnets = &api.ClusterSubnets{ + Public: privateSubnets("us-west-2a", "us-west-2b"), + } + cfg.VPC.ControlPlaneOnPrivateSubnets = api.Enabled() + err = cfg.ValidateVPCConfig() + Expect(err).To(MatchError("vpc.controlPlaneOnPrivateSubnets requires at least 2 private subnets, got 0")) + }) + }) + + When("it is enabled with only one private subnet", func() { + It("returns an error", func() { + cfg.VPC.Subnets = &api.ClusterSubnets{ + Private: privateSubnets("us-west-2a"), + } + cfg.VPC.ControlPlaneOnPrivateSubnets = api.Enabled() + err = cfg.ValidateVPCConfig() + Expect(err).To(MatchError("vpc.controlPlaneOnPrivateSubnets requires at least 2 private subnets, got 1")) + }) + }) + + When("it is enabled with two private subnets in the same AZ", func() { + It("returns an error, since EKS requires two availability zones", func() { + cfg.VPC.Subnets = &api.ClusterSubnets{ + Private: privateSubnets("us-west-2a", "us-west-2a"), + } + cfg.VPC.ControlPlaneOnPrivateSubnets = api.Enabled() + err = cfg.ValidateVPCConfig() + Expect(err).To(MatchError("vpc.controlPlaneOnPrivateSubnets requires private subnets in at least 2 availability zones, got 1 ([us-west-2a])")) + }) + }) + + When("private subnets are given only by ID", func() { + It("allows the config through, since their zones are resolved from EC2 later", func() { + subnets := api.NewAZSubnetMapping() + subnets.Set("alias-a", api.AZSubnetSpec{ID: "subnet-aaa"}) + subnets.Set("alias-b", api.AZSubnetSpec{ID: "subnet-bbb"}) + cfg.VPC.ID = "vpc-123" + cfg.VPC.Subnets = &api.ClusterSubnets{Private: subnets} + cfg.VPC.ControlPlaneOnPrivateSubnets = api.Enabled() + err = cfg.ValidateVPCConfig() + Expect(err).NotTo(HaveOccurred()) + }) + }) + + When("one private subnet is keyed by AZ and another repeats that AZ explicitly", func() { + It("returns an error", func() { + subnets := api.NewAZSubnetMapping() + subnets.Set("us-west-2a", api.AZSubnetSpec{ID: "subnet-aaa"}) + subnets.Set("alias-b", api.AZSubnetSpec{ID: "subnet-bbb", AZ: "us-west-2a"}) + cfg.VPC.ID = "vpc-123" + cfg.VPC.Subnets = &api.ClusterSubnets{Private: subnets} + cfg.VPC.ControlPlaneOnPrivateSubnets = api.Enabled() + err = cfg.ValidateVPCConfig() + Expect(err).To(MatchError("vpc.controlPlaneOnPrivateSubnets requires private subnets in at least 2 availability zones, got 1 ([us-west-2a])")) + }) + }) + + When("it is enabled on Outposts", func() { + It("does not enforce the multi-AZ requirement", func() { + cfg.VPC.Subnets = &api.ClusterSubnets{ + Private: privateSubnets("us-west-2a"), + } + cfg.VPC.ControlPlaneOnPrivateSubnets = api.Enabled() + cfg.Outpost = &api.Outpost{ + ControlPlaneOutpostARN: "arn:aws:outposts:us-west-2:1234:outpost/op-1234", + } + err = cfg.ValidateVPCConfig() + Expect(err).NotTo(HaveOccurred()) + }) + }) + }) + Context("ipv6 CIDRs", func() { When("IPv6Cidr or IPv6CidrPool is provided and ipv6 is not set", func() { It("returns an error", func() { diff --git a/pkg/apis/eksctl.io/v1alpha5/vpc.go b/pkg/apis/eksctl.io/v1alpha5/vpc.go index d22daf17ee..c9e6e7f3af 100644 --- a/pkg/apis/eksctl.io/v1alpha5/vpc.go +++ b/pkg/apis/eksctl.io/v1alpha5/vpc.go @@ -178,6 +178,14 @@ type ( // ControlPlaneSubnetIDs configures the subnets for the control plane. // +optional ControlPlaneSubnetIDs []string `json:"controlPlaneSubnetIDs,omitempty"` + // ControlPlaneOnPrivateSubnets restricts the control plane (the cross-account ENIs + // that EKS places in the cluster subnets) to private subnets only, excluding public + // subnets. It applies both when eksctl creates the VPC and when a pre-existing VPC + // is used. Cannot be combined with ControlPlaneSubnetIDs. Requires at least two + // private subnets spanning at least two availability zones, which must have NAT or + // the relevant VPC endpoints for nodes to reach the API server. + // +optional + ControlPlaneOnPrivateSubnets *bool `json:"controlPlaneOnPrivateSubnets,omitempty"` // ControlPlaneSecurityGroupIDs configures the security groups for the control plane. // +optional ControlPlaneSecurityGroupIDs []string `json:"controlPlaneSecurityGroupIDs,omitempty"` diff --git a/pkg/apis/eksctl.io/v1alpha5/zz_generated.deepcopy.go b/pkg/apis/eksctl.io/v1alpha5/zz_generated.deepcopy.go index d375bff55a..127202c2ee 100644 --- a/pkg/apis/eksctl.io/v1alpha5/zz_generated.deepcopy.go +++ b/pkg/apis/eksctl.io/v1alpha5/zz_generated.deepcopy.go @@ -1148,6 +1148,11 @@ func (in *ClusterVPC) DeepCopyInto(out *ClusterVPC) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.ControlPlaneOnPrivateSubnets != nil { + in, out := &in.ControlPlaneOnPrivateSubnets, &out.ControlPlaneOnPrivateSubnets + *out = new(bool) + **out = **in + } if in.ControlPlaneSecurityGroupIDs != nil { in, out := &in.ControlPlaneSecurityGroupIDs, &out.ControlPlaneSecurityGroupIDs *out = make([]string, len(*in)) diff --git a/pkg/cfn/builder/cluster_test.go b/pkg/cfn/builder/cluster_test.go index 82d904d546..1bfbe36f39 100644 --- a/pkg/cfn/builder/cluster_test.go +++ b/pkg/cfn/builder/cluster_test.go @@ -22,6 +22,7 @@ import ( "github.com/weaveworks/eksctl/pkg/cfn/builder" "github.com/weaveworks/eksctl/pkg/cfn/builder/fakes" "github.com/weaveworks/eksctl/pkg/testutils/mockprovider" + "github.com/weaveworks/eksctl/pkg/vpc" ) var _ = Describe("Cluster Template Builder", func() { @@ -100,6 +101,67 @@ var _ = Describe("Cluster Template Builder", func() { }) }) + Context("when VPC.ControlPlaneOnPrivateSubnets is true", func() { + BeforeEach(func() { + cfg.VPC.ControlPlaneOnPrivateSubnets = api.Enabled() + }) + + It("should add only the private subnets to the control plane's VPC config", func() { + subnetIDs := clusterTemplate.Resources["ControlPlane"].Properties.ResourcesVpcConfig.SubnetIDs + Expect(subnetIDs).To(ConsistOf( + map[string]interface{}{"Ref": "SubnetPrivateUSWEST2A"}, + map[string]interface{}{"Ref": "SubnetPrivateUSWEST2B"}, + )) + }) + + Context("and ControlPlaneSubnetIDs is also set", func() { + BeforeEach(func() { + cfg.VPC.ControlPlaneSubnetIDs = []string{"subnet-1234", "subnet-5678"} + }) + + It("should give ControlPlaneSubnetIDs precedence in the template", func() { + subnetIDs := clusterTemplate.Resources["ControlPlane"].Properties.ResourcesVpcConfig.SubnetIDs + Expect(subnetIDs).To(ConsistOf("subnet-1234", "subnet-5678")) + }) + }) + }) + + Context("when VPC.ControlPlaneOnPrivateSubnets is not set", func() { + It("should add both public and private subnets to the control plane's VPC config", func() { + subnetIDs := clusterTemplate.Resources["ControlPlane"].Properties.ResourcesVpcConfig.SubnetIDs + Expect(subnetIDs).To(ConsistOf( + map[string]interface{}{"Ref": "SubnetPublicUSWEST2A"}, + map[string]interface{}{"Ref": "SubnetPublicUSWEST2B"}, + map[string]interface{}{"Ref": "SubnetPrivateUSWEST2A"}, + map[string]interface{}{"Ref": "SubnetPrivateUSWEST2B"}, + )) + }) + }) + + Context("when subnets are derived from availabilityZones by vpc.SetSubnets", func() { + BeforeEach(func() { + // This is the primary path: the user supplies only availabilityZones and + // eksctl creates the VPC and subnets. + cfg.VPC = api.NewClusterVPC(false) + cfg.VPC.ClusterEndpoints = api.ClusterEndpointAccessDefaults() + cfg.VPC.ControlPlaneOnPrivateSubnets = api.Enabled() + Expect(vpc.SetSubnets(cfg.VPC, cfg.AvailabilityZones, nil)).To(Succeed()) + }) + + It("should add only the generated private subnets to the control plane's VPC config", func() { + Expect(addErr).NotTo(HaveOccurred()) + subnetIDs := clusterTemplate.Resources["ControlPlane"].Properties.ResourcesVpcConfig.SubnetIDs + Expect(subnetIDs).To(ConsistOf( + map[string]interface{}{"Ref": "SubnetPrivateUSWEST2A"}, + map[string]interface{}{"Ref": "SubnetPrivateUSWEST2B"}, + )) + + By("still creating the public subnets for NAT and load balancers") + Expect(clusterTemplate.Resources).To(HaveKey("SubnetPublicUSWEST2A")) + Expect(clusterTemplate.Resources).To(HaveKey("SubnetPublicUSWEST2B")) + }) + }) + Context("when control plane tier is set with SupportType", func() { BeforeEach(func() { cfg.ControlPlaneScalingConfig = &api.ControlPlaneScalingConfig{ diff --git a/pkg/cfn/builder/vpc_existing.go b/pkg/cfn/builder/vpc_existing.go index 4152508239..bda02b5c31 100644 --- a/pkg/cfn/builder/vpc_existing.go +++ b/pkg/cfn/builder/vpc_existing.go @@ -34,9 +34,8 @@ func NewExistingVPCResourceSet(rs *resourceSet, clusterConfig *api.ClusterConfig clusterConfig: clusterConfig, ec2API: ec2API, vpcID: gfnt.NewString(clusterConfig.VPC.ID), - subnetDetails: &SubnetDetails{ - controlPlaneOnOutposts: clusterConfig.IsControlPlaneOnOutposts(), - }, + // autoMode is not applied to a pre-existing VPC; see newSubnetDetails. + subnetDetails: newSubnetDetails(clusterConfig, false), } } diff --git a/pkg/cfn/builder/vpc_existing_test.go b/pkg/cfn/builder/vpc_existing_test.go index 412ac380f0..48b6330614 100644 --- a/pkg/cfn/builder/vpc_existing_test.go +++ b/pkg/cfn/builder/vpc_existing_test.go @@ -179,6 +179,50 @@ var _ = Describe("Existing VPC", func() { }) }) + Context("when vpc.controlPlaneOnPrivateSubnets is enabled", func() { + BeforeEach(func() { + cfg.VPC.ControlPlaneOnPrivateSubnets = api.Enabled() + }) + + It("restricts the control plane to the existing private subnets", func() { + Expect(addErr).NotTo(HaveOccurred()) + Expect(subnetDetails.ControlPlaneSubnetRefs()).To(ConsistOf( + gfnt.NewString(privateSubnet1), + gfnt.NewString(privateSubnet2), + )) + }) + }) + + Context("when vpc.controlPlaneOnPrivateSubnets is not set", func() { + It("uses both the existing public and private subnets for the control plane", func() { + Expect(addErr).NotTo(HaveOccurred()) + Expect(subnetDetails.ControlPlaneSubnetRefs()).To(ConsistOf( + gfnt.NewString(publicSubnet1), + gfnt.NewString(publicSubnet2), + gfnt.NewString(privateSubnet1), + gfnt.NewString(privateSubnet2), + )) + }) + }) + + Context("when Auto Mode is enabled", func() { + BeforeEach(func() { + cfg.AutoModeConfig = &api.AutoModeConfig{Enabled: api.Enabled()} + }) + + It("does not restrict the control plane to private subnets", func() { + // Auto Mode only restricts the control plane for VPCs that eksctl creates. + // Restricting a pre-existing VPC requires vpc.controlPlaneOnPrivateSubnets. + Expect(addErr).NotTo(HaveOccurred()) + Expect(subnetDetails.ControlPlaneSubnetRefs()).To(ConsistOf( + gfnt.NewString(publicSubnet1), + gfnt.NewString(publicSubnet2), + gfnt.NewString(privateSubnet1), + gfnt.NewString(privateSubnet2), + )) + }) + }) + Context("when ipv6 is true", func() { BeforeEach(func() { cfg.KubernetesNetworkConfig.IPFamily = api.IPV6Family diff --git a/pkg/cfn/builder/vpc_ipv4.go b/pkg/cfn/builder/vpc_ipv4.go index 1fd6202c79..0695c50b1b 100644 --- a/pkg/cfn/builder/vpc_ipv4.go +++ b/pkg/cfn/builder/vpc_ipv4.go @@ -49,20 +49,33 @@ type SubnetDetails struct { PrivateLocalZone []SubnetResource PublicLocalZone []SubnetResource - controlPlaneOnOutposts bool - autoMode bool + controlPlaneOnOutposts bool + controlPlaneOnPrivateSubnets bool + autoMode bool +} + +// newSubnetDetails returns a SubnetDetails with the control plane placement flags derived +// from clusterConfig. All construction sites must go through this to stay in sync as flags +// are added. +// +// autoMode is passed in rather than derived, because it only restricts the control plane to +// private subnets for VPCs that eksctl creates; the pre-existing VPC path has never applied +// it. Use vpc.controlPlaneOnPrivateSubnets to restrict a pre-existing VPC. +func newSubnetDetails(clusterConfig *api.ClusterConfig, autoMode bool) *SubnetDetails { + return &SubnetDetails{ + controlPlaneOnOutposts: clusterConfig.IsControlPlaneOnOutposts(), + controlPlaneOnPrivateSubnets: clusterConfig.IsControlPlaneOnPrivateSubnets(), + autoMode: autoMode, + } } // NewIPv4VPCResourceSet creates and returns a new VPCResourceSet func NewIPv4VPCResourceSet(rs *resourceSet, clusterConfig *api.ClusterConfig, ec2API awsapi.EC2, extendForOutposts bool) *IPv4VPCResourceSet { return &IPv4VPCResourceSet{ - rs: rs, - clusterConfig: clusterConfig, - ec2API: ec2API, - subnetDetails: &SubnetDetails{ - controlPlaneOnOutposts: clusterConfig.IsControlPlaneOnOutposts(), - autoMode: clusterConfig.IsAutoModeEnabled(), - }, + rs: rs, + clusterConfig: clusterConfig, + ec2API: ec2API, + subnetDetails: newSubnetDetails(clusterConfig, clusterConfig.IsAutoModeEnabled()), azToRTMap: make(map[string]*gfnt.Value), extendForOutposts: extendForOutposts, } @@ -157,7 +170,12 @@ func (s *SubnetDetails) ControlPlaneSubnetRefs() []*gfnt.Value { if s.controlPlaneOnOutposts && len(privateSubnetRefs) > 0 { return privateSubnetRefs } - if s.autoMode { + // Auto Mode and controlPlaneOnPrivateSubnets are independent settings that happen to + // require the same outcome; either one on its own restricts the control plane to + // private subnets. No length guard here: the subnets are validated by + // ValidateVPCConfig, and an empty set must surface as an error from the EKS API + // rather than silently falling back to public subnets. + if s.autoMode || s.controlPlaneOnPrivateSubnets { return privateSubnetRefs } return append(s.PublicSubnetRefs(), privateSubnetRefs...) diff --git a/pkg/cfn/builder/vpc_ipv4_test.go b/pkg/cfn/builder/vpc_ipv4_test.go index 9dff9ed43a..db64147501 100644 --- a/pkg/cfn/builder/vpc_ipv4_test.go +++ b/pkg/cfn/builder/vpc_ipv4_test.go @@ -443,6 +443,34 @@ var _ = Describe("VPC Template Builder", func() { Expect(refs).To(ContainElement(makePrimitive(privateSubnetRef2))) }) }) + + Describe("ControlPlaneSubnetRefs", func() { + It("returns both public and private subnet references by default", func() { + _, subnetDetails, err := vpcRs.CreateTemplate(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(subnetDetails.ControlPlaneSubnetRefs()).To(ConsistOf( + makePrimitive(publicSubnetRef1), + makePrimitive(publicSubnetRef2), + makePrimitive(privateSubnetRef1), + makePrimitive(privateSubnetRef2), + )) + }) + + Context("when vpc.controlPlaneOnPrivateSubnets is enabled", func() { + BeforeEach(func() { + cfg.VPC.ControlPlaneOnPrivateSubnets = api.Enabled() + }) + + It("returns only the private subnet references", func() { + _, subnetDetails, err := vpcRs.CreateTemplate(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(subnetDetails.ControlPlaneSubnetRefs()).To(ConsistOf( + makePrimitive(privateSubnetRef1), + makePrimitive(privateSubnetRef2), + )) + }) + }) + }) }) func makePrimitive(primitive string) *gfnt.Value { diff --git a/pkg/cfn/builder/vpc_ipv6.go b/pkg/cfn/builder/vpc_ipv6.go index 7ff9daa429..888d97fc81 100644 --- a/pkg/cfn/builder/vpc_ipv6.go +++ b/pkg/cfn/builder/vpc_ipv6.go @@ -75,9 +75,9 @@ func (v *IPv6VPCResourceSet) CreateTemplate(ctx context.Context) (*gfnt.Value, * addSubnetOutput(privateSubnetResourceRefs, v.clusterConfig.VPC.Subnets.Private, outputs.ClusterSubnetsPrivate) if v.clusterConfig.IsFullyPrivate() { - return vpcResourceRef, &SubnetDetails{ - Private: privateSubnets, - }, nil + subnetDetails := newSubnetDetails(v.clusterConfig, v.clusterConfig.IsAutoModeEnabled()) + subnetDetails.Private = privateSubnets + return vpcResourceRef, subnetDetails, nil } // add the rest of the public resources. @@ -153,11 +153,10 @@ func (v *IPv6VPCResourceSet) CreateTemplate(ctx context.Context) (*gfnt.Value, * } addSubnetOutput(publicSubnetResourceRefs, v.clusterConfig.VPC.Subnets.Public, outputs.ClusterSubnetsPublic) - return vpcResourceRef, &SubnetDetails{ - Private: privateSubnets, - Public: publicSubnets, - autoMode: v.clusterConfig.IsAutoModeEnabled(), - }, nil + subnetDetails := newSubnetDetails(v.clusterConfig, v.clusterConfig.IsAutoModeEnabled()) + subnetDetails.Private = privateSubnets + subnetDetails.Public = publicSubnets + return vpcResourceRef, subnetDetails, nil } func (v *IPv6VPCResourceSet) addIpv6CidrBlock() { diff --git a/pkg/cfn/builder/vpc_ipv6_test.go b/pkg/cfn/builder/vpc_ipv6_test.go index 9d2ca3a0b8..8ace330d2a 100644 --- a/pkg/cfn/builder/vpc_ipv6_test.go +++ b/pkg/cfn/builder/vpc_ipv6_test.go @@ -604,6 +604,31 @@ var _ = Describe("IPv6 VPC builder", func() { }) }) + Context("when vpc.controlPlaneOnPrivateSubnets is enabled", func() { + It("restricts the control plane to private subnets", func() { + cfg.VPC.ControlPlaneOnPrivateSubnets = api.Enabled() + vpcRs := builder.NewIPv6VPCResourceSet(builder.NewRS(), cfg, nil) + _, subnetDetails, err := vpcRs.CreateTemplate(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(subnetDetails.ControlPlaneSubnetRefs()).To(ConsistOf( + makePrimitive(builder.PrivateSubnetKey+azAFormatted), + makePrimitive(builder.PrivateSubnetKey+azBFormatted), + )) + }) + + It("is also honoured for a fully-private cluster", func() { + cfg.VPC.ControlPlaneOnPrivateSubnets = api.Enabled() + cfg.PrivateCluster = &api.PrivateCluster{Enabled: true} + vpcRs := builder.NewIPv6VPCResourceSet(builder.NewRS(), cfg, nil) + _, subnetDetails, err := vpcRs.CreateTemplate(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(subnetDetails.ControlPlaneSubnetRefs()).To(ConsistOf( + makePrimitive(builder.PrivateSubnetKey+azAFormatted), + makePrimitive(builder.PrivateSubnetKey+azBFormatted), + )) + }) + }) + Context("when there are 3 AZs", func() { BeforeEach(func() { cfg.AvailabilityZones = []string{azA, azB, azC} diff --git a/pkg/ctl/cmdutils/update_cluster_vpc.go b/pkg/ctl/cmdutils/update_cluster_vpc.go index a7f33b8c01..2e9cb36bd4 100644 --- a/pkg/ctl/cmdutils/update_cluster_vpc.go +++ b/pkg/ctl/cmdutils/update_cluster_vpc.go @@ -1,6 +1,7 @@ package cmdutils import ( + "errors" "fmt" "strings" @@ -85,6 +86,9 @@ func NewUpdateClusterVPCLoader(cmd *Cmd, options UpdateClusterVPCOptions) Cluste if l.ClusterConfig.VPC == nil { l.ClusterConfig.VPC = api.NewClusterVPC(false) } + if api.IsEnabled(l.ClusterConfig.VPC.ControlPlaneOnPrivateSubnets) { + return errors.New("vpc.controlPlaneOnPrivateSubnets is only supported when creating a cluster; to change the control plane subnets of an existing cluster, set vpc.controlPlaneSubnetIDs to the IDs of the private subnets") + } api.SetClusterEndpointAccessDefaults(l.ClusterConfig.VPC) return nil } diff --git a/pkg/ctl/utils/update_cluster_vpc_config_test.go b/pkg/ctl/utils/update_cluster_vpc_config_test.go index 7cad17b9cb..07987b5996 100644 --- a/pkg/ctl/utils/update_cluster_vpc_config_test.go +++ b/pkg/ctl/utils/update_cluster_vpc_config_test.go @@ -1,6 +1,9 @@ package utils_test import ( + "os" + "path/filepath" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -24,3 +27,37 @@ var _ = DescribeTable("invalid usage of update-cluster-vpc-config", func(e updat expectedErr: "at least one of these options must be specified: --private-access, --public-access, --public-access-cidrs, --control-plane-subnet-ids, --control-plane-security-group-ids", }), ) + +var _ = Describe("update-cluster-vpc-config with a config file", func() { + writeConfigFile := func(vpc string) string { + path := filepath.Join(GinkgoT().TempDir(), "config.yaml") + contents := `apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig +metadata: + name: test + region: us-west-2 +vpc: +` + vpc + Expect(os.WriteFile(path, []byte(contents), 0600)).To(Succeed()) + return path + } + + When("vpc.controlPlaneOnPrivateSubnets is set", func() { + It("returns an error instead of silently ignoring it", func() { + cmd := newMockCmd("update-cluster-vpc-config", "-f", writeConfigFile(" controlPlaneOnPrivateSubnets: true\n")) + _, err := cmd.execute() + Expect(err).To(MatchError(ContainSubstring("vpc.controlPlaneOnPrivateSubnets is only supported when creating a cluster"))) + }) + }) + + When("vpc.controlPlaneOnPrivateSubnets is explicitly false", func() { + It("does not return a validation error", func() { + cmd := newMockCmd("update-cluster-vpc-config", "-f", writeConfigFile(" controlPlaneOnPrivateSubnets: false\n controlPlaneSubnetIDs: [subnet-1234, subnet-5678]\n")) + _, err := cmd.execute() + // The command proceeds past validation and fails when reaching AWS. + if err != nil { + Expect(err.Error()).NotTo(ContainSubstring("controlPlaneOnPrivateSubnets")) + } + }) + }) +}) diff --git a/userdocs/src/usage/cluster-subnets-security-groups.md b/userdocs/src/usage/cluster-subnets-security-groups.md index 70f55d7d89..2f540ad129 100644 --- a/userdocs/src/usage/cluster-subnets-security-groups.md +++ b/userdocs/src/usage/cluster-subnets-security-groups.md @@ -31,6 +31,48 @@ eksctl utils update-cluster-vpc-config -f config.yaml Without the `--approve` flag, eksctl only logs the proposed changes. Once you are satisfied with the proposed changes, rerun the command with the `--approve` flag. +## Restricting the control plane to private subnets at creation time + +`eksctl utils update-cluster-vpc-config` changes the control plane subnets of an existing cluster, and it does so by +calling the EKS API directly rather than by updating the cluster's CloudFormation stack. That leaves the stack out of sync +with the actual configuration. + +To place the control plane on private subnets from the outset, set `vpc.controlPlaneOnPrivateSubnets` when creating the +cluster. Only the private subnets are then passed to the EKS API, so the cross-account ENIs are never created in public +subnets, and the CloudFormation stack reflects the intended configuration from the start: + +```yaml +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig +metadata: + name: cluster + region: us-west-2 + +availabilityZones: + - us-west-2a + - us-west-2b + +vpc: + controlPlaneOnPrivateSubnets: true +``` + +```console +eksctl create cluster -f config.yaml +``` + +Public subnets are still created and are still used for NAT gateways, internet-facing load balancers, and any nodegroup +that is not private, so this setting does not turn the cluster into a fully-private one. To create a cluster with no +public subnets at all, see [EKS Private Cluster without Outbound Internet Access](/usage/eks-private-cluster/). + +Note the following when using this setting: + +- At least two private subnets across at least two availability zones are required, because EKS places its cross-account + ENIs in a minimum of two zones. eksctl validates this before creating anything. +- The private subnets must have a NAT gateway or the relevant VPC endpoints, otherwise nodes cannot reach the API server. + When eksctl creates the VPC this is handled by the `vpc.nat` configuration. +- It cannot be combined with `vpc.controlPlaneSubnetIDs`; specify one or the other. +- It applies both when eksctl creates the VPC and when a pre-existing VPC is supplied via `vpc.id`. + ## Updating control plane security groups To manage traffic between the control plane and worker nodes, EKS supports passing additional security groups that are applied to the cross-account network interfaces provisioned by EKS. To update the security groups for the EKS control plane, run: