From 84042d461aa6b03e8d6247a7e70f4cea82219499 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Mon, 17 Aug 2026 21:39:28 -0400 Subject: [PATCH] feat(api): Redesign gateway CRDs around DSR/Maglev datapath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Full-NAT (DNAT+SNAT) makes the gateway node that picked a backend also rewrite and forward every reply, so it stays a stateful bottleneck: one node owns the connections it accepted, and the API had to track which node that was (PrimaryNode, EgressAddress/EgressSID) instead of just the LB's health. Direct Server Return removes the gateway from the reply path entirely — the backend answers the client directly. The schema changes below follow from that: - NetworkGatewayStatus: drop SRv6Address/EgressAddress/EgressSID. A DSR gateway node rewrites nothing and has no SNAT/masquerade address of its own to publish; the type is now a per-node marker + health object. - NetworkRuleStatus: drop PrimaryNode. Every NetworkGateway serves every accepted rule identically (anycast, via consistent hashing on the flow 5-tuple) — no single node owns backend selection anymore. - NetworkEgressPolicyStatus: drop AssignedGatewayNode. Tenant egress (backend -> internet) is a different traffic pattern from ingress and moves to its own sharded, stateful NAT66 tier (NAT66Shard, below) with its own placement ring, instead of pinning a tenant to one gateway node. - New ServiceVIPBinding CRD: the backend-node half of DSR — binds the VIP to a dummy interface for a veth/container backend, or registers a transparent tap-boundary address substitution for a VM backend, so the backend can answer on the VIP directly. - New NAT66Shard CRD: marks a node as a member of the sharded egress NAT66 tier and publishes a dedicated per-shard public address/uSID, so a reply is routed to the owning shard by ordinary unicast routing with no cross-shard hashing on the return path. - BGPVRFInstanceSpec: add optional NPTv6 (ULAPrefix/PublicPrefix), a stateless RFC 6296 mapping keyed by VRFID rather than by address, so two VRFs can share an identical ULAPrefix without collision. Breaking change, no back-compat shim — the dropped fields have no meaning under the new datapath. Co-Authored-By: Claude Sonnet 5 --- api/v1alpha1/egresspolicy_types.go | 50 ++-- api/v1alpha1/egresspolicy_types_test.go | 92 +----- api/v1alpha1/gateway_types.go | 81 ++---- api/v1alpha1/gateway_types_test.go | 28 +- api/v1alpha1/nat66shard_types.go | 89 ++++++ api/v1alpha1/rule_types.go | 30 +- api/v1alpha1/rule_types_test.go | 26 +- api/v1alpha1/vipbinding_types.go | 147 ++++++++++ api/v1alpha1/vrf_types.go | 35 +++ api/v1alpha1/zz_generated.deepcopy.go | 214 ++++++++++++++ config/crd/kustomization.yaml | 4 + ...network.datumapis.com_bgpvrfinstances.yaml | 41 +++ .../network.datumapis.com_nat66shards.yaml | 188 +++++++++++++ ...k.datumapis.com_networkegresspolicies.yaml | 52 ++-- ...network.datumapis.com_networkgateways.yaml | 93 ++----- .../network.datumapis.com_networkrules.yaml | 32 +-- ...work.datumapis.com_servicevipbindings.yaml | 241 ++++++++++++++++ docs/api/bgp.md | 194 +++++++++++++ docs/api/gateway.md | 263 +++++++++++++++--- 19 files changed, 1495 insertions(+), 405 deletions(-) create mode 100644 api/v1alpha1/nat66shard_types.go create mode 100644 api/v1alpha1/vipbinding_types.go create mode 100644 config/crd/network.datumapis.com_nat66shards.yaml create mode 100644 config/crd/network.datumapis.com_servicevipbindings.yaml diff --git a/api/v1alpha1/egresspolicy_types.go b/api/v1alpha1/egresspolicy_types.go index 32803f3..4cdc839 100644 --- a/api/v1alpha1/egresspolicy_types.go +++ b/api/v1alpha1/egresspolicy_types.go @@ -5,12 +5,12 @@ import ( ) // NetworkEgressPolicy enables internet egress for a single tenant -// VPC/VPCAttachment, served by the shared hyperconverged gateway engine's -// masquerade (SNAT/PAT) datapath. Unlike NetworkRule, it carries no -// VIP/backend/port: egress is on or off for a (vpcRef, vpcAttachmentRef) -// pair, existence-implies-enabled, not a per-flow rule — because the -// destination of an egress flow is an arbitrary internet address, not a -// pre-configured backend list. +// VPC/VPCAttachment, served by the sharded, stateful galactic-nat66 tier +// (see NAT66Shard). Unlike NetworkRule, it carries no VIP/backend/port: +// egress is on or off for a (vpcRef, vpcAttachmentRef) pair, +// existence-implies-enabled, not a per-flow rule — because the destination +// of an egress flow is an arbitrary internet address, not a pre-configured +// backend list. // // It is namespaced (deployed to galactic-system) and tenant-writable; like // NetworkRule, vpcRef/vpcAttachmentRef are opaque string identifiers because @@ -20,21 +20,22 @@ import ( // accepted — see the Accepted condition. // // Presence of an accepted NetworkEgressPolicy resolves only *enablement* -// (should this tenant reach the egress datapath at all) — a routing-layer -// decision (does the tenant's VRF have a default route toward the shared -// egress_sid locator), not a per-packet datapath lookup. *Isolation* -// (preventing two tenants with colliding ULA source addresses from -// colliding in the egress connection table) is a separate, datapath-level -// concern resolved by tagging each flow with the tenant/VRF identifier -// carried in the egress_sid locator's own Argument bits, not by anything in -// this spec. +// (should this tenant's VRF get a default route toward the shared NAT66 +// tier at all) — unlike this type's original design (superseded), there is +// no single "assigned gateway node" to compute or pin: any NAT66Shard may +// serve any tenant's flow, chosen by the shard-placement consistent-hash +// ring (internal/maglev, keyed on (tenant VRFID, backend, destination) — +// see NAT66Shard's doc comment), not by a per-tenant node assignment stored +// here. *Isolation* (preventing two tenants with colliding ULA source +// addresses from colliding in the egress connection table) is a separate, +// datapath-level concern resolved by tagging each flow with the VRFID +// carried in the tenant's own SRv6 Argument, not by anything in this spec. // // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Namespaced,shortName=netegress // +kubebuilder:printcolumn:name="VPC",type="string",JSONPath=".spec.vpcRef" // +kubebuilder:printcolumn:name="VPC-ATTACHMENT",type="string",JSONPath=".spec.vpcAttachmentRef" -// +kubebuilder:printcolumn:name="ASSIGNED-NODE",type="string",JSONPath=".status.assignedGatewayNode" // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" type NetworkEgressPolicy struct { metav1.TypeMeta `json:",inline"` @@ -71,25 +72,6 @@ type NetworkEgressPolicyStatus struct { // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` - // AssignedGatewayNode is the name of the NetworkGateway-backed gateway - // node this policy's tenant should route egress traffic through, - // mirroring NetworkRule's own status.primaryNode field and computed - // the same way: assigned_node = hash(vpcRef) % - // (design plan §4.5 — a tenant's egress node and its primary ingress - // node are the same node, by design, so both fields are computed by - // the identical AssignPrimaryNode function). The controller consuming - // this CRD sets this field exactly once, at creation. - // - // This value must never be silently recomputed by a reconciler once - // set, for the exact same reason NetworkRuleStatus.PrimaryNode's own - // doc comment gives: recomputing it on a later reconcile can flip - // which node a tenant's egress traffic routes through and cause an - // avoidable traffic flap; a reconciler that observes a stale or - // removed node here must surface that via a condition instead of - // overwriting the value. - // +optional - AssignedGatewayNode string `json:"assignedGatewayNode,omitempty"` - // Conditions contains the standard conditions for this resource, // including Accepted (see AcceptedReasonOwnershipVerified / // AcceptedReasonOwnershipDenied in rule_types.go, reused as-is here). diff --git a/api/v1alpha1/egresspolicy_types_test.go b/api/v1alpha1/egresspolicy_types_test.go index 5f45b69..6ae6857 100644 --- a/api/v1alpha1/egresspolicy_types_test.go +++ b/api/v1alpha1/egresspolicy_types_test.go @@ -128,73 +128,14 @@ func TestNetworkEgressPolicyFieldNames(t *testing.T) { } } -// TestNetworkGatewayEgressAddressFieldName verifies NetworkGatewayStatus's -// new EgressAddress field round-trips under the JSON key "egressAddress" -// and stays independently settable from SRv6Address (a gateway node may -// have one, both, or neither populated). -func TestNetworkGatewayEgressAddressFieldName(t *testing.T) { - gw := newTestGateway() - gw.Status.EgressAddress = "2001:db8:ffff::1" - - data, err := json.Marshal(gw) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - - var m map[string]any - if err := json.Unmarshal(data, &m); err != nil { - t.Fatalf("Unmarshal: %v", err) - } - - status, ok := m["status"].(map[string]any) - if !ok { - t.Fatalf("status not found or wrong type: %v", m["status"]) - } - if v, ok := status["egressAddress"]; !ok || v != "2001:db8:ffff::1" { - t.Errorf("expected status.egressAddress=%q, got %v", "2001:db8:ffff::1", status["egressAddress"]) - } - if v, ok := status["sRv6Address"]; !ok || v != "2001:db8:1::1" { - t.Errorf("expected status.sRv6Address unaffected, got %v", v) - } -} - -// TestNetworkGatewayEgressSIDFieldName verifies NetworkGatewayStatus's new -// EgressSID field round-trips under the JSON key "egressSID" (design plan -// §3.1/§4.3) and stays independently settable from SRv6Address/ -// EgressAddress. -func TestNetworkGatewayEgressSIDFieldName(t *testing.T) { - gw := newTestGateway() - gw.Status.EgressAddress = "2001:db8:ffff::1" - gw.Status.EgressSID = "2001:db8:eeee::1" - - data, err := json.Marshal(gw) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - - var m map[string]any - if err := json.Unmarshal(data, &m); err != nil { - t.Fatalf("Unmarshal: %v", err) - } - - status, ok := m["status"].(map[string]any) - if !ok { - t.Fatalf("status not found or wrong type: %v", m["status"]) - } - if v, ok := status["egressSID"]; !ok || v != "2001:db8:eeee::1" { - t.Errorf("expected status.egressSID=%q, got %v", "2001:db8:eeee::1", status["egressSID"]) - } - if v, ok := status["egressAddress"]; !ok || v != "2001:db8:ffff::1" { - t.Errorf("expected status.egressAddress unaffected, got %v", v) - } -} - -// TestNetworkEgressPolicyAssignedGatewayNodeFieldName verifies the new -// AssignedGatewayNode field round-trips under the JSON key -// "assignedGatewayNode" (design plan §4.5). -func TestNetworkEgressPolicyAssignedGatewayNodeFieldName(t *testing.T) { +// TestNetworkEgressPolicyStatusHasNoAssignedGatewayNode is a regression +// test: the earlier design pinned a policy to a single gateway node's +// masquerade datapath. The sharded galactic-nat66 tier has no such fixed +// assignment — any NAT66Shard may serve any tenant's flow, chosen by the +// shard-placement consistent-hash ring, not a per-tenant node stored here. +func TestNetworkEgressPolicyStatusHasNoAssignedGatewayNode(t *testing.T) { orig := newTestEgressPolicy() - orig.Status.AssignedGatewayNode = "gw-node-a" + orig.Status.Conditions = []metav1.Condition{{Type: ConditionTypeAccepted}} data, err := json.Marshal(orig) if err != nil { @@ -210,22 +151,7 @@ func TestNetworkEgressPolicyAssignedGatewayNodeFieldName(t *testing.T) { if !ok { t.Fatalf("status not found or wrong type: %v", m["status"]) } - if v, ok := status["assignedGatewayNode"]; !ok || v != "gw-node-a" { - t.Errorf("expected status.assignedGatewayNode=%q, got %v", "gw-node-a", status["assignedGatewayNode"]) - } -} - -// TestNetworkEgressPolicyDeepCopyIncludesAssignedGatewayNode extends -// TestNetworkEgressPolicyDeepCopy to cover the new field: mutating the copy -// must not affect the original. -func TestNetworkEgressPolicyDeepCopyIncludesAssignedGatewayNode(t *testing.T) { - orig := newTestEgressPolicy() - orig.Status.AssignedGatewayNode = "gw-node-a" - - dup := orig.DeepCopy() - dup.Status.AssignedGatewayNode = "gw-node-b" - - if orig.Status.AssignedGatewayNode != "gw-node-a" { - t.Errorf("AssignedGatewayNode mutated: got %q", orig.Status.AssignedGatewayNode) + if _, ok := status["assignedGatewayNode"]; ok { + t.Errorf("unexpected assignedGatewayNode field present in status: %v", status) } } diff --git a/api/v1alpha1/gateway_types.go b/api/v1alpha1/gateway_types.go index 4642ba7..fcb53b0 100644 --- a/api/v1alpha1/gateway_types.go +++ b/api/v1alpha1/gateway_types.go @@ -4,33 +4,29 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// NetworkGateway defines an XDP ingress NAT+LB gateway engine instance bound -// to a single dedicated gateway-role node. Exactly one NetworkGateway exists -// per gateway node (spec.targetRef.name is the Kubernetes node name), -// mirroring the BGPRouter node-scoped root object pattern. NetworkRule -// resources are assigned to a NetworkGateway via status.primaryNode. +// NetworkGateway marks a single dedicated gateway-role node as running the +// Maglev/DSR consistent-hash L4 load-balancer engine. Exactly one +// NetworkGateway exists per gateway node (spec.targetRef.name is the +// Kubernetes node name), mirroring the BGPRouter node-scoped root object +// pattern. NetworkRule resources are served by every NetworkGateway in the +// namespace equally (anycast — see NetworkRuleStatus's doc comment); this +// object's only job is to identify which nodes participate at all and +// surface each node's engine health via Conditions. // -// There is no tunnel overlay in this design (an earlier Geneve-based -// approach was superseded before this type shipped): the gateway's XDP -// program does Full-NAT (DNAT the VIP to a backend Pod's address, SNAT the -// client's source to status.sRv6Address) and pushes an SRv6 uSID outer -// header addressed to the backend's worker node directly, so return traffic -// (addressed to status.sRv6Address) arrives back at this same gateway node -// over the ordinary SRv6 fabric — no compute-node encap agent, no tunnel -// endpoint to publish. status.sRv6Address is advertised into BGP the same -// way any workload prefix is (a BGPAdvertisement naming it, /128, Argument -// 0 — the value PR #740 reserves and forbids registering into any tenant -// VRF, guaranteeing it never collides with a real tenant's Argument), so -// every other node learns a real kernel SEG6 route to it for free through -// the existing EVPN pipeline. +// This design does no address rewriting on the load-balancing path at all +// (DSR — Direct Server Return): the gateway's XDP program picks a backend +// via consistent hashing on the flow's 5-tuple and pushes an SRv6 uSID outer +// header addressed to the backend's worker node directly, untouched +// otherwise. The backend node answers the client directly (see +// ServiceVIPBinding) — reply traffic never re-enters this gateway node, so +// unlike the Full-NAT design this type originally described, a gateway node +// has no SNAT source address of its own to publish and nothing analogous to +// sRv6Address/egressAddress/egressSID belongs on this status anymore. // // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Namespaced,shortName=netgw // +kubebuilder:printcolumn:name="TARGET",type="string",JSONPath=".spec.targetRef.name" -// +kubebuilder:printcolumn:name="SRV6-ADDRESS",type="string",JSONPath=".status.sRv6Address" -// +kubebuilder:printcolumn:name="EGRESS-ADDRESS",type="string",JSONPath=".status.egressAddress" -// +kubebuilder:printcolumn:name="EGRESS-SID",type="string",JSONPath=".status.egressSID" // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" type NetworkGateway struct { metav1.TypeMeta `json:",inline"` @@ -53,49 +49,6 @@ type NetworkGatewayStatus struct { // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` - // SRv6Address is this gateway node's own SRv6-reachable IPv6 address, - // used as the Full-NAT SNAT source for every ingress flow this node - // translates. Backend Pods' replies are naturally routed back to it - // over the ordinary SRv6 fabric (the same mechanism that routes any - // other node's traffic), where this node's XDP program decapsulates - // and un-NATs them using its own conn_table — there is no separate - // tunnel endpoint or overlay device to publish. Populated by the - // engine once it has computed the address (a uFMT 48+16 uSID over this - // node's own BGPRouter locator/node-ID, at the reserved Argument 0) - // and advertised it into BGP. - // +optional - // +kubebuilder:validation:XValidation:rule="self == '' || isIP(self)",message="sRv6Address must be a valid IPv6 address" - SRv6Address string `json:"sRv6Address,omitempty"` - - // EgressAddress is this gateway node's own publicly-routable IPv6 - // address, used as the masquerade SNAT source for every egress flow - // this node translates on behalf of tenant VPC backends reaching the - // internet. Unlike SRv6Address (reachable only within the SRv6 fabric), - // this address must additionally be reachable from the public internet - // — an eBGP/uplink-peering concern outside this API. Operator-supplied - // via GALACTIC_GATEWAY_EGRESS_ADDRESS; there is no in-cluster - // derivation mechanism yet, the same gap SRv6Address itself has today. - // A gateway node not offering egress leaves this field empty. - // +optional - // +kubebuilder:validation:XValidation:rule="self == '' || isIP(self)",message="egressAddress must be a valid IPv6 address" - EgressAddress string `json:"egressAddress,omitempty"` - - // EgressSID is this gateway node's own egress_sid uSID *locator* - // (design plan §3.1) — the reserved Argument range's Block+Node-ID - // portion tenant VRF default routes encapsulate toward. Unlike - // EgressAddress (a plain, publicly-routable address, no uSID - // structure), this is a real uSID: other nodes need a kernel route to - // it before they can install a SEG6 encap route naming it as the - // destination (the same reason SRv6Address is advertised into BGP), - // so this is published and advertised the same way SRv6Address/ - // EgressAddress already are. Operator-supplied via - // GALACTIC_GATEWAY_EGRESS_SID; a gateway node not offering egress - // leaves this field empty, always paired with EgressAddress (both - // set, or neither). - // +optional - // +kubebuilder:validation:XValidation:rule="self == '' || isIP(self)",message="egressSID must be a valid IPv6 address" - EgressSID string `json:"egressSID,omitempty"` - // Conditions contains the standard conditions for this resource. // // +listType=map diff --git a/api/v1alpha1/gateway_types_test.go b/api/v1alpha1/gateway_types_test.go index 6c7624f..d99ab9e 100644 --- a/api/v1alpha1/gateway_types_test.go +++ b/api/v1alpha1/gateway_types_test.go @@ -17,9 +17,6 @@ func newTestGateway() *NetworkGateway { Spec: NetworkGatewaySpec{ TargetRef: TargetRef{Kind: "Node", Name: "gw-node-a"}, }, - Status: NetworkGatewayStatus{ - SRv6Address: "2001:db8:1::1", - }, } } @@ -30,15 +27,11 @@ func TestNetworkGatewayDeepCopy(t *testing.T) { dup := orig.DeepCopy() dup.Spec.TargetRef.Name = "gw-node-b" - dup.Status.SRv6Address = "2001:db8:1::2" dup.Status.Conditions = append(dup.Status.Conditions, metav1.Condition{Type: ConditionTypeReady}) if orig.Spec.TargetRef.Name != "gw-node-a" { t.Errorf("TargetRef.Name mutated: got %q", orig.Spec.TargetRef.Name) } - if orig.Status.SRv6Address != "2001:db8:1::1" { - t.Errorf("SRv6Address mutated: got %q", orig.Status.SRv6Address) - } if len(orig.Status.Conditions) != 0 { t.Errorf("Conditions mutated: got %v", orig.Status.Conditions) } @@ -73,9 +66,6 @@ func TestNetworkGatewayJSONRoundTrip(t *testing.T) { if got.Spec.TargetRef != orig.Spec.TargetRef { t.Errorf("TargetRef: got %+v, want %+v", got.Spec.TargetRef, orig.Spec.TargetRef) } - if got.Status.SRv6Address != orig.Status.SRv6Address { - t.Errorf("SRv6Address: got %q, want %q", got.Status.SRv6Address, orig.Status.SRv6Address) - } if len(got.Status.Conditions) != 1 { t.Fatalf("Conditions len: got %d, want 1", len(got.Status.Conditions)) } @@ -95,8 +85,10 @@ func TestNetworkGatewayListDeepCopy(t *testing.T) { } } -// TestNetworkGatewayFieldNames verifies the JSON keys for spec/status fields -// match the CRD schema. +// TestNetworkGatewayFieldNames verifies the JSON keys for spec fields match +// the CRD schema, and that no per-node SNAT-source fields survive from the +// superseded Full-NAT design — a DSR gateway node rewrites nothing, so it +// has no address of its own to publish here. func TestNetworkGatewayFieldNames(t *testing.T) { orig := newTestGateway() @@ -118,11 +110,11 @@ func TestNetworkGatewayFieldNames(t *testing.T) { t.Errorf("expected spec.targetRef field, got %v", spec) } - status, ok := m["status"].(map[string]any) - if !ok { - t.Fatalf("status not found or wrong type: %v", m["status"]) - } - if v, ok := status["sRv6Address"]; !ok || v != "2001:db8:1::1" { - t.Errorf("expected status.sRv6Address=%q, got %v", "2001:db8:1::1", status["sRv6Address"]) + if status, ok := m["status"].(map[string]any); ok { + for _, removed := range []string{"sRv6Address", "egressAddress", "egressSID"} { + if _, ok := status[removed]; ok { + t.Errorf("unexpected superseded field %q present in status: %v", removed, status) + } + } } } diff --git a/api/v1alpha1/nat66shard_types.go b/api/v1alpha1/nat66shard_types.go new file mode 100644 index 0000000..094bf3e --- /dev/null +++ b/api/v1alpha1/nat66shard_types.go @@ -0,0 +1,89 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NAT66Shard marks a single node as a member of the sharded, stateful NAT66 +// egress tier (galactic-nat66) — a component deliberately kept off the +// ingress load-balancer's own consistent-hash ring (see NetworkGateway): +// tenant egress traffic (backend -> arbitrary internet destination) is a +// different traffic pattern from ingress (fixed VIP, fixed backend pool) +// and needs its own placement ring, own per-flow state, and its own +// self-routing return path, entirely independent of any NetworkGateway node. +// +// Every shard owns a dedicated, BGP-advertised public IPv6 address +// (Status.ShardAddress) that a flow's allocated masquerade port lives +// within — so a reply is delivered to the correct shard by ordinary +// unicast SRv6/BGP routing alone, with no hashing or cross-shard lookup on +// the return path at all (the "any node can determine the owning shard from +// the tuple alone" property, satisfied by construction rather than by a +// replicated hash table). +// +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Namespaced,shortName=nat66shard +// +kubebuilder:printcolumn:name="TARGET",type="string",JSONPath=".spec.targetRef.name" +// +kubebuilder:printcolumn:name="SHARD-ADDRESS",type="string",JSONPath=".status.shardAddress" +// +kubebuilder:printcolumn:name="SHARD-SID",type="string",JSONPath=".status.shardSID" +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" +type NAT66Shard struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NAT66ShardSpec `json:"spec,omitempty"` + Status NAT66ShardStatus `json:"status,omitempty"` +} + +// NAT66ShardSpec defines the desired state of a NAT66Shard. +type NAT66ShardSpec struct { + // TargetRef identifies the Node this shard executes on. + // +kubebuilder:validation:Required + TargetRef TargetRef `json:"targetRef"` +} + +// NAT66ShardStatus defines the observed state of a NAT66Shard. +type NAT66ShardStatus struct { + // ObservedGeneration is the .metadata.generation this status was computed from. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // ShardAddress is this shard's own dedicated, publicly-routable IPv6 + // address — every masquerade port this shard allocates lives within it, + // so any node can route a reply to the correct shard using ordinary + // unicast routing on this address alone, with no per-flow state lookup + // anywhere but the owning shard itself. Operator-supplied per shard + // today (no in-cluster derivation mechanism yet — the same gap + // BGPRouter.Spec.SRv6Locator/NodeID assignment has today). + // +optional + // +kubebuilder:validation:XValidation:rule="self == '' || isIP(self)",message="shardAddress must be a valid IPv6 address" + ShardAddress string `json:"shardAddress,omitempty"` + + // ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike + // ShardAddress, a plain routable address), advertised into BGP the same + // way any other node-reachability route is (a /128 BGPAdvertisement, no + // VRFID/Function) so every other node learns a kernel SEG6 route toward + // it before installing a tenant VRF's default egress route against it. + // +optional + // +kubebuilder:validation:XValidation:rule="self == '' || isIP(self)",message="shardSID must be a valid IPv6 address" + ShardSID string `json:"shardSID,omitempty"` + + // Conditions contains the standard conditions for this resource. + // + // +listType=map + // +listMapKey=type + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// NAT66ShardList is a list of NAT66Shard resources. +// +kubebuilder:object:root=true +type NAT66ShardList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NAT66Shard `json:"items"` +} + +func init() { + SchemeBuilder.Register(&NAT66Shard{}, &NAT66ShardList{}) +} diff --git a/api/v1alpha1/rule_types.go b/api/v1alpha1/rule_types.go index 8c048bd..9b7743e 100644 --- a/api/v1alpha1/rule_types.go +++ b/api/v1alpha1/rule_types.go @@ -48,22 +48,30 @@ type NetworkRuleBackend struct { Port int32 `json:"port"` } -// NetworkRule defines ingress load-balancing and NAT for a single tenant -// VPC/VPCAttachment, served by the shared hyperconverged gateway engine. -// It is namespaced (deployed to galactic-system) and tenant-writable; the +// NetworkRule defines ingress load-balancing for a single tenant +// VPC/VPCAttachment, served by every NetworkGateway node identically +// (anycast Direct Server Return — see NetworkGateway's doc comment). It is +// namespaced (deployed to galactic-system) and tenant-writable; the // vpcRef/vpcAttachmentRef fields are opaque string identifiers because the // VPC API is owned by a separate companion operator, not this repo. An // admission webhook (implemented by the consuming controller) must verify // the requester is authorized for vpcRef/vpcAttachmentRef before a rule is // accepted — see the Accepted condition. // +// Unlike the earlier Full-NAT design this type originally described, there +// is no primary/secondary gateway node for a rule: every NetworkGateway +// advertises every accepted rule's vipAddresses at equal BGP preference, +// consistent-hashes the same backend list to the same backend for the same +// flow (internal/maglev), and forwards without rewriting anything — +// backend selection never needs a single "owning" node the way Full-NAT's +// SNAT-source model did. +// // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Namespaced,shortName=netrule // +kubebuilder:printcolumn:name="VPC",type="string",JSONPath=".spec.vpcRef" // +kubebuilder:printcolumn:name="PROTOCOL",type="string",JSONPath=".spec.protocol" // +kubebuilder:printcolumn:name="PORT",type="integer",JSONPath=".spec.port" -// +kubebuilder:printcolumn:name="PRIMARY-NODE",type="string",JSONPath=".status.primaryNode" // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" type NetworkRule struct { metav1.TypeMeta `json:",inline"` @@ -126,20 +134,6 @@ type NetworkRuleStatus struct { // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` - // PrimaryNode is the name of the NetworkGateway-backed gateway node - // assigned to advertise this rule's VIPAddresses at the preferred BGP - // local-preference, per the active-active model: primary_node = - // hash(vpcRef) % . The controller consuming this - // CRD sets this field exactly once, at creation. - // - // This value must never be silently recomputed by a reconciler once - // set. Recomputing it on a later reconcile can flip which node is - // primary for a live VIP and cause an avoidable traffic flap; a - // reconciler that observes a stale or removed node here must surface - // that via a condition instead of overwriting the value. - // +optional - PrimaryNode string `json:"primaryNode,omitempty"` - // Conditions contains the standard conditions for this resource. // // +listType=map diff --git a/api/v1alpha1/rule_types_test.go b/api/v1alpha1/rule_types_test.go index a48d2cc..d3a748f 100644 --- a/api/v1alpha1/rule_types_test.go +++ b/api/v1alpha1/rule_types_test.go @@ -24,9 +24,6 @@ func newTestRule() *NetworkRule { {Address: "fd00:10:1::1", Port: 8443}, }, }, - Status: NetworkRuleStatus{ - PrimaryNode: "gw-node-a", - }, } } @@ -38,7 +35,6 @@ func TestNetworkRuleDeepCopy(t *testing.T) { dup.Spec.VIPAddresses[0] = "2001:db8:1::20" dup.Spec.Backends[0].Address = "fd00:10:1::2" - dup.Status.PrimaryNode = "gw-node-b" if orig.Spec.VIPAddresses[0] != "2001:db8:1::10" { t.Errorf("VIPAddresses[0] mutated: got %q", orig.Spec.VIPAddresses[0]) @@ -46,9 +42,6 @@ func TestNetworkRuleDeepCopy(t *testing.T) { if orig.Spec.Backends[0].Address != "fd00:10:1::1" { t.Errorf("Backends[0].Address mutated: got %q", orig.Spec.Backends[0].Address) } - if orig.Status.PrimaryNode != "gw-node-a" { - t.Errorf("PrimaryNode mutated: got %q", orig.Status.PrimaryNode) - } } // TestNetworkRuleDeepCopyNil verifies DeepCopy on a nil pointer returns nil. @@ -87,9 +80,6 @@ func TestNetworkRuleJSONRoundTrip(t *testing.T) { if len(got.Spec.Backends) != 2 { t.Errorf("Backends len: got %d, want 2", len(got.Spec.Backends)) } - if got.Status.PrimaryNode != orig.Status.PrimaryNode { - t.Errorf("PrimaryNode: got %q, want %q", got.Status.PrimaryNode, orig.Status.PrimaryNode) - } if len(got.Status.Conditions) != 1 || got.Status.Conditions[0].Reason != AcceptedReasonOwnershipVerified { t.Errorf("Conditions: got %v", got.Status.Conditions) } @@ -102,16 +92,20 @@ func TestNetworkRuleListDeepCopy(t *testing.T) { Items: []NetworkRule{*newTestRule()}, } copied := list.DeepCopy() - copied.Items[0].Status.PrimaryNode = "gw-node-b" + copied.Items[0].Spec.VPCRef = "vpc-b" - if list.Items[0].Status.PrimaryNode != "gw-node-a" { + if list.Items[0].Spec.VPCRef != "vpc-a" { t.Errorf("original list item mutated via copy") } } -// TestNetworkRulePrimaryNodeFieldName verifies the JSON key is "primaryNode". -func TestNetworkRulePrimaryNodeFieldName(t *testing.T) { +// TestNetworkRuleStatusHasNoPrimaryNode is a regression test: the earlier +// active-passive Full-NAT design assigned a single primaryNode per rule. +// This design's anycast/DSR model has every NetworkGateway serve every rule +// identically, so no such field should exist any more. +func TestNetworkRuleStatusHasNoPrimaryNode(t *testing.T) { orig := newTestRule() + orig.Status.Conditions = []metav1.Condition{{Type: ConditionTypeAccepted}} data, err := json.Marshal(orig.Status) if err != nil { @@ -123,8 +117,8 @@ func TestNetworkRulePrimaryNodeFieldName(t *testing.T) { t.Fatalf("Unmarshal: %v", err) } - if v, ok := m["primaryNode"]; !ok || v != "gw-node-a" { - t.Errorf("unexpected primaryNode value: %v", m["primaryNode"]) + if _, ok := m["primaryNode"]; ok { + t.Errorf("unexpected primaryNode field present in status: %v", m) } } diff --git a/api/v1alpha1/vipbinding_types.go b/api/v1alpha1/vipbinding_types.go new file mode 100644 index 0000000..079ba75 --- /dev/null +++ b/api/v1alpha1/vipbinding_types.go @@ -0,0 +1,147 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ServiceVIPBinding drives one worker node's backend-side half of the +// DSR/Maglev load-balancer datapath: it tells the node which service VIP a +// specific local backend must be reachable on, so the backend can reply to +// clients directly (the "Direct Server Return" this design depends on — +// see NetworkGateway's doc comment). Written by the same controller that +// already resolves a NetworkRule's backends to worker nodes/SRv6 +// information (galactic-gateway's usidresolver.go), one object per +// (node, VIP, backend) triple; consumed by a per-node reconciler running +// inside galactic-router's tenant role. +// +// EgressKind decides which of two backend mechanisms this object drives, +// mirroring the same veth/tap fork the SRv6 uSID decap datapath already +// has (internal/plumbing/ebpf/usidmap's egress_kind field). Both mechanisms +// now converge on the same VIP-boundary substitution +// (BackendAddress:BackendPort for VIPAddress:Port at the SRv6 uSID TC-BPF +// boundary, usid_ingress's inbound half / usid_egress's outbound half) — +// required for both, not just tap, since a decapsulated ingress packet is +// delivered into the owning tenant's own VRF routing table, which has no +// route to an address bound outside that VRF (found live: see galactic's +// ServiceVIPBindingReconciler doc comment): +// +// - veth (container backend): the node ALSO binds VIPAddress on its own +// galactic-vip0 dummy interface (internal/plumbing/vip's +// Bind/Unbind/Verify in galactic) — this alone does not deliver +// anything to the backend pod (galactic-vip0 lives in the node's root +// namespace, not the tenant's VRF), but still lets the node itself +// verifiably answer on the VIP. +// - tap (VM backend): there is no guest-side configuration capability in +// this repo by design (internal/cnitap's own doc comment), so the +// substitution above is this kind's *only* delivery mechanism — the +// guest OS never needs to know the VIP exists at all. +// +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Namespaced,shortName=svcvip +// +kubebuilder:printcolumn:name="NODE",type="string",JSONPath=".spec.targetRef.name" +// +kubebuilder:printcolumn:name="VIP",type="string",JSONPath=".spec.vipAddress" +// +kubebuilder:printcolumn:name="BACKEND",type="string",JSONPath=".spec.backendAddress" +// +kubebuilder:printcolumn:name="KIND",type="string",JSONPath=".spec.egressKind" +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" +type ServiceVIPBinding struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ServiceVIPBindingSpec `json:"spec,omitempty"` + Status ServiceVIPBindingStatus `json:"status,omitempty"` +} + +// ConditionTypeBound indicates whether the backend is actually confirmed +// reachable on the VIP -- see ServiceVIPBindingStatus's doc comment. +const ConditionTypeBound string = "Bound" + +// ServiceVIPBindingEgressKind mirrors usidmap's EgressKindVeth/EgressKindTap +// constants at the API layer — see ServiceVIPBinding's doc comment. +// +// +kubebuilder:validation:Enum=veth;tap +type ServiceVIPBindingEgressKind string + +const ( + // ServiceVIPBindingEgressKindVeth selects the netns-bind mechanism. + ServiceVIPBindingEgressKindVeth ServiceVIPBindingEgressKind = "veth" + + // ServiceVIPBindingEgressKindTap selects the transparent tap-boundary + // translation mechanism. + ServiceVIPBindingEgressKindTap ServiceVIPBindingEgressKind = "tap" +) + +// ServiceVIPBindingSpec defines the desired VIP binding/translation state. +type ServiceVIPBindingSpec struct { + // TargetRef identifies the Node this binding applies to. + // +kubebuilder:validation:Required + TargetRef TargetRef `json:"targetRef"` + + // VIPAddress is the service VIP the backend must be reachable on. + // +kubebuilder:validation:Required + // +kubebuilder:validation:XValidation:rule="isIP(self)",message="vipAddress must be a valid IP address" + VIPAddress string `json:"vipAddress"` + + // Port is the VIP-facing port traffic arrives on. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + Port int32 `json:"port"` + + // Protocol is the transport protocol this binding applies to. + // +kubebuilder:validation:Required + Protocol NetworkRuleProtocol `json:"protocol"` + + // BackendAddress is the backend's own real address (its pod-netns + // address for a veth backend, or its actual guest-facing address for a + // tap backend) — the VIP-boundary substitution target for both kinds + // now (see EgressKind's own doc comment for why veth needs this too, + // not just tap). +optional at the API level for the same reason + // EgressKind itself carries no matching CEL requirement; the + // reconciler validates it's set for either kind before doing anything. + // +optional + // +kubebuilder:validation:XValidation:rule="self == '' || isIP(self)",message="backendAddress must be a valid IP address" + BackendAddress string `json:"backendAddress,omitempty"` + + // BackendPort is the backend's own real port, paired with + // BackendAddress — required for both kinds, see that field's doc + // comment. + // +optional + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + BackendPort int32 `json:"backendPort,omitempty"` + + // EgressKind selects which backend mechanism this binding drives. + // +kubebuilder:validation:Required + EgressKind ServiceVIPBindingEgressKind `json:"egressKind"` +} + +// ServiceVIPBindingStatus defines the observed state of a ServiceVIPBinding. +type ServiceVIPBindingStatus struct { + // ObservedGeneration is the .metadata.generation this status was computed from. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // Conditions contains the standard conditions for this resource, + // including Bound (set True once internal/plumbing/vip.Verify or the + // equivalent tap-translation-table check confirms the backend is + // actually reachable on VIPAddress, not merely that the bind/table-write + // call itself returned nil). + // + // +listType=map + // +listMapKey=type + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// ServiceVIPBindingList is a list of ServiceVIPBinding resources. +// +kubebuilder:object:root=true +type ServiceVIPBindingList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ServiceVIPBinding `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ServiceVIPBinding{}, &ServiceVIPBindingList{}) +} diff --git a/api/v1alpha1/vrf_types.go b/api/v1alpha1/vrf_types.go index 85731e3..3da8fa2 100644 --- a/api/v1alpha1/vrf_types.go +++ b/api/v1alpha1/vrf_types.go @@ -47,6 +47,41 @@ type BGPVRFInstanceSpec struct { // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=32 ExportRouteTargets []RouteTarget `json:"exportRouteTargets"` + + // NPTv6 configures stateless RFC 6296 Network Prefix Translation for + // this VRF: backends presenting an address within ULAPrefix are + // translated, checksum-neutrally and bidirectionally, to the + // corresponding address in PublicPrefix. Unset means this VRF's traffic + // crosses the SRv6 fabric with its own ULA source/destination + // unmodified — the common case. Keyed by this VRFID, never by address, + // specifically so two VRFs may configure the identical ULAPrefix (e.g. + // both using the same private range) without collision — each VRF's + // mapping is independent, looked up only after this VRF's own identity + // is already resolved from the SRv6 uSID Argument, never derived from + // address content alone. + // +optional + NPTv6 *NPTv6Spec `json:"nptv6,omitempty"` +} + +// NPTv6Spec is one VRF's stateless RFC 6296 prefix-translation mapping. +// ULAPrefix and PublicPrefix must share the same prefix length — RFC 6296 +// translation only ever rewrites the shared prefix, never the host bits. +// +// +kubebuilder:validation:XValidation:rule="self.ulaPrefix.split('/')[1] == self.publicPrefix.split('/')[1]",message="ulaPrefix and publicPrefix must share the same prefix length" +type NPTv6Spec struct { + // ULAPrefix is this VRF's tenant-facing IPv6 ULA prefix (e.g. + // "fd20:60::/64"), as presented by backends inside the VRF. + // +kubebuilder:validation:Required + // +kubebuilder:validation:XValidation:rule="isCIDR(self)",message="ulaPrefix must be a valid CIDR" + // +kubebuilder:validation:XValidation:rule="self.contains(':')",message="ulaPrefix must be an IPv6 CIDR" + ULAPrefix string `json:"ulaPrefix"` + + // PublicPrefix is the externally-routable IPv6 prefix ULAPrefix + // translates to/from, same prefix length as ULAPrefix. + // +kubebuilder:validation:Required + // +kubebuilder:validation:XValidation:rule="isCIDR(self)",message="publicPrefix must be a valid CIDR" + // +kubebuilder:validation:XValidation:rule="self.contains(':')",message="publicPrefix must be an IPv6 CIDR" + PublicPrefix string `json:"publicPrefix"` } // RouteTarget is a BGP extended community in "ASN:NN" or "IP:NN" format. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 402f5b3..2063b58 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1200,6 +1200,11 @@ func (in *BGPVRFInstanceSpec) DeepCopyInto(out *BGPVRFInstanceSpec) { *out = make([]RouteTarget, len(*in)) copy(*out, *in) } + if in.NPTv6 != nil { + in, out := &in.NPTv6, &out.NPTv6 + *out = new(NPTv6Spec) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BGPVRFInstanceSpec. @@ -1306,6 +1311,118 @@ func (in *LocalSecretRef) DeepCopy() *LocalSecretRef { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NAT66Shard) DeepCopyInto(out *NAT66Shard) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NAT66Shard. +func (in *NAT66Shard) DeepCopy() *NAT66Shard { + if in == nil { + return nil + } + out := new(NAT66Shard) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NAT66Shard) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NAT66ShardList) DeepCopyInto(out *NAT66ShardList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NAT66Shard, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NAT66ShardList. +func (in *NAT66ShardList) DeepCopy() *NAT66ShardList { + if in == nil { + return nil + } + out := new(NAT66ShardList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NAT66ShardList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NAT66ShardSpec) DeepCopyInto(out *NAT66ShardSpec) { + *out = *in + out.TargetRef = in.TargetRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NAT66ShardSpec. +func (in *NAT66ShardSpec) DeepCopy() *NAT66ShardSpec { + if in == nil { + return nil + } + out := new(NAT66ShardSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NAT66ShardStatus) DeepCopyInto(out *NAT66ShardStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NAT66ShardStatus. +func (in *NAT66ShardStatus) DeepCopy() *NAT66ShardStatus { + if in == nil { + return nil + } + out := new(NAT66ShardStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NPTv6Spec) DeepCopyInto(out *NPTv6Spec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NPTv6Spec. +func (in *NPTv6Spec) DeepCopy() *NPTv6Spec { + if in == nil { + return nil + } + out := new(NPTv6Spec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkEgressPolicy) DeepCopyInto(out *NetworkEgressPolicy) { *out = *in @@ -1781,6 +1898,103 @@ func (in *RouterTarget) DeepCopy() *RouterTarget { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceVIPBinding) DeepCopyInto(out *ServiceVIPBinding) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceVIPBinding. +func (in *ServiceVIPBinding) DeepCopy() *ServiceVIPBinding { + if in == nil { + return nil + } + out := new(ServiceVIPBinding) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ServiceVIPBinding) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceVIPBindingList) DeepCopyInto(out *ServiceVIPBindingList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ServiceVIPBinding, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceVIPBindingList. +func (in *ServiceVIPBindingList) DeepCopy() *ServiceVIPBindingList { + if in == nil { + return nil + } + out := new(ServiceVIPBindingList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ServiceVIPBindingList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceVIPBindingSpec) DeepCopyInto(out *ServiceVIPBindingSpec) { + *out = *in + out.TargetRef = in.TargetRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceVIPBindingSpec. +func (in *ServiceVIPBindingSpec) DeepCopy() *ServiceVIPBindingSpec { + if in == nil { + return nil + } + out := new(ServiceVIPBindingSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceVIPBindingStatus) DeepCopyInto(out *ServiceVIPBindingStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceVIPBindingStatus. +func (in *ServiceVIPBindingStatus) DeepCopy() *ServiceVIPBindingStatus { + if in == nil { + return nil + } + out := new(ServiceVIPBindingStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TargetRef) DeepCopyInto(out *TargetRef) { *out = *in diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 00a8e0b..3554f72 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -8,3 +8,7 @@ resources: - network.datumapis.com_bgprouters.yaml - network.datumapis.com_bgpvrfinstances.yaml - network.datumapis.com_networkegresspolicies.yaml + - network.datumapis.com_networkgateways.yaml + - network.datumapis.com_networkrules.yaml + - network.datumapis.com_servicevipbindings.yaml + - network.datumapis.com_nat66shards.yaml diff --git a/config/crd/network.datumapis.com_bgpvrfinstances.yaml b/config/crd/network.datumapis.com_bgpvrfinstances.yaml index 944ced7..dec5d35 100644 --- a/config/crd/network.datumapis.com_bgpvrfinstances.yaml +++ b/config/crd/network.datumapis.com_bgpvrfinstances.yaml @@ -98,6 +98,47 @@ spec: maxItems: 32 minItems: 1 type: array + nptv6: + description: |- + NPTv6 configures stateless RFC 6296 Network Prefix Translation for + this VRF: backends presenting an address within ULAPrefix are + translated, checksum-neutrally and bidirectionally, to the + corresponding address in PublicPrefix. Unset means this VRF's traffic + crosses the SRv6 fabric with its own ULA source/destination + unmodified — the common case. Keyed by this VRFID, never by address, + specifically so two VRFs may configure the identical ULAPrefix (e.g. + both using the same private range) without collision — each VRF's + mapping is independent, looked up only after this VRF's own identity + is already resolved from the SRv6 uSID Argument, never derived from + address content alone. + properties: + publicPrefix: + description: |- + PublicPrefix is the externally-routable IPv6 prefix ULAPrefix + translates to/from, same prefix length as ULAPrefix. + type: string + x-kubernetes-validations: + - message: publicPrefix must be a valid CIDR + rule: isCIDR(self) + - message: publicPrefix must be an IPv6 CIDR + rule: self.contains(':') + ulaPrefix: + description: |- + ULAPrefix is this VRF's tenant-facing IPv6 ULA prefix (e.g. + "fd20:60::/64"), as presented by backends inside the VRF. + type: string + x-kubernetes-validations: + - message: ulaPrefix must be a valid CIDR + rule: isCIDR(self) + - message: ulaPrefix must be an IPv6 CIDR + rule: self.contains(':') + required: + - publicPrefix + - ulaPrefix + type: object + x-kubernetes-validations: + - message: ulaPrefix and publicPrefix must share the same prefix length + rule: self.ulaPrefix.split('/')[1] == self.publicPrefix.split('/')[1] routerRef: description: |- RouterRef targets a single BGPRouter by name. diff --git a/config/crd/network.datumapis.com_nat66shards.yaml b/config/crd/network.datumapis.com_nat66shards.yaml new file mode 100644 index 0000000..e299012 --- /dev/null +++ b/config/crd/network.datumapis.com_nat66shards.yaml @@ -0,0 +1,188 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: nat66shards.network.datumapis.com +spec: + group: network.datumapis.com + names: + kind: NAT66Shard + listKind: NAT66ShardList + plural: nat66shards + shortNames: + - nat66shard + singular: nat66shard + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.targetRef.name + name: TARGET + type: string + - jsonPath: .status.shardAddress + name: SHARD-ADDRESS + type: string + - jsonPath: .status.shardSID + name: SHARD-SID + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + NAT66Shard marks a single node as a member of the sharded, stateful NAT66 + egress tier (galactic-nat66) — a component deliberately kept off the + ingress load-balancer's own consistent-hash ring (see NetworkGateway): + tenant egress traffic (backend -> arbitrary internet destination) is a + different traffic pattern from ingress (fixed VIP, fixed backend pool) + and needs its own placement ring, own per-flow state, and its own + self-routing return path, entirely independent of any NetworkGateway node. + + Every shard owns a dedicated, BGP-advertised public IPv6 address + (Status.ShardAddress) that a flow's allocated masquerade port lives + within — so a reply is delivered to the correct shard by ordinary + unicast SRv6/BGP routing alone, with no hashing or cross-shard lookup on + the return path at all (the "any node can determine the owning shard from + the tuple alone" property, satisfied by construction rather than by a + replicated hash table). + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NAT66ShardSpec defines the desired state of a NAT66Shard. + properties: + targetRef: + description: TargetRef identifies the Node this shard executes on. + properties: + kind: + description: Kind is the target resource kind (e.g. Node). + minLength: 1 + type: string + name: + description: Name is the name of the target resource. + minLength: 1 + type: string + required: + - kind + - name + type: object + required: + - targetRef + type: object + status: + description: NAT66ShardStatus defines the observed state of a NAT66Shard. + properties: + conditions: + description: Conditions contains the standard conditions for this + resource. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + description: ObservedGeneration is the .metadata.generation this status + was computed from. + format: int64 + type: integer + shardAddress: + description: |- + ShardAddress is this shard's own dedicated, publicly-routable IPv6 + address — every masquerade port this shard allocates lives within it, + so any node can route a reply to the correct shard using ordinary + unicast routing on this address alone, with no per-flow state lookup + anywhere but the owning shard itself. Operator-supplied per shard + today (no in-cluster derivation mechanism yet — the same gap + BGPRouter.Spec.SRv6Locator/NodeID assignment has today). + type: string + x-kubernetes-validations: + - message: shardAddress must be a valid IPv6 address + rule: self == '' || isIP(self) + shardSID: + description: |- + ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike + ShardAddress, a plain routable address), advertised into BGP the same + way any other node-reachability route is (a /128 BGPAdvertisement, no + VRFID/Function) so every other node learns a kernel SEG6 route toward + it before installing a tenant VRF's default egress route against it. + type: string + x-kubernetes-validations: + - message: shardSID must be a valid IPv6 address + rule: self == '' || isIP(self) + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/network.datumapis.com_networkegresspolicies.yaml b/config/crd/network.datumapis.com_networkegresspolicies.yaml index dffef23..2ce5517 100644 --- a/config/crd/network.datumapis.com_networkegresspolicies.yaml +++ b/config/crd/network.datumapis.com_networkegresspolicies.yaml @@ -23,9 +23,6 @@ spec: - jsonPath: .spec.vpcAttachmentRef name: VPC-ATTACHMENT type: string - - jsonPath: .status.assignedGatewayNode - name: ASSIGNED-NODE - type: string - jsonPath: .metadata.creationTimestamp name: AGE type: date @@ -34,12 +31,12 @@ spec: openAPIV3Schema: description: |- NetworkEgressPolicy enables internet egress for a single tenant - VPC/VPCAttachment, served by the shared hyperconverged gateway engine's - masquerade (SNAT/PAT) datapath. Unlike NetworkRule, it carries no - VIP/backend/port: egress is on or off for a (vpcRef, vpcAttachmentRef) - pair, existence-implies-enabled, not a per-flow rule — because the - destination of an egress flow is an arbitrary internet address, not a - pre-configured backend list. + VPC/VPCAttachment, served by the sharded, stateful galactic-nat66 tier + (see NAT66Shard). Unlike NetworkRule, it carries no VIP/backend/port: + egress is on or off for a (vpcRef, vpcAttachmentRef) pair, + existence-implies-enabled, not a per-flow rule — because the destination + of an egress flow is an arbitrary internet address, not a pre-configured + backend list. It is namespaced (deployed to galactic-system) and tenant-writable; like NetworkRule, vpcRef/vpcAttachmentRef are opaque string identifiers because @@ -49,14 +46,16 @@ spec: accepted — see the Accepted condition. Presence of an accepted NetworkEgressPolicy resolves only *enablement* - (should this tenant reach the egress datapath at all) — a routing-layer - decision (does the tenant's VRF have a default route toward the shared - egress_sid locator), not a per-packet datapath lookup. *Isolation* - (preventing two tenants with colliding ULA source addresses from - colliding in the egress connection table) is a separate, datapath-level - concern resolved by tagging each flow with the tenant/VRF identifier - carried in the egress_sid locator's own Argument bits, not by anything in - this spec. + (should this tenant's VRF get a default route toward the shared NAT66 + tier at all) — unlike this type's original design (superseded), there is + no single "assigned gateway node" to compute or pin: any NAT66Shard may + serve any tenant's flow, chosen by the shard-placement consistent-hash + ring (internal/maglev, keyed on (tenant VRFID, backend, destination) — + see NAT66Shard's doc comment), not by a per-tenant node assignment stored + here. *Isolation* (preventing two tenants with colliding ULA source + addresses from colliding in the egress connection table) is a separate, + datapath-level concern resolved by tagging each flow with the VRFID + carried in the tenant's own SRv6 Argument, not by anything in this spec. properties: apiVersion: description: |- @@ -105,25 +104,6 @@ spec: NetworkEgressPolicyStatus defines the observed state of a NetworkEgressPolicy. properties: - assignedGatewayNode: - description: |- - AssignedGatewayNode is the name of the NetworkGateway-backed gateway - node this policy's tenant should route egress traffic through, - mirroring NetworkRule's own status.primaryNode field and computed - the same way: assigned_node = hash(vpcRef) % - (design plan §4.5 — a tenant's egress node and its primary ingress - node are the same node, by design, so both fields are computed by - the identical AssignPrimaryNode function). The controller consuming - this CRD sets this field exactly once, at creation. - - This value must never be silently recomputed by a reconciler once - set, for the exact same reason NetworkRuleStatus.PrimaryNode's own - doc comment gives: recomputing it on a later reconcile can flip - which node a tenant's egress traffic routes through and cause an - avoidable traffic flap; a reconciler that observes a stale or - removed node here must surface that via a condition instead of - overwriting the value. - type: string conditions: description: |- Conditions contains the standard conditions for this resource, diff --git a/config/crd/network.datumapis.com_networkgateways.yaml b/config/crd/network.datumapis.com_networkgateways.yaml index 87dedc8..4d5515f 100644 --- a/config/crd/network.datumapis.com_networkgateways.yaml +++ b/config/crd/network.datumapis.com_networkgateways.yaml @@ -20,15 +20,6 @@ spec: - jsonPath: .spec.targetRef.name name: TARGET type: string - - jsonPath: .status.sRv6Address - name: SRV6-ADDRESS - type: string - - jsonPath: .status.egressAddress - name: EGRESS-ADDRESS - type: string - - jsonPath: .status.egressSID - name: EGRESS-SID - type: string - jsonPath: .metadata.creationTimestamp name: AGE type: date @@ -36,25 +27,24 @@ spec: schema: openAPIV3Schema: description: |- - NetworkGateway defines an XDP ingress NAT+LB gateway engine instance bound - to a single dedicated gateway-role node. Exactly one NetworkGateway exists - per gateway node (spec.targetRef.name is the Kubernetes node name), - mirroring the BGPRouter node-scoped root object pattern. NetworkRule - resources are assigned to a NetworkGateway via status.primaryNode. + NetworkGateway marks a single dedicated gateway-role node as running the + Maglev/DSR consistent-hash L4 load-balancer engine. Exactly one + NetworkGateway exists per gateway node (spec.targetRef.name is the + Kubernetes node name), mirroring the BGPRouter node-scoped root object + pattern. NetworkRule resources are served by every NetworkGateway in the + namespace equally (anycast — see NetworkRuleStatus's doc comment); this + object's only job is to identify which nodes participate at all and + surface each node's engine health via Conditions. - There is no tunnel overlay in this design (an earlier Geneve-based - approach was superseded before this type shipped): the gateway's XDP - program does Full-NAT (DNAT the VIP to a backend Pod's address, SNAT the - client's source to status.sRv6Address) and pushes an SRv6 uSID outer - header addressed to the backend's worker node directly, so return traffic - (addressed to status.sRv6Address) arrives back at this same gateway node - over the ordinary SRv6 fabric — no compute-node encap agent, no tunnel - endpoint to publish. status.sRv6Address is advertised into BGP the same - way any workload prefix is (a BGPAdvertisement naming it, /128, Argument - 0 — the value PR #740 reserves and forbids registering into any tenant - VRF, guaranteeing it never collides with a real tenant's Argument), so - every other node learns a real kernel SEG6 route to it for free through - the existing EVPN pipeline. + This design does no address rewriting on the load-balancing path at all + (DSR — Direct Server Return): the gateway's XDP program picks a backend + via consistent hashing on the flow's 5-tuple and pushes an SRv6 uSID outer + header addressed to the backend's worker node directly, untouched + otherwise. The backend node answers the client directly (see + ServiceVIPBinding) — reply traffic never re-enters this gateway node, so + unlike the Full-NAT design this type originally described, a gateway node + has no SNAT source address of its own to publish and nothing analogous to + sRv6Address/egressAddress/egressSID belongs on this status anymore. properties: apiVersion: description: |- @@ -159,60 +149,11 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map - egressAddress: - description: |- - EgressAddress is this gateway node's own publicly-routable IPv6 - address, used as the masquerade SNAT source for every egress flow - this node translates on behalf of tenant VPC backends reaching the - internet. Unlike SRv6Address (reachable only within the SRv6 fabric), - this address must additionally be reachable from the public internet - — an eBGP/uplink-peering concern outside this API. Operator-supplied - via GALACTIC_GATEWAY_EGRESS_ADDRESS; there is no in-cluster - derivation mechanism yet, the same gap SRv6Address itself has today. - A gateway node not offering egress leaves this field empty. - type: string - x-kubernetes-validations: - - message: egressAddress must be a valid IPv6 address - rule: self == '' || isIP(self) - egressSID: - description: |- - EgressSID is this gateway node's own egress_sid uSID *locator* - (design plan §3.1) — the reserved Argument range's Block+Node-ID - portion tenant VRF default routes encapsulate toward. Unlike - EgressAddress (a plain, publicly-routable address, no uSID - structure), this is a real uSID: other nodes need a kernel route to - it before they can install a SEG6 encap route naming it as the - destination (the same reason SRv6Address is advertised into BGP), - so this is published and advertised the same way SRv6Address/ - EgressAddress already are. Operator-supplied via - GALACTIC_GATEWAY_EGRESS_SID; a gateway node not offering egress - leaves this field empty, always paired with EgressAddress (both - set, or neither). - type: string - x-kubernetes-validations: - - message: egressSID must be a valid IPv6 address - rule: self == '' || isIP(self) observedGeneration: description: ObservedGeneration is the .metadata.generation this status was computed from. format: int64 type: integer - sRv6Address: - description: |- - SRv6Address is this gateway node's own SRv6-reachable IPv6 address, - used as the Full-NAT SNAT source for every ingress flow this node - translates. Backend Pods' replies are naturally routed back to it - over the ordinary SRv6 fabric (the same mechanism that routes any - other node's traffic), where this node's XDP program decapsulates - and un-NATs them using its own conn_table — there is no separate - tunnel endpoint or overlay device to publish. Populated by the - engine once it has computed the address (a uFMT 48+16 uSID over this - node's own BGPRouter locator/node-ID, at the reserved Argument 0) - and advertised it into BGP. - type: string - x-kubernetes-validations: - - message: sRv6Address must be a valid IPv6 address - rule: self == '' || isIP(self) type: object type: object served: true diff --git a/config/crd/network.datumapis.com_networkrules.yaml b/config/crd/network.datumapis.com_networkrules.yaml index e302fee..8b82b3e 100644 --- a/config/crd/network.datumapis.com_networkrules.yaml +++ b/config/crd/network.datumapis.com_networkrules.yaml @@ -26,9 +26,6 @@ spec: - jsonPath: .spec.port name: PORT type: integer - - jsonPath: .status.primaryNode - name: PRIMARY-NODE - type: string - jsonPath: .metadata.creationTimestamp name: AGE type: date @@ -36,14 +33,23 @@ spec: schema: openAPIV3Schema: description: |- - NetworkRule defines ingress load-balancing and NAT for a single tenant - VPC/VPCAttachment, served by the shared hyperconverged gateway engine. - It is namespaced (deployed to galactic-system) and tenant-writable; the + NetworkRule defines ingress load-balancing for a single tenant + VPC/VPCAttachment, served by every NetworkGateway node identically + (anycast Direct Server Return — see NetworkGateway's doc comment). It is + namespaced (deployed to galactic-system) and tenant-writable; the vpcRef/vpcAttachmentRef fields are opaque string identifiers because the VPC API is owned by a separate companion operator, not this repo. An admission webhook (implemented by the consuming controller) must verify the requester is authorized for vpcRef/vpcAttachmentRef before a rule is accepted — see the Accepted condition. + + Unlike the earlier Full-NAT design this type originally described, there + is no primary/secondary gateway node for a rule: every NetworkGateway + advertises every accepted rule's vipAddresses at equal BGP preference, + consistent-hashes the same backend list to the same backend for the same + flow (internal/maglev), and forwards without rewriting anything — + backend selection never needs a single "owning" node the way Full-NAT's + SNAT-source model did. properties: apiVersion: description: |- @@ -216,20 +222,6 @@ spec: was computed from. format: int64 type: integer - primaryNode: - description: |- - PrimaryNode is the name of the NetworkGateway-backed gateway node - assigned to advertise this rule's VIPAddresses at the preferred BGP - local-preference, per the active-active model: primary_node = - hash(vpcRef) % . The controller consuming this - CRD sets this field exactly once, at creation. - - This value must never be silently recomputed by a reconciler once - set. Recomputing it on a later reconcile can flip which node is - primary for a live VIP and cause an avoidable traffic flap; a - reconciler that observes a stale or removed node here must surface - that via a condition instead of overwriting the value. - type: string type: object type: object served: true diff --git a/config/crd/network.datumapis.com_servicevipbindings.yaml b/config/crd/network.datumapis.com_servicevipbindings.yaml new file mode 100644 index 0000000..3ca467e --- /dev/null +++ b/config/crd/network.datumapis.com_servicevipbindings.yaml @@ -0,0 +1,241 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: servicevipbindings.network.datumapis.com +spec: + group: network.datumapis.com + names: + kind: ServiceVIPBinding + listKind: ServiceVIPBindingList + plural: servicevipbindings + shortNames: + - svcvip + singular: servicevipbinding + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.targetRef.name + name: NODE + type: string + - jsonPath: .spec.vipAddress + name: VIP + type: string + - jsonPath: .spec.backendAddress + name: BACKEND + type: string + - jsonPath: .spec.egressKind + name: KIND + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + ServiceVIPBinding drives one worker node's backend-side half of the + DSR/Maglev load-balancer datapath: it tells the node which service VIP a + specific local backend must be reachable on, so the backend can reply to + clients directly (the "Direct Server Return" this design depends on — + see NetworkGateway's doc comment). Written by the same controller that + already resolves a NetworkRule's backends to worker nodes/SRv6 + information (galactic-gateway's usidresolver.go), one object per + (node, VIP, backend) triple; consumed by a per-node reconciler running + inside galactic-router's tenant role. + + EgressKind decides which of two backend mechanisms this object drives, + mirroring the same veth/tap fork the SRv6 uSID decap datapath already + has (internal/plumbing/ebpf/usidmap's egress_kind field). Both mechanisms + now converge on the same VIP-boundary substitution + (BackendAddress:BackendPort for VIPAddress:Port at the SRv6 uSID TC-BPF + boundary, usid_ingress's inbound half / usid_egress's outbound half) — + required for both, not just tap, since a decapsulated ingress packet is + delivered into the owning tenant's own VRF routing table, which has no + route to an address bound outside that VRF (found live: see galactic's + ServiceVIPBindingReconciler doc comment): + + - veth (container backend): the node ALSO binds VIPAddress on its own + galactic-vip0 dummy interface (internal/plumbing/vip's + Bind/Unbind/Verify in galactic) — this alone does not deliver + anything to the backend pod (galactic-vip0 lives in the node's root + namespace, not the tenant's VRF), but still lets the node itself + verifiably answer on the VIP. + - tap (VM backend): there is no guest-side configuration capability in + this repo by design (internal/cnitap's own doc comment), so the + substitution above is this kind's *only* delivery mechanism — the + guest OS never needs to know the VIP exists at all. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ServiceVIPBindingSpec defines the desired VIP binding/translation + state. + properties: + backendAddress: + description: |- + BackendAddress is the backend's own real address (its pod-netns + address for a veth backend, or its actual guest-facing address for a + tap backend) — the VIP-boundary substitution target for both kinds + now (see EgressKind's own doc comment for why veth needs this too, + not just tap). +optional at the API level for the same reason + EgressKind itself carries no matching CEL requirement; the + reconciler validates it's set for either kind before doing anything. + type: string + x-kubernetes-validations: + - message: backendAddress must be a valid IP address + rule: self == '' || isIP(self) + backendPort: + description: |- + BackendPort is the backend's own real port, paired with + BackendAddress — required for both kinds, see that field's doc + comment. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + egressKind: + description: EgressKind selects which backend mechanism this binding + drives. + enum: + - veth + - tap + type: string + port: + description: Port is the VIP-facing port traffic arrives on. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + description: Protocol is the transport protocol this binding applies + to. + enum: + - tcp + - udp + type: string + targetRef: + description: TargetRef identifies the Node this binding applies to. + properties: + kind: + description: Kind is the target resource kind (e.g. Node). + minLength: 1 + type: string + name: + description: Name is the name of the target resource. + minLength: 1 + type: string + required: + - kind + - name + type: object + vipAddress: + description: VIPAddress is the service VIP the backend must be reachable + on. + type: string + x-kubernetes-validations: + - message: vipAddress must be a valid IP address + rule: isIP(self) + required: + - egressKind + - port + - protocol + - targetRef + - vipAddress + type: object + status: + description: ServiceVIPBindingStatus defines the observed state of a ServiceVIPBinding. + properties: + conditions: + description: |- + Conditions contains the standard conditions for this resource, + including Bound (set True once internal/plumbing/vip.Verify or the + equivalent tap-translation-table check confirms the backend is + actually reachable on VIPAddress, not merely that the bind/table-write + call itself returned nil). + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + description: ObservedGeneration is the .metadata.generation this status + was computed from. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/docs/api/bgp.md b/docs/api/bgp.md index 11b33bb..1210e29 100644 --- a/docs/api/bgp.md +++ b/docs/api/bgp.md @@ -15,6 +15,8 @@ Package v1alpha1 contains API Schema definitions for the network.datumapis.com/v - [BGPPolicy](#bgppolicy) - [BGPRouter](#bgprouter) - [BGPVRFInstance](#bgpvrfinstance) +- [NAT66Shard](#nat66shard) +- [ServiceVIPBinding](#servicevipbinding) @@ -844,6 +846,7 @@ _Appears in:_ | `vrfID` _integer_ | VRFID is the 16-bit PoP-local VRF identifier used for RFC 9800 uSID
Argument addressing and to derive the RFC 4364 Type 1 Route
Distinguisher ("routerID:vrfID"). Unique per (VPC, PoP). Value 0 is
reserved. | | Maximum: 65535
Minimum: 1
Required: \{\}
| | `importRouteTargets` _[RouteTarget](#routetarget) array_ | ImportRouteTargets is the list of BGP extended community route targets
used to import routes into this VRF. | | MaxItems: 32
MinItems: 1
| | `exportRouteTargets` _[RouteTarget](#routetarget) array_ | ExportRouteTargets is the list of BGP extended community route targets
attached to routes exported from this VRF. | | MaxItems: 32
MinItems: 1
| +| `nptv6` _[NPTv6Spec](#nptv6spec)_ | NPTv6 configures stateless RFC 6296 Network Prefix Translation for
this VRF: backends presenting an address within ULAPrefix are
translated, checksum-neutrally and bidirectionally, to the
corresponding address in PublicPrefix. Unset means this VRF's traffic
crosses the SRv6 fabric with its own ULA source/destination
unmodified — the common case. Keyed by this VRFID, never by address,
specifically so two VRFs may configure the identical ULAPrefix (e.g.
both using the same private range) without collision — each VRF's
mapping is independent, looked up only after this VRF's own identity
is already resolved from the SRv6 uSID Argument, never derived from
address content alone. | | | #### BGPVRFInstanceStatus @@ -969,6 +972,95 @@ _Appears in:_ | `shutdown` | MaxPrefixShutdownActionShutdown tears down the BGP session when the limit is exceeded.
| +#### NAT66Shard + + + +NAT66Shard marks a single node as a member of the sharded, stateful NAT66 +egress tier (galactic-nat66) — a component deliberately kept off the +ingress load-balancer's own consistent-hash ring (see NetworkGateway): +tenant egress traffic (backend -> arbitrary internet destination) is a +different traffic pattern from ingress (fixed VIP, fixed backend pool) +and needs its own placement ring, own per-flow state, and its own +self-routing return path, entirely independent of any NetworkGateway node. + +Every shard owns a dedicated, BGP-advertised public IPv6 address +(Status.ShardAddress) that a flow's allocated masquerade port lives +within — so a reply is delivered to the correct shard by ordinary +unicast SRv6/BGP routing alone, with no hashing or cross-shard lookup on +the return path at all (the "any node can determine the owning shard from +the tuple alone" property, satisfied by construction rather than by a +replicated hash table). + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `network.datumapis.com/v1alpha1` | | | +| `kind` _string_ | `NAT66Shard` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[NAT66ShardSpec](#nat66shardspec)_ | | | | +| `status` _[NAT66ShardStatus](#nat66shardstatus)_ | | | | + + +#### NAT66ShardSpec + + + +NAT66ShardSpec defines the desired state of a NAT66Shard. + + + +_Appears in:_ +- [NAT66Shard](#nat66shard) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `targetRef` _[TargetRef](#targetref)_ | TargetRef identifies the Node this shard executes on. | | Required: \{\}
| + + +#### NAT66ShardStatus + + + +NAT66ShardStatus defines the observed state of a NAT66Shard. + + + +_Appears in:_ +- [NAT66Shard](#nat66shard) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `observedGeneration` _integer_ | ObservedGeneration is the .metadata.generation this status was computed from. | | | +| `shardAddress` _string_ | ShardAddress is this shard's own dedicated, publicly-routable IPv6
address — every masquerade port this shard allocates lives within it,
so any node can route a reply to the correct shard using ordinary
unicast routing on this address alone, with no per-flow state lookup
anywhere but the owning shard itself. Operator-supplied per shard
today (no in-cluster derivation mechanism yet — the same gap
BGPRouter.Spec.SRv6Locator/NodeID assignment has today). | | | +| `shardSID` _string_ | ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike
ShardAddress, a plain routable address), advertised into BGP the same
way any other node-reachability route is (a /128 BGPAdvertisement, no
VRFID/Function) so every other node learns a kernel SEG6 route toward
it before installing a tenant VRF's default egress route against it. | | | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource. | | | + + +#### NPTv6Spec + + + +NPTv6Spec is one VRF's stateless RFC 6296 prefix-translation mapping. +ULAPrefix and PublicPrefix must share the same prefix length — RFC 6296 +translation only ever rewrites the shared prefix, never the host bits. + + + +_Appears in:_ +- [BGPVRFInstanceSpec](#bgpvrfinstancespec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `ulaPrefix` _string_ | ULAPrefix is this VRF's tenant-facing IPv6 ULA prefix (e.g.
"fd20:60::/64"), as presented by backends inside the VRF. | | Required: \{\}
| +| `publicPrefix` _string_ | PublicPrefix is the externally-routable IPv6 prefix ULAPrefix
translates to/from, same prefix length as ULAPrefix. | | Required: \{\}
| + + #### NextHopSet @@ -1229,6 +1321,106 @@ _Appears in:_ | `both` | SendCommunityTypeBoth sends both standard and extended communities.
| +#### ServiceVIPBinding + + + +ServiceVIPBinding drives one worker node's backend-side half of the +DSR/Maglev load-balancer datapath: it tells the node which service VIP a +specific local backend must be reachable on, so the backend can reply to +clients directly (the "Direct Server Return" this design depends on — +see NetworkGateway's doc comment). Written by the same controller that +already resolves a NetworkRule's backends to worker nodes/SRv6 +information (galactic-gateway's usidresolver.go), one object per +(node, VIP, backend) triple; consumed by a per-node reconciler running +inside galactic-router's tenant role. + +EgressKind decides which of two entirely different backend mechanisms +this object drives, mirroring the same veth/tap fork the SRv6 uSID decap +datapath already has (internal/plumbing/ebpf/usidmap's egress_kind field): + + - veth (container backend): the node binds VIPAddress on its own + galactic-vip0 dummy interface and the backend answers on it from + inside its own pod netns — internal/plumbing/vip's Bind/Unbind/Verify. + - tap (VM backend): there is no guest-side configuration capability in + this repo by design (internal/cnitap's own doc comment) — instead the + node transparently substitutes VIPAddress:Port for + BackendAddress:BackendPort at the SRv6 uSID TC-BPF boundary + (usid_ingress's inbound half, a new usid_egress program's outbound + half), so the guest OS never needs to know the VIP exists at all. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `network.datumapis.com/v1alpha1` | | | +| `kind` _string_ | `ServiceVIPBinding` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[ServiceVIPBindingSpec](#servicevipbindingspec)_ | | | | +| `status` _[ServiceVIPBindingStatus](#servicevipbindingstatus)_ | | | | + + +#### ServiceVIPBindingEgressKind + +_Underlying type:_ _string_ + +ServiceVIPBindingEgressKind mirrors usidmap's EgressKindVeth/EgressKindTap +constants at the API layer — see ServiceVIPBinding's doc comment. + +_Validation:_ +- Enum: [veth tap] + +_Appears in:_ +- [ServiceVIPBindingSpec](#servicevipbindingspec) + +| Field | Description | +| --- | --- | +| `veth` | ServiceVIPBindingEgressKindVeth selects the netns-bind mechanism.
| +| `tap` | ServiceVIPBindingEgressKindTap selects the transparent tap-boundary
translation mechanism.
| + + +#### ServiceVIPBindingSpec + + + +ServiceVIPBindingSpec defines the desired VIP binding/translation state. + + + +_Appears in:_ +- [ServiceVIPBinding](#servicevipbinding) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `targetRef` _[TargetRef](#targetref)_ | TargetRef identifies the Node this binding applies to. | | Required: \{\}
| +| `vipAddress` _string_ | VIPAddress is the service VIP the backend must be reachable on. | | Required: \{\}
| +| `port` _integer_ | Port is the VIP-facing port traffic arrives on. | | Maximum: 65535
Minimum: 1
Required: \{\}
| +| `backendAddress` _string_ | BackendAddress is the backend's own real address (its pod-netns
address for a veth backend, or its actual guest-facing address for a
tap backend). Required only for EgressKindTap, where it is the
substitution target; a veth binding's backend answers on VIPAddress
itself once bound, so this field is ignored for that kind. | | | +| `backendPort` _integer_ | BackendPort is the backend's own real port, paired with
BackendAddress for the tap-translation case. Ignored for veth. | | Maximum: 65535
Minimum: 1
| +| `egressKind` _[ServiceVIPBindingEgressKind](#servicevipbindingegresskind)_ | EgressKind selects which backend mechanism this binding drives. | | Enum: [veth tap]
Required: \{\}
| + + +#### ServiceVIPBindingStatus + + + +ServiceVIPBindingStatus defines the observed state of a ServiceVIPBinding. + + + +_Appears in:_ +- [ServiceVIPBinding](#servicevipbinding) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `observedGeneration` _integer_ | ObservedGeneration is the .metadata.generation this status was computed from. | | | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource,
including Bound (set True once internal/plumbing/vip.Verify or the
equivalent tap-translation-table check confirms the backend is
actually reachable on VIPAddress, not merely that the bind/table-write
call itself returned nil). | | | + + #### TargetRef @@ -1240,6 +1432,8 @@ Supported values for kind: Node. _Appears in:_ - [BGPRouterSpec](#bgprouterspec) +- [NAT66ShardSpec](#nat66shardspec) +- [ServiceVIPBindingSpec](#servicevipbindingspec) | Field | Description | Default | Validation | | --- | --- | --- | --- | diff --git a/docs/api/gateway.md b/docs/api/gateway.md index cb32f8e..836e3d6 100644 --- a/docs/api/gateway.md +++ b/docs/api/gateway.md @@ -9,9 +9,83 @@ Package v1alpha1 contains API Schema definitions for the network.datumapis.com/v1alpha1 API group. ### Resource Types +- [NAT66Shard](#nat66shard) - [NetworkEgressPolicy](#networkegresspolicy) - [NetworkGateway](#networkgateway) - [NetworkRule](#networkrule) +- [ServiceVIPBinding](#servicevipbinding) + + + +#### NAT66Shard + + + +NAT66Shard marks a single node as a member of the sharded, stateful NAT66 +egress tier (galactic-nat66) — a component deliberately kept off the +ingress load-balancer's own consistent-hash ring (see NetworkGateway): +tenant egress traffic (backend -> arbitrary internet destination) is a +different traffic pattern from ingress (fixed VIP, fixed backend pool) +and needs its own placement ring, own per-flow state, and its own +self-routing return path, entirely independent of any NetworkGateway node. + +Every shard owns a dedicated, BGP-advertised public IPv6 address +(Status.ShardAddress) that a flow's allocated masquerade port lives +within — so a reply is delivered to the correct shard by ordinary +unicast SRv6/BGP routing alone, with no hashing or cross-shard lookup on +the return path at all (the "any node can determine the owning shard from +the tuple alone" property, satisfied by construction rather than by a +replicated hash table). + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `network.datumapis.com/v1alpha1` | | | +| `kind` _string_ | `NAT66Shard` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[NAT66ShardSpec](#nat66shardspec)_ | | | | +| `status` _[NAT66ShardStatus](#nat66shardstatus)_ | | | | + + +#### NAT66ShardSpec + + + +NAT66ShardSpec defines the desired state of a NAT66Shard. + + + +_Appears in:_ +- [NAT66Shard](#nat66shard) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `targetRef` _[TargetRef](#targetref)_ | TargetRef identifies the Node this shard executes on. | | Required: \{\}
| + + +#### NAT66ShardStatus + + + +NAT66ShardStatus defines the observed state of a NAT66Shard. + + + +_Appears in:_ +- [NAT66Shard](#nat66shard) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `observedGeneration` _integer_ | ObservedGeneration is the .metadata.generation this status was computed from. | | | +| `shardAddress` _string_ | ShardAddress is this shard's own dedicated, publicly-routable IPv6
address — every masquerade port this shard allocates lives within it,
so any node can route a reply to the correct shard using ordinary
unicast routing on this address alone, with no per-flow state lookup
anywhere but the owning shard itself. Operator-supplied per shard
today (no in-cluster derivation mechanism yet — the same gap
BGPRouter.Spec.SRv6Locator/NodeID assignment has today). | | | +| `shardSID` _string_ | ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike
ShardAddress, a plain routable address), advertised into BGP the same
way any other node-reachability route is (a /128 BGPAdvertisement, no
VRFID/Function) so every other node learns a kernel SEG6 route toward
it before installing a tenant VRF's default egress route against it. | | | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource. | | | + @@ -20,12 +94,12 @@ Package v1alpha1 contains API Schema definitions for the network.datumapis.com/v NetworkEgressPolicy enables internet egress for a single tenant -VPC/VPCAttachment, served by the shared hyperconverged gateway engine's -masquerade (SNAT/PAT) datapath. Unlike NetworkRule, it carries no -VIP/backend/port: egress is on or off for a (vpcRef, vpcAttachmentRef) -pair, existence-implies-enabled, not a per-flow rule — because the -destination of an egress flow is an arbitrary internet address, not a -pre-configured backend list. +VPC/VPCAttachment, served by the sharded, stateful galactic-nat66 tier +(see NAT66Shard). Unlike NetworkRule, it carries no VIP/backend/port: +egress is on or off for a (vpcRef, vpcAttachmentRef) pair, +existence-implies-enabled, not a per-flow rule — because the destination +of an egress flow is an arbitrary internet address, not a pre-configured +backend list. It is namespaced (deployed to galactic-system) and tenant-writable; like NetworkRule, vpcRef/vpcAttachmentRef are opaque string identifiers because @@ -35,14 +109,16 @@ the requester is authorized for vpcRef/vpcAttachmentRef before a policy is accepted — see the Accepted condition. Presence of an accepted NetworkEgressPolicy resolves only *enablement* -(should this tenant reach the egress datapath at all) — a routing-layer -decision (does the tenant's VRF have a default route toward the shared -egress_sid locator), not a per-packet datapath lookup. *Isolation* -(preventing two tenants with colliding ULA source addresses from -colliding in the egress connection table) is a separate, datapath-level -concern resolved by tagging each flow with the tenant/VRF identifier -carried in the egress_sid locator's own Argument bits, not by anything in -this spec. +(should this tenant's VRF get a default route toward the shared NAT66 +tier at all) — unlike this type's original design (superseded), there is +no single "assigned gateway node" to compute or pin: any NAT66Shard may +serve any tenant's flow, chosen by the shard-placement consistent-hash +ring (internal/maglev, keyed on (tenant VRFID, backend, destination) — +see NAT66Shard's doc comment), not by a per-tenant node assignment stored +here. *Isolation* (preventing two tenants with colliding ULA source +addresses from colliding in the egress connection table) is a separate, +datapath-level concern resolved by tagging each flow with the VRFID +carried in the tenant's own SRv6 Argument, not by anything in this spec. @@ -92,7 +168,6 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `observedGeneration` _integer_ | ObservedGeneration is the .metadata.generation this status was computed from. | | | -| `assignedGatewayNode` _string_ | AssignedGatewayNode is the name of the NetworkGateway-backed gateway
node this policy's tenant should route egress traffic through,
mirroring NetworkRule's own status.primaryNode field and computed
the same way: assigned_node = hash(vpcRef) %
(design plan §4.5 — a tenant's egress node and its primary ingress
node are the same node, by design, so both fields are computed by
the identical AssignPrimaryNode function). The controller consuming
this CRD sets this field exactly once, at creation.
This value must never be silently recomputed by a reconciler once
set, for the exact same reason NetworkRuleStatus.PrimaryNode's own
doc comment gives: recomputing it on a later reconcile can flip
which node a tenant's egress traffic routes through and cause an
avoidable traffic flap; a reconciler that observes a stale or
removed node here must surface that via a condition instead of
overwriting the value. | | | | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource,
including Accepted (see AcceptedReasonOwnershipVerified /
AcceptedReasonOwnershipDenied in rule_types.go, reused as-is here). | | | @@ -100,25 +175,24 @@ _Appears in:_ -NetworkGateway defines an XDP ingress NAT+LB gateway engine instance bound -to a single dedicated gateway-role node. Exactly one NetworkGateway exists -per gateway node (spec.targetRef.name is the Kubernetes node name), -mirroring the BGPRouter node-scoped root object pattern. NetworkRule -resources are assigned to a NetworkGateway via status.primaryNode. +NetworkGateway marks a single dedicated gateway-role node as running the +Maglev/DSR consistent-hash L4 load-balancer engine. Exactly one +NetworkGateway exists per gateway node (spec.targetRef.name is the +Kubernetes node name), mirroring the BGPRouter node-scoped root object +pattern. NetworkRule resources are served by every NetworkGateway in the +namespace equally (anycast — see NetworkRuleStatus's doc comment); this +object's only job is to identify which nodes participate at all and +surface each node's engine health via Conditions. -There is no tunnel overlay in this design (an earlier Geneve-based -approach was superseded before this type shipped): the gateway's XDP -program does Full-NAT (DNAT the VIP to a backend Pod's address, SNAT the -client's source to status.sRv6Address) and pushes an SRv6 uSID outer -header addressed to the backend's worker node directly, so return traffic -(addressed to status.sRv6Address) arrives back at this same gateway node -over the ordinary SRv6 fabric — no compute-node encap agent, no tunnel -endpoint to publish. status.sRv6Address is advertised into BGP the same -way any workload prefix is (a BGPAdvertisement naming it, /128, Argument -0 — the value PR #740 reserves and forbids registering into any tenant -VRF, guaranteeing it never collides with a real tenant's Argument), so -every other node learns a real kernel SEG6 route to it for free through -the existing EVPN pipeline. +This design does no address rewriting on the load-balancing path at all +(DSR — Direct Server Return): the gateway's XDP program picks a backend +via consistent hashing on the flow's 5-tuple and pushes an SRv6 uSID outer +header addressed to the backend's worker node directly, untouched +otherwise. The backend node answers the client directly (see +ServiceVIPBinding) — reply traffic never re-enters this gateway node, so +unlike the Full-NAT design this type originally described, a gateway node +has no SNAT source address of its own to publish and nothing analogous to +sRv6Address/egressAddress/egressSID belongs on this status anymore. @@ -165,9 +239,6 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `observedGeneration` _integer_ | ObservedGeneration is the .metadata.generation this status was computed from. | | | -| `sRv6Address` _string_ | SRv6Address is this gateway node's own SRv6-reachable IPv6 address,
used as the Full-NAT SNAT source for every ingress flow this node
translates. Backend Pods' replies are naturally routed back to it
over the ordinary SRv6 fabric (the same mechanism that routes any
other node's traffic), where this node's XDP program decapsulates
and un-NATs them using its own conn_table — there is no separate
tunnel endpoint or overlay device to publish. Populated by the
engine once it has computed the address (a uFMT 48+16 uSID over this
node's own BGPRouter locator/node-ID, at the reserved Argument 0)
and advertised it into BGP. | | | -| `egressAddress` _string_ | EgressAddress is this gateway node's own publicly-routable IPv6
address, used as the masquerade SNAT source for every egress flow
this node translates on behalf of tenant VPC backends reaching the
internet. Unlike SRv6Address (reachable only within the SRv6 fabric),
this address must additionally be reachable from the public internet
— an eBGP/uplink-peering concern outside this API. Operator-supplied
via GALACTIC_GATEWAY_EGRESS_ADDRESS; there is no in-cluster
derivation mechanism yet, the same gap SRv6Address itself has today.
A gateway node not offering egress leaves this field empty. | | | -| `egressSID` _string_ | EgressSID is this gateway node's own egress_sid uSID *locator*
(design plan §3.1) — the reserved Argument range's Block+Node-ID
portion tenant VRF default routes encapsulate toward. Unlike
EgressAddress (a plain, publicly-routable address, no uSID
structure), this is a real uSID: other nodes need a kernel route to
it before they can install a SEG6 encap route naming it as the
destination (the same reason SRv6Address is advertised into BGP),
so this is published and advertised the same way SRv6Address/
EgressAddress already are. Operator-supplied via
GALACTIC_GATEWAY_EGRESS_SID; a gateway node not offering egress
leaves this field empty, always paired with EgressAddress (both
set, or neither). | | | | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource. | | | @@ -175,15 +246,24 @@ _Appears in:_ -NetworkRule defines ingress load-balancing and NAT for a single tenant -VPC/VPCAttachment, served by the shared hyperconverged gateway engine. -It is namespaced (deployed to galactic-system) and tenant-writable; the +NetworkRule defines ingress load-balancing for a single tenant +VPC/VPCAttachment, served by every NetworkGateway node identically +(anycast Direct Server Return — see NetworkGateway's doc comment). It is +namespaced (deployed to galactic-system) and tenant-writable; the vpcRef/vpcAttachmentRef fields are opaque string identifiers because the VPC API is owned by a separate companion operator, not this repo. An admission webhook (implemented by the consuming controller) must verify the requester is authorized for vpcRef/vpcAttachmentRef before a rule is accepted — see the Accepted condition. +Unlike the earlier Full-NAT design this type originally described, there +is no primary/secondary gateway node for a rule: every NetworkGateway +advertises every accepted rule's vipAddresses at equal BGP preference, +consistent-hashes the same backend list to the same backend for the same +flow (internal/maglev), and forwards without rewriting anything — +backend selection never needs a single "owning" node the way Full-NAT's +SNAT-source model did. + @@ -229,6 +309,7 @@ _Validation:_ _Appears in:_ - [NetworkRuleSpec](#networkrulespec) +- [ServiceVIPBindingSpec](#servicevipbindingspec) | Field | Description | | --- | --- | @@ -272,10 +353,110 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `observedGeneration` _integer_ | ObservedGeneration is the .metadata.generation this status was computed from. | | | -| `primaryNode` _string_ | PrimaryNode is the name of the NetworkGateway-backed gateway node
assigned to advertise this rule's VIPAddresses at the preferred BGP
local-preference, per the active-active model: primary_node =
hash(vpcRef) % . The controller consuming this
CRD sets this field exactly once, at creation.
This value must never be silently recomputed by a reconciler once
set. Recomputing it on a later reconcile can flip which node is
primary for a live VIP and cause an avoidable traffic flap; a
reconciler that observes a stale or removed node here must surface
that via a condition instead of overwriting the value. | | | | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource. | | | +#### ServiceVIPBinding + + + +ServiceVIPBinding drives one worker node's backend-side half of the +DSR/Maglev load-balancer datapath: it tells the node which service VIP a +specific local backend must be reachable on, so the backend can reply to +clients directly (the "Direct Server Return" this design depends on — +see NetworkGateway's doc comment). Written by the same controller that +already resolves a NetworkRule's backends to worker nodes/SRv6 +information (galactic-gateway's usidresolver.go), one object per +(node, VIP, backend) triple; consumed by a per-node reconciler running +inside galactic-router's tenant role. + +EgressKind decides which of two entirely different backend mechanisms +this object drives, mirroring the same veth/tap fork the SRv6 uSID decap +datapath already has (internal/plumbing/ebpf/usidmap's egress_kind field): + + - veth (container backend): the node binds VIPAddress on its own + galactic-vip0 dummy interface and the backend answers on it from + inside its own pod netns — internal/plumbing/vip's Bind/Unbind/Verify. + - tap (VM backend): there is no guest-side configuration capability in + this repo by design (internal/cnitap's own doc comment) — instead the + node transparently substitutes VIPAddress:Port for + BackendAddress:BackendPort at the SRv6 uSID TC-BPF boundary + (usid_ingress's inbound half, a new usid_egress program's outbound + half), so the guest OS never needs to know the VIP exists at all. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `network.datumapis.com/v1alpha1` | | | +| `kind` _string_ | `ServiceVIPBinding` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[ServiceVIPBindingSpec](#servicevipbindingspec)_ | | | | +| `status` _[ServiceVIPBindingStatus](#servicevipbindingstatus)_ | | | | + + +#### ServiceVIPBindingEgressKind + +_Underlying type:_ _string_ + +ServiceVIPBindingEgressKind mirrors usidmap's EgressKindVeth/EgressKindTap +constants at the API layer — see ServiceVIPBinding's doc comment. + +_Validation:_ +- Enum: [veth tap] + +_Appears in:_ +- [ServiceVIPBindingSpec](#servicevipbindingspec) + +| Field | Description | +| --- | --- | +| `veth` | ServiceVIPBindingEgressKindVeth selects the netns-bind mechanism.
| +| `tap` | ServiceVIPBindingEgressKindTap selects the transparent tap-boundary
translation mechanism.
| + + +#### ServiceVIPBindingSpec + + + +ServiceVIPBindingSpec defines the desired VIP binding/translation state. + + + +_Appears in:_ +- [ServiceVIPBinding](#servicevipbinding) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `targetRef` _[TargetRef](#targetref)_ | TargetRef identifies the Node this binding applies to. | | Required: \{\}
| +| `vipAddress` _string_ | VIPAddress is the service VIP the backend must be reachable on. | | Required: \{\}
| +| `port` _integer_ | Port is the VIP-facing port traffic arrives on. | | Maximum: 65535
Minimum: 1
Required: \{\}
| +| `protocol` _[NetworkRuleProtocol](#networkruleprotocol)_ | Protocol is the transport protocol this binding applies to. | | Enum: [tcp udp]
Required: \{\}
| +| `backendAddress` _string_ | BackendAddress is the backend's own real address (its pod-netns
address for a veth backend, or its actual guest-facing address for a
tap backend). Required only for EgressKindTap, where it is the
substitution target; a veth binding's backend answers on VIPAddress
itself once bound, so this field is ignored for that kind. | | | +| `backendPort` _integer_ | BackendPort is the backend's own real port, paired with
BackendAddress for the tap-translation case. Ignored for veth. | | Maximum: 65535
Minimum: 1
| +| `egressKind` _[ServiceVIPBindingEgressKind](#servicevipbindingegresskind)_ | EgressKind selects which backend mechanism this binding drives. | | Enum: [veth tap]
Required: \{\}
| + + +#### ServiceVIPBindingStatus + + + +ServiceVIPBindingStatus defines the observed state of a ServiceVIPBinding. + + + +_Appears in:_ +- [ServiceVIPBinding](#servicevipbinding) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `observedGeneration` _integer_ | ObservedGeneration is the .metadata.generation this status was computed from. | | | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource,
including Bound (set True once internal/plumbing/vip.Verify or the
equivalent tap-translation-table check confirms the backend is
actually reachable on VIPAddress, not merely that the bind/table-write
call itself returned nil). | | | + + #### TargetRef @@ -286,7 +467,9 @@ Supported values for kind: Node. _Appears in:_ +- [NAT66ShardSpec](#nat66shardspec) - [NetworkGatewaySpec](#networkgatewayspec) +- [ServiceVIPBindingSpec](#servicevipbindingspec) | Field | Description | Default | Validation | | --- | --- | --- | --- |