From 826ee9af26c8a0c2cdbc35438d6f2f1e444d8c8b Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Mon, 10 Aug 2026 15:43:59 -0700 Subject: [PATCH 01/10] feat(networking): Windows L2Bridge egress with router-safe VFP ACLs (opt-in) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NAT cannot software-filter Windows container egress — VFP does not engage on an HNS NAT network, so per-endpoint Switch ACLs are inert there. This adds an opt-in L2Bridge path that binds containers to a host NIC via a NetAdapterName network policy, giving each container a VFP-managed vSwitch port where ACLs actually enforce. Proven on metal (static IP); DHCP is the v1 IPAM choice. The router-safe ACL ladder (buildL2BridgeEgressACLPolicies, a pure function) matches the Linux end-state: block ALL of 10/8, 172.16/12, 192.168/16, 169.254/16 — whole supernets, no gateway or own-subnet carve-out — and permit only the internet. VFP is default-DENY once any ACL is present, so the ladder is, by precedence (lower number wins): 90 Allow DHCP (UDP 67/68, Out+In) — lease/renew survives the block 95 Allow extra-allowed CIDRs (future use; default none) 100 Block the RFC1918 + link-local supernets, Out 65500 Allow 0.0.0.0/0 Out AND In — both mandatory (the In allow keeps TCP SYN-ACK return traffic alive; without the pair the port default-denies everything, internet included) DNS is set to public resolvers on the endpoint so the container never needs the LAN router for name resolution. The rule set is static (independent of the leased gateway/DNS) and applied at endpoint creation, before the container starts — fail-closed, no post-lease window. Any ApplyPolicy error tears down the endpoint and refuses the job. NAT stays the default and is untouched: buildEgressBlockPolicies and the NAT init/setup path are unchanged, and the new ladder is only reached when a pool sets network.l2bridge_egress = true. The NAT block-only builder is deliberately NOT repurposed — its whole-supernet block with no carve-out would blackhole the NAT gateway (10.88.0.1, inside 10/8) if VFP ever engaged. Config: network.l2bridge_egress (bool), network.host_nic (required when on), network.public_dns (default 1.1.1.1/8.8.8.8), network.extra_allowed_destinations. --- cmd/ephemerd/main.go | 32 +-- config.example.toml | 23 +++ pkg/config/config.go | 29 +++ pkg/networking/network_windows.go | 267 ++++++++++++++++++++++++- pkg/networking/network_windows_test.go | 172 ++++++++++++++++ pkg/networking/networking.go | 9 + 6 files changed, 514 insertions(+), 18 deletions(-) diff --git a/cmd/ephemerd/main.go b/cmd/ephemerd/main.go index d95710e..d52dec9 100644 --- a/cmd/ephemerd/main.go +++ b/cmd/ephemerd/main.go @@ -263,12 +263,16 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP controlPorts := []int{int(containerdTCPPort), int(containerdTCPPort) + 1, int(containerdTCPPort) + 2} net, err := networking.New(networking.Config{ - DataDir: configDir, - Subnet: cfg.Network.Subnet, - MTU: cfg.Network.MTU, - CNIBinDir: cm.Dir(), - ControlPorts: controlPorts, - Log: log, + DataDir: configDir, + Subnet: cfg.Network.Subnet, + MTU: cfg.Network.MTU, + CNIBinDir: cm.Dir(), + ControlPorts: controlPorts, + L2BridgeEgress: cfg.Network.L2BridgeEgress, + HostNIC: cfg.Network.HostNIC, + PublicDNS: cfg.Network.PublicDNS, + ExtraAllowedCIDRs: cfg.Network.ExtraAllowedDestinations, + Log: log, }) if err != nil { return fmt.Errorf("initializing networking: %w", err) @@ -427,12 +431,16 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP // Initialize container networking net, err := networking.New(networking.Config{ - DataDir: configDir, - Subnet: cfg.Network.Subnet, - MTU: cfg.Network.MTU, - CNIBinDir: cm.Dir(), - GatewayPorts: gatewayPorts, - Log: log, + DataDir: configDir, + Subnet: cfg.Network.Subnet, + MTU: cfg.Network.MTU, + CNIBinDir: cm.Dir(), + GatewayPorts: gatewayPorts, + L2BridgeEgress: cfg.Network.L2BridgeEgress, + HostNIC: cfg.Network.HostNIC, + PublicDNS: cfg.Network.PublicDNS, + ExtraAllowedCIDRs: cfg.Network.ExtraAllowedDestinations, + Log: log, }) if err != nil { return fmt.Errorf("initializing networking: %w", err) diff --git a/config.example.toml b/config.example.toml index f47fcd0..d28de01 100644 --- a/config.example.toml +++ b/config.example.toml @@ -87,6 +87,29 @@ owner = "your-org" # Override if containers have connectivity issues on unusual networks. # mtu = 1500 +# --- Windows L2Bridge egress (opt-in; Windows hosts only) ------------------- +# By default Windows containers use an HNS NAT network. NAT cannot +# software-filter Windows container egress (VFP does not engage on a NAT +# network), so egress enforcement there leans on the Hyper-V firewall. Setting +# l2bridge_egress = true instead puts containers on an L2Bridge with a +# VFP-managed vSwitch port, where per-endpoint ACLs actually enforce: all of +# RFC1918 (10/8, 172.16/12, 192.168/16, 169.254/16) — including the LAN router +# — is blocked, and only the internet is reachable. Leave false to keep NAT. +# l2bridge_egress = false +# +# Host NIC the L2Bridge binds onto. REQUIRED when l2bridge_egress = true; there +# is no default (the correct adapter name is host-specific — do NOT assume +# "Ethernet 2"). Ignored when l2bridge_egress is false. +# host_nic = "Ethernet" +# +# Public DNS resolvers handed to L2Bridge containers so name resolution goes to +# the internet rather than the (blocked) LAN router. Default 1.1.1.1, 8.8.8.8. +# public_dns = ["1.1.1.1", "8.8.8.8"] +# +# Extra destination CIDRs permitted through the L2Bridge egress ACLs, ABOVE the +# RFC1918 block (reserved for future use; default none — the strict posture). +# extra_allowed_destinations = [] + [runner] # Max concurrent jobs diff --git a/pkg/config/config.go b/pkg/config/config.go index bd29453..f97eccb 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -215,6 +215,35 @@ func (w *WebhookConfig) ResolvedReconcileInterval() time.Duration { type NetworkConfig struct { Subnet string `toml:"subnet"` // container subnet (auto-selected if empty) MTU int `toml:"mtu"` // bridge MTU (auto-detected from host if 0) + + // L2BridgeEgress opts a Windows pool into L2Bridge container networking + // with VFP-enforced egress filtering, instead of the default HNS NAT. + // NAT cannot software-filter Windows container egress (VFP does not engage + // on a NAT network); L2Bridge puts the container on a VFP-managed vSwitch + // port so per-endpoint ACLs actually enforce. Windows only; ignored on + // Linux/macOS. Default false — NAT stays the default and this flag flips + // nothing until an operator opts a pool in. + L2BridgeEgress bool `toml:"l2bridge_egress"` + + // HostNIC is the host network adapter name the L2Bridge binds onto + // (e.g. "Ethernet"). REQUIRED when L2BridgeEgress is true — the bridge + // has no uplink without it. There is no default: the correct NIC name is + // host-specific (do not assume "Ethernet 2", which was a spike's + // hot-added test NIC). Ignored when L2BridgeEgress is false. + HostNIC string `toml:"host_nic"` + + // PublicDNS is the DNS resolver list handed to L2Bridge containers. Public + // resolvers keep container DNS off the LAN router (which the egress ACLs + // block along with the rest of RFC1918). Empty falls back to a built-in + // public default (1.1.1.1, 8.8.8.8). Only consulted on the L2Bridge path. + PublicDNS []string `toml:"public_dns"` + + // ExtraAllowedDestinations are additional CIDRs permitted through the + // L2Bridge egress ACLs at a precedence ABOVE the RFC1918 block (so a + // listed destination wins over the block). Reserved for future use — + // default empty, which reproduces the strict Linux end-state (no RFC1918 + // carve-outs at all). Only consulted on the L2Bridge path. + ExtraAllowedDestinations []string `toml:"extra_allowed_destinations"` } type ContainerdConfig struct { diff --git a/pkg/networking/network_windows.go b/pkg/networking/network_windows.go index e963637..a8bb17c 100644 --- a/pkg/networking/network_windows.go +++ b/pkg/networking/network_windows.go @@ -15,8 +15,19 @@ import ( const ( networkName = "ephemerd" defaultGateway = "10.88.0.1" + + // l2BridgeNetworkName is the HNS network created on the L2Bridge egress + // path. It is deliberately distinct from networkName so a host that + // previously ran the NAT path (network "ephemerd") does not collide with, + // or get mistaken for, the L2Bridge network. + l2BridgeNetworkName = "ephemerd-l2bridge" ) +// defaultPublicDNS is the DNS resolver list handed to L2Bridge containers when +// Config.PublicDNS is empty. Public resolvers so container DNS never needs the +// LAN router — which the egress ACLs block along with the rest of RFC1918. +var defaultPublicDNS = []string{"1.1.1.1", "8.8.8.8"} + type windowsNetworking struct { cfg Config network *hcn.HostComputeNetwork @@ -30,6 +41,12 @@ func newPlatformNetworking() platformNetworking { func (w *windowsNetworking) init(cfg Config) error { w.cfg = cfg + // L2Bridge egress path (opt-in). NAT stays the default: only a pool that + // explicitly sets L2BridgeEgress reaches VFP-enforced egress filtering. + if cfg.L2BridgeEgress { + return w.initL2Bridge(cfg) + } + // Check if network already exists (from previous run) existing, err := hcn.GetNetworkByName(networkName) if err == nil { @@ -77,16 +94,98 @@ func (w *windowsNetworking) init(cfg Config) error { return nil } +// publicDNS returns the configured public resolver list, or the built-in +// default when none is configured. +func (w *windowsNetworking) publicDNS() []string { + if len(w.cfg.PublicDNS) > 0 { + return w.cfg.PublicDNS + } + return defaultPublicDNS +} + +// initL2Bridge creates (or adopts) the L2Bridge HNS network bound to the +// configured host NIC. Unlike the NAT network, this puts containers on a +// VFP-managed vSwitch port so the per-endpoint Switch ACLs applied in setup() +// actually enforce. +// +// IPAM is DHCP: the network declares NO static subnet or routes, so HNS does +// not pin an address and the container's vNIC leases its IP, default gateway, +// and (LAN) DNS from the LAN DHCP server. DNS is overridden to the public +// resolvers at the network and endpoint level so the container never needs the +// LAN router for name resolution — the egress ACLs block the router. +func (w *windowsNetworking) initL2Bridge(cfg Config) error { + if cfg.HostNIC == "" { + // Fail closed: without an uplink NIC the bridge has no path off-host, + // and silently falling back to NAT would defeat the egress guarantee + // the operator opted into. + return fmt.Errorf("L2Bridge egress enabled but no host_nic configured (set network.host_nic to the host adapter name to bridge onto)") + } + + if existing, err := hcn.GetNetworkByName(l2BridgeNetworkName); err == nil { + w.network = existing + cfg.Log.Info("HCN L2Bridge network found", "name", l2BridgeNetworkName, "id", existing.Id, "host_nic", cfg.HostNIC) + return nil + } + + adapterPol, err := json.Marshal(hcn.NetAdapterNameNetworkPolicySetting{ + NetworkAdapterName: cfg.HostNIC, + }) + if err != nil { + return fmt.Errorf("marshaling NetAdapterName policy for %q: %w", cfg.HostNIC, err) + } + + network := &hcn.HostComputeNetwork{ + Name: l2BridgeNetworkName, + Type: hcn.L2Bridge, + // No Ipams: DHCP IPAM. HNS does not assign an address; the container + // vNIC DHCPs on the LAN for its IP, gateway, and default route. + Policies: []hcn.NetworkPolicy{ + { + Type: hcn.NetAdapterName, // binds the L2Bridge to the physical NIC + Settings: adapterPol, + }, + }, + Dns: hcn.Dns{ + ServerList: w.publicDNS(), + }, + SchemaVersion: hcn.SchemaVersion{ + Major: 2, + Minor: 0, + }, + } + + created, err := network.Create() + if err != nil { + return fmt.Errorf("creating HCN L2Bridge network on %q: %w", cfg.HostNIC, err) + } + w.network = created + + cfg.Log.Info("HCN L2Bridge network created", "name", l2BridgeNetworkName, "id", created.Id, "host_nic", cfg.HostNIC) + return nil +} + func (w *windowsNetworking) setup(ctx context.Context, id string, netns string) (*SetupResult, error) { w.mu.Lock() defer w.mu.Unlock() - // Create endpoint on the network + // DNS: on L2Bridge, hand the container the configured PUBLIC resolvers so + // name resolution goes to the internet (permitted by the low-precedence + // allow-any ACL) and never to the LAN router (blocked by the RFC1918 ACLs). + // On NAT, DNS is the gateway's forwarder as before. + dnsServers := []string{"8.8.8.8", "8.8.4.4"} + if w.cfg.L2BridgeEgress { + dnsServers = w.publicDNS() + } + + // Create endpoint on the network. On the L2Bridge path we deliberately set + // NO IpConfigurations so the container leases its address, gateway, and + // default route from the LAN DHCP server (DHCP IPAM). On NAT, HNS assigns + // from the NAT subnet as before. endpoint := &hcn.HostComputeEndpoint{ Name: id + "-ep", HostComputeNetwork: w.network.Id, Dns: hcn.Dns{ - ServerList: []string{"8.8.8.8", "8.8.4.4"}, + ServerList: dnsServers, }, SchemaVersion: hcn.SchemaVersion{ Major: 2, @@ -106,6 +205,12 @@ func (w *windowsNetworking) setup(ctx context.Context, id string, netns string) // RFC1918 services, and link-local metadata endpoints. Fail CLOSED: tear // down the endpoint we just created and refuse the job rather than start a // container we cannot firewall. + // + // The ACLs are applied to the endpoint BEFORE the container is started + // (setup runs ahead of task creation), and the L2Bridge rule set is STATIC + // — it does not depend on the leased gateway or DNS (DNS is public, the + // router gets no allow) — so there is no post-lease window in which the + // container has LAN connectivity but no filters. if err := w.applyACLPolicies(created); err != nil { if delErr := created.Delete(); delErr != nil { w.cfg.Log.Warn("failed to delete endpoint after ACL failure", "id", id, "error", delErr) @@ -216,11 +321,161 @@ func buildEgressBlockPolicies() ([]hcn.EndpointPolicy, error) { return policies, nil } -// applyACLPolicies blocks container traffic to RFC 1918 and link-local ranges. -// The full rule set is built up front and applied atomically; any failure is -// returned so the caller (setup) can treat it as fatal for the job. +// VFP Switch-ACL precedence for the L2Bridge egress model. VFP evaluates ACLs +// by Priority, LOWER number = HIGHER precedence (evaluated first, first match +// wins). The ladder is: DHCP allow (top) > operator carve-outs > RFC1918 block +// > allow-any (bottom). The two allow-any rules at the bottom are what stop the +// port from default-denying everything — VFP is default-DENY the moment any +// ACL is present. +const ( + // aclPriorityDHCP lets the container lease/renew even with the rest of + // RFC1918 blocked. Above the block so the DHCP server (which lives on the + // LAN, inside a blocked supernet) stays reachable. + aclPriorityDHCP uint16 = 90 + // aclPriorityExtraAllow carves configured destinations out ABOVE the + // RFC1918 block. Unused by default (no carve-outs) — reserved for future + // operator-allowed destinations. + aclPriorityExtraAllow uint16 = 95 + // aclPriorityBlock denies the RFC1918 + link-local supernets, whole. No + // gateway/own-subnet carve-out: on L2Bridge the container is a LAN peer, + // so carving the subnet would expose the management plane and the router. + aclPriorityBlock uint16 = 100 + // aclPriorityAllowAny permits everything not blocked above (the internet) + // and, crucially, inbound return traffic. Lowest precedence. + aclPriorityAllowAny uint16 = 65500 +) + +// aclAnyProtocol matches any IP protocol in an HNS ACL ("256"), used for the +// allow-any and block rules per the proven metal run. The DHCP rules use UDP. +const ( + aclAnyProtocol = "256" + aclUDPProtocol = "17" +) + +// marshalACL serializes one AclPolicySetting into an EndpointPolicy, failing +// closed on a marshal error (a rule we cannot serialize is a rule we cannot +// enforce; never skip it and continue with a weaker set). +func marshalACL(acl hcn.AclPolicySetting) (hcn.EndpointPolicy, error) { + settings, err := json.Marshal(acl) + if err != nil { + return hcn.EndpointPolicy{}, fmt.Errorf("marshaling ACL %+v: %w", acl, err) + } + return hcn.EndpointPolicy{Type: hcn.ACL, Settings: settings}, nil +} + +// buildL2BridgeEgressACLPolicies constructs the router-safe VFP Switch-ACL set +// for an L2Bridge endpoint. It is a pure function (no HCN calls) so the exact +// emitted policy set — actions, directions, remotes, protocols, priorities — is +// unit-testable. Fails closed on any marshal error. +// +// The model matches the Linux end-state (firewall_linux.go): block ALL of +// 10/8, 172.16/12, 192.168/16, 169.254/16 — including the LAN router and the +// container's own subnet — and permit only the internet. DNS is handled out of +// band (public resolvers on the endpoint), so unlike Linux there is no DNS or +// gateway carve-out here, and no container-to-container allow (that would be +// LAN access, which we block). extraAllowed carves additional CIDRs out above +// the block for future use; empty (the default) reproduces the strict posture. +func buildL2BridgeEgressACLPolicies(extraAllowed []string) ([]hcn.EndpointPolicy, error) { + var policies []hcn.EndpointPolicy + add := func(acl hcn.AclPolicySetting) error { + p, err := marshalACL(acl) + if err != nil { + return err + } + policies = append(policies, p) + return nil + } + + // Tier: DHCP. Allow UDP to/from the BOOTP/DHCP ports so the container can + // obtain and renew its lease even though the DHCP server sits inside a + // blocked supernet. Out matches the request (dst 67), In matches the reply + // (dst 68); no address scope (the DHCP server is discovered). + if err := add(hcn.AclPolicySetting{ + Protocols: aclUDPProtocol, + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeOut, + RemotePorts: "67,68", + RuleType: hcn.RuleTypeSwitch, + Priority: aclPriorityDHCP, + }); err != nil { + return nil, err + } + if err := add(hcn.AclPolicySetting{ + Protocols: aclUDPProtocol, + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeIn, + LocalPorts: "67,68", + RuleType: hcn.RuleTypeSwitch, + Priority: aclPriorityDHCP, + }); err != nil { + return nil, err + } + + // Tier: operator carve-outs (future use). Allowed ABOVE the block so a + // listed destination wins. Empty by default. + for _, cidr := range extraAllowed { + if err := add(hcn.AclPolicySetting{ + Protocols: aclAnyProtocol, + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeOut, + RemoteAddresses: cidr, + RuleType: hcn.RuleTypeSwitch, + Priority: aclPriorityExtraAllow, + }); err != nil { + return nil, err + } + } + + // Tier: block the RFC1918 + link-local supernets, whole. No carve-out. + for _, cidr := range egressBlockedCIDRs { + if err := add(hcn.AclPolicySetting{ + Protocols: aclAnyProtocol, + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeOut, + RemoteAddresses: cidr, + RuleType: hcn.RuleTypeSwitch, + Priority: aclPriorityBlock, + }); err != nil { + return nil, err + } + } + + // Tier: allow-any Out AND In (lowest precedence). BOTH are mandatory: + // without them the port default-denies everything (internet included), and + // the INBOUND allow is required or TCP return traffic (SYN-ACK) is dropped + // and even permitted destinations fail. + for _, dir := range []hcn.DirectionType{hcn.DirectionTypeOut, hcn.DirectionTypeIn} { + if err := add(hcn.AclPolicySetting{ + Protocols: aclAnyProtocol, + Action: hcn.ActionTypeAllow, + Direction: dir, + RemoteAddresses: "0.0.0.0/0", + RuleType: hcn.RuleTypeSwitch, + Priority: aclPriorityAllowAny, + }); err != nil { + return nil, err + } + } + + return policies, nil +} + +// applyACLPolicies applies the per-endpoint egress ACLs. On the L2Bridge path +// it applies the router-safe VFP ladder (buildL2BridgeEgressACLPolicies); on +// NAT it applies the existing block-only set (buildEgressBlockPolicies), which +// is left untouched so the default NAT path behaves exactly as before. The full +// rule set is built up front and applied atomically; any failure is returned so +// the caller (setup) can treat it as fatal for the job. func (w *windowsNetworking) applyACLPolicies(endpoint *hcn.HostComputeEndpoint) error { - policies, err := buildEgressBlockPolicies() + var ( + policies []hcn.EndpointPolicy + err error + ) + if w.cfg.L2BridgeEgress { + policies, err = buildL2BridgeEgressACLPolicies(w.cfg.ExtraAllowedCIDRs) + } else { + policies, err = buildEgressBlockPolicies() + } if err != nil { return err } diff --git a/pkg/networking/network_windows_test.go b/pkg/networking/network_windows_test.go index d6642fc..5ea9eb2 100644 --- a/pkg/networking/network_windows_test.go +++ b/pkg/networking/network_windows_test.go @@ -4,6 +4,7 @@ package networking import ( "encoding/json" + "net" "testing" "github.com/Microsoft/hcsshim/hcn" @@ -53,3 +54,174 @@ func TestBuildEgressBlockPolicies(t *testing.T) { t.Error("link-local/metadata range 169.254.0.0/16 not blocked") } } + +// decodeACLs unmarshals every EndpointPolicy back into an AclPolicySetting, +// failing the test on a non-ACL policy or a malformed setting. +func decodeACLs(t *testing.T, policies []hcn.EndpointPolicy) []hcn.AclPolicySetting { + t.Helper() + acls := make([]hcn.AclPolicySetting, 0, len(policies)) + for i, p := range policies { + if p.Type != hcn.ACL { + t.Errorf("policy[%d] type = %v, want ACL", i, p.Type) + } + var acl hcn.AclPolicySetting + if err := json.Unmarshal(p.Settings, &acl); err != nil { + t.Fatalf("unmarshal ACL[%d]: %v", i, err) + } + acls = append(acls, acl) + } + return acls +} + +// TestL2BridgeEgressACLPolicies_LadderShape pins the full router-safe VFP +// ladder: the two mandatory allow-any rules (Out+In), the DHCP allows, and a +// whole-supernet block for every RFC1918 + link-local range — with the exact +// actions, directions, protocols, and priorities that were proven on metal. +func TestL2BridgeEgressACLPolicies_LadderShape(t *testing.T) { + acls := decodeACLs(t, mustBuildL2Bridge(t, nil)) + + var ( + allowAnyOut, allowAnyIn bool + dhcpOut, dhcpIn bool + blockedSupernets = map[string]hcn.AclPolicySetting{} + ) + for _, a := range acls { + switch { + case a.Action == hcn.ActionTypeAllow && a.RemoteAddresses == "0.0.0.0/0" && a.Direction == hcn.DirectionTypeOut: + allowAnyOut = true + assertACL(t, "allow-any-out", a, aclAnyProtocol, aclPriorityAllowAny) + case a.Action == hcn.ActionTypeAllow && a.RemoteAddresses == "0.0.0.0/0" && a.Direction == hcn.DirectionTypeIn: + allowAnyIn = true + assertACL(t, "allow-any-in", a, aclAnyProtocol, aclPriorityAllowAny) + case a.Action == hcn.ActionTypeAllow && a.Protocols == aclUDPProtocol && a.Direction == hcn.DirectionTypeOut: + dhcpOut = true + if a.RemotePorts != "67,68" || a.Priority != aclPriorityDHCP { + t.Errorf("dhcp-out = %+v, want RemotePorts 67,68 priority %d", a, aclPriorityDHCP) + } + case a.Action == hcn.ActionTypeAllow && a.Protocols == aclUDPProtocol && a.Direction == hcn.DirectionTypeIn: + dhcpIn = true + if a.LocalPorts != "67,68" || a.Priority != aclPriorityDHCP { + t.Errorf("dhcp-in = %+v, want LocalPorts 67,68 priority %d", a, aclPriorityDHCP) + } + case a.Action == hcn.ActionTypeBlock: + blockedSupernets[a.RemoteAddresses] = a + } + } + + if !allowAnyOut || !allowAnyIn { + t.Errorf("missing mandatory allow-any rule: out=%v in=%v (both required or the port default-denies / drops SYN-ACK)", allowAnyOut, allowAnyIn) + } + if !dhcpOut || !dhcpIn { + t.Errorf("missing DHCP allow: out=%v in=%v", dhcpOut, dhcpIn) + } + + // Every RFC1918 + link-local supernet must be blocked, whole, Out, at the + // block priority. + for _, cidr := range egressBlockedCIDRs { + a, ok := blockedSupernets[cidr] + if !ok { + t.Errorf("supernet %s is not blocked", cidr) + continue + } + assertACL(t, "block "+cidr, a, aclAnyProtocol, aclPriorityBlock) + if a.Direction != hcn.DirectionTypeOut { + t.Errorf("block %s direction = %v, want Out", cidr, a.Direction) + } + } + if len(blockedSupernets) != len(egressBlockedCIDRs) { + t.Errorf("got %d block rules, want %d (one per supernet, no extras)", len(blockedSupernets), len(egressBlockedCIDRs)) + } +} + +// TestL2BridgeEgressACLPolicies_NoGatewayOrSubnetCarveOut pins the load-bearing +// safety property that differs from NAT: on L2Bridge the container is a LAN +// peer, so there must be NO carve-out — no allow for any RFC1918 address, and no +// block that excludes a subnet. The router and the management plane stay +// unreachable, matching the Linux end-state. +func TestL2BridgeEgressACLPolicies_NoGatewayOrSubnetCarveOut(t *testing.T) { + acls := decodeACLs(t, mustBuildL2Bridge(t, nil)) + + for _, a := range acls { + // The only allows permitted with default (empty) extraAllowed are the + // two allow-any (0.0.0.0/0) rules and the two DHCP rules (no address). + if a.Action != hcn.ActionTypeAllow { + continue + } + if a.Protocols == aclUDPProtocol { + continue // DHCP allow — no address scope + } + if a.RemoteAddresses != "0.0.0.0/0" { + t.Errorf("unexpected allow carve-out to %q (only 0.0.0.0/0 and DHCP allows are permitted with no extra-allowed configured)", a.RemoteAddresses) + } + } + + // Blocks must be the whole supernets, never a range/exclusion. + for _, a := range acls { + if a.Action == hcn.ActionTypeBlock && a.RemoteAddresses != "" { + if _, _, err := net.ParseCIDR(a.RemoteAddresses); err != nil { + t.Errorf("block RemoteAddresses %q is not a plain CIDR (a carve-out range leaked in): %v", a.RemoteAddresses, err) + } + } + } +} + +// TestL2BridgeEgressACLPolicies_Precedence pins the priority ladder: DHCP and +// operator carve-outs must sit ABOVE the RFC1918 block (lower number = higher +// precedence), which must sit above the allow-any floor. +func TestL2BridgeEgressACLPolicies_Precedence(t *testing.T) { + if !(aclPriorityDHCP < aclPriorityBlock && + aclPriorityExtraAllow < aclPriorityBlock && + aclPriorityBlock < aclPriorityAllowAny) { + t.Fatalf("priority ladder broken: dhcp=%d extra=%d block=%d allowany=%d (want dhcp,extra < block < allowany)", + aclPriorityDHCP, aclPriorityExtraAllow, aclPriorityBlock, aclPriorityAllowAny) + } +} + +// TestL2BridgeEgressACLPolicies_ExtraAllowed verifies configured carve-outs are +// emitted as Out allows ABOVE the block so they win over the RFC1918 deny. +func TestL2BridgeEgressACLPolicies_ExtraAllowed(t *testing.T) { + extra := []string{"192.168.50.10/32", "172.20.0.0/16"} + acls := decodeACLs(t, mustBuildL2Bridge(t, extra)) + + found := map[string]hcn.AclPolicySetting{} + for _, a := range acls { + if a.Action == hcn.ActionTypeAllow && a.RemoteAddresses != "" && a.RemoteAddresses != "0.0.0.0/0" { + found[a.RemoteAddresses] = a + } + } + for _, cidr := range extra { + a, ok := found[cidr] + if !ok { + t.Errorf("extra-allowed %s not emitted", cidr) + continue + } + if a.Direction != hcn.DirectionTypeOut || a.Priority != aclPriorityExtraAllow { + t.Errorf("extra-allowed %s = %+v, want Out priority %d", cidr, a, aclPriorityExtraAllow) + } + if a.Priority >= aclPriorityBlock { + t.Errorf("extra-allowed %s priority %d not above block %d (would not win)", cidr, a.Priority, aclPriorityBlock) + } + } +} + +func mustBuildL2Bridge(t *testing.T, extra []string) []hcn.EndpointPolicy { + t.Helper() + policies, err := buildL2BridgeEgressACLPolicies(extra) + if err != nil { + t.Fatalf("buildL2BridgeEgressACLPolicies: %v", err) + } + return policies +} + +func assertACL(t *testing.T, name string, a hcn.AclPolicySetting, wantProto string, wantPrio uint16) { + t.Helper() + if a.Protocols != wantProto { + t.Errorf("%s protocol = %q, want %q", name, a.Protocols, wantProto) + } + if a.Priority != wantPrio { + t.Errorf("%s priority = %d, want %d", name, a.Priority, wantPrio) + } + if a.RuleType != hcn.RuleTypeSwitch { + t.Errorf("%s ruletype = %q, want Switch", name, a.RuleType) + } +} diff --git a/pkg/networking/networking.go b/pkg/networking/networking.go index 1874cc1..838d4e3 100644 --- a/pkg/networking/networking.go +++ b/pkg/networking/networking.go @@ -29,6 +29,15 @@ type Config struct { // dispatch server listening on the bridge). ControlPorts []int + // L2BridgeEgress, HostNIC, PublicDNS, and ExtraAllowedCIDRs configure the + // Windows L2Bridge egress path (see network_windows.go). They are ignored + // on Linux/macOS. When L2BridgeEgress is false (the default), Windows uses + // the HNS NAT network and these fields are unused. + L2BridgeEgress bool + HostNIC string + PublicDNS []string + ExtraAllowedCIDRs []string + Log *slog.Logger } From a61d49169c44fd75500c2b39558d6216c0742007 Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Wed, 12 Aug 2026 01:28:29 -0700 Subject: [PATCH 02/10] fix(networking): drop the DHCP ACLs that blackhole the L2Bridge port MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On-metal validation of this branch on mfl-win-amd64-101 (Server 2025 26100) found the router-safe ladder blocked EVERYTHING — the internet included — rather than only RFC1918. Bisecting the rule set on the same endpoint and the same code path, with only these two rules varying: blocks + allow-any + UDP 67/68 allows -> every probe fails: 1.1.1.1:443, 8.8.8.8:53, DNS, public HTTPS, and all RFC1918 targets blocks + allow-any -> the intended posture exactly: Grafana 192.168.10.45:3000, Incus 192.168.12.113:8443, Proxmox 192.168.5.1/.2:8006, the LAN router 192.168.1.1:80/443 and the ephemerd host all blocked, while 1.1.1.1:443, 8.8.8.8:53, DNS-by-name and public HTTPS all work Controls run alongside: the two allow-any rules alone leave everything reachable (so Allow rules do work and the port is not default-denied by their mere presence), and the original by-hand 7-policy set still enforces selectively (so the mechanism is intact). The only variable that turns a working ladder into a total blackout is the pair of port-scoped DHCP allows. HNS accepts them — ApplyPolicy returns success — but the VFP rule set it produces drops all traffic, and an explicit higher-precedence Allow for the gateway does not survive it either. This fails closed (breaks jobs) rather than open (leaks), but it makes the feature unusable. Nothing on this path needs them: the endpoint is addressed by HNS IPAM, not by a DHCP client inside the container. Removed, with a regression test asserting every rule in the ladder carries an address scope and none carries a port scope. Also corrects two comments that no longer match reality: installFirewallRules is not a no-op (firewall_windows.go programs Hyper-V firewall rules, though hyperVEgressRules builds them for DefaultSubnet/defaultGateway, so on L2Bridge they match nothing), and the DHCP-IPAM claim on initL2Bridge is wrong — HNS rejects an L2Bridge network with no Ipams outright: hcnCreateNetwork failed in Win32: The network does not have a subnet for this endpoint. (0x803b0005) / ErrorCode 2151350277 so no network, endpoint or container can be created while that path is taken. That is recorded as UNRESOLVED rather than fixed here: giving the network a subnet works (HNS then assigns endpoint addresses itself, and the rest of the path was verified on metal), but it means handing containers real LAN addresses, which needs an allocation range the LAN's DHCP server will not also hand out. That is a design decision, not a code fix. --- pkg/networking/network_windows.go | 124 ++++++++++++++----------- pkg/networking/network_windows_test.go | 51 +++++----- 2 files changed, 96 insertions(+), 79 deletions(-) diff --git a/pkg/networking/network_windows.go b/pkg/networking/network_windows.go index a8bb17c..774ea78 100644 --- a/pkg/networking/network_windows.go +++ b/pkg/networking/network_windows.go @@ -108,11 +108,26 @@ func (w *windowsNetworking) publicDNS() []string { // VFP-managed vSwitch port so the per-endpoint Switch ACLs applied in setup() // actually enforce. // -// IPAM is DHCP: the network declares NO static subnet or routes, so HNS does -// not pin an address and the container's vNIC leases its IP, default gateway, -// and (LAN) DNS from the LAN DHCP server. DNS is overridden to the public -// resolvers at the network and endpoint level so the container never needs the -// LAN router for name resolution — the egress ACLs block the router. +// DNS is overridden to the public resolvers at the network and endpoint level +// so the container never needs the LAN router for name resolution — the egress +// ACLs block the router. +// +// UNRESOLVED — IPAM. This declares NO Ipams, intending the container's vNIC to +// DHCP on the LAN. HNS does not support that: on metal (Server 2025 26100) +// Create() fails outright with +// +// hcnCreateNetwork failed in Win32: The network does not have a subnet for +// this endpoint. (0x803b0005) / ErrorCode 2151350277 +// +// so no L2Bridge network, endpoint, or container can be created at all while +// this path is taken. An L2Bridge network requires an Ipam subnet with a +// default route; with one present HNS assigns endpoint addresses itself (no +// IpConfigurations needed) and the rest of this path — endpoint creation, +// namespace attach, and the egress ACL ladder — was verified working on metal. +// +// Adopting that means ephemerd hands containers addresses out of the real LAN +// subnet, which needs a range the LAN's DHCP server will not also hand out. +// That allocation decision is why this is not simply switched over here. func (w *windowsNetworking) initL2Bridge(cfg Config) error { if cfg.HostNIC == "" { // Fail closed: without an uplink NIC the bridge has no path off-host, @@ -137,8 +152,8 @@ func (w *windowsNetworking) initL2Bridge(cfg Config) error { network := &hcn.HostComputeNetwork{ Name: l2BridgeNetworkName, Type: hcn.L2Bridge, - // No Ipams: DHCP IPAM. HNS does not assign an address; the container - // vNIC DHCPs on the LAN for its IP, gateway, and default route. + // No Ipams. See the UNRESOLVED note above: HNS rejects this + // (0x803b0005) — an L2Bridge network must carry a subnet. Policies: []hcn.NetworkPolicy{ { Type: hcn.NetAdapterName, // binds the L2Bridge to the physical NIC @@ -198,18 +213,21 @@ func (w *windowsNetworking) setup(ctx context.Context, id string, netns string) return nil, fmt.Errorf("creating HCN endpoint for %s: %w", id, err) } - // Apply ACL policies to block private network access. This is the ONLY - // egress restriction on Windows (there is no global firewall backstop — - // installFirewallRules is a no-op), so a failure here means the container - // would otherwise run with unrestricted egress to the host LAN, other - // RFC1918 services, and link-local metadata endpoints. Fail CLOSED: tear - // down the endpoint we just created and refuse the job rather than start a - // container we cannot firewall. + // Apply ACL policies to block private network access. These are the only + // egress restriction that actually enforces on this path: installFirewallRules + // is NOT a no-op (firewall_windows.go programs Hyper-V firewall rules), but + // those rules are built for the NAT subnet — hyperVEgressRules is called with + // DefaultSubnet/defaultGateway — so on L2Bridge, where the container holds a + // LAN address, they match nothing. A failure here therefore means the + // container would run with unrestricted egress to the host LAN, other RFC1918 + // services, and link-local metadata endpoints. Fail CLOSED: tear down the + // endpoint we just created and refuse the job rather than start a container we + // cannot firewall. // // The ACLs are applied to the endpoint BEFORE the container is started // (setup runs ahead of task creation), and the L2Bridge rule set is STATIC - // — it does not depend on the leased gateway or DNS (DNS is public, the - // router gets no allow) — so there is no post-lease window in which the + // — it does not depend on the gateway or DNS (DNS is public, the + // router gets no allow) — so there is no window in which the // container has LAN connectivity but no filters. if err := w.applyACLPolicies(created); err != nil { if delErr := created.Delete(); delErr != nil { @@ -323,15 +341,16 @@ func buildEgressBlockPolicies() ([]hcn.EndpointPolicy, error) { // VFP Switch-ACL precedence for the L2Bridge egress model. VFP evaluates ACLs // by Priority, LOWER number = HIGHER precedence (evaluated first, first match -// wins). The ladder is: DHCP allow (top) > operator carve-outs > RFC1918 block -// > allow-any (bottom). The two allow-any rules at the bottom are what stop the -// port from default-denying everything — VFP is default-DENY the moment any -// ACL is present. +// wins). The ladder is: operator carve-outs (top) > RFC1918 block > allow-any +// (bottom). The two allow-any rules at the bottom are what stop the port from +// default-denying everything — VFP is default-DENY the moment any ACL is +// present. +// +// Every rule in this ladder is address-scoped. See the note in +// buildL2BridgeEgressACLPolicies: mixing in a port-scoped rule with no address +// scope (the removed UDP 67/68 DHCP allows) blackholes the entire port on +// metal. const ( - // aclPriorityDHCP lets the container lease/renew even with the rest of - // RFC1918 blocked. Above the block so the DHCP server (which lives on the - // LAN, inside a blocked supernet) stays reachable. - aclPriorityDHCP uint16 = 90 // aclPriorityExtraAllow carves configured destinations out ABOVE the // RFC1918 block. Unused by default (no carve-outs) — reserved for future // operator-allowed destinations. @@ -345,12 +364,9 @@ const ( aclPriorityAllowAny uint16 = 65500 ) -// aclAnyProtocol matches any IP protocol in an HNS ACL ("256"), used for the -// allow-any and block rules per the proven metal run. The DHCP rules use UDP. -const ( - aclAnyProtocol = "256" - aclUDPProtocol = "17" -) +// aclAnyProtocol matches any IP protocol in an HNS ACL ("256"). Every rule in +// the ladder uses it, per the proven metal run. +const aclAnyProtocol = "256" // marshalACL serializes one AclPolicySetting into an EndpointPolicy, failing // closed on a marshal error (a rule we cannot serialize is a rule we cannot @@ -386,30 +402,30 @@ func buildL2BridgeEgressACLPolicies(extraAllowed []string) ([]hcn.EndpointPolicy return nil } - // Tier: DHCP. Allow UDP to/from the BOOTP/DHCP ports so the container can - // obtain and renew its lease even though the DHCP server sits inside a - // blocked supernet. Out matches the request (dst 67), In matches the reply - // (dst 68); no address scope (the DHCP server is discovered). - if err := add(hcn.AclPolicySetting{ - Protocols: aclUDPProtocol, - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeOut, - RemotePorts: "67,68", - RuleType: hcn.RuleTypeSwitch, - Priority: aclPriorityDHCP, - }); err != nil { - return nil, err - } - if err := add(hcn.AclPolicySetting{ - Protocols: aclUDPProtocol, - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeIn, - LocalPorts: "67,68", - RuleType: hcn.RuleTypeSwitch, - Priority: aclPriorityDHCP, - }); err != nil { - return nil, err - } + // NO DHCP tier. An earlier revision allowed UDP 67/68 (Out with + // RemotePorts, In with LocalPorts, no address scope) at a precedence above + // the block, so a lease/renew could survive the RFC1918 deny. On metal + // those two rules BLACKHOLE THE PORT: with them present nothing egresses at + // all — not the internet, not even a destination carved out by an explicit + // higher-precedence Allow. + // + // Measured on mfl-win-amd64-101 (Server 2025 26100), same endpoint, same + // code path, only these two rules varying: + // + // blocks + allow-any + DHCP rules -> ALL probes fail (1.1.1.1, 8.8.8.8, + // DNS, and every RFC1918 target) + // blocks + allow-any -> exactly the intended posture: + // Grafana/Incus/Proxmox/router/host + // blocked, 1.1.1.1 + 8.8.8.8 + DNS + + // public HTTPS all reachable + // + // HNS accepts the rules (ApplyPolicy returns success) but the resulting VFP + // rule set drops everything, so this fails CLOSED rather than open — it + // breaks jobs instead of leaking. Port-scoped ACLs carrying no address + // scope must not be mixed into this ladder. + // + // Nothing here needs DHCP: the endpoint is addressed by HNS IPAM, not by a + // DHCP client in the container. // Tier: operator carve-outs (future use). Allowed ABOVE the block so a // listed destination wins. Empty by default. diff --git a/pkg/networking/network_windows_test.go b/pkg/networking/network_windows_test.go index 5ea9eb2..32487c6 100644 --- a/pkg/networking/network_windows_test.go +++ b/pkg/networking/network_windows_test.go @@ -82,7 +82,6 @@ func TestL2BridgeEgressACLPolicies_LadderShape(t *testing.T) { var ( allowAnyOut, allowAnyIn bool - dhcpOut, dhcpIn bool blockedSupernets = map[string]hcn.AclPolicySetting{} ) for _, a := range acls { @@ -93,16 +92,6 @@ func TestL2BridgeEgressACLPolicies_LadderShape(t *testing.T) { case a.Action == hcn.ActionTypeAllow && a.RemoteAddresses == "0.0.0.0/0" && a.Direction == hcn.DirectionTypeIn: allowAnyIn = true assertACL(t, "allow-any-in", a, aclAnyProtocol, aclPriorityAllowAny) - case a.Action == hcn.ActionTypeAllow && a.Protocols == aclUDPProtocol && a.Direction == hcn.DirectionTypeOut: - dhcpOut = true - if a.RemotePorts != "67,68" || a.Priority != aclPriorityDHCP { - t.Errorf("dhcp-out = %+v, want RemotePorts 67,68 priority %d", a, aclPriorityDHCP) - } - case a.Action == hcn.ActionTypeAllow && a.Protocols == aclUDPProtocol && a.Direction == hcn.DirectionTypeIn: - dhcpIn = true - if a.LocalPorts != "67,68" || a.Priority != aclPriorityDHCP { - t.Errorf("dhcp-in = %+v, want LocalPorts 67,68 priority %d", a, aclPriorityDHCP) - } case a.Action == hcn.ActionTypeBlock: blockedSupernets[a.RemoteAddresses] = a } @@ -111,9 +100,6 @@ func TestL2BridgeEgressACLPolicies_LadderShape(t *testing.T) { if !allowAnyOut || !allowAnyIn { t.Errorf("missing mandatory allow-any rule: out=%v in=%v (both required or the port default-denies / drops SYN-ACK)", allowAnyOut, allowAnyIn) } - if !dhcpOut || !dhcpIn { - t.Errorf("missing DHCP allow: out=%v in=%v", dhcpOut, dhcpIn) - } // Every RFC1918 + link-local supernet must be blocked, whole, Out, at the // block priority. @@ -143,15 +129,12 @@ func TestL2BridgeEgressACLPolicies_NoGatewayOrSubnetCarveOut(t *testing.T) { for _, a := range acls { // The only allows permitted with default (empty) extraAllowed are the - // two allow-any (0.0.0.0/0) rules and the two DHCP rules (no address). + // two allow-any (0.0.0.0/0) rules. if a.Action != hcn.ActionTypeAllow { continue } - if a.Protocols == aclUDPProtocol { - continue // DHCP allow — no address scope - } if a.RemoteAddresses != "0.0.0.0/0" { - t.Errorf("unexpected allow carve-out to %q (only 0.0.0.0/0 and DHCP allows are permitted with no extra-allowed configured)", a.RemoteAddresses) + t.Errorf("unexpected allow carve-out to %q (only 0.0.0.0/0 is permitted with no extra-allowed configured)", a.RemoteAddresses) } } @@ -165,15 +148,33 @@ func TestL2BridgeEgressACLPolicies_NoGatewayOrSubnetCarveOut(t *testing.T) { } } -// TestL2BridgeEgressACLPolicies_Precedence pins the priority ladder: DHCP and -// operator carve-outs must sit ABOVE the RFC1918 block (lower number = higher +// TestL2BridgeEgressACLPolicies_Precedence pins the priority ladder: operator +// carve-outs must sit ABOVE the RFC1918 block (lower number = higher // precedence), which must sit above the allow-any floor. func TestL2BridgeEgressACLPolicies_Precedence(t *testing.T) { - if !(aclPriorityDHCP < aclPriorityBlock && - aclPriorityExtraAllow < aclPriorityBlock && + if !(aclPriorityExtraAllow < aclPriorityBlock && aclPriorityBlock < aclPriorityAllowAny) { - t.Fatalf("priority ladder broken: dhcp=%d extra=%d block=%d allowany=%d (want dhcp,extra < block < allowany)", - aclPriorityDHCP, aclPriorityExtraAllow, aclPriorityBlock, aclPriorityAllowAny) + t.Fatalf("priority ladder broken: extra=%d block=%d allowany=%d (want extra < block < allowany)", + aclPriorityExtraAllow, aclPriorityBlock, aclPriorityAllowAny) + } +} + +// TestL2BridgeEgressACLPolicies_EveryRuleIsAddressScoped is a regression guard +// for the metal finding that motivated removing the DHCP allows: a Switch ACL +// carrying only a port scope (no RemoteAddresses/LocalAddresses) blackholes the +// entire VFP port. HNS accepts such a rule, then nothing egresses at all — not +// the internet, not even an explicitly allowed destination. Keep every rule in +// this ladder address-scoped. +func TestL2BridgeEgressACLPolicies_EveryRuleIsAddressScoped(t *testing.T) { + for _, extra := range [][]string{nil, {"203.0.113.0/24"}} { + for _, a := range decodeACLs(t, mustBuildL2Bridge(t, extra)) { + if a.RemoteAddresses == "" && a.LocalAddresses == "" { + t.Errorf("ACL with no address scope: %+v (blackholes the whole port on metal)", a) + } + if a.RemotePorts != "" || a.LocalPorts != "" { + t.Errorf("port-scoped ACL in the L2Bridge ladder: %+v", a) + } + } } } From 8d60c33f5384e415bae9de1f18c085c4ea530214 Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Wed, 12 Aug 2026 22:17:54 -0700 Subject: [PATCH 03/10] feat(networking): operator-declared ip_pool IPAM for L2Bridge egress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DHCP IPAM is not available on an HNS L2Bridge network. Declaring no Ipams fails at network creation on metal (Server 2025 26100): hcnCreateNetwork failed in Win32: The network does not have a subnet for this endpoint. (0x803b0005) so no network, no endpoint, no container. The network must carry a subnet — and once it has one, HNS assigns endpoint addresses itself, from anywhere inside that prefix. On a real LAN that is the site DHCP server's scope: four container runs landed on four unrelated addresses. So ephemerd allocates the addresses. A new required key, network.ip_pool, declares the range the operator's DHCP server never leases; each endpoint is pinned with an explicit IpConfiguration out of that pool, released on teardown, and endpoints left by a previous run are reserved on adopt. There is deliberately no default pool and nothing assumes any particular network: subnet comes from the address on host_nic, gateway from that adapter's IPv4 default route, and either can be pinned. Missing ip_pool or host_nic fails at config load, naming the key; a pool outside the subnet, or one swallowing the host or the router, fails at networking init. Also fixes container provisioning on this path. Manager.GatewayIP() drove the per-job dind listener and the Go module proxy to the NAT gateway 10.88.0.1, which exists on no interface once the NAT network is gone — both would fail to bind and every job would fail to provision. The platform now reports the host's own L2Bridge address instead, and the ACL ladder gains a single address-scoped /32 allow for it, emitted only when dind or the module proxy is enabled. That allow opens every port the host listens on, because a port-scoped Switch ACL blackholes the whole VFP port; the control-plane ports are fenced back off with inbound host firewall rules scoped to the pool, which match here only because L2Bridge does not NAT (the reason that approach failed under NAT in #136). The proven ACL ladder is otherwise unchanged: whole-supernet RFC1918 + link-local blocks with no gateway or own-subnet carve-out, over the mandatory allow-any Out+In floor, every rule address-scoped. Still opt-in, NAT still the default, and no live node touched. Switching an existing node needs a reboot, not a daemon restart. --- cmd/ephemerd/main.go | 21 ++ config.example.toml | 48 ++- pkg/config/config.go | 82 ++++- pkg/config/network_test.go | 129 ++++++++ pkg/networking/firewall_windows.go | 140 +++++++-- pkg/networking/l2bridge.go | 419 +++++++++++++++++++++++++ pkg/networking/l2bridge_test.go | 342 ++++++++++++++++++++ pkg/networking/network_darwin.go | 3 + pkg/networking/network_linux.go | 3 + pkg/networking/network_windows.go | 256 ++++++++++++--- pkg/networking/network_windows_test.go | 179 ++++++++++- pkg/networking/networking.go | 59 +++- 12 files changed, 1601 insertions(+), 80 deletions(-) create mode 100644 pkg/config/network_test.go create mode 100644 pkg/networking/l2bridge.go create mode 100644 pkg/networking/l2bridge_test.go diff --git a/cmd/ephemerd/main.go b/cmd/ephemerd/main.go index d52dec9..f78fc8a 100644 --- a/cmd/ephemerd/main.go +++ b/cmd/ephemerd/main.go @@ -270,8 +270,11 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP ControlPorts: controlPorts, L2BridgeEgress: cfg.Network.L2BridgeEgress, HostNIC: cfg.Network.HostNIC, + IPPool: cfg.Network.IPPool, + Gateway: cfg.Network.Gateway, PublicDNS: cfg.Network.PublicDNS, ExtraAllowedCIDRs: cfg.Network.ExtraAllowedDestinations, + AllowHostAccess: needsHostAccess(cfg), Log: log, }) if err != nil { @@ -438,8 +441,11 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP GatewayPorts: gatewayPorts, L2BridgeEgress: cfg.Network.L2BridgeEgress, HostNIC: cfg.Network.HostNIC, + IPPool: cfg.Network.IPPool, + Gateway: cfg.Network.Gateway, PublicDNS: cfg.Network.PublicDNS, ExtraAllowedCIDRs: cfg.Network.ExtraAllowedDestinations, + AllowHostAccess: needsHostAccess(cfg), Log: log, }) if err != nil { @@ -1145,6 +1151,21 @@ func crictlCmd() *cli.Command { } } +// needsHostAccess reports whether ephemerd runs anything that job containers +// must be able to reach over the network on the host address: +// +// - dind: the per-job Docker API listener, which on Windows is a TCP listener +// on the host address handed to the job as DOCKER_HOST. +// - the Go module proxy: bound to the same address and injected as GOPROXY. +// +// It only affects the Windows L2Bridge egress path, where the ACL ladder +// otherwise blocks the host along with the rest of RFC1918 — with neither +// feature enabled the strictest posture (host unreachable) applies. On NAT and +// on Linux the gateway is already reachable and this changes nothing. +func needsHostAccess(cfg *config.Config) bool { + return cfg.Dind.Enabled || cfg.ModuleProxy.Enabled +} + func joinPath(parts ...string) string { result := parts[0] for _, p := range parts[1:] { diff --git a/config.example.toml b/config.example.toml index d28de01..dd4a8c8 100644 --- a/config.example.toml +++ b/config.example.toml @@ -81,6 +81,7 @@ owner = "your-org" [network] # Container subnet. Ephemerd auto-picks a free subnet if this conflicts with # an existing network (e.g. Podman, Docker). You don't need to change this. +# On the Windows L2Bridge path below it means something different — see there. # subnet = "10.88.0.0/16" # Bridge MTU. Auto-detected from the host's default interface. @@ -95,13 +96,42 @@ owner = "your-org" # VFP-managed vSwitch port, where per-endpoint ACLs actually enforce: all of # RFC1918 (10/8, 172.16/12, 192.168/16, 169.254/16) — including the LAN router # — is blocked, and only the internet is reachable. Leave false to keep NAT. +# +# TRADE-OFF: on L2Bridge your job containers are no longer behind NAT. They are +# peers on this host's LAN with real LAN addresses, which is exactly why the +# ACLs can enforce — and why you must reserve addresses for them (ip_pool). # l2bridge_egress = false # # Host NIC the L2Bridge binds onto. REQUIRED when l2bridge_egress = true; there -# is no default (the correct adapter name is host-specific — do NOT assume -# "Ethernet 2"). Ignored when l2bridge_egress is false. +# is no default (the correct adapter name is host-specific). Use the name shown +# by `Get-NetAdapter`. Ignored when l2bridge_egress is false. # host_nic = "Ethernet" # +# Addresses ephemerd may assign to job containers. REQUIRED when +# l2bridge_egress = true, with NO default. +# +# Why you have to set this: an L2Bridge HNS network must declare a subnet (HNS +# refuses to create one without), and left to itself HNS then picks endpoint +# addresses from anywhere inside that subnet — i.e. straight into your DHCP +# server's scope. ephemerd instead allocates from this pool and pins each +# container to the address it picked. Reserve the range on your DHCP server (or +# place it outside the DHCP scope) BEFORE enabling this, or you will get +# duplicate-address conflicts on your LAN. +# +# Either form works. Size it for at least runner.max_concurrent containers. +# The examples below use RFC 5737 documentation addresses — replace them. +# ip_pool = "192.0.2.192/27" # CIDR: .193-.222 (network/broadcast excluded) +# ip_pool = "192.0.2.200-192.0.2.230" # or an inclusive range +# +# The LAN itself. Both are auto-derived from host_nic at startup — subnet from +# the address configured on the adapter, gateway from its IPv4 default route — +# and ephemerd logs what it inferred. Set them only if the adapter has no +# default route of its own, or carries a prefix other than the LAN you want +# declared. ip_pool must lie inside subnet and must not contain either the +# host's own address or the gateway; startup refuses the config if it does. +# subnet = "192.0.2.0/24" +# gateway = "192.0.2.1" +# # Public DNS resolvers handed to L2Bridge containers so name resolution goes to # the internet rather than the (blocked) LAN router. Default 1.1.1.1, 8.8.8.8. # public_dns = ["1.1.1.1", "8.8.8.8"] @@ -109,6 +139,20 @@ owner = "your-org" # Extra destination CIDRs permitted through the L2Bridge egress ACLs, ABOVE the # RFC1918 block (reserved for future use; default none — the strict posture). # extra_allowed_destinations = [] +# +# NOTE — host reachability. When dind or the Go module proxy is enabled, +# ephemerd adds a single /32 allow for its own host address, because both serve +# job containers over the network (DOCKER_HOST / GOPROXY) and would otherwise be +# blocked with the rest of RFC1918. HNS Switch ACLs cannot be port-scoped +# without blackholing the whole port, so the control-plane ports are fenced off +# at the host firewall instead. With neither feature enabled, this host is +# unreachable from job containers like every other RFC1918 address. +# +# NOTE — migration. Switching an existing Windows node from NAT to L2Bridge +# needs a REBOOT, not just a daemon restart: creating an L2Bridge network beside +# a live HNS NAT network has been observed to wedge HNS (containers start with +# no egress, runners register then go offline, service stop hangs on drain). +# Drain the node, set the keys, reboot. [runner] diff --git a/pkg/config/config.go b/pkg/config/config.go index f97eccb..18be3e8 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -213,8 +213,22 @@ func (w *WebhookConfig) ResolvedReconcileInterval() time.Duration { // NetworkConfig configures container networking. type NetworkConfig struct { - Subnet string `toml:"subnet"` // container subnet (auto-selected if empty) - MTU int `toml:"mtu"` // bridge MTU (auto-detected from host if 0) + // Subnet is the container subnet. + // + // Linux (CNI bridge): auto-selected when empty, avoiding ranges already in + // use on the host. + // + // Windows L2Bridge (l2bridge_egress = true): the CIDR of the LAN the bridge + // is attached to — containers are peers on it, not behind NAT — declared as + // the HNS network's Ipam subnet. Auto-derived from the address configured on + // host_nic when empty, which is the expected setting. Pin it only when the + // adapter carries a prefix that differs from the LAN you want declared. + // + // Windows NAT (the default): not consulted; the HNS NAT network always uses + // the built-in 10.88.0.0/16. + Subnet string `toml:"subnet"` + + MTU int `toml:"mtu"` // bridge MTU (auto-detected from host if 0) // L2BridgeEgress opts a Windows pool into L2Bridge container networking // with VFP-enforced egress filtering, instead of the default HNS NAT. @@ -232,6 +246,33 @@ type NetworkConfig struct { // hot-added test NIC). Ignored when L2BridgeEgress is false. HostNIC string `toml:"host_nic"` + // IPPool is the range of LAN addresses ephemerd may assign to job + // containers on the L2Bridge path. REQUIRED when L2BridgeEgress is true, + // with no default, and validated at load time. + // + // Why it cannot be inferred: an L2Bridge network must declare a subnet (HNS + // rejects a subnet-less one outright), and once it has one HNS will assign + // endpoint addresses from anywhere inside it — on a real LAN, straight into + // the site DHCP server's scope. ephemerd therefore allocates addresses + // itself, and only the operator knows which slice of their LAN the DHCP + // server is configured never to lease. + // + // Accepts a CIDR ("192.0.2.192/27" — network and broadcast excluded) or an + // inclusive range ("192.0.2.200-192.0.2.230"). Must lie inside Subnet and + // must not contain the host's own address or the LAN gateway. Size it for + // at least runner.max_concurrent addresses. Ignored when L2BridgeEgress is + // false. + IPPool string `toml:"ip_pool"` + + // Gateway is the LAN router the L2Bridge default route points at (the HNS + // Ipam route next hop). Auto-derived from the default route on HostNIC when + // empty, which is the expected setting; pin it only when the adapter has no + // default route of its own or carries more than one. + // + // Containers route THROUGH this address while the egress ACLs stop them + // ADDRESSING it — the gateway gets no allow rule. Windows L2Bridge only. + Gateway string `toml:"gateway"` + // PublicDNS is the DNS resolver list handed to L2Bridge containers. Public // resolvers keep container DNS off the LAN router (which the egress ACLs // block along with the rest of RFC1918). Empty falls back to a built-in @@ -246,6 +287,39 @@ type NetworkConfig struct { ExtraAllowedDestinations []string `toml:"extra_allowed_destinations"` } +// validate rejects an L2Bridge configuration that cannot be completed safely at +// runtime. It runs at config load, on every platform, so a node whose config was +// rendered wrong dies at startup with a specific message instead of coming up +// and quietly mis-addressing containers. +// +// Only the two settings that cannot be inferred are required. subnet, gateway, +// and public_dns are all derived from the host's real primary adapter when +// unset, so nothing here assumes any particular network. ip_pool deliberately +// has NO default: on L2Bridge, containers take addresses on the operator's own +// LAN, and a built-in guess would hand out addresses the site's DHCP server is +// also leasing. Semantic checks that need the host (does the pool fit inside the +// adapter's subnet, does it swallow the gateway) happen in pkg/networking once +// the adapter has been read. +func (n *NetworkConfig) validate() error { + if !n.L2BridgeEgress { + return nil + } + if strings.TrimSpace(n.HostNIC) == "" { + return fmt.Errorf(`network.host_nic is required when network.l2bridge_egress = true: ` + + `set it to the host adapter the bridge attaches to — the name shown by ` + "`Get-NetAdapter`" + `, ` + + `e.g. host_nic = "Ethernet". There is no default; the correct adapter is host-specific`) + } + if strings.TrimSpace(n.IPPool) == "" { + return fmt.Errorf(`network.ip_pool is required when network.l2bridge_egress = true: ` + + `on L2Bridge, job containers are addressed on this host's own LAN rather than behind NAT, ` + + `so ephemerd must be told which addresses it may hand out. Set it to a range your DHCP server ` + + `is configured never to lease — either a CIDR (ip_pool = "192.0.2.192/27") or an inclusive ` + + `range (ip_pool = "192.0.2.200-192.0.2.230") — sized for at least runner.max_concurrent ` + + `containers. There is no default: any built-in guess would collide with live DHCP leases`) + } + return nil +} + type ContainerdConfig struct { // Reserved for future containerd-specific settings (e.g. snapshotter overrides) } @@ -1363,6 +1437,10 @@ func (c *Config) validate() error { } } + if err := c.Network.validate(); err != nil { + return err + } + // Webhook secret handling depends on who owns the tunnel: // - "external": ingress and the GitHub webhook are configured elsewhere, // so the secret must match that external config — we cannot invent one. diff --git a/pkg/config/network_test.go b/pkg/config/network_test.go new file mode 100644 index 0000000..054e49f --- /dev/null +++ b/pkg/config/network_test.go @@ -0,0 +1,129 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// The addresses here are RFC 5737 documentation ranges. Neither these tests nor +// the shipped defaults encode any real site's LAN — network.ip_pool has no +// default at all, which is the point of most of this file. + +// loadNetworkTOML writes a minimal valid config with the given [network] block +// and runs it through Load, returning the validation error (if any). +func loadNetworkTOML(t *testing.T, networkBlock string) (*Config, error) { + t.Helper() + t.Setenv("GITHUB_TOKEN", "ghp_test123") + + path := filepath.Join(t.TempDir(), "config.toml") + body := "[github]\nowner = \"testorg\"\n\n[network]\n" + networkBlock + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return Load(path) +} + +// TestNetworkValidate_L2BridgeRequiresIPPool is the fail-fast the whole L2Bridge +// IPAM design rests on. On L2Bridge, containers take addresses on the operator's +// own LAN; ephemerd must never guess a range, because a guess collides with live +// DHCP leases. Opting in without ip_pool has to stop the daemon at startup with +// a message naming the key. +func TestNetworkValidate_L2BridgeRequiresIPPool(t *testing.T) { + _, err := loadNetworkTOML(t, "l2bridge_egress = true\nhost_nic = \"Ethernet\"\n") + if err == nil { + t.Fatal("Load accepted l2bridge_egress = true with no ip_pool; want a startup failure") + } + msg := err.Error() + for _, want := range []string{"network.ip_pool", "l2bridge_egress", "DHCP"} { + if !strings.Contains(msg, want) { + t.Errorf("error %q does not mention %q", msg, want) + } + } + // The message must show the operator what a value looks like, in + // documentation address space — never a real LAN. + if !strings.Contains(msg, "192.0.2.") { + t.Errorf("error %q gives no example value", msg) + } +} + +// TestNetworkValidate_L2BridgeRequiresHostNIC covers the other setting that +// cannot be inferred: which adapter to bridge onto. +func TestNetworkValidate_L2BridgeRequiresHostNIC(t *testing.T) { + _, err := loadNetworkTOML(t, "l2bridge_egress = true\nip_pool = \"192.0.2.192/27\"\n") + if err == nil { + t.Fatal("Load accepted l2bridge_egress = true with no host_nic") + } + if !strings.Contains(err.Error(), "network.host_nic") { + t.Errorf("error %q does not name network.host_nic", err) + } +} + +// TestNetworkValidate_L2BridgeMinimalConfig verifies the intended surface: two +// required keys, everything else derived from the host at runtime. +func TestNetworkValidate_L2BridgeMinimalConfig(t *testing.T) { + cfg, err := loadNetworkTOML(t, "l2bridge_egress = true\nhost_nic = \"Ethernet\"\nip_pool = \"192.0.2.192/27\"\n") + if err != nil { + t.Fatalf("Load rejected a minimal valid L2Bridge config: %v", err) + } + if !cfg.Network.L2BridgeEgress || cfg.Network.HostNIC != "Ethernet" || cfg.Network.IPPool != "192.0.2.192/27" { + t.Errorf("parsed [network] = %+v, want the three keys round-tripped", cfg.Network) + } + // Everything optional stays empty so the runtime derives it from the host's + // real adapter. A non-empty default here would be a hardcoded network. + if cfg.Network.Subnet != "" || cfg.Network.Gateway != "" || len(cfg.Network.PublicDNS) != 0 { + t.Errorf("optional keys defaulted to %q / %q / %v, want empty (derived at runtime)", + cfg.Network.Subnet, cfg.Network.Gateway, cfg.Network.PublicDNS) + } +} + +// TestNetworkValidate_L2BridgeOptionalOverrides verifies the escape hatches +// parse when an operator needs to pin them. +func TestNetworkValidate_L2BridgeOptionalOverrides(t *testing.T) { + cfg, err := loadNetworkTOML(t, `l2bridge_egress = true +host_nic = "Ethernet" +ip_pool = "192.0.2.200-192.0.2.230" +subnet = "192.0.2.0/24" +gateway = "192.0.2.1" +public_dns = ["9.9.9.9"] +extra_allowed_destinations = ["203.0.113.0/24"] +`) + if err != nil { + t.Fatalf("Load: %v", err) + } + n := cfg.Network + if n.IPPool != "192.0.2.200-192.0.2.230" || n.Subnet != "192.0.2.0/24" || n.Gateway != "192.0.2.1" { + t.Errorf("address keys = %+v, want the configured values", n) + } + if len(n.PublicDNS) != 1 || n.PublicDNS[0] != "9.9.9.9" { + t.Errorf("public_dns = %v, want [9.9.9.9]", n.PublicDNS) + } + if len(n.ExtraAllowedDestinations) != 1 || n.ExtraAllowedDestinations[0] != "203.0.113.0/24" { + t.Errorf("extra_allowed_destinations = %v", n.ExtraAllowedDestinations) + } +} + +// TestNetworkValidate_NotRequiredWhenOptedOut pins that the default path — NAT, +// and every Linux/macOS node — is untouched by any of this. +func TestNetworkValidate_NotRequiredWhenOptedOut(t *testing.T) { + if _, err := loadNetworkTOML(t, "mtu = 1400\n"); err != nil { + t.Fatalf("Load rejected a config that never opts into L2Bridge: %v", err) + } + if _, err := loadNetworkTOML(t, "l2bridge_egress = false\n"); err != nil { + t.Fatalf("Load rejected l2bridge_egress = false: %v", err) + } +} + +// TestNetworkValidate_BlankStringsAreNotSet guards against a rendered config +// that emits the keys with empty values passing validation. +func TestNetworkValidate_BlankStringsAreNotSet(t *testing.T) { + _, err := loadNetworkTOML(t, "l2bridge_egress = true\nhost_nic = \" \"\nip_pool = \"192.0.2.192/27\"\n") + if err == nil { + t.Error("whitespace-only host_nic accepted") + } + _, err = loadNetworkTOML(t, "l2bridge_egress = true\nhost_nic = \"Ethernet\"\nip_pool = \" \"\n") + if err == nil { + t.Error("whitespace-only ip_pool accepted") + } +} diff --git a/pkg/networking/firewall_windows.go b/pkg/networking/firewall_windows.go index 5a5f3db..43d722b 100644 --- a/pkg/networking/firewall_windows.go +++ b/pkg/networking/firewall_windows.go @@ -3,9 +3,7 @@ package networking import ( - "encoding/binary" "fmt" - "net" "os/exec" "strconv" "strings" @@ -299,7 +297,120 @@ func removeByPrefixScript() string { ) } +// ------------------------------------------------------------------------- +// L2Bridge host-firewall backstop +// ------------------------------------------------------------------------- +// +// On the L2Bridge path the primary egress enforcement is the per-endpoint VFP +// ACL ladder (buildL2BridgeEgressACLPolicies), which was proven on metal. The +// Hyper-V firewall blocks installed for NAT are deliberately NOT reused there: +// hyperVEgressRules subtracts the container subnet from every blocked range, and +// on L2Bridge the container subnet IS the management LAN — the subtraction would +// carve the management plane straight back out of the deny. +// +// What the host firewall CAN do here, and could not on NAT, is match on the +// container's source address. #136 established that host MPSSVC rules cannot +// filter NATed container egress, because the host sees that traffic post-NAT +// with its own address as the source. L2Bridge does not NAT: a container's +// packet reaches the host carrying the container's own pool address, as ordinary +// inbound LAN traffic. So an INBOUND rule scoped remoteip= matches +// exactly the job containers and nothing else. +// +// That is used for one job: closing the control-plane ports back off after the +// VFP ladder's host /32 allow opens the host (AllowHostAccess). The allow has to +// be address-scoped — a port-scoped Switch ACL blackholes the entire VFP port — +// so the port precision lives here instead. + +// l2BridgeControlPlaneRules returns the inbound host-firewall blocks for +// container -> ephemerd control-plane traffic on the L2Bridge path: one rule per +// control port, scoped to the host address on the bridged LAN and to the source +// range containers are allocated from. +// +// Pure (no side effects) so the exact rule set is unit-testable without netsh. +func l2BridgeControlPlaneRules(hostIP, ipPool string, controlPorts []int) []winFirewallRule { + if hostIP == "" || ipPool == "" { + return nil + } + rules := make([]winFirewallRule, 0, len(controlPorts)) + for _, port := range controlPorts { + rules = append(rules, winFirewallRule{ + name: fmt.Sprintf("%s-l2b-control-%d", firewallRulePrefix, port), + spec: []string{ + "dir=in", + "action=block", + "protocol=TCP", + "localip=" + hostIP, + "localport=" + strconv.Itoa(port), + "remoteip=" + ipPool, + "profile=any", + "enable=yes", + }, + }) + } + return rules +} + +// installL2BridgeFirewallRules programs the L2Bridge backstop. Best-effort like +// the rest of installFirewallRules: the VFP ladder is the enforcement point and +// a host that cannot program netsh must not fail daemon startup. +func (w *windowsNetworking) installL2BridgeFirewallRules() error { + if w.plan == nil { + return nil + } + rules := l2BridgeControlPlaneRules(w.plan.HostIP, w.plan.PoolSpec, w.cfg.ControlPorts) + if len(rules) == 0 { + w.cfg.Log.Info("L2Bridge egress: no control ports to fence off at the host firewall") + return nil + } + for _, r := range rules { + _ = netsh(r.deleteArgs()...) // idempotent: clear any rule of this name first + w.cfg.Log.Info("adding L2Bridge control-plane firewall rule", "rule", r.name) + if err := netsh(r.addArgs()...); err != nil { + w.cfg.Log.Warn("failed to add L2Bridge control-plane firewall rule", "rule", r.name, "error", err) + } + } + w.cfg.Log.Info("L2Bridge control-plane firewall rules installed", "rules", len(rules)) + return nil +} + +// removeL2BridgeFirewallRules deletes the backstop rules by name. +func (w *windowsNetworking) removeL2BridgeFirewallRules() { + if w.plan == nil { + return + } + for _, r := range l2BridgeControlPlaneRules(w.plan.HostIP, w.plan.PoolSpec, w.cfg.ControlPorts) { + if err := netsh(r.deleteArgs()...); err != nil { + w.cfg.Log.Debug("failed to remove L2Bridge control-plane firewall rule", "rule", r.name, "error", err) + } + } +} + +// defaultGatewayForAdapter returns the IPv4 default-route next hop reachable via +// the named adapter, used to fill in network.gateway when the operator has not +// pinned it. The lowest-metric route wins, matching what Windows itself would +// pick. Returns an empty string (no error) when the adapter has no default +// route, which the caller turns into a "set network.gateway" message. +func defaultGatewayForAdapter(name string) (string, error) { + out, err := powershellOutput(fmt.Sprintf( + "(Get-NetRoute -InterfaceAlias %s -DestinationPrefix '0.0.0.0/0' -ErrorAction SilentlyContinue "+ + "| Sort-Object -Property RouteMetric | Select-Object -First 1).NextHop", + psQuote(name), + )) + if err != nil { + return "", err + } + return strings.TrimSpace(out), nil +} + func (w *windowsNetworking) installFirewallRules() error { + // L2Bridge: the VFP ACL ladder is the enforcement point. Installing the NAT + // Hyper-V/netsh rule set here would be worse than useless — it is scoped to + // the 10.88/16 NAT subnet (matches nothing) and its subtract-the-container- + // subnet logic would carve the management LAN out of the deny. + if w.cfg.L2BridgeEgress { + return w.installL2BridgeFirewallRules() + } + // Degrade gracefully at every step: a host that cannot program the // Hyper-V firewall must never fail daemon startup. It falls back to the // netsh host-firewall rules (weaker, but better than nothing on builds @@ -356,9 +467,10 @@ func (w *windowsNetworking) installFirewallRules() error { } func (w *windowsNetworking) removeFirewallRules() { - // Always attempt to remove the netsh fallback rules too — harmless if they - // were never installed — so a host that switched paths between runs does - // not leak the other path's rules. + // Always attempt to remove every rule set ephemerd can install — harmless + // if a given one was never installed — so a host that switched paths + // between runs does not leak the other path's rules. + w.removeL2BridgeFirewallRules() w.removeNetshFirewallRules() if !hyperVFirewallAvailable() { @@ -490,27 +602,15 @@ func subtractCIDR(cidr, exclude string) ([]string, error) { // v4Range returns the first and last address of an IPv4 CIDR as uint32. func v4Range(cidr string) (lo, hi uint32, err error) { - _, ipnet, err := net.ParseCIDR(cidr) + r, _, err := cidrRange(cidr) if err != nil { return 0, 0, fmt.Errorf("parsing %s: %w", cidr, err) } - ip4 := ipnet.IP.To4() - if ip4 == nil { - return 0, 0, fmt.Errorf("parsing %s: not an IPv4 CIDR", cidr) - } - ones, bits := ipnet.Mask.Size() - if bits != 32 { - return 0, 0, fmt.Errorf("parsing %s: not an IPv4 mask", cidr) - } - lo = binary.BigEndian.Uint32(ip4) - hi = lo | (1<<(32-ones) - 1) - return lo, hi, nil + return r.lo, r.hi, nil } // u32ToIP renders a uint32 back to dotted-quad form. -func u32ToIP(v uint32) string { - return net.IPv4(byte(v>>24), byte(v>>16), byte(v>>8), byte(v)).String() -} +func u32ToIP(v uint32) string { return u32ToIPv4(v) } func (w *windowsNetworking) installNetshFirewallRules() error { rules, err := hostFirewallRules(DefaultSubnet, defaultGateway, w.cfg.ControlPorts) diff --git a/pkg/networking/l2bridge.go b/pkg/networking/l2bridge.go new file mode 100644 index 0000000..5b08729 --- /dev/null +++ b/pkg/networking/l2bridge.go @@ -0,0 +1,419 @@ +package networking + +import ( + "encoding/binary" + "fmt" + "net" + "strings" + "sync" +) + +// Address planning for the Windows L2Bridge egress path. +// +// This file is deliberately platform-independent: it holds the pure address +// arithmetic (pool parsing, allocation, host-network derivation, validation) so +// the whole plan is unit-testable on any OS, while network_windows.go keeps the +// HNS calls. +// +// WHY AN EXPLICIT POOL IS REQUIRED +// +// On L2Bridge, containers are not behind NAT — they are peers on the host's own +// LAN. HNS refuses to create an L2Bridge network that declares no Ipam subnet: +// +// hcnCreateNetwork failed in Win32: The network does not have a subnet for +// this endpoint. (0x803b0005) +// +// so "just DHCP" is not an option; the network must carry a subnet. Given one, +// HNS will happily assign endpoint addresses itself — from anywhere in that +// subnet, which on a real LAN means colliding with the site DHCP server's +// scope. Measured on metal: four container runs landed on four unrelated +// addresses scattered across the declared prefix. +// +// So ephemerd allocates the addresses itself, out of an operator-declared range +// (network.ip_pool) that the site's DHCP server is configured not to hand out, +// and pins each endpoint with an explicit IpConfiguration. There is deliberately +// NO default pool: any built-in guess would either be wrong for the operator's +// LAN or would quietly hand out addresses that DHCP also leases. Missing config +// fails fast at startup instead. + +// ipRange is an inclusive IPv4 address range, held as host-order uint32 so pool +// arithmetic is exact and cheap. +type ipRange struct { + lo, hi uint32 +} + +func (r ipRange) size() uint64 { return uint64(r.hi) - uint64(r.lo) + 1 } + +func (r ipRange) contains(v uint32) bool { return v >= r.lo && v <= r.hi } + +// containsRange reports whether other lies entirely inside r. +func (r ipRange) containsRange(other ipRange) bool { + return other.lo >= r.lo && other.hi <= r.hi +} + +func (r ipRange) String() string { return u32ToIPv4(r.lo) + "-" + u32ToIPv4(r.hi) } + +// ipv4ToU32 converts a dotted-quad to host-order uint32. +func ipv4ToU32(ip net.IP) (uint32, bool) { + v4 := ip.To4() + if v4 == nil { + return 0, false + } + return binary.BigEndian.Uint32(v4), true +} + +// u32ToIPv4 renders a host-order uint32 back to dotted-quad form. +func u32ToIPv4(v uint32) string { + return net.IPv4(byte(v>>24), byte(v>>16), byte(v>>8), byte(v)).String() +} + +// parseIPPool parses an address pool in either supported form: +// +// "192.0.2.32-192.0.2.63" explicit inclusive start-end range +// "192.0.2.32/27" CIDR (network and broadcast addresses excluded) +// +// Both forms are IPv4 only — the Windows container stack ephemerd drives has no +// IPv6 path. A CIDR of /31 or /32 has no network/broadcast to reserve and is +// used whole. +func parseIPPool(spec string) (ipRange, error) { + s := strings.TrimSpace(spec) + if s == "" { + return ipRange{}, fmt.Errorf("empty address pool") + } + + if start, end, ok := strings.Cut(s, "-"); ok { + lo, err := parseIPv4(strings.TrimSpace(start)) + if err != nil { + return ipRange{}, fmt.Errorf("pool %q: start address: %w", spec, err) + } + hi, err := parseIPv4(strings.TrimSpace(end)) + if err != nil { + return ipRange{}, fmt.Errorf("pool %q: end address: %w", spec, err) + } + if hi < lo { + return ipRange{}, fmt.Errorf("pool %q: end address is below the start address", spec) + } + return ipRange{lo: lo, hi: hi}, nil + } + + if !strings.Contains(s, "/") { + return ipRange{}, fmt.Errorf("pool %q: want a CIDR (192.0.2.32/27) or a start-end range (192.0.2.32-192.0.2.63)", spec) + } + + _, ipnet, err := net.ParseCIDR(s) + if err != nil { + return ipRange{}, fmt.Errorf("pool %q: %w", spec, err) + } + base, ok := ipv4ToU32(ipnet.IP) + if !ok { + return ipRange{}, fmt.Errorf("pool %q: not an IPv4 CIDR", spec) + } + ones, bits := ipnet.Mask.Size() + if bits != 32 { + return ipRange{}, fmt.Errorf("pool %q: not an IPv4 mask", spec) + } + lo, hi := base, base|(1<<(32-ones)-1) + if ones <= 30 { + // Skip the network and broadcast addresses; handing either to a + // container would be a guaranteed-broken endpoint. + lo, hi = lo+1, hi-1 + } + return ipRange{lo: lo, hi: hi}, nil +} + +// parseIPv4 parses a bare dotted-quad address. +func parseIPv4(s string) (uint32, error) { + ip := net.ParseIP(s) + if ip == nil { + return 0, fmt.Errorf("%q is not an IP address", s) + } + v, ok := ipv4ToU32(ip) + if !ok { + return 0, fmt.Errorf("%q is not an IPv4 address", s) + } + return v, nil +} + +// ipAllocator hands out addresses from an ipRange, one per container endpoint. +// +// ephemerd owns allocation on the L2Bridge path (rather than letting HNS pick) +// precisely so every container address stays inside the operator's reserved +// range. Allocation is lowest-free-first and idempotent per id, so a retried +// setup for the same container reuses its address instead of leaking one. +type ipAllocator struct { + mu sync.Mutex + pool ipRange + reserved map[uint32]struct{} // never handed out (host, gateway, adopted endpoints) + byID map[string]uint32 + inUse map[uint32]string +} + +// newIPAllocator builds an allocator over pool. Addresses in reserved are +// permanently withheld; malformed entries are ignored (callers pass addresses +// they have already validated). +func newIPAllocator(pool ipRange, reserved ...string) *ipAllocator { + a := &ipAllocator{ + pool: pool, + reserved: map[uint32]struct{}{}, + byID: map[string]uint32{}, + inUse: map[uint32]string{}, + } + for _, r := range reserved { + a.reserve(r) + } + return a +} + +// reserve withholds an address from future allocations. Used for the host and +// gateway addresses, and for endpoints adopted from a previous daemon run. +// A non-IPv4 or out-of-pool address is a no-op. +func (a *ipAllocator) reserve(addr string) { + v, err := parseIPv4(strings.TrimSpace(addr)) + if err != nil { + return + } + a.mu.Lock() + defer a.mu.Unlock() + a.reserved[v] = struct{}{} +} + +// allocate returns the lowest free address in the pool for id, or an error when +// the pool is exhausted. Callers must treat exhaustion as fatal for the job: +// starting a container without an address of our choosing means either no +// network or an HNS-picked address that may collide with a DHCP lease. +func (a *ipAllocator) allocate(id string) (string, error) { + a.mu.Lock() + defer a.mu.Unlock() + + if v, ok := a.byID[id]; ok { + return u32ToIPv4(v), nil + } + + for v := a.pool.lo; ; v++ { + _, isReserved := a.reserved[v] + _, isUsed := a.inUse[v] + if !isReserved && !isUsed { + a.byID[id] = v + a.inUse[v] = id + return u32ToIPv4(v), nil + } + if v == a.pool.hi { + break // written this way so a pool ending at 255.255.255.255 cannot wrap + } + } + + return "", fmt.Errorf("address pool %s exhausted (%d in use): raise network.ip_pool or lower runner.max_concurrent", a.pool, len(a.inUse)) +} + +// release returns id's address to the pool. Safe to call for an unknown id. +func (a *ipAllocator) release(id string) { + a.mu.Lock() + defer a.mu.Unlock() + if v, ok := a.byID[id]; ok { + delete(a.byID, id) + delete(a.inUse, v) + } +} + +// l2BridgePlan is the fully resolved address plan for the L2Bridge network: +// everything HNS needs, plus the host address containers use to reach services +// ephemerd hosts (dind, the module proxy). +type l2BridgePlan struct { + // Subnet is the LAN CIDR the HNS network declares in its Ipam. Required — + // HNS rejects a subnet-less L2Bridge network outright (0x803b0005). + Subnet string + // PrefixLen is Subnet's prefix length, stamped onto each endpoint's + // IpConfiguration so the container's route table matches the real LAN. + PrefixLen int + // Gateway is the LAN router, used as the Ipam default-route next hop. + // Containers route THROUGH it while the egress ACLs stop them ADDRESSING + // it — that combination was verified on metal. + Gateway string + // HostIP is the ephemerd host's own address on HostNIC. + HostIP string + // Pool is the operator-declared range container addresses come from. + Pool ipRange + // PoolSpec is the pool exactly as configured, kept for firewall rules and + // log lines that should echo the operator's own words. + PoolSpec string + // DerivedSubnet / DerivedGateway record whether each value was read off the + // host rather than configured, so startup can log what it inferred. + DerivedSubnet bool + DerivedGateway bool +} + +// hostNetLookup abstracts the two host queries the plan needs, so +// resolveL2BridgePlan is testable without a Windows host or a real NIC. +type hostNetLookup struct { + // iface returns the host's IPv4 address and its network on the named + // adapter. + iface func(name string) (net.IP, *net.IPNet, error) + // gateway returns the default-route next hop reachable via the named + // adapter. + gateway func(name string) (string, error) +} + +// resolveL2BridgePlan turns the operator's [network] config plus the host's +// actual NIC state into a complete address plan, or a specific error naming the +// key that needs setting. +// +// Nothing here defaults to a particular network. subnet and gateway are read off +// the host's real primary adapter when the operator has not pinned them; ip_pool +// has no default at all, because only the operator knows which addresses their +// DHCP server is configured never to lease. +func resolveL2BridgePlan(cfg Config, look hostNetLookup) (*l2BridgePlan, error) { + nic := strings.TrimSpace(cfg.HostNIC) + if nic == "" { + return nil, fmt.Errorf("network.host_nic is required when network.l2bridge_egress = true: " + + "set it to the host adapter to bridge onto (the name shown by `Get-NetAdapter`, e.g. \"Ethernet\"). " + + "There is no default — the correct adapter is host-specific") + } + if strings.TrimSpace(cfg.IPPool) == "" { + return nil, errIPPoolRequired + } + + hostIP, hostNet, err := look.iface(nic) + if err != nil { + return nil, err + } + hostU32, ok := ipv4ToU32(hostIP) + if !ok { + return nil, fmt.Errorf("adapter %q: host address %s is not IPv4 (network.l2bridge_egress is IPv4 only)", nic, hostIP) + } + + plan := &l2BridgePlan{HostIP: u32ToIPv4(hostU32)} + + // Subnet: the operator's if pinned, otherwise the adapter's own prefix. + subnetSpec := strings.TrimSpace(cfg.Subnet) + if subnetSpec == "" { + subnetSpec = hostNet.String() + plan.DerivedSubnet = true + } + subnetRange, prefixLen, err := cidrRange(subnetSpec) + if err != nil { + return nil, fmt.Errorf("network.subnet %q: %w", subnetSpec, err) + } + if !subnetRange.contains(hostU32) { + return nil, fmt.Errorf("network.subnet %s does not contain the address %s configured on adapter %q: "+ + "either clear network.subnet to derive it from the adapter, or correct it to the LAN the adapter is on", + subnetSpec, plan.HostIP, nic) + } + plan.Subnet = subnetSpec + plan.PrefixLen = prefixLen + + // Gateway: the operator's if pinned, otherwise the adapter's default route. + gwSpec := strings.TrimSpace(cfg.Gateway) + if gwSpec == "" { + if look.gateway == nil { + return nil, errGatewayRequired(nic, fmt.Errorf("no default-route lookup available on this platform")) + } + gwSpec, err = look.gateway(nic) + if err != nil { + return nil, errGatewayRequired(nic, err) + } + gwSpec = strings.TrimSpace(gwSpec) + if gwSpec == "" { + return nil, errGatewayRequired(nic, fmt.Errorf("adapter has no IPv4 default route")) + } + plan.DerivedGateway = true + } + gwU32, err := parseIPv4(gwSpec) + if err != nil { + return nil, fmt.Errorf("network.gateway: %w", err) + } + if !subnetRange.contains(gwU32) { + return nil, fmt.Errorf("network.gateway %s is outside network.subnet %s: the L2Bridge default route "+ + "next hop must be an on-link address of the bridged LAN", gwSpec, plan.Subnet) + } + plan.Gateway = u32ToIPv4(gwU32) + + // Pool: must be inside the subnet and must not swallow the host or router. + pool, err := parseIPPool(cfg.IPPool) + if err != nil { + return nil, fmt.Errorf("network.ip_pool: %w", err) + } + if !subnetRange.containsRange(pool) { + return nil, fmt.Errorf("network.ip_pool %s (%s) is not inside network.subnet %s: containers are LAN peers "+ + "on L2Bridge, so the pool must be a reserved slice of the bridged LAN", cfg.IPPool, pool, plan.Subnet) + } + if pool.contains(hostU32) { + return nil, fmt.Errorf("network.ip_pool %s contains this host's own address %s: pick a range that excludes it", + cfg.IPPool, plan.HostIP) + } + if pool.contains(gwU32) { + return nil, fmt.Errorf("network.ip_pool %s contains the LAN gateway %s: pick a range that excludes it", + cfg.IPPool, plan.Gateway) + } + plan.Pool = pool + plan.PoolSpec = strings.TrimSpace(cfg.IPPool) + + return plan, nil +} + +// errIPPoolRequired is the backstop for the one setting that cannot be +// inferred. pkg/config rejects a missing ip_pool at load time, before any HNS +// object exists, with the full explanation; this catches a networking.Config +// assembled by any other caller. +var errIPPoolRequired = fmt.Errorf( + `network.ip_pool is required when network.l2bridge_egress = true: ` + + `set it to a slice of your LAN that DHCP never leases, as a CIDR ("192.0.2.192/27") ` + + `or an inclusive range ("192.0.2.200-192.0.2.230"). There is no default`) + +// errGatewayRequired reports that the LAN default route could not be derived and +// tells the operator which key pins it manually. +func errGatewayRequired(nic string, cause error) error { + return fmt.Errorf("could not derive the LAN default gateway for adapter %q (%w): "+ + "set network.gateway explicitly to the router address the L2Bridge should route through", nic, cause) +} + +// cidrRange returns a CIDR's inclusive address range and prefix length. +func cidrRange(cidr string) (ipRange, int, error) { + _, ipnet, err := net.ParseCIDR(strings.TrimSpace(cidr)) + if err != nil { + return ipRange{}, 0, err + } + base, ok := ipv4ToU32(ipnet.IP) + if !ok { + return ipRange{}, 0, fmt.Errorf("not an IPv4 CIDR") + } + ones, bits := ipnet.Mask.Size() + if bits != 32 { + return ipRange{}, 0, fmt.Errorf("not an IPv4 mask") + } + return ipRange{lo: base, hi: base | (1<<(32-ones) - 1)}, ones, nil +} + +// hostIPv4OnInterface returns the first routable IPv4 address configured on the +// named adapter, together with its network. On Windows the adapter name is the +// friendly name shown by Get-NetAdapter, which is also what the HNS +// NetAdapterName network policy expects — the same string in both places. +func hostIPv4OnInterface(name string) (net.IP, *net.IPNet, error) { + iface, err := net.InterfaceByName(name) + if err != nil { + return nil, nil, fmt.Errorf("network.host_nic %q: no adapter with that name on this host "+ + "(check `Get-NetAdapter`): %w", name, err) + } + addrs, err := iface.Addrs() + if err != nil { + return nil, nil, fmt.Errorf("network.host_nic %q: reading adapter addresses: %w", name, err) + } + for _, a := range addrs { + ipnet, ok := a.(*net.IPNet) + if !ok { + continue + } + v4 := ipnet.IP.To4() + if v4 == nil || v4.IsLoopback() || v4.IsLinkLocalUnicast() { + continue + } + // Normalize to a 4-byte IP and a /n IPv4 mask. + ones, bits := ipnet.Mask.Size() + if bits != 32 { + continue + } + return v4, &net.IPNet{IP: v4.Mask(ipnet.Mask), Mask: net.CIDRMask(ones, 32)}, nil + } + return nil, nil, fmt.Errorf("network.host_nic %q: adapter has no routable IPv4 address "+ + "(an APIPA/link-local-only adapter cannot bridge); give the adapter a static or DHCP address, "+ + "or set network.host_nic to the adapter that carries the LAN", name) +} diff --git a/pkg/networking/l2bridge_test.go b/pkg/networking/l2bridge_test.go new file mode 100644 index 0000000..cbd9d68 --- /dev/null +++ b/pkg/networking/l2bridge_test.go @@ -0,0 +1,342 @@ +package networking + +import ( + "fmt" + "net" + "strings" + "testing" +) + +// Addresses in these tests come from RFC 5737 documentation ranges +// (192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24). Nothing here — and nothing in +// the shipped defaults — encodes a real site's LAN. + +func mustPool(t *testing.T, spec string) ipRange { + t.Helper() + p, err := parseIPPool(spec) + if err != nil { + t.Fatalf("parseIPPool(%q): %v", spec, err) + } + return p +} + +func TestParseIPPool_Forms(t *testing.T) { + tests := []struct { + spec string + wantLo string + wantHi string + wantCount uint64 + }{ + // Inclusive start-end range, used verbatim. + {"192.0.2.200-192.0.2.230", "192.0.2.200", "192.0.2.230", 31}, + {" 192.0.2.10 - 192.0.2.10 ", "192.0.2.10", "192.0.2.10", 1}, + // CIDR: network and broadcast addresses are not handed to containers. + {"192.0.2.192/27", "192.0.2.193", "192.0.2.222", 30}, + {"192.0.2.0/24", "192.0.2.1", "192.0.2.254", 254}, + // /31 and /32 have no network/broadcast to reserve. + {"192.0.2.8/31", "192.0.2.8", "192.0.2.9", 2}, + {"192.0.2.8/32", "192.0.2.8", "192.0.2.8", 1}, + } + for _, tt := range tests { + got := mustPool(t, tt.spec) + if u32ToIPv4(got.lo) != tt.wantLo || u32ToIPv4(got.hi) != tt.wantHi { + t.Errorf("parseIPPool(%q) = %s-%s, want %s-%s", + tt.spec, u32ToIPv4(got.lo), u32ToIPv4(got.hi), tt.wantLo, tt.wantHi) + } + if got.size() != tt.wantCount { + t.Errorf("parseIPPool(%q).size() = %d, want %d", tt.spec, got.size(), tt.wantCount) + } + } +} + +func TestParseIPPool_Rejects(t *testing.T) { + for _, spec := range []string{ + "", // unset + "192.0.2.5", // bare address, neither form + "192.0.2.230-192.0.2.200", // reversed + "192.0.2.999-192.0.2.1000", // not addresses + "not-an-address", // garbage + "2001:db8::/64", // IPv6: no v6 path on this stack + "192.0.2.0/33", // impossible mask + "192.0.2.200-", // half a range + } { + if _, err := parseIPPool(spec); err == nil { + t.Errorf("parseIPPool(%q) accepted an invalid pool", spec) + } + } +} + +// TestIPAllocator_AllocatesWithinPool is the core guarantee: every address a +// container gets comes out of the operator's reserved range, so it can never +// collide with an address the site's DHCP server hands out. +func TestIPAllocator_AllocatesWithinPool(t *testing.T) { + pool := mustPool(t, "192.0.2.200-192.0.2.203") + a := newIPAllocator(pool) + + seen := map[string]bool{} + for i := range 4 { + got, err := a.allocate(fmt.Sprintf("job-%d", i)) + if err != nil { + t.Fatalf("allocate #%d: %v", i, err) + } + v, err := parseIPv4(got) + if err != nil { + t.Fatalf("allocate returned %q: %v", got, err) + } + if !pool.contains(v) { + t.Errorf("allocated %s outside the pool %s", got, pool) + } + if seen[got] { + t.Errorf("allocated %s twice", got) + } + seen[got] = true + } + + // Exhaustion must be an error, never a silent fallback to an + // HNS-chosen address that could collide with a DHCP lease. + if _, err := a.allocate("job-overflow"); err == nil { + t.Fatal("allocate succeeded on an exhausted pool; want an error so the job is refused") + } else if !strings.Contains(err.Error(), "network.ip_pool") { + t.Errorf("exhaustion error %q does not name the config key to raise", err) + } +} + +func TestIPAllocator_ReservesHostAndGateway(t *testing.T) { + pool := mustPool(t, "192.0.2.1-192.0.2.3") + a := newIPAllocator(pool, "192.0.2.1", "192.0.2.2") + + got, err := a.allocate("job") + if err != nil { + t.Fatalf("allocate: %v", err) + } + if got != "192.0.2.3" { + t.Errorf("allocate = %s, want 192.0.2.3 (the reserved host and gateway must be skipped)", got) + } +} + +func TestIPAllocator_ReleaseAndIdempotence(t *testing.T) { + a := newIPAllocator(mustPool(t, "192.0.2.10-192.0.2.11")) + + first, err := a.allocate("job-a") + if err != nil { + t.Fatalf("allocate: %v", err) + } + // Re-allocating for the same id must return the same address, so a retried + // setup does not burn a second one. + again, err := a.allocate("job-a") + if err != nil { + t.Fatalf("re-allocate: %v", err) + } + if again != first { + t.Errorf("re-allocate for the same id = %s, want the original %s", again, first) + } + + if _, err := a.allocate("job-b"); err != nil { + t.Fatalf("allocate second: %v", err) + } + if _, err := a.allocate("job-c"); err == nil { + t.Fatal("pool of 2 handed out a third address") + } + + // After release the address returns to the pool — otherwise a long-lived + // daemon leaks the pool one job at a time. + a.release("job-a") + reused, err := a.allocate("job-c") + if err != nil { + t.Fatalf("allocate after release: %v", err) + } + if reused != first { + t.Errorf("allocate after release = %s, want the freed %s", reused, first) + } + + a.release("never-allocated") // must not panic +} + +// fakeLookup builds a hostNetLookup that reports a fixed adapter address and +// default route, so the plan can be exercised without a Windows host. +func fakeLookup(hostCIDR, gateway string) hostNetLookup { + return hostNetLookup{ + iface: func(string) (net.IP, *net.IPNet, error) { + ip, ipnet, err := net.ParseCIDR(hostCIDR) + if err != nil { + return nil, nil, err + } + return ip.To4(), ipnet, nil + }, + gateway: func(string) (string, error) { return gateway, nil }, + } +} + +// TestResolveL2BridgePlan_DerivesFromHost verifies the auto-derivation contract: +// with only host_nic and ip_pool set, the subnet and gateway come off the host's +// real adapter. Nothing is defaulted to a fixed network. +func TestResolveL2BridgePlan_DerivesFromHost(t *testing.T) { + plan, err := resolveL2BridgePlan( + Config{HostNIC: "Ethernet", IPPool: "198.51.100.200-198.51.100.230"}, + fakeLookup("198.51.100.7/24", "198.51.100.1"), + ) + if err != nil { + t.Fatalf("resolveL2BridgePlan: %v", err) + } + if plan.Subnet != "198.51.100.0/24" || !plan.DerivedSubnet { + t.Errorf("subnet = %q derived=%v, want 198.51.100.0/24 derived", plan.Subnet, plan.DerivedSubnet) + } + if plan.PrefixLen != 24 { + t.Errorf("prefix len = %d, want 24", plan.PrefixLen) + } + if plan.Gateway != "198.51.100.1" || !plan.DerivedGateway { + t.Errorf("gateway = %q derived=%v, want 198.51.100.1 derived", plan.Gateway, plan.DerivedGateway) + } + if plan.HostIP != "198.51.100.7" { + t.Errorf("host ip = %q, want 198.51.100.7", plan.HostIP) + } + if plan.PoolSpec != "198.51.100.200-198.51.100.230" { + t.Errorf("pool spec = %q, want the operator's own string", plan.PoolSpec) + } +} + +// TestResolveL2BridgePlan_ExplicitOverridesWin verifies a pinned subnet/gateway +// is used as-is and reported as not derived. +func TestResolveL2BridgePlan_ExplicitOverridesWin(t *testing.T) { + plan, err := resolveL2BridgePlan( + Config{ + HostNIC: "Ethernet", + IPPool: "198.51.100.96/29", + Subnet: "198.51.100.0/25", + Gateway: "198.51.100.9", + }, + fakeLookup("198.51.100.7/24", "198.51.100.1"), + ) + if err != nil { + t.Fatalf("resolveL2BridgePlan: %v", err) + } + if plan.Subnet != "198.51.100.0/25" || plan.DerivedSubnet { + t.Errorf("subnet = %q derived=%v, want the configured 198.51.100.0/25", plan.Subnet, plan.DerivedSubnet) + } + if plan.Gateway != "198.51.100.9" || plan.DerivedGateway { + t.Errorf("gateway = %q derived=%v, want the configured 198.51.100.9", plan.Gateway, plan.DerivedGateway) + } +} + +// TestResolveL2BridgePlan_Failures walks every way the plan can be incomplete. +// Each must fail fast with a message naming the key to fix — never fall back to +// a guessed pool, which would hand containers addresses DHCP is also leasing. +func TestResolveL2BridgePlan_Failures(t *testing.T) { + tests := []struct { + name string + cfg Config + look hostNetLookup + wantText string + }{ + { + name: "no host_nic", + cfg: Config{IPPool: "198.51.100.200-198.51.100.230"}, + look: fakeLookup("198.51.100.7/24", "198.51.100.1"), + wantText: "network.host_nic", + }, + { + name: "no ip_pool", + cfg: Config{HostNIC: "Ethernet"}, + look: fakeLookup("198.51.100.7/24", "198.51.100.1"), + wantText: "network.ip_pool", + }, + { + name: "adapter not found", + cfg: Config{HostNIC: "Nope", IPPool: "198.51.100.200-198.51.100.230"}, + look: hostNetLookup{ + iface: func(string) (net.IP, *net.IPNet, error) { return nil, nil, fmt.Errorf("no such adapter") }, + gateway: func(string) (string, error) { return "198.51.100.1", nil }, + }, + wantText: "no such adapter", + }, + { + name: "no default route on the adapter", + cfg: Config{HostNIC: "Ethernet", IPPool: "198.51.100.200-198.51.100.230"}, + look: hostNetLookup{ + iface: fakeLookup("198.51.100.7/24", "").iface, + gateway: func(string) (string, error) { return "", nil }, + }, + wantText: "network.gateway", + }, + { + name: "pool outside the subnet", + cfg: Config{HostNIC: "Ethernet", IPPool: "203.0.113.200-203.0.113.230"}, + look: fakeLookup("198.51.100.7/24", "198.51.100.1"), + wantText: "not inside network.subnet", + }, + { + name: "pool swallows this host", + cfg: Config{HostNIC: "Ethernet", IPPool: "198.51.100.1-198.51.100.50"}, + look: fakeLookup("198.51.100.7/24", "198.51.100.1"), + wantText: "own address", + }, + { + name: "pool swallows the gateway", + cfg: Config{HostNIC: "Ethernet", IPPool: "198.51.100.200-198.51.100.254"}, + look: fakeLookup("198.51.100.7/24", "198.51.100.201"), + wantText: "LAN gateway", + }, + { + name: "gateway outside the subnet", + cfg: Config{HostNIC: "Ethernet", IPPool: "198.51.100.200-198.51.100.230", Gateway: "203.0.113.1"}, + look: fakeLookup("198.51.100.7/24", "198.51.100.1"), + wantText: "outside network.subnet", + }, + { + name: "pinned subnet does not contain the adapter address", + cfg: Config{HostNIC: "Ethernet", IPPool: "203.0.113.200-203.0.113.230", Subnet: "203.0.113.0/24"}, + look: fakeLookup("198.51.100.7/24", "198.51.100.1"), + wantText: "does not contain the address", + }, + { + name: "malformed pool", + cfg: Config{HostNIC: "Ethernet", IPPool: "banana"}, + look: fakeLookup("198.51.100.7/24", "198.51.100.1"), + wantText: "network.ip_pool", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + plan, err := resolveL2BridgePlan(tt.cfg, tt.look) + if err == nil { + t.Fatalf("resolveL2BridgePlan succeeded, want an error; got plan %+v", plan) + } + if !strings.Contains(err.Error(), tt.wantText) { + t.Errorf("error %q does not mention %q, so an operator cannot tell what to fix", err, tt.wantText) + } + }) + } +} + +// TestResolveL2BridgePlan_NoBuiltInNetwork is the guard against re-introducing a +// hardcoded LAN. With no adapter information available the plan must fail, not +// silently fall back to some built-in subnet, gateway, or pool. +func TestResolveL2BridgePlan_NoBuiltInNetwork(t *testing.T) { + _, err := resolveL2BridgePlan( + Config{HostNIC: "Ethernet", IPPool: "198.51.100.200-198.51.100.230"}, + hostNetLookup{ + iface: func(string) (net.IP, *net.IPNet, error) { return nil, nil, fmt.Errorf("adapter has no address") }, + gateway: func(string) (string, error) { return "", fmt.Errorf("no route") }, + }, + ) + if err == nil { + t.Fatal("resolveL2BridgePlan invented an address plan with no host information") + } +} + +// TestResolveL2BridgePlan_PoolFitsMaxConcurrent documents the sizing +// relationship an operator has to satisfy, using the shipped default +// runner.max_concurrent of 4. +func TestResolveL2BridgePlan_PoolFitsMaxConcurrent(t *testing.T) { + plan, err := resolveL2BridgePlan( + Config{HostNIC: "Ethernet", IPPool: "198.51.100.200-198.51.100.203"}, + fakeLookup("198.51.100.7/24", "198.51.100.1"), + ) + if err != nil { + t.Fatalf("resolveL2BridgePlan: %v", err) + } + if plan.Pool.size() < 4 { + t.Errorf("pool holds %d addresses, too few for the default max_concurrent of 4", plan.Pool.size()) + } +} diff --git a/pkg/networking/network_darwin.go b/pkg/networking/network_darwin.go index 48fab79..6daeb62 100644 --- a/pkg/networking/network_darwin.go +++ b/pkg/networking/network_darwin.go @@ -51,3 +51,6 @@ func (d *darwinNetworking) removeFirewallRules() {} func (d *darwinNetworking) cleanup() {} func cleanStaleBridge(_ *slog.Logger) {} // no-op on macOS + +// hostAddr: no L2Bridge on macOS — the generic subnet derivation applies. +func (d *darwinNetworking) hostAddr() string { return "" } diff --git a/pkg/networking/network_linux.go b/pkg/networking/network_linux.go index 6e13c45..c94e732 100644 --- a/pkg/networking/network_linux.go +++ b/pkg/networking/network_linux.go @@ -104,6 +104,9 @@ func (l *linuxNetworking) teardown(ctx context.Context, id string, netns string) return nil } +// hostAddr: no L2Bridge on Linux — the generic subnet derivation applies. +func (l *linuxNetworking) hostAddr() string { return "" } + func (l *linuxNetworking) cleanup() { log := l.cfg.Log diff --git a/pkg/networking/network_windows.go b/pkg/networking/network_windows.go index 774ea78..95450f3 100644 --- a/pkg/networking/network_windows.go +++ b/pkg/networking/network_windows.go @@ -32,6 +32,12 @@ type windowsNetworking struct { cfg Config network *hcn.HostComputeNetwork mu sync.Mutex + + // plan and ipam are set only on the L2Bridge path: the resolved LAN + // address plan, and the allocator that hands each endpoint an address out + // of the operator's reserved pool. + plan *l2BridgePlan + ipam *ipAllocator } func newPlatformNetworking() platformNetworking { @@ -112,32 +118,52 @@ func (w *windowsNetworking) publicDNS() []string { // so the container never needs the LAN router for name resolution — the egress // ACLs block the router. // -// UNRESOLVED — IPAM. This declares NO Ipams, intending the container's vNIC to -// DHCP on the LAN. HNS does not support that: on metal (Server 2025 26100) -// Create() fails outright with +// IPAM — why the network declares a subnet and ephemerd owns allocation. +// DHCP IPAM is not available. Declaring no Ipams fails on metal (Server 2025 +// 26100) at network creation: // // hcnCreateNetwork failed in Win32: The network does not have a subnet for // this endpoint. (0x803b0005) / ErrorCode 2151350277 // -// so no L2Bridge network, endpoint, or container can be created at all while -// this path is taken. An L2Bridge network requires an Ipam subnet with a -// default route; with one present HNS assigns endpoint addresses itself (no -// IpConfigurations needed) and the rest of this path — endpoint creation, -// namespace attach, and the egress ACL ladder — was verified working on metal. -// -// Adopting that means ephemerd hands containers addresses out of the real LAN -// subnet, which needs a range the LAN's DHCP server will not also hand out. -// That allocation decision is why this is not simply switched over here. +// so an L2Bridge network must carry a subnet plus a default route. Given one, +// HNS assigns endpoint addresses on its own — scattered anywhere across the +// declared prefix, which on a real LAN means straight into the site DHCP +// server's scope. ephemerd therefore pins each endpoint to an address it +// allocated out of the operator's reserved network.ip_pool (see setup and +// l2bridge.go). The subnet and the default-route next hop are read off the host +// adapter at runtime unless the operator pinned them; nothing here assumes any +// particular network. func (w *windowsNetworking) initL2Bridge(cfg Config) error { - if cfg.HostNIC == "" { - // Fail closed: without an uplink NIC the bridge has no path off-host, - // and silently falling back to NAT would defeat the egress guarantee - // the operator opted into. - return fmt.Errorf("L2Bridge egress enabled but no host_nic configured (set network.host_nic to the host adapter name to bridge onto)") + // Fail closed on an incomplete plan: silently falling back to NAT would + // defeat the egress guarantee the operator opted into, and guessing a pool + // would hand out addresses that collide with live DHCP leases. + plan, err := resolveL2BridgePlan(cfg, hostNetLookup{ + iface: hostIPv4OnInterface, + gateway: defaultGatewayForAdapter, + }) + if err != nil { + return fmt.Errorf("L2Bridge egress enabled but the address plan is incomplete: %w", err) + } + w.plan = plan + w.ipam = newIPAllocator(plan.Pool, plan.HostIP, plan.Gateway) + + cfg.Log.Info("L2Bridge address plan resolved", + "host_nic", cfg.HostNIC, + "subnet", plan.Subnet, "subnet_derived", plan.DerivedSubnet, + "gateway", plan.Gateway, "gateway_derived", plan.DerivedGateway, + "host_ip", plan.HostIP, + "ip_pool", plan.PoolSpec, "pool_size", plan.Pool.size()) + + if cfg.AllowHostAccess { + // Worth a warning on its own line: this is the one rule that lets a job + // container address the ephemerd host at all. + cfg.Log.Warn("L2Bridge egress permits containers to reach this host (required by dind / the module proxy)", + "host_ip", plan.HostIP, "control_ports_blocked", cfg.ControlPorts) } if existing, err := hcn.GetNetworkByName(l2BridgeNetworkName); err == nil { w.network = existing + w.adoptExistingEndpoints(existing) cfg.Log.Info("HCN L2Bridge network found", "name", l2BridgeNetworkName, "id", existing.Id, "host_nic", cfg.HostNIC) return nil } @@ -152,8 +178,28 @@ func (w *windowsNetworking) initL2Bridge(cfg Config) error { network := &hcn.HostComputeNetwork{ Name: l2BridgeNetworkName, Type: hcn.L2Bridge, - // No Ipams. See the UNRESOLVED note above: HNS rejects this - // (0x803b0005) — an L2Bridge network must carry a subnet. + Ipams: []hcn.Ipam{ + { + // "Static" means HNS does not run a DHCP client for the + // endpoints; the addresses come from this subnet. ephemerd + // narrows that further by pinning each endpoint itself. + Type: "Static", + Subnets: []hcn.Subnet{ + { + IpAddressPrefix: plan.Subnet, + Routes: []hcn.Route{ + { + // The LAN router. Containers route THROUGH it + // while the egress ACLs stop them ADDRESSING + // it — verified working on metal. + NextHop: plan.Gateway, + DestinationPrefix: "0.0.0.0/0", + }, + }, + }, + }, + }, + }, Policies: []hcn.NetworkPolicy{ { Type: hcn.NetAdapterName, // binds the L2Bridge to the physical NIC @@ -171,14 +217,60 @@ func (w *windowsNetworking) initL2Bridge(cfg Config) error { created, err := network.Create() if err != nil { - return fmt.Errorf("creating HCN L2Bridge network on %q: %w", cfg.HostNIC, err) + return fmt.Errorf("creating HCN L2Bridge network on %q (subnet %s, gateway %s): %w", + cfg.HostNIC, plan.Subnet, plan.Gateway, err) } w.network = created - cfg.Log.Info("HCN L2Bridge network created", "name", l2BridgeNetworkName, "id", created.Id, "host_nic", cfg.HostNIC) + cfg.Log.Info("HCN L2Bridge network created", + "name", l2BridgeNetworkName, "id", created.Id, "host_nic", cfg.HostNIC, + "subnet", plan.Subnet, "gateway", plan.Gateway) return nil } +// adoptExistingEndpoints marks the addresses of endpoints already present on an +// adopted network as in use, so a daemon restart that finds leftover endpoints +// does not hand the same address to a new container. +// +// These reservations are held for the life of the process: ephemerd did not +// allocate them, so it has no id to release them under. The addresses return to +// the pool at the next restart, once the stale endpoints are gone. Size the pool +// with a little headroom rather than exactly runner.max_concurrent. +// +// Best-effort: a failure to enumerate is logged, not fatal — HNS rejects a +// duplicate address at endpoint creation anyway, which fails the job closed. +func (w *windowsNetworking) adoptExistingEndpoints(network *hcn.HostComputeNetwork) { + eps, err := hcn.ListEndpointsOfNetwork(network.Id) + if err != nil { + w.cfg.Log.Warn("could not enumerate existing L2Bridge endpoints; pool may briefly double-allocate", "error", err) + return + } + for _, ep := range eps { + for _, ipc := range ep.IpConfigurations { + if ipc.IpAddress != "" { + w.ipam.reserve(ipc.IpAddress) + } + } + } + if len(eps) > 0 { + w.cfg.Log.Info("reserved addresses of pre-existing L2Bridge endpoints", "endpoints", len(eps)) + } +} + +// hostAddr reports the address containers use to reach services ephemerd hosts. +// On L2Bridge that is the host's own LAN address — there is no bridge gateway to +// bind to, and the NAT path's hard-coded 10.88.0.1 exists on no interface once +// the NAT network is out of the picture. Returning it here is what keeps the +// per-job dind listener (pkg/dind/listen_windows.go) and the Go module proxy +// bindable, and therefore jobs provisionable. +func (w *windowsNetworking) hostAddr() string { + if w.cfg.L2BridgeEgress && w.plan != nil { + return w.plan.HostIP + } + // NAT path: unchanged — the generic subnet derivation yields defaultGateway. + return "" +} + func (w *windowsNetworking) setup(ctx context.Context, id string, netns string) (*SetupResult, error) { w.mu.Lock() defer w.mu.Unlock() @@ -192,10 +284,16 @@ func (w *windowsNetworking) setup(ctx context.Context, id string, netns string) dnsServers = w.publicDNS() } - // Create endpoint on the network. On the L2Bridge path we deliberately set - // NO IpConfigurations so the container leases its address, gateway, and - // default route from the LAN DHCP server (DHCP IPAM). On NAT, HNS assigns - // from the NAT subnet as before. + // Create endpoint on the network. + // + // On NAT, HNS assigns from the NAT subnet as before (no IpConfigurations). + // + // On L2Bridge the container is a peer on the host's LAN, so leaving the + // choice to HNS means an address anywhere in the declared subnet — which on + // a real LAN overlaps the site DHCP server's scope. ephemerd pins the + // endpoint to an address it allocated from the operator's reserved + // network.ip_pool instead, at the LAN's own prefix length so the container's + // route table matches its neighbours'. endpoint := &hcn.HostComputeEndpoint{ Name: id + "-ep", HostComputeNetwork: w.network.Id, @@ -208,21 +306,44 @@ func (w *windowsNetworking) setup(ctx context.Context, id string, netns string) }, } + var allocatedIP string + if w.cfg.L2BridgeEgress { + if w.ipam == nil || w.plan == nil { + return nil, fmt.Errorf("L2Bridge egress enabled but no address plan was resolved for %s (refusing to let HNS pick an address that may collide with a DHCP lease)", id) + } + ip, err := w.ipam.allocate(id) + if err != nil { + return nil, fmt.Errorf("allocating an address for %s: %w", id, err) + } + allocatedIP = ip + endpoint.IpConfigurations = []hcn.IpConfig{ + { + IpAddress: allocatedIP, + // PrefixLen came from net.IPMask.Size() on an IPv4 mask, so it + // is 0-32 and always fits. + PrefixLength: uint8(w.plan.PrefixLen), + }, + } + } + created, err := w.network.CreateEndpoint(endpoint) if err != nil { + if allocatedIP != "" { + w.ipam.release(id) + } return nil, fmt.Errorf("creating HCN endpoint for %s: %w", id, err) } - // Apply ACL policies to block private network access. These are the only - // egress restriction that actually enforces on this path: installFirewallRules - // is NOT a no-op (firewall_windows.go programs Hyper-V firewall rules), but - // those rules are built for the NAT subnet — hyperVEgressRules is called with - // DefaultSubnet/defaultGateway — so on L2Bridge, where the container holds a - // LAN address, they match nothing. A failure here therefore means the - // container would run with unrestricted egress to the host LAN, other RFC1918 - // services, and link-local metadata endpoints. Fail CLOSED: tear down the - // endpoint we just created and refuse the job rather than start a container we - // cannot firewall. + // Apply ACL policies to block private network access. On L2Bridge these are + // the only egress restriction that enforces: the Hyper-V firewall rule set + // in firewall_windows.go is built for the NAT subnet and is deliberately not + // installed on this path (its subtract-the-container-subnet logic would + // carve the management LAN back out of the deny — see the L2Bridge backstop + // note there). A failure here therefore means the container would run with + // unrestricted egress to the host LAN, other RFC1918 services, and + // link-local metadata endpoints. Fail CLOSED: tear down the endpoint we just + // created and refuse the job rather than start a container we cannot + // firewall. // // The ACLs are applied to the endpoint BEFORE the container is started // (setup runs ahead of task creation), and the L2Bridge rule set is STATIC @@ -233,6 +354,7 @@ func (w *windowsNetworking) setup(ctx context.Context, id string, netns string) if delErr := created.Delete(); delErr != nil { w.cfg.Log.Warn("failed to delete endpoint after ACL failure", "id", id, "error", delErr) } + w.releaseIP(id) return nil, fmt.Errorf("applying egress ACL policies for %s (refusing to start unfirewalled): %w", id, err) } @@ -244,23 +366,38 @@ func (w *windowsNetworking) setup(ctx context.Context, id string, netns string) ns, err = ns.Create() if err != nil { _ = created.Delete() + w.releaseIP(id) return nil, fmt.Errorf("creating HCN namespace for %s: %w", id, err) } if err := hcn.AddNamespaceEndpoint(ns.Id, created.Id); err != nil { _ = ns.Delete() _ = created.Delete() + w.releaseIP(id) return nil, fmt.Errorf("attaching endpoint to namespace for %s: %w", id, err) } - w.cfg.Log.Debug("HCN endpoint created", "id", id, "endpoint", created.Id, "namespace", ns.Id) - return &SetupResult{NetNS: ns.Id, EndpointID: created.Id}, nil + w.cfg.Log.Debug("HCN endpoint created", "id", id, "endpoint", created.Id, "namespace", ns.Id, "ip", allocatedIP) + return &SetupResult{NetNS: ns.Id, EndpointID: created.Id, IP: allocatedIP}, nil +} + +// releaseIP returns a container's pool address, if it holds one. Safe on the NAT +// path (no allocator) and for containers that never got an address. +func (w *windowsNetworking) releaseIP(id string) { + if w.ipam != nil { + w.ipam.release(id) + } } func (w *windowsNetworking) teardown(ctx context.Context, id string, netns string) error { w.mu.Lock() defer w.mu.Unlock() + // Return the pool address whatever happens below: an endpoint that no + // longer exists (or one we fail to delete) must not strand its address, or + // a long-lived daemon would leak the pool one job at a time. + defer w.releaseIP(id) + // Find endpoint by name endpoint, err := hcn.GetEndpointByName(id + "-ep") if err != nil { @@ -351,6 +488,10 @@ func buildEgressBlockPolicies() ([]hcn.EndpointPolicy, error) { // scope (the removed UDP 67/68 DHCP allows) blackholes the entire port on // metal. const ( + // aclPriorityHostAllow carves the ephemerd host's own /32 out ABOVE the + // RFC1918 block. Emitted only when Config.AllowHostAccess is set, which is + // what makes the per-job dind Docker API and the module proxy reachable. + aclPriorityHostAllow uint16 = 90 // aclPriorityExtraAllow carves configured destinations out ABOVE the // RFC1918 block. Unused by default (no carve-outs) — reserved for future // operator-allowed destinations. @@ -391,7 +532,18 @@ func marshalACL(acl hcn.AclPolicySetting) (hcn.EndpointPolicy, error) { // gateway carve-out here, and no container-to-container allow (that would be // LAN access, which we block). extraAllowed carves additional CIDRs out above // the block for future use; empty (the default) reproduces the strict posture. -func buildL2BridgeEgressACLPolicies(extraAllowed []string) ([]hcn.EndpointPolicy, error) { +// +// hostIP, when non-empty, adds one further carve-out: a /32 allow for the +// ephemerd host itself. Nothing ephemerd serves TO containers over the network +// works without it — the per-job dind Docker API and the Go module proxy both +// listen on the host address, and a container that cannot address the host +// cannot use either. It is deliberately an address-scoped /32 and not a +// port-scoped rule: see the port-scoping note above; scoping it to the dind +// port would blackhole the whole VFP port. The exposure that opens (every port +// the host has listening, not just ephemerd's) is closed back down at the host +// firewall by l2BridgeControlPlaneRules, which CAN match the container source +// on this path because L2Bridge does not NAT. +func buildL2BridgeEgressACLPolicies(extraAllowed []string, hostIP string) ([]hcn.EndpointPolicy, error) { var policies []hcn.EndpointPolicy add := func(acl hcn.AclPolicySetting) error { p, err := marshalACL(acl) @@ -427,6 +579,21 @@ func buildL2BridgeEgressACLPolicies(extraAllowed []string) ([]hcn.EndpointPolicy // Nothing here needs DHCP: the endpoint is addressed by HNS IPAM, not by a // DHCP client in the container. + // Tier: the ephemerd host's own /32, when something it serves must be + // reachable (dind, the module proxy). Highest precedence in the ladder. + if hostIP != "" { + if err := add(hcn.AclPolicySetting{ + Protocols: aclAnyProtocol, + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeOut, + RemoteAddresses: hostIP + "/32", + RuleType: hcn.RuleTypeSwitch, + Priority: aclPriorityHostAllow, + }); err != nil { + return nil, err + } + } + // Tier: operator carve-outs (future use). Allowed ABOVE the block so a // listed destination wins. Empty by default. for _, cidr := range extraAllowed { @@ -488,7 +655,7 @@ func (w *windowsNetworking) applyACLPolicies(endpoint *hcn.HostComputeEndpoint) err error ) if w.cfg.L2BridgeEgress { - policies, err = buildL2BridgeEgressACLPolicies(w.cfg.ExtraAllowedCIDRs) + policies, err = buildL2BridgeEgressACLPolicies(w.cfg.ExtraAllowedCIDRs, w.hostAllowIP()) } else { policies, err = buildEgressBlockPolicies() } @@ -501,6 +668,17 @@ func (w *windowsNetworking) applyACLPolicies(endpoint *hcn.HostComputeEndpoint) }) } +// hostAllowIP returns the host address to carve out of the egress block, or "" +// for the strict posture in which the host is unreachable like the rest of +// RFC1918. Non-empty only when the operator runs something containers must +// reach (dind, the module proxy) AND the address plan resolved. +func (w *windowsNetworking) hostAllowIP() string { + if w.cfg.AllowHostAccess && w.plan != nil { + return w.plan.HostIP + } + return "" +} + // installFirewallRules and removeFirewallRules live in firewall_windows.go // (mirroring firewall_linux.go): the host-global Windows Firewall backstop // that complements the per-endpoint ACLs applied above. diff --git a/pkg/networking/network_windows_test.go b/pkg/networking/network_windows_test.go index 32487c6..c220763 100644 --- a/pkg/networking/network_windows_test.go +++ b/pkg/networking/network_windows_test.go @@ -5,6 +5,7 @@ package networking import ( "encoding/json" "net" + "strings" "testing" "github.com/Microsoft/hcsshim/hcn" @@ -74,11 +75,11 @@ func decodeACLs(t *testing.T, policies []hcn.EndpointPolicy) []hcn.AclPolicySett } // TestL2BridgeEgressACLPolicies_LadderShape pins the full router-safe VFP -// ladder: the two mandatory allow-any rules (Out+In), the DHCP allows, and a -// whole-supernet block for every RFC1918 + link-local range — with the exact -// actions, directions, protocols, and priorities that were proven on metal. +// ladder: the two mandatory allow-any rules (Out+In) and a whole-supernet block +// for every RFC1918 + link-local range — with the exact actions, directions, +// protocols, and priorities that were proven on metal. func TestL2BridgeEgressACLPolicies_LadderShape(t *testing.T) { - acls := decodeACLs(t, mustBuildL2Bridge(t, nil)) + acls := decodeACLs(t, mustBuildL2Bridge(t, nil, "")) var ( allowAnyOut, allowAnyIn bool @@ -125,7 +126,7 @@ func TestL2BridgeEgressACLPolicies_LadderShape(t *testing.T) { // block that excludes a subnet. The router and the management plane stay // unreachable, matching the Linux end-state. func TestL2BridgeEgressACLPolicies_NoGatewayOrSubnetCarveOut(t *testing.T) { - acls := decodeACLs(t, mustBuildL2Bridge(t, nil)) + acls := decodeACLs(t, mustBuildL2Bridge(t, nil, "")) for _, a := range acls { // The only allows permitted with default (empty) extraAllowed are the @@ -167,7 +168,7 @@ func TestL2BridgeEgressACLPolicies_Precedence(t *testing.T) { // this ladder address-scoped. func TestL2BridgeEgressACLPolicies_EveryRuleIsAddressScoped(t *testing.T) { for _, extra := range [][]string{nil, {"203.0.113.0/24"}} { - for _, a := range decodeACLs(t, mustBuildL2Bridge(t, extra)) { + for _, a := range decodeACLs(t, mustBuildL2Bridge(t, extra, "198.51.100.7")) { if a.RemoteAddresses == "" && a.LocalAddresses == "" { t.Errorf("ACL with no address scope: %+v (blackholes the whole port on metal)", a) } @@ -182,7 +183,7 @@ func TestL2BridgeEgressACLPolicies_EveryRuleIsAddressScoped(t *testing.T) { // emitted as Out allows ABOVE the block so they win over the RFC1918 deny. func TestL2BridgeEgressACLPolicies_ExtraAllowed(t *testing.T) { extra := []string{"192.168.50.10/32", "172.20.0.0/16"} - acls := decodeACLs(t, mustBuildL2Bridge(t, extra)) + acls := decodeACLs(t, mustBuildL2Bridge(t, extra, "")) found := map[string]hcn.AclPolicySetting{} for _, a := range acls { @@ -205,9 +206,9 @@ func TestL2BridgeEgressACLPolicies_ExtraAllowed(t *testing.T) { } } -func mustBuildL2Bridge(t *testing.T, extra []string) []hcn.EndpointPolicy { +func mustBuildL2Bridge(t *testing.T, extra []string, hostIP string) []hcn.EndpointPolicy { t.Helper() - policies, err := buildL2BridgeEgressACLPolicies(extra) + policies, err := buildL2BridgeEgressACLPolicies(extra, hostIP) if err != nil { t.Fatalf("buildL2BridgeEgressACLPolicies: %v", err) } @@ -226,3 +227,163 @@ func assertACL(t *testing.T, name string, a hcn.AclPolicySetting, wantProto stri t.Errorf("%s ruletype = %q, want Switch", name, a.RuleType) } } + +// TestL2BridgeEgressACLPolicies_HostAllow verifies the one carve-out that keeps +// jobs provisionable: with a host address supplied, the ladder emits an +// address-scoped /32 Allow ABOVE the RFC1918 block. Without it the container +// cannot reach the per-job dind Docker API (DOCKER_HOST) or the module proxy, +// both of which listen on the host address — the host is inside 192.168/16 or +// 10/8 like every other LAN peer. +func TestL2BridgeEgressACLPolicies_HostAllow(t *testing.T) { + const hostIP = "198.51.100.7" + acls := decodeACLs(t, mustBuildL2Bridge(t, nil, hostIP)) + + var found *hcn.AclPolicySetting + for i, a := range acls { + if a.Action == hcn.ActionTypeAllow && a.RemoteAddresses == hostIP+"/32" { + found = &acls[i] + } + } + if found == nil { + t.Fatalf("no /32 allow for the host address %s (dind and the module proxy would be unreachable)", hostIP) + } + assertACL(t, "host-allow", *found, aclAnyProtocol, aclPriorityHostAllow) + if found.Direction != hcn.DirectionTypeOut { + t.Errorf("host allow direction = %v, want Out", found.Direction) + } + if found.Priority >= aclPriorityBlock { + t.Errorf("host allow priority %d is not above the block priority %d (it would never win)", found.Priority, aclPriorityBlock) + } + // It must stay address-scoped. Narrowing it to the dind port would look + // safer and would in fact blackhole the whole VFP port (see the metal note + // on the removed DHCP rules). + if found.RemotePorts != "" || found.LocalPorts != "" { + t.Errorf("host allow is port-scoped (%+v); port-scoped Switch ACLs blackhole the VFP port", *found) + } +} + +// TestL2BridgeEgressACLPolicies_NoHostAllowByDefault pins the strict default: +// with no host address supplied (nothing ephemerd serves to containers), the +// ephemerd host is blocked along with the rest of RFC1918, exactly as verified +// on metal. +func TestL2BridgeEgressACLPolicies_NoHostAllowByDefault(t *testing.T) { + for _, a := range decodeACLs(t, mustBuildL2Bridge(t, nil, "")) { + if a.Action == hcn.ActionTypeAllow && a.RemoteAddresses != "0.0.0.0/0" { + t.Errorf("unexpected allow %q with no host access configured (want only 0.0.0.0/0)", a.RemoteAddresses) + } + } +} + +// TestL2BridgeControlPlaneRules verifies the host-firewall backstop that closes +// the control-plane ports back off after the host /32 allow opens the host. +// These are INBOUND rules scoped to the container pool as the remote — which +// only matches because L2Bridge does not NAT (on NAT the host sees its own +// address as the source; that is what sank #136). +func TestL2BridgeControlPlaneRules(t *testing.T) { + const ( + hostIP = "198.51.100.7" + pool = "198.51.100.200-198.51.100.230" + ) + rules := l2BridgeControlPlaneRules(hostIP, pool, []int{10000, 10001, 10002}) + if len(rules) != 3 { + t.Fatalf("got %d rules, want 3 (one per control port)", len(rules)) + } + for _, r := range rules { + if !strings.HasPrefix(r.name, firewallRulePrefix) { + t.Errorf("rule %q lacks the %q prefix, so Cleanup would not find it", r.name, firewallRulePrefix) + } + if specValue(r, "dir") != "in" { + t.Errorf("rule %s dir = %q, want in (container->host is inbound at the host)", r.name, specValue(r, "dir")) + } + if specValue(r, "action") != "block" { + t.Errorf("rule %s action = %q, want block", r.name, specValue(r, "action")) + } + if specValue(r, "localip") != hostIP { + t.Errorf("rule %s localip = %q, want %s", r.name, specValue(r, "localip"), hostIP) + } + if specValue(r, "remoteip") != pool { + t.Errorf("rule %s remoteip = %q, want the container pool %s", r.name, specValue(r, "remoteip"), pool) + } + if specValue(r, "localport") == "" { + t.Errorf("rule %s has no localport; a portless block would cut the host off the container entirely", r.name) + } + } + + // Rule names must not collide with the NAT netsh rules, or removing one set + // would delete the other. + natRules, err := hostFirewallRules(DefaultSubnet, defaultGateway, []int{10000, 10001, 10002}) + if err != nil { + t.Fatalf("hostFirewallRules: %v", err) + } + natNames := map[string]bool{} + for _, r := range natRules { + natNames[r.name] = true + } + for _, r := range rules { + if natNames[r.name] { + t.Errorf("L2Bridge rule name %q collides with a NAT rule name", r.name) + } + } +} + +// TestL2BridgeControlPlaneRules_NoPlanNoRules verifies the backstop stays silent +// when there is nothing to scope it to, rather than emitting a rule with an +// empty address that would match everything. +func TestL2BridgeControlPlaneRules_NoPlanNoRules(t *testing.T) { + if rules := l2BridgeControlPlaneRules("", "198.51.100.0/24", []int{10000}); rules != nil { + t.Errorf("rules emitted with no host address: %+v", rules) + } + if rules := l2BridgeControlPlaneRules("198.51.100.7", "", []int{10000}); rules != nil { + t.Errorf("rules emitted with no pool: %+v", rules) + } + if rules := l2BridgeControlPlaneRules("198.51.100.7", "198.51.100.0/24", nil); len(rules) != 0 { + t.Errorf("rules emitted with no control ports: %+v", rules) + } +} + +// TestWindowsHostAddr_L2BridgeReportsHostIP is the regression guard for the +// binding that took down provisioning on the ICS experiment: on L2Bridge there +// is no 10.88.0.1 on any interface, so anything that binds GatewayIP() — the +// per-job dind listener and the Go module proxy — must be handed the host's own +// LAN address instead. +func TestWindowsHostAddr_L2BridgeReportsHostIP(t *testing.T) { + w := &windowsNetworking{ + cfg: Config{L2BridgeEgress: true}, + plan: &l2BridgePlan{HostIP: "198.51.100.7"}, + } + if got := w.hostAddr(); got != "198.51.100.7" { + t.Errorf("hostAddr() = %q, want the host's L2Bridge address", got) + } + m := &Manager{cfg: Config{}, platform: w} + if got := m.GatewayIP(); got != "198.51.100.7" { + t.Errorf("GatewayIP() = %q, want the host's L2Bridge address (10.88.0.1 binds to nothing on L2Bridge)", got) + } + + // NAT path unchanged: no plan, no override, the old gateway still applies. + nat := &windowsNetworking{cfg: Config{}} + if got := nat.hostAddr(); got != "" { + t.Errorf("NAT hostAddr() = %q, want empty (generic derivation)", got) + } + if got := (&Manager{cfg: Config{}, platform: nat}).GatewayIP(); got != defaultGateway { + t.Errorf("NAT GatewayIP() = %q, want %q", got, defaultGateway) + } +} + +// TestWindowsHostAllowIP_GatedOnAllowHostAccess pins that the host carve-out is +// opt-in: it appears only when ephemerd actually serves something to containers. +func TestWindowsHostAllowIP_GatedOnAllowHostAccess(t *testing.T) { + plan := &l2BridgePlan{HostIP: "198.51.100.7"} + off := &windowsNetworking{cfg: Config{L2BridgeEgress: true}, plan: plan} + if got := off.hostAllowIP(); got != "" { + t.Errorf("hostAllowIP() = %q with AllowHostAccess=false, want empty (strict posture)", got) + } + on := &windowsNetworking{cfg: Config{L2BridgeEgress: true, AllowHostAccess: true}, plan: plan} + if got := on.hostAllowIP(); got != "198.51.100.7" { + t.Errorf("hostAllowIP() = %q with AllowHostAccess=true, want the host address", got) + } + // No plan (init failed or NAT) must never produce a carve-out. + none := &windowsNetworking{cfg: Config{L2BridgeEgress: true, AllowHostAccess: true}} + if got := none.hostAllowIP(); got != "" { + t.Errorf("hostAllowIP() = %q with no resolved plan, want empty", got) + } +} diff --git a/pkg/networking/networking.go b/pkg/networking/networking.go index 838d4e3..2d2b04d 100644 --- a/pkg/networking/networking.go +++ b/pkg/networking/networking.go @@ -29,15 +29,37 @@ type Config struct { // dispatch server listening on the bridge). ControlPorts []int - // L2BridgeEgress, HostNIC, PublicDNS, and ExtraAllowedCIDRs configure the - // Windows L2Bridge egress path (see network_windows.go). They are ignored - // on Linux/macOS. When L2BridgeEgress is false (the default), Windows uses - // the HNS NAT network and these fields are unused. + // The fields below configure the Windows L2Bridge egress path (see + // network_windows.go and l2bridge.go). They are ignored on Linux/macOS. + // When L2BridgeEgress is false (the default), Windows uses the HNS NAT + // network and none of them are consulted. + // + // L2BridgeEgress is the opt-in. HostNIC (the host adapter to bridge onto) + // and IPPool (the reserved LAN range container addresses come from) are + // REQUIRED when it is set; Subnet and Gateway are derived from HostNIC when + // empty. PublicDNS defaults to public resolvers so container DNS never needs + // the blocked LAN router. ExtraAllowedCIDRs carves destinations out above + // the RFC1918 block. L2BridgeEgress bool HostNIC string + IPPool string + Gateway string PublicDNS []string ExtraAllowedCIDRs []string + // AllowHostAccess permits job containers to address the ephemerd host + // itself on the L2Bridge path. Required by anything ephemerd serves TO + // containers over the network — the per-job dind Docker API and the Go + // module proxy both listen on the host address — because the egress ACLs + // otherwise block the host along with the rest of RFC1918. + // + // It is an address-scoped /32 allow, so it opens every port the host has + // listening, not just ephemerd's. The control-plane ports are blocked back + // off at the host firewall (see l2BridgeControlPlaneRules), which CAN match + // on the container source here because L2Bridge does not NAT. Left false + // when nothing needs to be reachable, which is the strictest posture. + AllowHostAccess bool + Log *slog.Logger } @@ -143,6 +165,15 @@ type platformNetworking interface { installFirewallRules() error removeFirewallRules() cleanup() + + // hostAddr returns the host address containers reach ephemerd's own + // services on, when the platform knows it better than GatewayIP's + // subnet arithmetic does. Empty means "use the generic derivation". + // + // This exists for the Windows L2Bridge path, where containers are LAN + // peers and the reachable host address is the host's own adapter address + // — not the .1 of any container subnet. + hostAddr() string } // New creates and initializes the networking manager for the current platform. @@ -168,11 +199,23 @@ func (m *Manager) Teardown(ctx context.Context, id string, netns string) error { return m.platform.teardown(ctx, id, netns) } -// GatewayIP returns the bridge gateway IP address (e.g., "10.88.0.1"). -// This is the first usable IP in the container subnet, reachable from -// inside containers. Used by services that need to be accessible to jobs -// (e.g., Go module proxy, DNS). +// GatewayIP returns the host address that services ephemerd runs for jobs must +// bind to in order to be reachable from inside containers — the Go module proxy +// and, on Windows, the per-job dind Docker API listener. +// +// Normally that is the bridge gateway (e.g. "10.88.0.1"), the first usable +// address of the container subnet. On the Windows L2Bridge path there is no +// such bridge gateway: containers are peers on the host's LAN, so the platform +// reports the host's own adapter address instead. Binding to the old hard-coded +// 10.88.0.1 there would fail outright — no interface holds that address once the +// NAT network is out of the picture — and take dind provisioning down with it. func (m *Manager) GatewayIP() string { + if m.platform != nil { + if addr := m.platform.hostAddr(); addr != "" { + return addr + } + } + subnet := m.cfg.Subnet if subnet == "" { subnet = DefaultSubnet From 40d69af3e82b87ffe9fcf8a7df3594c7c3b29fa5 Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Wed, 12 Aug 2026 22:21:09 -0700 Subject: [PATCH 04/10] fix(networking): move hostIPv4OnInterface to a Windows-only file It is only called from network_windows.go, so on non-Windows builds it tripped the unused linter and failed CI. The address arithmetic it feeds stays in l2bridge.go, untagged, so it keeps building and running under test on every platform. --- pkg/networking/l2bridge.go | 35 ---------------------- pkg/networking/l2bridge_windows.go | 47 ++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 35 deletions(-) create mode 100644 pkg/networking/l2bridge_windows.go diff --git a/pkg/networking/l2bridge.go b/pkg/networking/l2bridge.go index 5b08729..cdfa717 100644 --- a/pkg/networking/l2bridge.go +++ b/pkg/networking/l2bridge.go @@ -382,38 +382,3 @@ func cidrRange(cidr string) (ipRange, int, error) { } return ipRange{lo: base, hi: base | (1<<(32-ones) - 1)}, ones, nil } - -// hostIPv4OnInterface returns the first routable IPv4 address configured on the -// named adapter, together with its network. On Windows the adapter name is the -// friendly name shown by Get-NetAdapter, which is also what the HNS -// NetAdapterName network policy expects — the same string in both places. -func hostIPv4OnInterface(name string) (net.IP, *net.IPNet, error) { - iface, err := net.InterfaceByName(name) - if err != nil { - return nil, nil, fmt.Errorf("network.host_nic %q: no adapter with that name on this host "+ - "(check `Get-NetAdapter`): %w", name, err) - } - addrs, err := iface.Addrs() - if err != nil { - return nil, nil, fmt.Errorf("network.host_nic %q: reading adapter addresses: %w", name, err) - } - for _, a := range addrs { - ipnet, ok := a.(*net.IPNet) - if !ok { - continue - } - v4 := ipnet.IP.To4() - if v4 == nil || v4.IsLoopback() || v4.IsLinkLocalUnicast() { - continue - } - // Normalize to a 4-byte IP and a /n IPv4 mask. - ones, bits := ipnet.Mask.Size() - if bits != 32 { - continue - } - return v4, &net.IPNet{IP: v4.Mask(ipnet.Mask), Mask: net.CIDRMask(ones, 32)}, nil - } - return nil, nil, fmt.Errorf("network.host_nic %q: adapter has no routable IPv4 address "+ - "(an APIPA/link-local-only adapter cannot bridge); give the adapter a static or DHCP address, "+ - "or set network.host_nic to the adapter that carries the LAN", name) -} diff --git a/pkg/networking/l2bridge_windows.go b/pkg/networking/l2bridge_windows.go new file mode 100644 index 0000000..5116193 --- /dev/null +++ b/pkg/networking/l2bridge_windows.go @@ -0,0 +1,47 @@ +//go:build windows + +package networking + +import ( + "fmt" + "net" +) + +// hostIPv4OnInterface returns the first routable IPv4 address configured on the +// named adapter, together with its network. On Windows the adapter name is the +// friendly name shown by Get-NetAdapter, which is also what the HNS +// NetAdapterName network policy expects — the same string in both places. +// +// This lives in a Windows-only file because it reads the live host; the address +// arithmetic it feeds (resolveL2BridgePlan and friends) stays in l2bridge.go so +// it builds and is tested on every platform. +func hostIPv4OnInterface(name string) (net.IP, *net.IPNet, error) { + iface, err := net.InterfaceByName(name) + if err != nil { + return nil, nil, fmt.Errorf("network.host_nic %q: no adapter with that name on this host "+ + "(check `Get-NetAdapter`): %w", name, err) + } + addrs, err := iface.Addrs() + if err != nil { + return nil, nil, fmt.Errorf("network.host_nic %q: reading adapter addresses: %w", name, err) + } + for _, a := range addrs { + ipnet, ok := a.(*net.IPNet) + if !ok { + continue + } + v4 := ipnet.IP.To4() + if v4 == nil || v4.IsLoopback() || v4.IsLinkLocalUnicast() { + continue + } + // Normalize to a 4-byte IP and a /n IPv4 mask. + ones, bits := ipnet.Mask.Size() + if bits != 32 { + continue + } + return v4, &net.IPNet{IP: v4.Mask(ipnet.Mask), Mask: net.CIDRMask(ones, 32)}, nil + } + return nil, nil, fmt.Errorf("network.host_nic %q: adapter has no routable IPv4 address "+ + "(an APIPA/link-local-only adapter cannot bridge); give the adapter a static or DHCP address, "+ + "or set network.host_nic to the adapter that carries the LAN", name) +} From 616d5c05815a13bcdb5ed5f2d30ee9b0c513e0a2 Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Thu, 13 Aug 2026 20:58:19 -0700 Subject: [PATCH 05/10] fix(run): use the host's network config instead of building its own `ephemerd run` constructed its own networking.Config, so a local run on a host configured for L2Bridge egress got the default instead: an HNS NAT network plus the NAT-era netsh host-firewall rules. Two consequences, both seen on a live node. The job landed UNFILTERED on NAT even though the host was deliberately configured to filter, and the netsh rules block RFC1918 host-wide -- on a host whose DNS resolver is a LAN address that severs the host's own name resolution, which showed up as image pulls failing with "no such host" and then hanging at zero bytes. The run now loads config.toml once and passes the [network] settings through workflow.Runner, matching what serve does field for field, including AllowHostAccess via needsHostAccess so dind keeps working. On L2Bridge it adopts the existing network and reserves the addresses of endpoints already on it, so it cannot hand a job an address the service is already using. A missing or malformed config is still not fatal: the run falls back to the built-in default network and, on Windows, warns that the job's egress is unfiltered. Also documents the platform split honestly in the security guide. It previously claimed Windows enforces the same block list via HCN ACLs, which is false on the NAT path -- those ACLs are a VFP construct and VFP is not engaged on a NAT switch. The guide now states plainly that the Windows default does not enforce, why no software mechanism can on that stack, and what L2Bridge requires: a wired adapter, because a Wi-Fi station cannot carry container MACs, and a reserved ip_pool with no default because any guess would collide with live DHCP leases. --- cmd/ephemerd/run.go | 81 +++++++++++++++++++++++----- cmd/ephemerd/run_test.go | 114 ++++++++++++++++++++++++++++++++++++--- docs/guides/security.md | 56 +++++++++++++++++-- pkg/workflow/runner.go | 41 ++++++++++++-- 4 files changed, 266 insertions(+), 26 deletions(-) diff --git a/cmd/ephemerd/run.go b/cmd/ephemerd/run.go index c5f60dd..cf85747 100644 --- a/cmd/ephemerd/run.go +++ b/cmd/ephemerd/run.go @@ -157,34 +157,91 @@ func runWorkflow(ctx context.Context, workflowPath string, jobFilter string, ima socketPath = `\\.\pipe\ephemerd-run-` + filepath.Base(tmpDir) } - image := resolveRunImage(imageFlag, platform) + // Load the service config once. A local run must use the SAME container + // network the host is configured for — building its own would put the job + // on an unfiltered network on a host that was deliberately configured to + // filter, and on Windows would also install the NAT-era netsh rules that + // block RFC1918 host-wide. A missing or unreadable config is not fatal: + // the run falls back to the built-in default network. + cfg := loadRunConfig() runner := &workflow.Runner{ DataDir: tmpDir, SocketPath: socketPath, - Image: image, + Image: resolveRunImage(imageFlag, platform, cfg), + Network: runNetworkOptions(cfg, log), Log: log, } return runner.RunJob(ctx, jobName, job, repoDir) } +// loadRunConfig reads the service config for a local run. A missing or +// malformed config.toml is not an error here — the run simply falls back to the +// built-in defaults for both image and network — so it returns nil rather than +// failing the command. +func loadRunConfig() *config.Config { + cfg, err := config.Load(filepath.Join(configDir, "config.toml")) + if err != nil { + return nil + } + return cfg +} + +// runNetworkOptions maps the host's [network] config onto a local run and warns +// when the job will land on the default, unfilterable container network. +// +// On Windows the default is an HNS NAT network on the Hyper-V vSwitch. Container +// egress there CANNOT be filtered in software — WFP, the Hyper-V firewall, VFP +// on a NAT switch and netsh have all been ruled out on metal, because WinNAT's +// translation path never presents the packet to an inspectable filtering layer. +// A job on that network can reach anything the host can, including the LAN and +// any management planes on it. network.l2bridge_egress is the only path that +// actually enforces. See docs/guides/security.md ("Network Firewall"). +func runNetworkOptions(cfg *config.Config, log *slog.Logger) workflow.NetworkOptions { + if cfg == nil { + warnUnfilteredRunNetwork(log, "no readable config.toml") + return workflow.NetworkOptions{} + } + + opts := workflow.NetworkOptions{ + Subnet: cfg.Network.Subnet, + MTU: cfg.Network.MTU, + L2BridgeEgress: cfg.Network.L2BridgeEgress, + HostNIC: cfg.Network.HostNIC, + IPPool: cfg.Network.IPPool, + Gateway: cfg.Network.Gateway, + PublicDNS: cfg.Network.PublicDNS, + ExtraAllowedCIDRs: cfg.Network.ExtraAllowedDestinations, + AllowHostAccess: needsHostAccess(cfg), + } + + if runtime.GOOS == "windows" && !opts.L2BridgeEgress { + warnUnfilteredRunNetwork(log, "network.l2bridge_egress is not enabled") + } + return opts +} + +func warnUnfilteredRunNetwork(log *slog.Logger, reason string) { + if runtime.GOOS != "windows" { + return + } + log.Warn("this job's container egress is NOT filtered: it runs on the default Hyper-V vSwitch (HNS NAT), "+ + "where container egress cannot be filtered in software — the job can reach your LAN and anything on it. "+ + "Set network.l2bridge_egress (requires a WIRED adapter and a reserved network.ip_pool) to enforce egress", + "reason", reason) +} + // resolveRunImage determines the container image for a run job. // Priority: --image flag → service config.toml → empty (caller applies the // built-in default — see workflow.Runner.RunJob, which substitutes // defaultImage when this returns ""). -func resolveRunImage(flagValue string, platform workflow.TargetPlatform) string { +func resolveRunImage(flagValue string, platform workflow.TargetPlatform, cfg *config.Config) string { if flagValue != "" { return flagValue } - - osName := platform.String() - cfgPath := filepath.Join(configDir, "config.toml") - if cfg, err := config.Load(cfgPath); err == nil { - if img := cfg.GitHub.DefaultImageFor(osName); img != "" { - return img - } + if cfg == nil { + return "" } - - return "" + return cfg.GitHub.DefaultImageFor(platform.String()) } diff --git a/cmd/ephemerd/run_test.go b/cmd/ephemerd/run_test.go index 6bb5bc1..91fd3d6 100644 --- a/cmd/ephemerd/run_test.go +++ b/cmd/ephemerd/run_test.go @@ -41,7 +41,7 @@ default_image_windows = "ghcr.io/from-config:windows" configDir = dir t.Setenv("GITHUB_TOKEN", "ghp_test") - if got := resolveRunImage("ghcr.io/explicit:v1", workflow.PlatformLinux); got != "ghcr.io/explicit:v1" { + if got := resolveRunImage("ghcr.io/explicit:v1", workflow.PlatformLinux, loadRunConfig()); got != "ghcr.io/explicit:v1" { t.Errorf("flag-wins: got %q, want explicit override", got) } } @@ -58,7 +58,7 @@ default_image_windows = "ghcr.io/from-config:windows" configDir = dir t.Setenv("GITHUB_TOKEN", "ghp_test") - got := resolveRunImage("", workflow.PlatformLinux) + got := resolveRunImage("", workflow.PlatformLinux, loadRunConfig()) if got != "ghcr.io/from-config:linux" { t.Errorf("config-wins linux: got %q, want %q", got, "ghcr.io/from-config:linux") } @@ -76,7 +76,7 @@ default_image_windows = "ghcr.io/from-config:windows" configDir = dir t.Setenv("GITHUB_TOKEN", "ghp_test") - got := resolveRunImage("", workflow.PlatformWindows) + got := resolveRunImage("", workflow.PlatformWindows, loadRunConfig()) if got != "ghcr.io/from-config:windows" { t.Errorf("config-wins windows: got %q, want %q", got, "ghcr.io/from-config:windows") } @@ -88,7 +88,7 @@ func TestResolveRunImage_NoConfigFile(t *testing.T) { defer configDirGuard(t)() configDir = t.TempDir() - if got := resolveRunImage("", workflow.PlatformLinux); got != "" { + if got := resolveRunImage("", workflow.PlatformLinux, loadRunConfig()); got != "" { t.Errorf("no-config: got %q, want empty (caller defaults)", got) } } @@ -103,7 +103,7 @@ func TestResolveRunImage_ConfigParseError(t *testing.T) { writeConfig(t, dir, "this is not valid TOML [\n") configDir = dir - if got := resolveRunImage("", workflow.PlatformLinux); got != "" { + if got := resolveRunImage("", workflow.PlatformLinux, loadRunConfig()); got != "" { t.Errorf("config-parse-error: got %q, want empty fallback", got) } } @@ -123,7 +123,109 @@ owner = "testorg" // Windows has no built-in default image (the runtime picks one from // the host build number), so DefaultImageFor("windows") returns "" — // resolver must propagate the empty string. - if got := resolveRunImage("", workflow.PlatformWindows); got != "" { + if got := resolveRunImage("", workflow.PlatformWindows, loadRunConfig()); got != "" { t.Errorf("no-windows-override: got %q, want empty (caller defaults)", got) } } + +// A local run must inherit the host's L2Bridge settings. Before this, `ephemerd +// run` built its own default network: on Windows an HNS NAT network plus the +// NAT-era netsh rules that block RFC1918 host-wide — which severs the host's own +// DNS when its resolver is a LAN address — and put the job on an UNFILTERED +// network on a host deliberately configured to filter. +func TestRunNetworkOptions_CarriesL2BridgeConfig(t *testing.T) { + defer configDirGuard(t)() + dir := t.TempDir() + writeConfig(t, dir, ` +[github] +owner = "testorg" + +[network] +l2bridge_egress = true +host_nic = "Ethernet" +ip_pool = "192.0.2.192/27" +public_dns = ["9.9.9.9"] +extra_allowed_destinations = ["198.51.100.0/24"] +`) + configDir = dir + t.Setenv("GITHUB_TOKEN", "ghp_test") + + opts := runNetworkOptions(loadRunConfig(), quietLog()) + + if !opts.L2BridgeEgress { + t.Error("L2BridgeEgress not carried into the run; the job would land on the unfiltered NAT network") + } + if opts.HostNIC != "Ethernet" { + t.Errorf("HostNIC: got %q, want %q", opts.HostNIC, "Ethernet") + } + if opts.IPPool != "192.0.2.192/27" { + t.Errorf("IPPool: got %q, want %q", opts.IPPool, "192.0.2.192/27") + } + if len(opts.PublicDNS) != 1 || opts.PublicDNS[0] != "9.9.9.9" { + t.Errorf("PublicDNS: got %v, want [9.9.9.9]", opts.PublicDNS) + } + if len(opts.ExtraAllowedCIDRs) != 1 || opts.ExtraAllowedCIDRs[0] != "198.51.100.0/24" { + t.Errorf("ExtraAllowedCIDRs: got %v, want [198.51.100.0/24]", opts.ExtraAllowedCIDRs) + } +} + +// AllowHostAccess must follow the same rule serve uses (needsHostAccess): the +// host /32 allow exists only because dind and the module proxy serve job +// containers over the network. With neither enabled the strict posture applies. +func TestRunNetworkOptions_AllowHostAccessFollowsDind(t *testing.T) { + defer configDirGuard(t)() + dir := t.TempDir() + writeConfig(t, dir, ` +[github] +owner = "testorg" + +[network] +l2bridge_egress = true +host_nic = "Ethernet" +ip_pool = "192.0.2.192/27" + +[dind] +enabled = true +`) + configDir = dir + t.Setenv("GITHUB_TOKEN", "ghp_test") + + if opts := runNetworkOptions(loadRunConfig(), quietLog()); !opts.AllowHostAccess { + t.Error("AllowHostAccess must be set when dind is enabled, or dind cannot reach the host and jobs fail to provision") + } +} + +func TestRunNetworkOptions_StrictWhenNothingServesContainers(t *testing.T) { + defer configDirGuard(t)() + dir := t.TempDir() + writeConfig(t, dir, ` +[github] +owner = "testorg" + +[network] +l2bridge_egress = true +host_nic = "Ethernet" +ip_pool = "192.0.2.192/27" + +[dind] +enabled = false +`) + configDir = dir + t.Setenv("GITHUB_TOKEN", "ghp_test") + + if opts := runNetworkOptions(loadRunConfig(), quietLog()); opts.AllowHostAccess { + t.Error("AllowHostAccess must stay false when nothing serves containers — that is the strictest posture") + } +} + +// No config at all is not fatal: the run falls back to the built-in default +// network (and warns on Windows that egress is unfiltered). +func TestRunNetworkOptions_NoConfigFallsBack(t *testing.T) { + defer configDirGuard(t)() + configDir = t.TempDir() + + opts := runNetworkOptions(loadRunConfig(), quietLog()) + if opts.L2BridgeEgress || opts.HostNIC != "" || opts.IPPool != "" { + t.Errorf("no-config: expected zero-value options, got %+v", opts) + } +} diff --git a/docs/guides/security.md b/docs/guides/security.md index 7211e18..e7a333c 100644 --- a/docs/guides/security.md +++ b/docs/guides/security.md @@ -27,18 +27,66 @@ macOS jobs run in full virtual machines via Apple's Virtualization.framework. Ea ## Network Firewall -By default, containers are blocked from reaching private network ranges: +The intent is that containers cannot reach private network ranges: - `10.0.0.0/8` (RFC 1918) - `172.16.0.0/12` (RFC 1918) - `192.168.0.0/16` (RFC 1918) - `169.254.0.0/16` (link-local) -This prevents jobs from scanning or accessing other machines on your local network, cloud metadata services (169.254.169.254), or other containers. Outbound internet access is allowed so jobs can fetch dependencies, push artifacts, and interact with external APIs. +This keeps jobs from scanning or reaching other machines on your LAN, cloud metadata services (169.254.169.254), or other containers, while outbound internet access stays open so jobs can fetch dependencies and talk to external APIs. -On Linux, these rules are enforced via iptables in the CNI bridge configuration. On Windows, per-endpoint HCN ACL policies block the same ranges. +**How well that intent holds depends on the platform. Read the Windows section — the default there does not enforce.** -The container's own subnet (default `10.88.0.0/16`) is excluded from the block list so containers can communicate with their gateway for outbound NAT. +### Linux and macOS — enforced + +On Linux this is enforced with iptables rules in the CNI bridge configuration. The container's own subnet (default `10.88.0.0/16`) is excluded so containers can reach their gateway for DNS and outbound NAT. + +macOS jobs run inside a Linux VM sidecar and get the identical in-VM iptables stack, and the sidecar is itself NAT-hidden behind the host. + +### Windows default (HNS NAT) — NOT enforced + +> **Job containers on the default Windows network can reach your entire LAN, including any management interfaces on it.** Treat a Windows runner on the default network as if it were an unfiltered host on your network. + +By default, Windows job containers attach to an HNS **NAT** network on the Hyper-V vSwitch. ephemerd still applies per-endpoint HCN ACL policies there, but **they do not take effect**, and no software mechanism on that stack does. This was established by exhaustive testing on real hardware, not inferred: + +- **Host WFP filters** (`netsh`, the WFP API, every layer including IPFORWARD and OUTBOUND) never see container egress. WinNAT's translation path does not present the packet to an inspectable filtering layer, even though the packet does traverse `tcpip.sys`. +- **HNS Switch ACLs** are a VFP construct, and VFP is not engaged on a NAT switch. They apply successfully and do nothing. +- **Enabling VFP** on the NAT switch default-denies everything — HNS only programs selective VFP policy for L2Bridge and Overlay. +- **The Hyper-V firewall** (`New-NetFirewallHyperVRule`) has nothing to bind to: Hyper-V-isolated containers on the NAT switch never register a VM creator, so `Get-NetFirewallHyperVVMCreator` returns nothing. +- **`netsh` host rules** are post-NAT. Container traffic is indistinguishable from the host's own by then, so any rule broad enough to catch a container also blackholes the host — including the host's own DNS if your resolver is a LAN address. + +There is no configuration that makes the NAT path filterable. If you need enforced egress on Windows, use L2Bridge. + +### Windows L2Bridge — the enforcing path + +Setting `network.l2bridge_egress = true` moves job containers onto an L2Bridge network, where VFP *is* engaged and the ACL ladder genuinely enforces. This is the same stack Kubernetes Windows CNIs (Calico, Antrea) use in production. + +```toml +[network] +l2bridge_egress = true +host_nic = "Ethernet" # wired adapter to bridge onto +ip_pool = "192.0.2.192/27" # reserved range, see below +``` + +**Requirements — both are hard:** + +- **A wired Ethernet adapter.** L2Bridge puts container MAC addresses on the physical segment. A Wi-Fi adapter operating as a station cannot carry additional MACs, so 802.11 links cannot host this path at all. This is a limitation of wireless, not of ephemerd. +- **A reserved `ip_pool` your DHCP server will never lease.** On L2Bridge, containers are addressed on your LAN rather than behind NAT, so ephemerd must be told which addresses it may hand out. There is deliberately **no default** — any built-in guess would collide with live DHCP leases. Size it for at least `runner.max_concurrent` containers, and add the matching exclusion on your DHCP server *before* enabling. + +`subnet`, `gateway`, and DNS are derived from `host_nic` at startup; set them explicitly only to override. Startup fails fast with a message naming the offending key rather than guessing. + +**Understand the trade-offs before enabling:** + +- Containers hold **real LAN addresses** and are routable L2 peers of your network. The ACLs are load-bearing — a mis-scoped pool or block list means full LAN access, a worse blast radius than NAT. +- The block list covers the container's own subnet and its default gateway. Containers still *route* through the gateway but cannot *address* it. +- If `dind` or the Go module proxy is enabled, containers must be able to reach the host (that is how they reach the Docker API and `GOPROXY`), so a `/32` allow for the host is added automatically. Because a port-scoped Switch ACL disables the whole VFP port, that allow covers **all** host ports — so do not run anything on a Windows runner host that you would not expose to job containers. With both features disabled, the host stays blocked. +- **Migrating an existing node requires a reboot**, not just a service restart: creating an L2Bridge network beside a live NAT network leaves HNS in a broken state. Reserve the pool, drain the node, set the keys, then reboot. +- Anti-spoofing is not currently enforced — a container can forge a source address on the segment. + +### Local runs + +`ephemerd run` uses whatever network the host's `config.toml` specifies, so a local run on an L2Bridge host is filtered the same way a real job is. With no config, or on a Windows host without `l2bridge_egress`, it falls back to the default NAT network and logs a warning that the job's egress is unfiltered. ## Capability Restrictions diff --git a/pkg/workflow/runner.go b/pkg/workflow/runner.go index 23a36b0..911a798 100644 --- a/pkg/workflow/runner.go +++ b/pkg/workflow/runner.go @@ -24,11 +24,32 @@ const ( defaultImage = "ghcr.io/actions/actions-runner:latest" ) +// NetworkOptions carries the host's [network] configuration into a local run. +// +// Without it a local run silently built its own default network — on Windows an +// HNS NAT network plus the NAT-era netsh host-firewall rules — even on a host +// configured for L2Bridge egress. That was wrong twice over: the container came +// up unfiltered on NAT rather than on the enforcing L2Bridge path, and the netsh +// rules block RFC1918 host-wide, which severs the host's own DNS when its +// resolver is a LAN address. Mirrors the fields serve passes to networking.New. +type NetworkOptions struct { + Subnet string + MTU int + L2BridgeEgress bool + HostNIC string + IPPool string + Gateway string + PublicDNS []string + ExtraAllowedCIDRs []string + AllowHostAccess bool +} + // Runner executes workflow jobs locally using embedded containerd. type Runner struct { DataDir string SocketPath string // optional: containerd socket override for isolation from the service Image string // container image; empty falls back to defaultImage + Network NetworkOptions Log *slog.Logger } @@ -67,11 +88,23 @@ func (r *Runner) RunJob(ctx context.Context, jobName string, job Job, repoDir st return fmt.Errorf("extracting CNI plugins: %w", err) } - // Initialize networking + // Initialize networking. A local run joins whatever network the host is + // configured for rather than inventing its own — on the L2Bridge path it + // adopts the existing network and reserves the addresses of endpoints + // already on it, so it cannot hand a job an address the service is using. net, err := networking.New(networking.Config{ - DataDir: r.DataDir, - CNIBinDir: cm.Dir(), - Log: r.Log, + DataDir: r.DataDir, + CNIBinDir: cm.Dir(), + Subnet: r.Network.Subnet, + MTU: r.Network.MTU, + L2BridgeEgress: r.Network.L2BridgeEgress, + HostNIC: r.Network.HostNIC, + IPPool: r.Network.IPPool, + Gateway: r.Network.Gateway, + PublicDNS: r.Network.PublicDNS, + ExtraAllowedCIDRs: r.Network.ExtraAllowedCIDRs, + AllowHostAccess: r.Network.AllowHostAccess, + Log: r.Log, }) if err != nil { return fmt.Errorf("initializing networking: %w", err) From c197490a9b18b038af6f73a6d68d10a34dc50f6d Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Fri, 14 Aug 2026 01:37:40 -0700 Subject: [PATCH 06/10] fix(networking): L2Bridge ACL priorities below 100 kill the VFP port The 2026-08-13 deployment died with every container unreachable: runners registered, then sat offline while their jobs stayed queued. pktmon told the whole story -- the guest was configured and ARPing for its gateway, the router's reply arrived at the NIC, and the vSwitch dropped it as "Invalid Packet" before the container port, every single time. A bisect harness (recreating the proven 2026-08-12 hand run through the daemon's containerd, one field varied per run) isolated the cause to a single fact about this HNS build (Server 2025, 26100): a Switch-rule ACL with Priority below 100 silently kills the endpoint's entire VFP dataplane. It is independent of the rule's action, direction, and address -- an irrelevant Block of a documentation IP at priority 90 reproduces the dead port, and the identical ladder with every priority at or above 100 works. HNS accepts the policy without error either way. The ladder's host-allow tier sat at priority 90 (and the extra-allow tier at 95), so any endpoint with dind enabled came up dead. The earlier "port-scoped DHCP rules blackhole the port" finding was in the same band (90/95) and was likely this trap as well. Re-tier the ladder to 100 (host allow) / 150 (extra allows) / 200 (RFC1918+link-local blocks) / 65500 (allow-any Out+In). Same precedence order, nothing below 100. A regression test pins both the constants and every emitted rule above the floor. Two more fixes from the same investigation: - Creating the L2Bridge network consumes the host NIC: it re-enumerates as "vEthernet ()", so every daemon restart after the first failed plan resolution and crash-looped. Adapter lookup (address and gateway both) now tries the vEthernet-renamed form, and the unwrapped form for operators who configured the vEthernet name. - Endpoint IpConfigurations no longer set PrefixLength. HNS derives it from the network's subnet; the proven hand run and Microsoft's own sdnbridge CNI both omit it. Validated live on the win-amd64 node: with the re-tiered ladder a real CI job's runner came online in under a minute and the job executed on the L2Bridge network, ACLs enforcing (harness probe: internet up, management plane blocked). Same node, same jobs, old ladder: dead. --- pkg/networking/firewall_windows.go | 25 ++++++++++----- pkg/networking/l2bridge_windows.go | 43 +++++++++++++++++++++++--- pkg/networking/network_windows.go | 32 +++++++++++++++---- pkg/networking/network_windows_test.go | 32 +++++++++++++++++++ 4 files changed, 113 insertions(+), 19 deletions(-) diff --git a/pkg/networking/firewall_windows.go b/pkg/networking/firewall_windows.go index 43d722b..5d0524d 100644 --- a/pkg/networking/firewall_windows.go +++ b/pkg/networking/firewall_windows.go @@ -391,15 +391,24 @@ func (w *windowsNetworking) removeL2BridgeFirewallRules() { // pick. Returns an empty string (no error) when the adapter has no default // route, which the caller turns into a "set network.gateway" message. func defaultGatewayForAdapter(name string) (string, error) { - out, err := powershellOutput(fmt.Sprintf( - "(Get-NetRoute -InterfaceAlias %s -DestinationPrefix '0.0.0.0/0' -ErrorAction SilentlyContinue "+ - "| Sort-Object -Property RouteMetric | Select-Object -First 1).NextHop", - psQuote(name), - )) - if err != nil { - return "", err + // Try the configured name and its vEthernet-renamed form (creating the + // L2Bridge network renames the NIC — see adapterNameCandidates). + var lastErr error + for _, candidate := range adapterNameCandidates(name) { + out, err := powershellOutput(fmt.Sprintf( + "(Get-NetRoute -InterfaceAlias %s -DestinationPrefix '0.0.0.0/0' -ErrorAction SilentlyContinue "+ + "| Sort-Object -Property RouteMetric | Select-Object -First 1).NextHop", + psQuote(candidate), + )) + if err != nil { + lastErr = err + continue + } + if hop := strings.TrimSpace(out); hop != "" { + return hop, nil + } } - return strings.TrimSpace(out), nil + return "", lastErr } func (w *windowsNetworking) installFirewallRules() error { diff --git a/pkg/networking/l2bridge_windows.go b/pkg/networking/l2bridge_windows.go index 5116193..fd9437d 100644 --- a/pkg/networking/l2bridge_windows.go +++ b/pkg/networking/l2bridge_windows.go @@ -5,21 +5,54 @@ package networking import ( "fmt" "net" + "strings" ) +// adapterNameCandidates returns the adapter names to try for a configured +// network.host_nic, in order. +// +// Creating the L2Bridge network CONSUMES the configured NIC: HNS binds an +// external vSwitch to it and the host's side of it re-enumerates as +// "vEthernet ()". So on every daemon start after the first, the raw +// configured name no longer exists — resolving only the literal name made the +// daemon crash-loop on restart ("no adapter with that name on this host", +// found on metal 2026-08-13). Deleting the network reverses the rename, so +// the unwrapped form must be tried too when the operator configured the +// vEthernet name. +func adapterNameCandidates(name string) []string { + candidates := []string{name} + if inner, ok := strings.CutPrefix(name, "vEthernet ("); ok { + if inner, ok := strings.CutSuffix(inner, ")"); ok { + candidates = append(candidates, inner) + } + } else { + candidates = append(candidates, "vEthernet ("+name+")") + } + return candidates +} + // hostIPv4OnInterface returns the first routable IPv4 address configured on the // named adapter, together with its network. On Windows the adapter name is the // friendly name shown by Get-NetAdapter, which is also what the HNS // NetAdapterName network policy expects — the same string in both places. +// The vEthernet-renamed form of the adapter is accepted too (see +// adapterNameCandidates). // // This lives in a Windows-only file because it reads the live host; the address // arithmetic it feeds (resolveL2BridgePlan and friends) stays in l2bridge.go so // it builds and is tested on every platform. func hostIPv4OnInterface(name string) (net.IP, *net.IPNet, error) { - iface, err := net.InterfaceByName(name) - if err != nil { + var iface *net.Interface + var err error + for _, candidate := range adapterNameCandidates(name) { + iface, err = net.InterfaceByName(candidate) + if err == nil { + break + } + } + if iface == nil { return nil, nil, fmt.Errorf("network.host_nic %q: no adapter with that name on this host "+ - "(check `Get-NetAdapter`): %w", name, err) + "(also tried %v; check `Get-NetAdapter`): %w", name, adapterNameCandidates(name)[1:], err) } addrs, err := iface.Addrs() if err != nil { @@ -41,7 +74,7 @@ func hostIPv4OnInterface(name string) (net.IP, *net.IPNet, error) { } return v4, &net.IPNet{IP: v4.Mask(ipnet.Mask), Mask: net.CIDRMask(ones, 32)}, nil } - return nil, nil, fmt.Errorf("network.host_nic %q: adapter has no routable IPv4 address "+ + return nil, nil, fmt.Errorf("network.host_nic %q (resolved adapter %q): adapter has no routable IPv4 address "+ "(an APIPA/link-local-only adapter cannot bridge); give the adapter a static or DHCP address, "+ - "or set network.host_nic to the adapter that carries the LAN", name) + "or set network.host_nic to the adapter that carries the LAN", name, iface.Name) } diff --git a/pkg/networking/network_windows.go b/pkg/networking/network_windows.go index 95450f3..bebd841 100644 --- a/pkg/networking/network_windows.go +++ b/pkg/networking/network_windows.go @@ -316,12 +316,14 @@ func (w *windowsNetworking) setup(ctx context.Context, id string, netns string) return nil, fmt.Errorf("allocating an address for %s: %w", id, err) } allocatedIP = ip + // PrefixLength is deliberately NOT set: HNS derives it from the + // network's declared subnet. The 2026-08-12 hand proof pinned exactly + // this shape and worked; Microsoft's own sdnbridge CNI also omits it. + // Setting it was the delta in the dead deployment (gateway ARP replies + // dropped by the vSwitch as "Invalid Packet" before reaching the port). endpoint.IpConfigurations = []hcn.IpConfig{ { IpAddress: allocatedIP, - // PrefixLen came from net.IPMask.Size() on an IPv4 mask, so it - // is 0-32 and always fits. - PrefixLength: uint8(w.plan.PrefixLen), }, } } @@ -487,19 +489,37 @@ func buildEgressBlockPolicies() ([]hcn.EndpointPolicy, error) { // buildL2BridgeEgressACLPolicies: mixing in a port-scoped rule with no address // scope (the removed UDP 67/68 DHCP allows) blackholes the entire port on // metal. +// +// CRITICAL — priorities below 100 kill the port (proven on metal 2026-08-14, +// l2test bisect harness, Server 2025 build 26100). A Switch-rule ACL with +// Priority < 100 silently kills the endpoint's ENTIRE VFP dataplane: the port +// drops every inbound frame — including the gateway's ARP reply (pktmon shows +// "Invalid Packet" at the vSwitch before the port) — so the container can +// never resolve its next hop and has no connectivity at all, while HNS reports +// the endpoint healthy and applies the policy without error. This is +// independent of the rule's action, direction, or address: an irrelevant +// Block 203.0.113.1/32 at priority 90 reproduces it, and the identical ladder +// with every priority >= 100 works (internet up, RFC1918 blocked). This is +// what broke the 2026-08-13 production deployment (host-allow at 90), and the +// earlier "port-scoped DHCP rules blackhole the port" incident (rules at +// 90/95) was in the same band. NO RULE MAY EVER CARRY Priority < 100; the +// regression tests enforce it. const ( + // aclPriorityMinimum is the lowest Switch-ACL priority that is safe on + // metal. Rules below this kill the port — see the block comment above. + aclPriorityMinimum uint16 = 100 // aclPriorityHostAllow carves the ephemerd host's own /32 out ABOVE the // RFC1918 block. Emitted only when Config.AllowHostAccess is set, which is // what makes the per-job dind Docker API and the module proxy reachable. - aclPriorityHostAllow uint16 = 90 + aclPriorityHostAllow uint16 = 100 // aclPriorityExtraAllow carves configured destinations out ABOVE the // RFC1918 block. Unused by default (no carve-outs) — reserved for future // operator-allowed destinations. - aclPriorityExtraAllow uint16 = 95 + aclPriorityExtraAllow uint16 = 150 // aclPriorityBlock denies the RFC1918 + link-local supernets, whole. No // gateway/own-subnet carve-out: on L2Bridge the container is a LAN peer, // so carving the subnet would expose the management plane and the router. - aclPriorityBlock uint16 = 100 + aclPriorityBlock uint16 = 200 // aclPriorityAllowAny permits everything not blocked above (the internet) // and, crucially, inbound return traffic. Lowest precedence. aclPriorityAllowAny uint16 = 65500 diff --git a/pkg/networking/network_windows_test.go b/pkg/networking/network_windows_test.go index c220763..8f05dac 100644 --- a/pkg/networking/network_windows_test.go +++ b/pkg/networking/network_windows_test.go @@ -158,6 +158,38 @@ func TestL2BridgeEgressACLPolicies_Precedence(t *testing.T) { t.Fatalf("priority ladder broken: extra=%d block=%d allowany=%d (want extra < block < allowany)", aclPriorityExtraAllow, aclPriorityBlock, aclPriorityAllowAny) } + if aclPriorityHostAllow >= aclPriorityBlock { + t.Fatalf("host allow (%d) must outrank the RFC1918 block (%d) or dind/module-proxy traffic is denied", + aclPriorityHostAllow, aclPriorityBlock) + } +} + +// TestL2BridgeEgressACLPolicies_NoRuleBelowPriority100 is a regression guard +// for the metal finding that killed the 2026-08-13 production deployment: a +// Switch-rule ACL with Priority < 100 silently kills the endpoint's ENTIRE +// VFP dataplane on Server 2025 (build 26100). The port drops every inbound +// frame — including the gateway's ARP reply — so the container never resolves +// its next hop and has no connectivity, while HNS applies the policy without +// error and reports the endpoint healthy. Proven independent of the rule's +// action, direction, and address by the l2test bisect harness (an irrelevant +// Block 203.0.113.1/32 at priority 90 reproduces it; the identical ladder +// with every priority >= 100 works). NO rule may ever carry Priority < 100. +func TestL2BridgeEgressACLPolicies_NoRuleBelowPriority100(t *testing.T) { + for _, p := range []uint16{aclPriorityHostAllow, aclPriorityExtraAllow, aclPriorityBlock, aclPriorityAllowAny} { + if p < aclPriorityMinimum { + t.Fatalf("ladder constant %d is below the safe minimum %d (kills the VFP port on metal)", p, aclPriorityMinimum) + } + } + for _, extra := range [][]string{nil, {"203.0.113.0/24"}} { + for _, hostIP := range []string{"", "198.51.100.7"} { + for _, a := range decodeACLs(t, mustBuildL2Bridge(t, extra, hostIP)) { + if a.Priority < aclPriorityMinimum { + t.Errorf("emitted ACL with priority %d < %d (kills the VFP port on metal): %+v", + a.Priority, aclPriorityMinimum, a) + } + } + } + } } // TestL2BridgeEgressACLPolicies_EveryRuleIsAddressScoped is a regression guard From 6b69629463e5b5f65d42f5fdc3a2d76dde9201e6 Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Fri, 14 Aug 2026 07:39:22 -0700 Subject: [PATCH 07/10] fix(runtime): don't let the orphan sweep delete a provisioning job's dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The orphan sweep decides "orphan" by the absence of a containerd container. But Create copies the ~200MB runner dir (job-) and can then spend minutes pulling a cold Windows image before it calls NewContainer — the whole time, the job has on-disk state but no container. A sweep firing in that window (startup CleanOrphans racing the startup poll, or the periodic SweepOrphans) deleted a live job's runner dir out from under it, half- removing node.exe and leaving the runner in a "path not found" self-update loop. Observed on the win-amd64 node 2026-08-14. Track in-flight IDs on the Runtime: Create registers its ID before the copy and clears it on return; SweepOrphans unions those IDs into its keep set. Closes the window regardless of provisioning duration. CleanOrphans stays nil-keyed — it is startup-only, before any provisioning begins. Also documents the dedicated-NIC recommendation (creating the L2Bridge migrates the host IP onto a vEthernet adapter; doing that on a remote node's only NIC risks unreachability) in config.example.toml and the security guide, and corrects the security guide's wired-adapter claim: the egress ACLs rewrite container source MACs to the host NIC's, so the "Wi-Fi can't carry extra MACs" reasoning was wrong. Wi-Fi is now called out as untested rather than impossible. --- config.example.toml | 11 ++++++- docs/guides/security.md | 7 ++--- pkg/runtime/provisioning_test.go | 54 ++++++++++++++++++++++++++++++++ pkg/runtime/runtime.go | 49 +++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 5 deletions(-) create mode 100644 pkg/runtime/provisioning_test.go diff --git a/config.example.toml b/config.example.toml index dd4a8c8..36b8075 100644 --- a/config.example.toml +++ b/config.example.toml @@ -105,7 +105,16 @@ owner = "your-org" # Host NIC the L2Bridge binds onto. REQUIRED when l2bridge_egress = true; there # is no default (the correct adapter name is host-specific). Use the name shown # by `Get-NetAdapter`. Ignored when l2bridge_egress is false. -# host_nic = "Ethernet" +# +# STRONGLY RECOMMENDED: bind a DEDICATED NIC, not the host's management NIC. +# Creating the L2Bridge builds an external Hyper-V vSwitch on this adapter and +# migrates its IP onto a "vEthernet ()" adapter. On a host reached over +# that same NIC there is a brief connectivity blip during creation, and any +# failure mid-creation can leave the box unreachable — a hazard for remote +# nodes. A second NIC dedicated to container traffic keeps the management path +# untouched. (ephemerd tolerates the vSwitch rename either way, so restarts are +# safe once the network exists.) +# host_nic = "Ethernet 2" # # Addresses ephemerd may assign to job containers. REQUIRED when # l2bridge_egress = true, with NO default. diff --git a/docs/guides/security.md b/docs/guides/security.md index e7a333c..41f1e05 100644 --- a/docs/guides/security.md +++ b/docs/guides/security.md @@ -65,14 +65,13 @@ Setting `network.l2bridge_egress = true` moves job containers onto an L2Bridge n ```toml [network] l2bridge_egress = true -host_nic = "Ethernet" # wired adapter to bridge onto +host_nic = "Ethernet 2" # dedicated NIC to bridge onto (see below) ip_pool = "192.0.2.192/27" # reserved range, see below ``` -**Requirements — both are hard:** +**A reserved `ip_pool` your DHCP server will never lease is mandatory.** On L2Bridge, containers are addressed on your LAN rather than behind NAT, so ephemerd must be told which addresses it may hand out. There is deliberately **no default** — any built-in guess would collide with live DHCP leases. Size it for at least `runner.max_concurrent` containers, and add the matching exclusion on your DHCP server *before* enabling. Container egress ACLs rewrite each container's source MAC to the host NIC's, so the containers appear on the segment as the host itself. -- **A wired Ethernet adapter.** L2Bridge puts container MAC addresses on the physical segment. A Wi-Fi adapter operating as a station cannot carry additional MACs, so 802.11 links cannot host this path at all. This is a limitation of wireless, not of ephemerd. -- **A reserved `ip_pool` your DHCP server will never lease.** On L2Bridge, containers are addressed on your LAN rather than behind NAT, so ephemerd must be told which addresses it may hand out. There is deliberately **no default** — any built-in guess would collide with live DHCP leases. Size it for at least `runner.max_concurrent` containers, and add the matching exclusion on your DHCP server *before* enabling. +**A dedicated NIC is strongly recommended.** Creating the L2Bridge builds an external Hyper-V vSwitch on `host_nic` and migrates its IP onto a `vEthernet ()` adapter. If that is the host's only NIC — the one you administer it over — there is a connectivity blip during creation, and a failure mid-creation can leave a remote node unreachable. A second NIC dedicated to container traffic keeps the management path untouched. ephemerd tolerates the vSwitch rename, so once the network exists, daemon restarts are safe. Wi-Fi adapters are untested and not recommended: Hyper-V external switches on 802.11 historically require bridging workarounds. `subnet`, `gateway`, and DNS are derived from `host_nic` at startup; set them explicitly only to override. Startup fails fast with a message naming the offending key rather than guessing. diff --git a/pkg/runtime/provisioning_test.go b/pkg/runtime/provisioning_test.go new file mode 100644 index 0000000..5c33366 --- /dev/null +++ b/pkg/runtime/provisioning_test.go @@ -0,0 +1,54 @@ +package runtime + +import "testing" + +// TestProvisioning_InFlightIDsSurviveSweep is the regression guard for the +// 2026-08-14 metal race: the orphan sweep decides "orphan" by the absence of a +// containerd container, but a job's runner-dir copy and workdir exist for the +// whole provisioning window (copyDirForJob → NewContainer), which on a cold +// Windows image pull is minutes long. A sweep firing in that window must NOT +// treat the in-flight job as an orphan, or it deletes a live job's runner dir +// and corrupts it into a self-update loop. +func TestProvisioning_InFlightIDsSurviveSweep(t *testing.T) { + r := &Runtime{} + const id = "ephemerd-github-ephpm-live_shannon" + + done := r.beginProvisioning(id) + + // The sweep builds its keep set from live containerd containers (none here, + // mirroring the window before NewContainer runs) and then unions in-flight + // provisioning IDs. + keep := map[string]struct{}{} + r.addProvisioning(keep) + if _, ok := keep[id]; !ok { + t.Fatalf("in-flight provisioning id %q missing from the sweep keep set — its runner dir would be deleted mid-provision", id) + } + + // After provisioning completes the container exists in containerd and the + // in-flight guard is released; the ID no longer needs the provisioning set. + done() + keep2 := map[string]struct{}{} + r.addProvisioning(keep2) + if _, ok := keep2[id]; ok { + t.Errorf("id %q still marked in-flight after done() — leak", id) + } +} + +// TestProvisioning_ConcurrentJobsIndependent ensures one job finishing +// provisioning does not unguard another still in flight. +func TestProvisioning_ConcurrentJobsIndependent(t *testing.T) { + r := &Runtime{} + doneA := r.beginProvisioning("job-a") + _ = r.beginProvisioning("job-b") // still in flight + + doneA() + + keep := map[string]struct{}{} + r.addProvisioning(keep) + if _, ok := keep["job-a"]; ok { + t.Errorf("job-a should be released") + } + if _, ok := keep["job-b"]; !ok { + t.Errorf("job-b must still be guarded while in flight") + } +} diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 7f540fb..a37369d 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -138,6 +138,45 @@ type Runtime struct { cfg Config client *client.Client pullMu sync.Mutex // serializes image pulls to avoid content store contention + + // provisioning holds the IDs of jobs whose on-disk state (runner-dir copy, + // job workdir, snapshot) exists but whose container is not yet registered + // in containerd — i.e. jobs somewhere between the copyDirForJob and + // NewContainer calls in Create. The orphan sweep decides "orphan" by the + // absence of a containerd container, so without this a sweep that fires + // during the provisioning window (a Windows image pull can take minutes) + // deletes a live job's runner dir out from under it, corrupting it into a + // self-update loop (observed on metal 2026-08-14). SweepOrphans unions + // these IDs into its keep set. + provMu sync.Mutex + provisioning map[string]struct{} +} + +// beginProvisioning marks id as in-flight so the orphan sweep will not reclaim +// its on-disk state before its container exists. The returned func clears it +// and must be deferred by the caller. +func (r *Runtime) beginProvisioning(id string) func() { + r.provMu.Lock() + if r.provisioning == nil { + r.provisioning = make(map[string]struct{}) + } + r.provisioning[id] = struct{}{} + r.provMu.Unlock() + return func() { + r.provMu.Lock() + delete(r.provisioning, id) + r.provMu.Unlock() + } +} + +// addProvisioning inserts the in-flight provisioning IDs into keep so a sweep +// preserves their on-disk state. +func (r *Runtime) addProvisioning(keep map[string]struct{}) { + r.provMu.Lock() + defer r.provMu.Unlock() + for id := range r.provisioning { + keep[id] = struct{}{} + } } // Client returns the underlying containerd client. Used by the in-VM @@ -277,6 +316,8 @@ func (r *Runtime) SweepOrphans(ctx context.Context) error { for _, c := range containers { live[c.ID()] = struct{}{} } + // Jobs mid-provision have on-disk state but no container yet: keep them. + r.addProvisioning(live) return r.sweepOrphanState(ctx, live) } @@ -671,6 +712,14 @@ func (r *Runtime) Create(ctx context.Context, cfg CreateConfig) (*RunnerEnv, err jitConfig := cfg.JITConfig ctx = namespaces.WithNamespace(ctx, namespace) + // Protect this job's on-disk state (runner-dir copy, job workdir, snapshot) + // from the orphan sweep for the whole provisioning window — from here until + // Create returns. Until NewContainer runs there is no containerd container + // for the sweep to key off, so without this a sweep firing mid-provision + // (an image pull can take minutes) would delete a live job's runner dir. + doneProvisioning := r.beginProvisioning(id) + defer doneProvisioning() + // Use a default image when no custom image is specified. // If runner.default_image is set in config, use that. // Otherwise: Linux uses the official GHA runner image, From 2bcf2224e2d6298200db587aef1ba20ce4bb3a89 Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Fri, 14 Aug 2026 07:45:48 -0700 Subject: [PATCH 08/10] fix(dind): open the dind listener port to the container pool on L2Bridge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An adversarial in-job probe on the win-amd64 node showed egress containment working perfectly (LAN + every management plane blocked, internet up) but `docker version` timing out. The VFP host /32 allow lets a container's packet leave its port toward the host, but the host's OWN inbound Windows Firewall default-denies it, so the per-job dind Docker API listener — bound to the host's LAN address — was never reachable. A timeout, not a refusal, confirmed the drop was at the host firewall, not the socket. Add a scoped inbound allow, opened when the Windows dind listener binds and removed when the job's server stops: dir=in action=allow protocol=TCP localip= localport= remoteip=. It opens exactly that one dynamic port to exactly the container pool — a blanket host allow would expose RDP/SMB/RPC to job containers, which the strict posture must not. Plumbed through networking.Manager.OpenHostPort/CloseHostPort; no-op on NAT and on Linux/macOS (containers reach the bridge gateway directly there). A unit test pins the rule's scope so a future change can't widen it to localport=any. --- pkg/dind/dind.go | 8 +++++ pkg/dind/listen_windows.go | 15 ++++++++ pkg/networking/firewall_windows.go | 48 +++++++++++++++++++++++++ pkg/networking/hostport_windows_test.go | 41 +++++++++++++++++++++ pkg/networking/network_darwin.go | 5 +++ pkg/networking/network_linux.go | 5 +++ pkg/networking/networking.go | 30 ++++++++++++++++ 7 files changed, 152 insertions(+) create mode 100644 pkg/networking/hostport_windows_test.go diff --git a/pkg/dind/dind.go b/pkg/dind/dind.go index e86e985..a0ea8cd 100644 --- a/pkg/dind/dind.go +++ b/pkg/dind/dind.go @@ -53,6 +53,9 @@ type Server struct { server *http.Server client *client.Client network *networking.Manager + // hostPort is the TCP port the Windows listener opened in the host + // firewall for the container pool (L2Bridge only); 0 means none open. + hostPort int buildkit *buildkit.Server // shared embedded BuildKit solver (nil → fall back to platform default) runnerNetNS string // path to runner container's net namespace; used to install DNAT rules for port bindings allowPrivileged bool // gate for docker run --privileged / --cap-add; see config.DindConfig.AllowPrivileged @@ -348,6 +351,11 @@ func (s *Server) Stop() { s.log.Debug("closing listener", "error", err) } } + // Remove the L2Bridge host-firewall allow opened for this job's listener. + if s.hostPort != 0 && s.network != nil { + s.network.CloseHostPort(s.hostPort) + s.hostPort = 0 + } // Clean up the socket and job docker directory dockerDir := filepath.Dir(s.sockPath) diff --git a/pkg/dind/listen_windows.go b/pkg/dind/listen_windows.go index 34d8f50..68e1a99 100644 --- a/pkg/dind/listen_windows.go +++ b/pkg/dind/listen_windows.go @@ -37,6 +37,21 @@ func (s *Server) listen() (net.Listener, error) { _ = ln.Close() return nil, fmt.Errorf("unexpected listener address type: %T", ln.Addr()) } + + // On the L2Bridge path the container reaches this listener at the host's + // LAN address, where the host's own inbound Windows Firewall default-denies + // it (the VFP host /32 allow only governs the container's egress port). Open + // a scoped inbound allow for exactly this port from the container pool. No-op + // on NAT. Failure is not fatal: log and continue — dind then simply is not + // reachable, which the job surfaces, rather than taking provisioning down. + if s.network != nil { + if err := s.network.OpenHostPort(tcpAddr.Port); err != nil { + s.log.Warn("failed to open dind host port for the container pool; docker may be unreachable on L2Bridge", "port", tcpAddr.Port, "error", err) + } else { + s.hostPort = tcpAddr.Port + } + } + s.endpoint = fmt.Sprintf("tcp://%s:%d", host, tcpAddr.Port) return ln, nil } diff --git a/pkg/networking/firewall_windows.go b/pkg/networking/firewall_windows.go index 5d0524d..a28da6e 100644 --- a/pkg/networking/firewall_windows.go +++ b/pkg/networking/firewall_windows.go @@ -373,6 +373,54 @@ func (w *windowsNetworking) installL2BridgeFirewallRules() error { return nil } +// hostPortAllowRule is the scoped inbound allow that makes one host TCP port +// reachable from the container pool on the L2Bridge path (see openHostPort). +func hostPortAllowRule(hostIP, ipPool string, port int) winFirewallRule { + return winFirewallRule{ + name: fmt.Sprintf("%s-l2b-hostport-%d", firewallRulePrefix, port), + spec: []string{ + "dir=in", + "action=allow", + "protocol=TCP", + "localip=" + hostIP, + "localport=" + strconv.Itoa(port), + "remoteip=" + ipPool, + "profile=any", + "enable=yes", + }, + } +} + +// openHostPort adds a scoped inbound allow so job containers can reach one host +// TCP port (a per-job dind Docker API listener, or the module proxy). Without +// it the host's default inbound deny drops the connection even though the VFP +// host /32 allow permits the container to send. Scoped to remoteip= and +// localport=, so nothing else on the host opens. No-op unless the +// L2Bridge egress path is active with a resolved plan. +func (w *windowsNetworking) openHostPort(port int) error { + if !w.cfg.L2BridgeEgress || w.plan == nil || w.plan.HostIP == "" || w.plan.PoolSpec == "" { + return nil + } + r := hostPortAllowRule(w.plan.HostIP, w.plan.PoolSpec, port) + _ = netsh(r.deleteArgs()...) // idempotent + if err := netsh(r.addArgs()...); err != nil { + return fmt.Errorf("opening host port %d for the container pool: %w", port, err) + } + w.cfg.Log.Info("opened L2Bridge host port for container pool", "port", port, "pool", w.plan.PoolSpec) + return nil +} + +// closeHostPort removes the allow added by openHostPort. +func (w *windowsNetworking) closeHostPort(port int) { + if !w.cfg.L2BridgeEgress || w.plan == nil || w.plan.HostIP == "" || w.plan.PoolSpec == "" { + return + } + r := hostPortAllowRule(w.plan.HostIP, w.plan.PoolSpec, port) + if err := netsh(r.deleteArgs()...); err != nil { + w.cfg.Log.Debug("failed to remove L2Bridge host-port allow", "port", port, "error", err) + } +} + // removeL2BridgeFirewallRules deletes the backstop rules by name. func (w *windowsNetworking) removeL2BridgeFirewallRules() { if w.plan == nil { diff --git a/pkg/networking/hostport_windows_test.go b/pkg/networking/hostport_windows_test.go new file mode 100644 index 0000000..5fd3fe5 --- /dev/null +++ b/pkg/networking/hostport_windows_test.go @@ -0,0 +1,41 @@ +//go:build windows + +package networking + +import ( + "strings" + "testing" +) + +// TestHostPortAllowRule_ScopedToPoolAndPort proves the dind-over-L2Bridge fix +// opens exactly one host port to exactly the container pool. The VFP host /32 +// allow lets a container's packet leave toward the host, but the host's own +// inbound Windows Firewall default-denies it — so the per-job dind listener is +// unreachable without this rule. It must stay tightly scoped: a broad inbound +// allow would expose RDP/SMB/RPC on the host to job containers. +func TestHostPortAllowRule_ScopedToPoolAndPort(t *testing.T) { + const ( + hostIP = "192.0.2.10" + pool = "192.0.2.192/27" + port = 63933 + ) + r := hostPortAllowRule(hostIP, pool, port) + spec := strings.Join(r.spec, " ") + + for _, want := range []string{ + "dir=in", "action=allow", "protocol=TCP", + "localip=" + hostIP, "localport=63933", "remoteip=" + pool, + } { + if !strings.Contains(spec, want) { + t.Errorf("host-port allow rule missing %q; spec = %q", want, spec) + } + } + // Must be port-scoped, not a blanket host allow. + if strings.Contains(spec, "localport=any") || !strings.Contains(spec, "localport=63933") { + t.Errorf("rule is not scoped to the single dind port: %q", spec) + } + // The rule name must carry the port so concurrent jobs get distinct rules. + if !strings.Contains(r.name, "63933") { + t.Errorf("rule name %q must be port-specific so concurrent jobs do not collide", r.name) + } +} diff --git a/pkg/networking/network_darwin.go b/pkg/networking/network_darwin.go index 6daeb62..b6291aa 100644 --- a/pkg/networking/network_darwin.go +++ b/pkg/networking/network_darwin.go @@ -54,3 +54,8 @@ func cleanStaleBridge(_ *slog.Logger) {} // no-op on macOS // hostAddr: no L2Bridge on macOS — the generic subnet derivation applies. func (d *darwinNetworking) hostAddr() string { return "" } + +// openHostPort/closeHostPort are Windows-L2Bridge-only; the macOS path +// delegates networking to the in-VM Linux stack. +func (d *darwinNetworking) openHostPort(int) error { return nil } +func (d *darwinNetworking) closeHostPort(int) {} diff --git a/pkg/networking/network_linux.go b/pkg/networking/network_linux.go index c94e732..1eee219 100644 --- a/pkg/networking/network_linux.go +++ b/pkg/networking/network_linux.go @@ -107,6 +107,11 @@ func (l *linuxNetworking) teardown(ctx context.Context, id string, netns string) // hostAddr: no L2Bridge on Linux — the generic subnet derivation applies. func (l *linuxNetworking) hostAddr() string { return "" } +// openHostPort/closeHostPort are Windows-L2Bridge-only; Linux containers reach +// the bridge gateway without a host-firewall carve-out. +func (l *linuxNetworking) openHostPort(int) error { return nil } +func (l *linuxNetworking) closeHostPort(int) {} + func (l *linuxNetworking) cleanup() { log := l.cfg.Log diff --git a/pkg/networking/networking.go b/pkg/networking/networking.go index 2d2b04d..7397413 100644 --- a/pkg/networking/networking.go +++ b/pkg/networking/networking.go @@ -174,6 +174,17 @@ type platformNetworking interface { // peers and the reachable host address is the host's own adapter address // — not the .1 of any container subnet. hostAddr() string + + // openHostPort / closeHostPort open and close a scoped host-firewall + // inbound allow for one TCP port, from the container pool to the host. + // Needed only on the Windows L2Bridge path: the VFP host /32 allow lets a + // container's packet leave its port toward the host, but the host's own + // inbound Windows Firewall default-denies it, so per-job services ephemerd + // binds on the host (the dind Docker API, the module proxy) are otherwise + // unreachable. Scoped to remoteip=, localport= so ONLY that + // service opens — RDP/SMB/RPC stay blocked. No-op on NAT and non-Windows. + openHostPort(port int) error + closeHostPort(port int) } // New creates and initializes the networking manager for the current platform. @@ -237,6 +248,25 @@ func (m *Manager) InstallFirewallRules() error { return m.platform.installFirewallRules() } +// OpenHostPort opens a scoped host-firewall inbound allow for one TCP port from +// the container pool to the host, so a per-job service ephemerd binds on the +// host (dind's Docker API, the module proxy) is reachable from job containers. +// Only the Windows L2Bridge path does anything; elsewhere it is a no-op. Pair +// with CloseHostPort on teardown. +func (m *Manager) OpenHostPort(port int) error { + if m.platform == nil { + return nil + } + return m.platform.openHostPort(port) +} + +// CloseHostPort removes an allow previously added by OpenHostPort. +func (m *Manager) CloseHostPort(port int) { + if m.platform != nil { + m.platform.closeHostPort(port) + } +} + // Cleanup removes all networking state: firewall rules, bridge interface, // CNI config, IP allocations, and DNS files. Called on shutdown. func (m *Manager) Cleanup() { From 546798a36dcac9ec1a1bae529a17ae1e00dfba52 Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Fri, 14 Aug 2026 11:28:31 -0700 Subject: [PATCH 09/10] fix(networking): prefix-sweep leaked L2Bridge host-port allows on shutdown Per-job dind host-port allows are removed by dind's CloseHostPort on a graceful job stop, but a hard kill (Stop-Process) skips that, and removeL2BridgeFirewallRules only deleted the control-plane rules by their computed names -- so a hard-killed job left its inbound allow behind, and they accumulated. Sweep the ephemerd-egress-l2b-hostport-* prefix via the firewall cmdlets (netsh delete-by-name has no wildcard) on every removeFirewallRules, so both shutdown and the next startup Cleanup reclaim any leaked allows. Runs regardless of whether a plan currently resolves. --- pkg/networking/firewall_windows.go | 26 ++++++++++++++++++------- pkg/networking/hostport_windows_test.go | 5 +++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/pkg/networking/firewall_windows.go b/pkg/networking/firewall_windows.go index a28da6e..480aa98 100644 --- a/pkg/networking/firewall_windows.go +++ b/pkg/networking/firewall_windows.go @@ -421,16 +421,28 @@ func (w *windowsNetworking) closeHostPort(port int) { } } -// removeL2BridgeFirewallRules deletes the backstop rules by name. +// hostPortRulePrefix is the DisplayName prefix of every per-job host-port allow +// (see hostPortAllowRule). Swept by prefix on shutdown so a hard kill — which +// skips dind's per-job CloseHostPort — cannot leak stale inbound allows. +const hostPortRulePrefix = firewallRulePrefix + "-l2b-hostport-" + +// removeL2BridgeFirewallRules deletes the backstop rules by name, and sweeps any +// leaked per-job host-port allows by prefix. func (w *windowsNetworking) removeL2BridgeFirewallRules() { - if w.plan == nil { - return - } - for _, r := range l2BridgeControlPlaneRules(w.plan.HostIP, w.plan.PoolSpec, w.cfg.ControlPorts) { - if err := netsh(r.deleteArgs()...); err != nil { - w.cfg.Log.Debug("failed to remove L2Bridge control-plane firewall rule", "rule", r.name, "error", err) + if w.plan != nil { + for _, r := range l2BridgeControlPlaneRules(w.plan.HostIP, w.plan.PoolSpec, w.cfg.ControlPorts) { + if err := netsh(r.deleteArgs()...); err != nil { + w.cfg.Log.Debug("failed to remove L2Bridge control-plane firewall rule", "rule", r.name, "error", err) + } } } + // Prefix-sweep the per-job host-port allows. netsh delete-by-name has no + // wildcard, so go through the firewall cmdlets. Runs regardless of w.plan — + // a hard kill can leave these behind for a pool that no longer resolves, and + // startup Cleanup must still reclaim them. + if err := powershell("Get-NetFirewallRule -DisplayName '" + hostPortRulePrefix + "*' -ErrorAction SilentlyContinue | Remove-NetFirewallRule -ErrorAction SilentlyContinue"); err != nil { + w.cfg.Log.Debug("failed to prefix-sweep L2Bridge host-port allows", "error", err) + } } // defaultGatewayForAdapter returns the IPv4 default-route next hop reachable via diff --git a/pkg/networking/hostport_windows_test.go b/pkg/networking/hostport_windows_test.go index 5fd3fe5..180264a 100644 --- a/pkg/networking/hostport_windows_test.go +++ b/pkg/networking/hostport_windows_test.go @@ -38,4 +38,9 @@ func TestHostPortAllowRule_ScopedToPoolAndPort(t *testing.T) { if !strings.Contains(r.name, "63933") { t.Errorf("rule name %q must be port-specific so concurrent jobs do not collide", r.name) } + // The name must fall under the prefix the shutdown sweep matches, or a + // hard-killed job leaks its allow forever. + if !strings.HasPrefix(r.name, hostPortRulePrefix) { + t.Errorf("rule name %q must start with the sweep prefix %q", r.name, hostPortRulePrefix) + } } From c7859f5a1d4f92d843ce2f91e272f376622e5b9d Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Fri, 14 Aug 2026 13:34:14 -0700 Subject: [PATCH 10/10] refactor(networking): stop pretending NAT can filter Windows egress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default NAT Windows path installed a Hyper-V-firewall rule set and a netsh host-firewall fallback, plus a block-only per-endpoint ACL set. All three were proven ineffective on real hardware: runhcs NAT containers register no Hyper-V VMCreator (so the rules bind to nothing), the host firewall sees NAT'd egress post-NAT (so a container-source-scoped rule matches nothing), and VFP does not engage on a NAT vSwitch (so the ACLs are inert). It was enforcement theater — a NAT node looked protected while its containers reached the whole LAN. Delete it. On the default NAT network ephemerd now installs nothing and logs plainly that container egress is NOT filtered, pointing the operator at network.l2bridge_egress (the only path that actually enforces, via HNS L2Bridge + per-endpoint VFP Switch ACLs). No security is lost — none of the removed code ever blocked anything. Removes ~975 lines: hyperVRule/hyperVEgressRules/discoverContainerVMCreators/ enableHyperVFirewallScript/removeByPrefixScript/hyperVFirewallAvailable and the WSL creator const; hostFirewallRules/installNetshFirewallRules/ removeNetshFirewallRules and their CIDR-subtraction helpers; and buildEgressBlockPolicies with its NAT applyACLPolicies branch. The L2Bridge path is untouched — the two branch cleanly on cfg.L2BridgeEgress — and the shared helpers it depends on (psQuote, powershell, netsh, winFirewallRule, egressBlockedCIDRs) are retained. Stale tests for the deleted code removed. Also corrects the docs the strip made honest: the firewall_windows.go header no longer frames the Hyper-V firewall as the "primary path," and docs/arch/windows-egress-wfp-investigation.md gains an addendum noting a host-side software path (L2Bridge VFP ACLs) was later found and shipped, so its "network-level only" conclusion applies to the NAT stack, not universally. --- docs/arch/windows-egress-wfp-investigation.md | 10 + .../firewall_hyperv_windows_test.go | 245 --------- pkg/networking/firewall_windows.go | 513 ++---------------- pkg/networking/firewall_windows_test.go | 213 +------- pkg/networking/network_windows.go | 90 +-- pkg/networking/network_windows_test.go | 61 --- 6 files changed, 79 insertions(+), 1053 deletions(-) delete mode 100644 pkg/networking/firewall_hyperv_windows_test.go diff --git a/docs/arch/windows-egress-wfp-investigation.md b/docs/arch/windows-egress-wfp-investigation.md index b790e28..f52d616 100644 --- a/docs/arch/windows-egress-wfp-investigation.md +++ b/docs/arch/windows-egress-wfp-investigation.md @@ -1,5 +1,15 @@ # Windows container egress: WFP IPFORWARD investigation (negative result) +> **Addendum (2026-08, resolved).** A host-side software path was subsequently +> found and shipped: HNS **L2Bridge** networks with per-endpoint **VFP Switch +> ACLs** (`network.l2bridge_egress`), proven on metal. The container is a peer on +> the host LAN (no NAT), so the VFP dataplane filters its egress at the vSwitch +> port. The "network-level isolation only" conclusion below therefore applies +> specifically to the default **NAT** stack this investigation used — where it +> still holds, and where ephemerd now installs nothing and logs that egress is +> unfiltered — **not universally**. See `docs/guides/security.md`. The body below +> is preserved unchanged as the historical record for the NAT stack. + ## TL;DR On Windows Server 2025 (build 26100) with a Hyper-V-isolated job container on an diff --git a/pkg/networking/firewall_hyperv_windows_test.go b/pkg/networking/firewall_hyperv_windows_test.go deleted file mode 100644 index ed4e740..0000000 --- a/pkg/networking/firewall_hyperv_windows_test.go +++ /dev/null @@ -1,245 +0,0 @@ -//go:build windows - -package networking - -import ( - "slices" - "strings" - "testing" -) - -// A concrete container VMCreatorId for the tests. The real value is discovered -// at runtime from Get-NetFirewallHyperVVMCreator (excluding WSL); the rule -// construction is identical whatever GUID it resolves to. -const testContainerVMCreatorID = "{9E9E4CB2-1B2C-4D3E-8F90-ABCDEF012345}" - -// TestHyperVEgressRules_BlocksEveryDeniedRange verifies each RFC1918 + -// link-local range produces exactly one outbound Block rule scoped to the -// container VMCreatorId; a missing range would let a job reach that slice of -// the LAN. -func TestHyperVEgressRules_BlocksEveryDeniedRange(t *testing.T) { - rules, err := hyperVEgressRules(testContainerVMCreatorID, DefaultSubnet, defaultGateway, nil) - if err != nil { - t.Fatalf("hyperVEgressRules: %v", err) - } - if len(rules) != len(egressBlockedCIDRs) { - t.Fatalf("got %d rules, want %d (one outbound block per denied range)", len(rules), len(egressBlockedCIDRs)) - } - for i, cidr := range egressBlockedCIDRs { - r := rules[i] - if r.direction != "Outbound" || r.action != "Block" { - t.Errorf("rule %s is not an outbound block: dir=%s action=%s", r.name, r.direction, r.action) - } - if r.vmCreatorID != testContainerVMCreatorID { - t.Errorf("rule %s not scoped to the container creator: %q", r.name, r.vmCreatorID) - } - if len(r.remoteAddrs) == 0 { - t.Errorf("rule %s has no RemoteAddresses scope", r.name) - } - wantName := firewallRulePrefix + "-" + creatorTag(testContainerVMCreatorID) + "-block-" + strings.ReplaceAll(cidr, "/", "_") - if r.name != wantName { - t.Errorf("rule[%d].name = %q, want %q", i, r.name, wantName) - } - } -} - -// TestHyperVEgressRules_GatewayAndSubnetNeverBlocked pins the safety property: -// the container subnet — which contains the NAT gateway (DNS, default route, -// GatewayPorts) and the other containers — must never appear inside a block -// rule's RemoteAddresses. Blocking the gateway would brick all container -// networking. The 10/8 block must be split exactly around the subnet. -func TestHyperVEgressRules_GatewayAndSubnetNeverBlocked(t *testing.T) { - rules, err := hyperVEgressRules(testContainerVMCreatorID, "10.88.0.0/16", "10.88.0.1", nil) - if err != nil { - t.Fatalf("hyperVEgressRules: %v", err) - } - - for _, r := range rules { - for _, addr := range r.remoteAddrs { - if strings.Contains(addr, "10.88.") { - t.Errorf("rule %s blocks the container subnet: RemoteAddresses contains %q", r.name, addr) - } - } - } - - tag := creatorTag(testContainerVMCreatorID) - wantName := firewallRulePrefix + "-" + tag + "-block-10.0.0.0_8" - want := []string{"10.0.0.0-10.87.255.255", "10.89.0.0-10.255.255.255"} - for _, r := range rules { - if r.name == wantName { - if !slices.Equal(r.remoteAddrs, want) { - t.Errorf("10/8 block RemoteAddresses = %v, want %v", r.remoteAddrs, want) - } - return - } - } - t.Errorf("no block rule found for 10.0.0.0/8 (name %q)", wantName) -} - -// TestHyperVEgressRules_ControlPortRules confirms the container->gateway -// control-plane blocks mirror the Linux drops: outbound, TCP, one specific port -// each, RemoteAddresses = gateway only — never a blanket gateway block and -// never port 53 (DNS must survive). -func TestHyperVEgressRules_ControlPortRules(t *testing.T) { - ports := []int{10000, 10001, 10002} // containerd, dispatch, debug exec - rules, err := hyperVEgressRules(testContainerVMCreatorID, DefaultSubnet, defaultGateway, ports) - if err != nil { - t.Fatalf("hyperVEgressRules: %v", err) - } - - var control []hyperVRule - for _, r := range rules { - if len(r.remotePorts) > 0 { - control = append(control, r) - } - } - if len(control) != len(ports) { - t.Fatalf("got %d control rules, want %d (one per control port)", len(control), len(ports)) - } - - for i, port := range []string{"10000", "10001", "10002"} { - r := control[i] - if r.direction != "Outbound" || r.action != "Block" || r.protocol != "TCP" { - t.Errorf("rule %s is not an outbound TCP block: dir=%s action=%s proto=%s", r.name, r.direction, r.action, r.protocol) - } - if !slices.Equal(r.remotePorts, []string{port}) { - t.Errorf("rule %s RemotePorts = %v, want [%s]", r.name, r.remotePorts, port) - } - if !slices.Equal(r.remoteAddrs, []string{defaultGateway}) { - t.Errorf("rule %s RemoteAddresses = %v, want [%s]", r.name, r.remoteAddrs, defaultGateway) - } - if port == "53" { - t.Errorf("rule %s blocks DNS (port 53) — must not", r.name) - } - } -} - -// TestHyperVRuleCommand_Rendering pins the exact PowerShell the install path -// runs: New-NetFirewallHyperVRule with the VMCreatorId scoping, Outbound/Block, -// and RemoteAddresses/RemotePorts rendered as quoted PowerShell arrays (so a -// string[] parameter receives distinct elements, not one comma-joined string). -func TestHyperVRuleCommand_Rendering(t *testing.T) { - block := hyperVRule{ - name: "ephemerd-egress-9e9e4cb2-block-10.0.0.0_8", - displayName: "ephemerd egress block 10.0.0.0/8", - direction: "Outbound", - action: "Block", - vmCreatorID: testContainerVMCreatorID, - remoteAddrs: []string{"10.0.0.0-10.87.255.255", "10.89.0.0-10.255.255.255"}, - } - got := block.command() - for _, want := range []string{ - "New-NetFirewallHyperVRule", - "-Name 'ephemerd-egress-9e9e4cb2-block-10.0.0.0_8'", - "-DisplayName 'ephemerd egress block 10.0.0.0/8'", - "-Direction Outbound", - "-Action Block", - "-VMCreatorId '{9E9E4CB2-1B2C-4D3E-8F90-ABCDEF012345}'", - "-RemoteAddresses '10.0.0.0-10.87.255.255','10.89.0.0-10.255.255.255'", - } { - if !strings.Contains(got, want) { - t.Errorf("command() = %q\n missing %q", got, want) - } - } - // An all-protocol block must not emit -Protocol (default Any) and must not - // emit -RemotePorts. - if strings.Contains(got, "-Protocol") { - t.Errorf("all-protocol block should omit -Protocol: %q", got) - } - if strings.Contains(got, "-RemotePorts") { - t.Errorf("block without ports should omit -RemotePorts: %q", got) - } - - control := hyperVRule{ - name: "ephemerd-egress-9e9e4cb2-control-10000", - displayName: "ephemerd egress block control tcp/10000", - direction: "Outbound", - action: "Block", - vmCreatorID: testContainerVMCreatorID, - protocol: "TCP", - remoteAddrs: []string{"10.88.0.1"}, - remotePorts: []string{"10000"}, - } - gotC := control.command() - for _, want := range []string{ - "-Protocol TCP", - "-RemoteAddresses '10.88.0.1'", - "-RemotePorts '10000'", - } { - if !strings.Contains(gotC, want) { - t.Errorf("control command() = %q\n missing %q", gotC, want) - } - } -} - -// TestHyperVRuleRemoveCommand verifies removal targets exactly the name add -// created (that is what makes remove-before-add idempotent) and stays quiet on -// a fresh host. -func TestHyperVRuleRemoveCommand(t *testing.T) { - r := hyperVRule{name: "ephemerd-egress-9e9e4cb2-block-192.168.0.0_16"} - got := r.removeCommand() - want := "Remove-NetFirewallHyperVRule -Name 'ephemerd-egress-9e9e4cb2-block-192.168.0.0_16' -ErrorAction SilentlyContinue" - if got != want { - t.Errorf("removeCommand() = %q, want %q", got, want) - } -} - -// TestHyperVRuleNames pins the naming contract: every rule carries the ephemerd -// prefix (so the set is findable and removable by removeByPrefixScript) and is -// scoped to the creator via a short tag so multiple creators do not collide. -func TestHyperVRuleNames(t *testing.T) { - rules, err := hyperVEgressRules(testContainerVMCreatorID, DefaultSubnet, defaultGateway, []int{10000}) - if err != nil { - t.Fatalf("hyperVEgressRules: %v", err) - } - tag := creatorTag(testContainerVMCreatorID) - if tag != "9e9e4cb2" { - t.Errorf("creatorTag = %q, want %q", tag, "9e9e4cb2") - } - for _, r := range rules { - if !strings.HasPrefix(r.name, firewallRulePrefix+"-") { - t.Errorf("rule name %q missing %q prefix", r.name, firewallRulePrefix) - } - if !strings.Contains(r.name, tag) { - t.Errorf("rule name %q missing creator tag %q", r.name, tag) - } - if r.displayName == "" { - t.Errorf("rule %q has empty DisplayName (mandatory for New-NetFirewallHyperVRule)", r.name) - } - } - - // removeByPrefixScript must match those names. - if !strings.Contains(removeByPrefixScript(), firewallRulePrefix+"-*") { - t.Errorf("removeByPrefixScript does not match the rule-name prefix: %q", removeByPrefixScript()) - } -} - -// TestCreatorTag covers normalization: hex-only, lowercased, first 8, with a -// safe fallback for a GUID that yields no hex. -func TestCreatorTag(t *testing.T) { - tests := []struct { - in, want string - }{ - {"{9E9E4CB2-1B2C-4D3E-8F90-ABCDEF012345}", "9e9e4cb2"}, - {"{40E0AC32-46A5-438A-A0B2-2B479E8F2E90}", "40e0ac32"}, - {"{GGGG}", "any"}, - {"", "any"}, - } - for _, tt := range tests { - if got := creatorTag(tt.in); got != tt.want { - t.Errorf("creatorTag(%q) = %q, want %q", tt.in, got, tt.want) - } - } -} - -// TestPSArrayAndQuote covers the PowerShell rendering helpers, including the -// embedded-quote escape that keeps a crafted value from breaking out of the -// argument. -func TestPSArrayAndQuote(t *testing.T) { - if got := psQuote("a'b"); got != "'a''b'" { - t.Errorf("psQuote = %q, want %q", got, "'a''b'") - } - if got := psArray([]string{"x", "y"}); got != "'x','y'" { - t.Errorf("psArray = %q, want %q", got, "'x','y'") - } -} diff --git a/pkg/networking/firewall_windows.go b/pkg/networking/firewall_windows.go index 480aa98..8d7a554 100644 --- a/pkg/networking/firewall_windows.go +++ b/pkg/networking/firewall_windows.go @@ -11,216 +11,47 @@ import ( // Egress firewall for Windows job containers. // -// Two layers restrict what a Windows job can reach: +// The ONLY working host-side egress enforcement on Windows is the L2Bridge VFP +// Switch-ACL path (opt-in via network.l2bridge_egress): per-endpoint ACLs on +// the container's vSwitch port, built by buildL2BridgeEgressACLPolicies in +// network_windows.go and applied in setup(), plus the small host-firewall +// backstop below that fences the control-plane ports back off. This was proven +// on metal. // -// 1. Per-endpoint HNS ACL policies (applyACLPolicies in network_windows.go) — -// VFP rules on the container's vSwitch port, applied in setup(). -// 2. The Hyper-V firewall rules installed here, programmed with -// New-NetFirewallHyperVRule and scoped to the container VMCreatorId. +// The default NAT network CANNOT be egress-filtered by any host-side mechanism, +// and ephemerd does not try. Every candidate was disproven on real hardware: // -// Layer 2 exists because layer 1 alone left the fleet reachable in practice: -// the containment suite (.github/workflows/containment.yml, "Fleet management -// planes must be unreachable") reached the Incus daemon and Grafana from a -// Hyper-V-isolated job (#135), so per-endpoint vSwitch ACLs cannot be the only -// line of defense. +// - Host Windows Defender Firewall (netsh/MPSSVC): evaluates the forwarded +// container->LAN traffic POST-NAT at the host endpoint, where the source is +// the host's own LAN address, so a rule scoped to the container subnet +// matches nothing (verified on a live Windows node — every management-plane +// probe still succeeded). +// - The Hyper-V firewall (New-NetFirewallHyperVRule, -VMCreatorId scoped): +// runhcs NAT containers register NO Hyper-V VM creator, so there is no +// creator to scope a rule to; the rules install but filter nothing. +// - A host-side WFP filter at IPFORWARD_V4 (the Linux FORWARD analogue): the +// container's forwarded + SNAT'd egress is never classified at ANY host +// IPv4 WFP layer — the packets travel the Hyper-V vSwitch/HNS datapath, +// out-of-band of the host tcpip.sys WFP hooks. Full evidence: +// docs/arch/windows-egress-wfp-investigation.md. // -// Why NOT the host Windows Defender Firewall (the netsh approach of #136). -// #136 installed host MPSSVC rules scoped localip=, on the -// theory that Windows Firewall would see the container's source IP on the -// WinNAT-forwarded flow. On real hardware it does not: the host firewall -// (WFP/MPSSVC) evaluates the forwarded container->LAN traffic POST-NAT at the -// host endpoint, where the source is the host's own LAN address, so a rule -// scoped to the 10.88/16 container source matches nothing. Verified against a -// live v0.1.6 Windows node: every management-plane probe still succeeded. The -// host firewall is the wrong enforcement point for NATed container egress. -// -// The Hyper-V firewall filters at the container's vNIC boundary, BEFORE NAT, -// where the packet still carries the container's own address — the correct -// enforcement point for Hyper-V-isolated Windows containers. Rules are scoped -// by -VMCreatorId so they apply to container ports, not to the host or to -// unrelated Hyper-V workloads (regular Hyper-V VMs are not filtered by the -// Hyper-V firewall at all; it governs container-class workloads — WSL, Windows -// Sandbox, and Windows containers). -// -// Why NOT a host-side WFP filter at IPFORWARD_V4 (the Linux FORWARD analogue). -// Tempting, since firewall_linux.go enforces in FORWARD. It was built and driven -// on a live Server 2025 node (a pure-Go tailscale/wf spike, no callout driver) -// and does not work: the container's forwarded + SNAT'd egress is never -// classified at ANY host IPv4 WFP layer — not IPFORWARD_V4, not -// INBOUND_IPPACKET_V4 (arrival-interface scoped or not), not OUTBOUND_IPPACKET_V4 -// (dest-only, post-NAT). Positive controls prove WFP itself is enforced on the -// host (blocking the host's own 1.1.1.1 at ALE and OUTBOUND both worked), so the -// packets simply travel the Hyper-V vSwitch/HNS datapath, out-of-band of the -// host tcpip.sys WFP hooks (Get-NetNat is empty and per-interface Forwarding is -// Disabled). Full evidence: docs/arch/windows-egress-wfp-investigation.md. -// The definitive fix for this stack is network-level: an isolated VLAN whose -// uplink denies RFC1918, not a host-side software filter. -// -// Gateway safety (identical reasoning to #136, and to firewall_linux.go). -// The container subnet contains the NAT gateway (DNS, the default route, -// module-proxy GatewayPorts) and the other containers. Blocking the gateway -// would brick all container networking. Rather than rely on Hyper-V firewall -// rule-priority ordering (allow-above-deny) to rescue the gateway — an -// unverified semantic, and unverified assumptions are exactly what sank #136 — -// the container subnet is subtracted from every blocked range up front -// (subtractCIDR), so the gateway and the container-to-container range never -// appear inside any Block rule in the first place. Everything outside the -// blocked ranges — the internet, including GitHub/Docker Hub/registries — is -// untouched: the creator's default outbound action stays Allow and only -// RFC1918 + link-local is denied. +// So on the NAT path ephemerd installs nothing and logs that container egress +// is unfiltered, pointing the operator at network.l2bridge_egress. Enforcing +// NAT egress requires a network-level control (an isolated VLAN whose uplink +// denies RFC1918), not a host-side software filter. // // IPv4 only, deliberately: the HCN NAT network is IPv4-only (no v6 IPAM), so // containers have no IPv6 path. -// firewallRulePrefix names every rule ephemerd installs — Hyper-V firewall -// rules and the netsh fallback rules alike — so the set is findable and -// removable on Cleanup. +// firewallRulePrefix names every rule ephemerd installs (the L2Bridge +// host-firewall backstop) so the set is findable and removable on Cleanup. const firewallRulePrefix = "ephemerd-egress" -// wslVMCreatorID is the well-known Hyper-V firewall VMCreatorId for the Windows -// Subsystem for Linux (documented by Microsoft). WSL is not an ephemerd job -// container, so it is excluded from egress filtering — narrowing the blast -// radius to the container runtime's own creator(s). -const wslVMCreatorID = "{40E0AC32-46A5-438A-A0B2-2B479E8F2E90}" - -// ------------------------------------------------------------------------- -// Hyper-V firewall (primary path) -// ------------------------------------------------------------------------- - -// hyperVRule is one New-NetFirewallHyperVRule invocation. Kept as structured -// fields (rather than a flat argv) so both the rendered PowerShell command and -// the unit tests derive from the same source, and so array-valued parameters -// (-RemoteAddresses, -RemotePorts) render as real PowerShell arrays. -type hyperVRule struct { - name string // -Name; unique, ephemerd-prefixed; used for idempotent remove-before-add and Cleanup - displayName string // -DisplayName; mandatory, human-facing - direction string // -Direction (Outbound) - action string // -Action (Block) - vmCreatorID string // -VMCreatorId '{GUID}' - protocol string // -Protocol; "" omits it (matches Any) - remoteAddrs []string // -RemoteAddresses; addresses/CIDRs/ranges; empty omits it (matches Any) - remotePorts []string // -RemotePorts; empty omits it (matches Any) -} - // psQuote single-quotes a value for PowerShell, doubling embedded quotes. func psQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") + "'" } -// psArray renders a slice as a PowerShell array literal of quoted strings -// ('a','b'), so a string[] parameter receives distinct elements rather than one -// comma-joined string. -func psArray(vals []string) string { - q := make([]string, len(vals)) - for i, v := range vals { - q[i] = psQuote(v) - } - return strings.Join(q, ",") -} - -// command renders the New-NetFirewallHyperVRule command that creates the rule. -func (r hyperVRule) command() string { - var b strings.Builder - b.WriteString("New-NetFirewallHyperVRule") - b.WriteString(" -Name " + psQuote(r.name)) - b.WriteString(" -DisplayName " + psQuote(r.displayName)) - b.WriteString(" -Direction " + r.direction) - b.WriteString(" -Action " + r.action) - b.WriteString(" -VMCreatorId " + psQuote(r.vmCreatorID)) - if r.protocol != "" { - b.WriteString(" -Protocol " + r.protocol) - } - if len(r.remoteAddrs) > 0 { - b.WriteString(" -RemoteAddresses " + psArray(r.remoteAddrs)) - } - if len(r.remotePorts) > 0 { - b.WriteString(" -RemotePorts " + psArray(r.remotePorts)) - } - return b.String() -} - -// removeCommand renders the idempotent delete for this rule by name. -func (r hyperVRule) removeCommand() string { - return "Remove-NetFirewallHyperVRule -Name " + psQuote(r.name) + " -ErrorAction SilentlyContinue" -} - -// creatorTag reduces a VMCreatorId GUID to a short, filesystem-safe token -// (first 8 hex digits, lowercased) used to keep rule names unique per creator -// when more than one container creator is present. -func creatorTag(vmCreatorID string) string { - var b strings.Builder - for _, r := range vmCreatorID { - switch { - case r >= '0' && r <= '9', r >= 'a' && r <= 'f': - b.WriteRune(r) - case r >= 'A' && r <= 'F': - b.WriteRune(r + ('a' - 'A')) - } - if b.Len() >= 8 { - break - } - } - if b.Len() == 0 { - return "any" - } - return b.String() -} - -// hyperVEgressRules returns the Hyper-V firewall rule set for one container -// VMCreatorId: an outbound Block for every denied range (with the container -// subnet — and thus the gateway — carved out) plus one outbound Block per -// control port for container->gateway control-plane traffic. -// -// Exposed as a pure function (no side effects) so tests can assert the exact -// rule set without invoking PowerShell. -func hyperVEgressRules(vmCreatorID, subnet, gateway string, controlPorts []int) ([]hyperVRule, error) { - tag := creatorTag(vmCreatorID) - var rules []hyperVRule - - // Outbound RFC1918 + link-local blocks. subtractCIDR removes the container - // subnet from any overlapping range so the gateway (DNS/NAT/default route) - // and the container-to-container range never appear inside a Block. The - // internet is never in these ranges, so it is left fully open. - for _, cidr := range egressBlockedCIDRs { - remote, err := subtractCIDR(cidr, subnet) - if err != nil { - return nil, fmt.Errorf("computing blocked ranges for %s: %w", cidr, err) - } - if len(remote) == 0 { - continue // fully covered by the container subnet - } - rules = append(rules, hyperVRule{ - name: fmt.Sprintf("%s-%s-block-%s", firewallRulePrefix, tag, strings.ReplaceAll(cidr, "/", "_")), - displayName: "ephemerd egress block " + cidr, - direction: "Outbound", - action: "Block", - vmCreatorID: vmCreatorID, - remoteAddrs: remote, - }) - } - - // Control-plane blocks: container -> gateway on the ephemerd control ports - // (containerd, dispatch gRPC, debug exec). Intentionally narrow — the - // gateway address, one TCP port each — so DNS (53), NAT, and the other - // gateway services stay reachable. Mirrors controlPlaneInputRules on Linux; - // evaluated Outbound here because the Hyper-V firewall filters at the - // container vNIC, where container->gateway is egress. - for _, port := range controlPorts { - rules = append(rules, hyperVRule{ - name: fmt.Sprintf("%s-%s-control-%d", firewallRulePrefix, tag, port), - displayName: fmt.Sprintf("ephemerd egress block control tcp/%d", port), - direction: "Outbound", - action: "Block", - vmCreatorID: vmCreatorID, - protocol: "TCP", - remoteAddrs: []string{gateway}, - remotePorts: []string{strconv.Itoa(port)}, - }) - } - - return rules, nil -} - // powershellArgs wraps a script for non-interactive execution. func powershellArgs(script string) []string { return []string{"-NonInteractive", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", script} @@ -244,77 +75,20 @@ func powershellOutput(script string) (string, error) { return string(out), nil } -// hyperVFirewallAvailable reports whether the Hyper-V firewall cmdlets exist on -// this host. They ship with Windows 11 22H2 / Windows Server 2025 and are -// absent on older builds, where the netsh fallback is used instead. -func hyperVFirewallAvailable() bool { - return powershell("if (Get-Command New-NetFirewallHyperVRule -ErrorAction SilentlyContinue) { exit 0 } else { exit 1 }") == nil -} - -// discoverContainerVMCreators returns the VMCreatorIds of the Hyper-V firewall -// VM creators present on the host, excluding WSL. On a Windows container host -// the remaining creator(s) are the container runtime's, which is what job -// containers run under. Returns an empty slice (not an error) when no creators -// are registered yet. -func discoverContainerVMCreators() ([]string, error) { - out, err := powershellOutput("Get-NetFirewallHyperVVMCreator | Select-Object -ExpandProperty VMCreatorId") - if err != nil { - return nil, err - } - var creators []string - for _, line := range strings.Split(out, "\n") { - id := strings.TrimSpace(line) - if id == "" { - continue - } - if strings.EqualFold(id, wslVMCreatorID) { - continue // not an ephemerd job container - } - creators = append(creators, id) - } - return creators, nil -} - -// enableHyperVFirewallScript ensures the Hyper-V firewall is enabled for a -// creator with permissive defaults: internet, the gateway (DNS/NAT), and -// host<->container control traffic all stay open, and only our explicit Block -// rules restrict RFC1918. Without this the rules would exist but not enforce on -// a creator whose firewall was never enabled. Best-effort — logged, not fatal. -func enableHyperVFirewallScript(creator string) string { - return fmt.Sprintf( - "Set-NetFirewallHyperVVMSetting -Name %s -Enabled True -DefaultInboundAction Allow -DefaultOutboundAction Allow -ErrorAction Stop", - psQuote(creator), - ) -} - -// removeByPrefixScript deletes every Hyper-V firewall rule ephemerd installed, -// across all creators, matched by the ephemerd- name prefix. Catches stale -// rules leaked by a crashed run as well as the current set. -func removeByPrefixScript() string { - return fmt.Sprintf( - "Get-NetFirewallHyperVRule | Where-Object { $_.Name -like '%s-*' } | Remove-NetFirewallHyperVRule -ErrorAction SilentlyContinue", - firewallRulePrefix, - ) -} - // ------------------------------------------------------------------------- // L2Bridge host-firewall backstop // ------------------------------------------------------------------------- // // On the L2Bridge path the primary egress enforcement is the per-endpoint VFP -// ACL ladder (buildL2BridgeEgressACLPolicies), which was proven on metal. The -// Hyper-V firewall blocks installed for NAT are deliberately NOT reused there: -// hyperVEgressRules subtracts the container subnet from every blocked range, and -// on L2Bridge the container subnet IS the management LAN — the subtraction would -// carve the management plane straight back out of the deny. +// ACL ladder (buildL2BridgeEgressACLPolicies), which was proven on metal. // // What the host firewall CAN do here, and could not on NAT, is match on the -// container's source address. #136 established that host MPSSVC rules cannot -// filter NATed container egress, because the host sees that traffic post-NAT -// with its own address as the source. L2Bridge does not NAT: a container's -// packet reaches the host carrying the container's own pool address, as ordinary -// inbound LAN traffic. So an INBOUND rule scoped remoteip= matches -// exactly the job containers and nothing else. +// container's source address. On NAT the host sees container egress post-NAT +// with its own address as the source, so a host rule scoped to the container +// source matches nothing. L2Bridge does not NAT: a container's packet reaches +// the host carrying the container's own pool address, as ordinary inbound LAN +// traffic. So an INBOUND rule scoped remoteip= matches exactly the job +// containers and nothing else. // // That is used for one job: closing the control-plane ports back off after the // VFP ladder's host /32 allow opens the host (AllowHostAccess). The allow has to @@ -472,97 +246,31 @@ func defaultGatewayForAdapter(name string) (string, error) { } func (w *windowsNetworking) installFirewallRules() error { - // L2Bridge: the VFP ACL ladder is the enforcement point. Installing the NAT - // Hyper-V/netsh rule set here would be worse than useless — it is scoped to - // the 10.88/16 NAT subnet (matches nothing) and its subtract-the-container- - // subnet logic would carve the management LAN out of the deny. + // L2Bridge: the VFP ACL ladder (applied per-endpoint in setup()) is the + // enforcement point; this installs only the host-firewall backstop that + // fences the control-plane ports back off. if w.cfg.L2BridgeEgress { return w.installL2BridgeFirewallRules() } - // Degrade gracefully at every step: a host that cannot program the - // Hyper-V firewall must never fail daemon startup. It falls back to the - // netsh host-firewall rules (weaker, but better than nothing on builds - // without the Hyper-V firewall) or, failing that, to the per-endpoint HNS - // ACLs already applied in setup(). - if !hyperVFirewallAvailable() { - w.cfg.Log.Warn("New-NetFirewallHyperVRule unavailable on this host; falling back to netsh host-firewall rules") - return w.installNetshFirewallRules() - } - - creators, err := discoverContainerVMCreators() - if err != nil { - w.cfg.Log.Warn("failed to enumerate Hyper-V firewall VM creators; falling back to netsh host-firewall rules", "error", err) - return w.installNetshFirewallRules() - } - if len(creators) == 0 { - w.cfg.Log.Warn("no container Hyper-V firewall VM creator registered yet; falling back to netsh host-firewall rules") - return w.installNetshFirewallRules() - } - - // init() always creates the HCN network on DefaultSubnet with - // defaultGateway (cfg.Subnet is not consulted on Windows), so the firewall - // must match those. - installed := 0 - for _, creator := range creators { - rules, err := hyperVEgressRules(creator, DefaultSubnet, defaultGateway, w.cfg.ControlPorts) - if err != nil { - w.cfg.Log.Warn("failed to build Hyper-V firewall rules", "creator", creator, "error", err) - continue - } - - if err := powershell(enableHyperVFirewallScript(creator)); err != nil { - w.cfg.Log.Warn("failed to enable Hyper-V firewall for creator (rules may not enforce)", "creator", creator, "error", err) - } - - for _, r := range rules { - // Idempotent: remove any rule carrying this name from a previous - // run before adding, so re-running install never duplicates. - _ = powershell(r.removeCommand()) - - w.cfg.Log.Info("adding Hyper-V firewall rule", "rule", r.name, "creator", creator) - if err := powershell(r.command()); err != nil { - // Non-fatal: one failed rule degrades to the remaining rules - // plus the per-endpoint ACLs, never to refusing to start. - w.cfg.Log.Warn("failed to add Hyper-V firewall rule", "rule", r.name, "error", err) - continue - } - installed++ - } - } - - w.cfg.Log.Info("Hyper-V firewall rules installed", "rules", installed, "creators", len(creators)) + // NAT path: there is no host-side mechanism that can filter container egress + // on this stack (WFP, the Hyper-V firewall, and netsh were all disproven on + // metal — see the file header). Installing rules here would be security + // theater: they match nothing. Log the gap and install nothing. + w.cfg.Log.Warn("Windows container egress is NOT filtered on the default NAT network — no host-side mechanism can filter it on this stack; set network.l2bridge_egress to enforce egress (see docs/guides/security.md)") return nil } func (w *windowsNetworking) removeFirewallRules() { - // Always attempt to remove every rule set ephemerd can install — harmless - // if a given one was never installed — so a host that switched paths - // between runs does not leak the other path's rules. + // removeL2BridgeFirewallRules is safe on either path — on NAT nothing + // matches — and also sweeps any leaked per-job host-port allows by prefix, + // so it must run unconditionally regardless of which path installed rules. w.removeL2BridgeFirewallRules() - w.removeNetshFirewallRules() - - if !hyperVFirewallAvailable() { - return - } - // Remove by prefix: catches every ephemerd rule across all creators, - // including stale ones, without needing to recompute per-creator names. - if err := powershell(removeByPrefixScript()); err != nil { - w.cfg.Log.Debug("failed to remove Hyper-V firewall rules", "error", err) - } } // ------------------------------------------------------------------------- -// netsh host firewall (fallback for hosts without the Hyper-V firewall) +// netsh host-firewall helpers (shared by the L2Bridge backstop above) // ------------------------------------------------------------------------- -// -// Retained only as a degraded fallback for Windows builds that lack -// New-NetFirewallHyperVRule (pre-Server 2025 / Windows 11 22H2). On such hosts -// these host-global rules are strictly better than nothing, even though — as -// #136 proved on Server 2025 — the host firewall evaluates NATed container -// egress post-NAT and cannot match on the container source. The outbound -// blocks are scoped localip= so a mis-scoped rule degrades to -// a no-op rather than cutting the host off its own management LAN. func netsh(args ...string) error { out, err := exec.Command("netsh", args...).CombinedOutput() @@ -589,132 +297,3 @@ func (r winFirewallRule) addArgs() []string { func (r winFirewallRule) deleteArgs() []string { return []string{"advfirewall", "firewall", "delete", "rule", "name=" + r.name} } - -// hostFirewallRules returns the netsh fallback rule set for the given container -// subnet, gateway, and control-plane ports: outbound blocks for every denied -// range (with the container subnet carved out) plus inbound blocks for -// container->gateway traffic on the control ports. -// -// Exposed as a pure function (no side effects) so tests can assert the exact -// rule set without invoking netsh. -func hostFirewallRules(subnet, gateway string, controlPorts []int) ([]winFirewallRule, error) { - var rules []winFirewallRule - - for _, cidr := range egressBlockedCIDRs { - remote, err := subtractCIDR(cidr, subnet) - if err != nil { - return nil, fmt.Errorf("computing blocked ranges for %s: %w", cidr, err) - } - if len(remote) == 0 { - continue // fully covered by the container subnet - } - rules = append(rules, winFirewallRule{ - name: firewallRulePrefix + "-block-" + strings.ReplaceAll(cidr, "/", "_"), - spec: []string{ - "dir=out", - "action=block", - "protocol=any", - "localip=" + subnet, - "remoteip=" + strings.Join(remote, ","), - "profile=any", - "enable=yes", - }, - }) - } - - for _, port := range controlPorts { - rules = append(rules, winFirewallRule{ - name: fmt.Sprintf("%s-control-%d", firewallRulePrefix, port), - spec: []string{ - "dir=in", - "action=block", - "protocol=TCP", - "localip=" + gateway, - "localport=" + strconv.Itoa(port), - "remoteip=" + subnet, - "profile=any", - "enable=yes", - }, - }) - } - - return rules, nil -} - -// subtractCIDR removes exclude from cidr and renders the remainder in -// address-range syntax: the original CIDR when the two do not overlap, -// otherwise up to two "start-end" ranges. Returns an empty slice when exclude -// covers cidr entirely. IPv4 only — the HCN NAT network has no IPv6 IPAM. -func subtractCIDR(cidr, exclude string) ([]string, error) { - clo, chi, err := v4Range(cidr) - if err != nil { - return nil, err - } - xlo, xhi, err := v4Range(exclude) - if err != nil { - return nil, err - } - - if xhi < clo || xlo > chi { - return []string{cidr}, nil // no overlap — keep the CIDR as-is - } - - var out []string - if xlo > clo { - out = append(out, u32ToIP(clo)+"-"+u32ToIP(xlo-1)) - } - if xhi < chi { - out = append(out, u32ToIP(xhi+1)+"-"+u32ToIP(chi)) - } - return out, nil -} - -// v4Range returns the first and last address of an IPv4 CIDR as uint32. -func v4Range(cidr string) (lo, hi uint32, err error) { - r, _, err := cidrRange(cidr) - if err != nil { - return 0, 0, fmt.Errorf("parsing %s: %w", cidr, err) - } - return r.lo, r.hi, nil -} - -// u32ToIP renders a uint32 back to dotted-quad form. -func u32ToIP(v uint32) string { return u32ToIPv4(v) } - -func (w *windowsNetworking) installNetshFirewallRules() error { - rules, err := hostFirewallRules(DefaultSubnet, defaultGateway, w.cfg.ControlPorts) - if err != nil { - w.cfg.Log.Warn("building netsh host firewall rules", "error", err) - return nil - } - - for _, r := range rules { - // Idempotent: delete any rule carrying this name from a previous run - // before adding. netsh delete removes every rule matching the name; - // "no rules match" on a fresh host is expected and ignored. - _ = netsh(r.deleteArgs()...) - - w.cfg.Log.Info("adding netsh firewall rule", "rule", r.name) - if err := netsh(r.addArgs()...); err != nil { - // Non-fatal: degrade to the per-endpoint ACLs rather than refusing - // to start. - w.cfg.Log.Warn("failed to add netsh firewall rule", "rule", r.name, "error", err) - } - } - - w.cfg.Log.Info("netsh host firewall rules installed", "rules", len(rules)) - return nil -} - -func (w *windowsNetworking) removeNetshFirewallRules() { - rules, err := hostFirewallRules(DefaultSubnet, defaultGateway, w.cfg.ControlPorts) - if err != nil { - w.cfg.Log.Debug("failed to rebuild netsh firewall rule set for removal", "error", err) - return - } - for _, r := range rules { - if err := netsh(r.deleteArgs()...); err != nil { - w.cfg.Log.Debug("failed to remove netsh firewall rule", "rule", r.name, "error", err) - } - } -} diff --git a/pkg/networking/firewall_windows_test.go b/pkg/networking/firewall_windows_test.go index 4a86170..7542fc3 100644 --- a/pkg/networking/firewall_windows_test.go +++ b/pkg/networking/firewall_windows_test.go @@ -3,13 +3,11 @@ package networking import ( - "slices" "strings" - "testing" ) // specValue extracts the value of a key=value pair from a rule spec, or "" -// when the key is absent. +// when the key is absent. Shared by the L2Bridge host-firewall backstop tests. func specValue(r winFirewallRule, key string) string { for _, s := range r.spec { if v, ok := strings.CutPrefix(s, key+"="); ok { @@ -18,212 +16,3 @@ func specValue(r winFirewallRule, key string) string { } return "" } - -func TestSubtractCIDR(t *testing.T) { - tests := []struct { - name string - cidr string - exclude string - want []string - wantErr bool - }{ - { - name: "subnet inside range splits it", - cidr: "10.0.0.0/8", - exclude: "10.88.0.0/16", - want: []string{"10.0.0.0-10.87.255.255", "10.89.0.0-10.255.255.255"}, - }, - { - name: "no overlap keeps the CIDR", - cidr: "172.16.0.0/12", - exclude: "10.88.0.0/16", - want: []string{"172.16.0.0/12"}, - }, - { - name: "exclude equals the range", - cidr: "10.88.0.0/16", - exclude: "10.88.0.0/16", - want: nil, - }, - { - name: "exclude covers the range", - cidr: "10.88.0.0/16", - exclude: "10.0.0.0/8", - want: nil, - }, - { - name: "exclude at the start leaves one range", - cidr: "10.0.0.0/8", - exclude: "10.0.0.0/16", - want: []string{"10.1.0.0-10.255.255.255"}, - }, - { - name: "exclude at the end leaves one range", - cidr: "10.0.0.0/8", - exclude: "10.255.0.0/16", - want: []string{"10.0.0.0-10.254.255.255"}, - }, - { - name: "malformed CIDR errors", - cidr: "not-a-cidr", - exclude: "10.88.0.0/16", - wantErr: true, - }, - { - name: "IPv6 CIDR errors", - cidr: "fc00::/7", - exclude: "10.88.0.0/16", - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := subtractCIDR(tt.cidr, tt.exclude) - if tt.wantErr { - if err == nil { - t.Fatalf("subtractCIDR(%q, %q) = %v, want error", tt.cidr, tt.exclude, got) - } - return - } - if err != nil { - t.Fatalf("subtractCIDR(%q, %q): %v", tt.cidr, tt.exclude, err) - } - if !slices.Equal(got, tt.want) { - t.Errorf("subtractCIDR(%q, %q) = %v, want %v", tt.cidr, tt.exclude, got, tt.want) - } - }) - } -} - -// TestHostFirewallRules_BlocksEveryDeniedRange verifies each RFC1918 + -// link-local range produces an outbound block rule; a missing range would let -// a job reach that slice of the LAN. -func TestHostFirewallRules_BlocksEveryDeniedRange(t *testing.T) { - rules, err := hostFirewallRules(DefaultSubnet, defaultGateway, nil) - if err != nil { - t.Fatalf("hostFirewallRules: %v", err) - } - if len(rules) != len(egressBlockedCIDRs) { - t.Fatalf("got %d rules, want %d (one outbound block per denied range)", len(rules), len(egressBlockedCIDRs)) - } - for i, cidr := range egressBlockedCIDRs { - r := rules[i] - wantName := firewallRulePrefix + "-block-" + strings.ReplaceAll(cidr, "/", "_") - if r.name != wantName { - t.Errorf("rule[%d].name = %q, want %q", i, r.name, wantName) - } - if specValue(r, "dir") != "out" || specValue(r, "action") != "block" { - t.Errorf("rule %s is not an outbound block: %v", r.name, r.spec) - } - if specValue(r, "remoteip") == "" { - t.Errorf("rule %s has no remoteip scope", r.name) - } - } -} - -// TestHostFirewallRules_GatewayAndSubnetNeverBlocked pins the safety property: -// the container subnet — which contains the NAT gateway (DNS, default route, -// GatewayPorts) and the other containers — must never appear inside a blocked -// range. Windows Firewall has no rule ordering and Block beats Allow, so a -// blocked gateway could not be rescued by an allow rule: it would brick all -// container networking. -func TestHostFirewallRules_GatewayAndSubnetNeverBlocked(t *testing.T) { - rules, err := hostFirewallRules("10.88.0.0/16", "10.88.0.1", nil) - if err != nil { - t.Fatalf("hostFirewallRules: %v", err) - } - - for _, r := range rules { - remote := specValue(r, "remoteip") - if strings.Contains(remote, "10.88.") { - t.Errorf("rule %s blocks the container subnet: remoteip=%s", r.name, remote) - } - // Outbound blocks must be scoped to container-sourced traffic so the - // host's own LAN access can never match. - if specValue(r, "localip") != "10.88.0.0/16" { - t.Errorf("rule %s not scoped to the container subnet: localip=%s", r.name, specValue(r, "localip")) - } - } - - // The 10.0.0.0/8 block must be split exactly around the subnet. - want := "10.0.0.0-10.87.255.255,10.89.0.0-10.255.255.255" - for _, r := range rules { - if r.name == firewallRulePrefix+"-block-10.0.0.0_8" { - if got := specValue(r, "remoteip"); got != want { - t.Errorf("10/8 block remoteip = %q, want %q", got, want) - } - return - } - } - t.Error("no block rule found for 10.0.0.0/8") -} - -// TestHostFirewallRules_ControlPortRules confirms the container→gateway -// control-plane blocks mirror the Linux INPUT drops: inbound, TCP, one -// specific port each, source = container subnet, destination = gateway — -// never a blanket gateway block and never port 53. -func TestHostFirewallRules_ControlPortRules(t *testing.T) { - ports := []int{10000, 10001, 10002} // containerd, dispatch, debug exec - rules, err := hostFirewallRules(DefaultSubnet, defaultGateway, ports) - if err != nil { - t.Fatalf("hostFirewallRules: %v", err) - } - - var control []winFirewallRule - for _, r := range rules { - if specValue(r, "dir") == "in" { - control = append(control, r) - } - } - if len(control) != len(ports) { - t.Fatalf("got %d inbound rules, want %d (one per control port)", len(control), len(ports)) - } - - for i, port := range []string{"10000", "10001", "10002"} { - r := control[i] - if specValue(r, "action") != "block" || specValue(r, "protocol") != "TCP" { - t.Errorf("rule %s is not a TCP block: %v", r.name, r.spec) - } - if specValue(r, "localport") != port { - t.Errorf("rule %s localport = %s, want %s", r.name, specValue(r, "localport"), port) - } - if specValue(r, "remoteip") != DefaultSubnet { - t.Errorf("rule %s missing source-subnet scope: %v", r.name, r.spec) - } - if specValue(r, "localip") != defaultGateway { - t.Errorf("rule %s missing gateway-dest scope: %v", r.name, r.spec) - } - if specValue(r, "localport") == "53" { - t.Errorf("rule %s blocks DNS (port 53) — must not", r.name) - } - } -} - -// TestHostFirewallRules_NamesAndArgs pins the naming and argv contract: every -// rule carries the ephemerd prefix (so the set is findable and removable), -// and delete targets exactly the name add created (that is what makes -// delete-before-add idempotent). -func TestHostFirewallRules_NamesAndArgs(t *testing.T) { - rules, err := hostFirewallRules(DefaultSubnet, defaultGateway, []int{10000}) - if err != nil { - t.Fatalf("hostFirewallRules: %v", err) - } - for _, r := range rules { - if !strings.HasPrefix(r.name, firewallRulePrefix) { - t.Errorf("rule name %q missing %q prefix", r.name, firewallRulePrefix) - } - add := r.addArgs() - wantAdd := []string{"advfirewall", "firewall", "add", "rule", "name=" + r.name} - if !slices.Equal(add[:5], wantAdd) { - t.Errorf("addArgs()[:5] = %v, want %v", add[:5], wantAdd) - } - if !slices.Equal(add[5:], r.spec) { - t.Errorf("addArgs() spec = %v, want %v", add[5:], r.spec) - } - wantDel := []string{"advfirewall", "firewall", "delete", "rule", "name=" + r.name} - if !slices.Equal(r.deleteArgs(), wantDel) { - t.Errorf("deleteArgs() = %v, want %v", r.deleteArgs(), wantDel) - } - } -} diff --git a/pkg/networking/network_windows.go b/pkg/networking/network_windows.go index bebd841..36d5d6d 100644 --- a/pkg/networking/network_windows.go +++ b/pkg/networking/network_windows.go @@ -336,16 +336,14 @@ func (w *windowsNetworking) setup(ctx context.Context, id string, netns string) return nil, fmt.Errorf("creating HCN endpoint for %s: %w", id, err) } - // Apply ACL policies to block private network access. On L2Bridge these are - // the only egress restriction that enforces: the Hyper-V firewall rule set - // in firewall_windows.go is built for the NAT subnet and is deliberately not - // installed on this path (its subtract-the-container-subnet logic would - // carve the management LAN back out of the deny — see the L2Bridge backstop - // note there). A failure here therefore means the container would run with - // unrestricted egress to the host LAN, other RFC1918 services, and - // link-local metadata endpoints. Fail CLOSED: tear down the endpoint we just - // created and refuse the job rather than start a container we cannot - // firewall. + // Apply ACL policies to block private network access. On L2Bridge the VFP + // Switch-ACL ladder applied here is the ONLY egress restriction that + // enforces — there is no host-side firewall mechanism that can filter this + // traffic (see the header in firewall_windows.go). A failure here therefore + // means the container would run with unrestricted egress to the host LAN, + // other RFC1918 services, and link-local metadata endpoints. Fail CLOSED: + // tear down the endpoint we just created and refuse the job rather than start + // a container we cannot firewall. // // The ACLs are applied to the endpoint BEFORE the container is started // (setup runs ahead of task creation), and the L2Bridge rule set is STATIC @@ -434,50 +432,6 @@ var egressBlockedCIDRs = []string{ "169.254.0.0/16", } -// buildEgressBlockPolicies constructs the per-endpoint block ACLs from -// egressBlockedCIDRs. It fails closed: a marshal error on any rule, or an empty -// resulting set, is an error rather than a silently weaker rule set. Split out -// from applyACLPolicies so the (pure) rule construction is unit-testable -// without a live HCN endpoint. -func buildEgressBlockPolicies() ([]hcn.EndpointPolicy, error) { - var policies []hcn.EndpointPolicy - - for _, cidr := range egressBlockedCIDRs { - if cidr == DefaultSubnet { - continue - } - - acl := hcn.AclPolicySetting{ - Protocols: "6,17", // TCP + UDP - Action: hcn.ActionTypeBlock, - Direction: hcn.DirectionTypeOut, - RemoteAddresses: cidr, - RuleType: hcn.RuleTypeSwitch, - Priority: 100, - } - - settings, err := json.Marshal(acl) - if err != nil { - // Fail closed: a rule we cannot serialize is a rule we cannot - // enforce. Do not skip it and continue with a weaker rule set. - return nil, fmt.Errorf("marshaling egress block ACL for %s: %w", cidr, err) - } - - policies = append(policies, hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: settings, - }) - } - - if len(policies) == 0 { - // Nothing to block would mean no egress restriction at all — treat as - // an error so the caller refuses to start an unfirewalled container. - return nil, fmt.Errorf("no egress block ACLs constructed (would run unfirewalled)") - } - - return policies, nil -} - // VFP Switch-ACL precedence for the L2Bridge egress model. VFP evaluates ACLs // by Priority, LOWER number = HIGHER precedence (evaluated first, first match // wins). The ladder is: operator carve-outs (top) > RFC1918 block > allow-any @@ -663,22 +617,22 @@ func buildL2BridgeEgressACLPolicies(extraAllowed []string, hostIP string) ([]hcn return policies, nil } -// applyACLPolicies applies the per-endpoint egress ACLs. On the L2Bridge path -// it applies the router-safe VFP ladder (buildL2BridgeEgressACLPolicies); on -// NAT it applies the existing block-only set (buildEgressBlockPolicies), which -// is left untouched so the default NAT path behaves exactly as before. The full -// rule set is built up front and applied atomically; any failure is returned so -// the caller (setup) can treat it as fatal for the job. +// applyACLPolicies applies the per-endpoint egress ACLs. Only the L2Bridge path +// has a working enforcement point: it applies the router-safe VFP ladder +// (buildL2BridgeEgressACLPolicies), built up front and applied atomically, with +// any failure returned so the caller (setup) can treat it as fatal for the job. +// +// On the NAT path there is nothing to apply: block ACLs on the NAT vSwitch port +// were inert on metal (they never filtered the NAT'd egress), and no host-side +// mechanism can filter NAT container egress on this stack (see the header in +// firewall_windows.go). NAT egress is unfiltered by design — installFirewallRules +// logs that gap and points the operator at network.l2bridge_egress. func (w *windowsNetworking) applyACLPolicies(endpoint *hcn.HostComputeEndpoint) error { - var ( - policies []hcn.EndpointPolicy - err error - ) - if w.cfg.L2BridgeEgress { - policies, err = buildL2BridgeEgressACLPolicies(w.cfg.ExtraAllowedCIDRs, w.hostAllowIP()) - } else { - policies, err = buildEgressBlockPolicies() + if !w.cfg.L2BridgeEgress { + return nil } + + policies, err := buildL2BridgeEgressACLPolicies(w.cfg.ExtraAllowedCIDRs, w.hostAllowIP()) if err != nil { return err } diff --git a/pkg/networking/network_windows_test.go b/pkg/networking/network_windows_test.go index 8f05dac..bf26e62 100644 --- a/pkg/networking/network_windows_test.go +++ b/pkg/networking/network_windows_test.go @@ -11,51 +11,6 @@ import ( "github.com/Microsoft/hcsshim/hcn" ) -// TestBuildEgressBlockPolicies verifies the fail-closed egress rule set (WIN-4). -// Every RFC1918 + link-local range must produce an outbound Block ACL; a -// partial or empty set would let a job reach the host LAN / other tenants / -// cloud metadata. -func TestBuildEgressBlockPolicies(t *testing.T) { - policies, err := buildEgressBlockPolicies() - if err != nil { - t.Fatalf("buildEgressBlockPolicies: %v", err) - } - - // Collect the CIDRs that actually became block rules. - got := map[string]bool{} - for _, p := range policies { - if p.Type != hcn.ACL { - t.Errorf("policy type = %v, want ACL", p.Type) - } - var acl hcn.AclPolicySetting - if err := json.Unmarshal(p.Settings, &acl); err != nil { - t.Fatalf("unmarshal ACL setting: %v", err) - } - if acl.Action != hcn.ActionTypeBlock { - t.Errorf("CIDR %s action = %v, want Block", acl.RemoteAddresses, acl.Action) - } - if acl.Direction != hcn.DirectionTypeOut { - t.Errorf("CIDR %s direction = %v, want Out", acl.RemoteAddresses, acl.Direction) - } - got[acl.RemoteAddresses] = true - } - - // Every configured range (minus the container's own subnet) must be blocked. - for _, cidr := range egressBlockedCIDRs { - if cidr == DefaultSubnet { - continue - } - if !got[cidr] { - t.Errorf("egress range %s was not turned into a block rule", cidr) - } - } - - // Link-local / metadata range must always be present. - if !got["169.254.0.0/16"] { - t.Error("link-local/metadata range 169.254.0.0/16 not blocked") - } -} - // decodeACLs unmarshals every EndpointPolicy back into an AclPolicySetting, // failing the test on a non-ACL policy or a malformed setting. func decodeACLs(t *testing.T, policies []hcn.EndpointPolicy) []hcn.AclPolicySetting { @@ -340,22 +295,6 @@ func TestL2BridgeControlPlaneRules(t *testing.T) { t.Errorf("rule %s has no localport; a portless block would cut the host off the container entirely", r.name) } } - - // Rule names must not collide with the NAT netsh rules, or removing one set - // would delete the other. - natRules, err := hostFirewallRules(DefaultSubnet, defaultGateway, []int{10000, 10001, 10002}) - if err != nil { - t.Fatalf("hostFirewallRules: %v", err) - } - natNames := map[string]bool{} - for _, r := range natRules { - natNames[r.name] = true - } - for _, r := range rules { - if natNames[r.name] { - t.Errorf("L2Bridge rule name %q collides with a NAT rule name", r.name) - } - } } // TestL2BridgeControlPlaneRules_NoPlanNoRules verifies the backstop stays silent