diff --git a/cmd/ephemerd/main.go b/cmd/ephemerd/main.go index d95710ef..f78fc8a0 100644 --- a/cmd/ephemerd/main.go +++ b/cmd/ephemerd/main.go @@ -263,12 +263,19 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP controlPorts := []int{int(containerdTCPPort), int(containerdTCPPort) + 1, int(containerdTCPPort) + 2} net, err := networking.New(networking.Config{ - DataDir: configDir, - Subnet: cfg.Network.Subnet, - MTU: cfg.Network.MTU, - CNIBinDir: cm.Dir(), - ControlPorts: controlPorts, - Log: log, + DataDir: configDir, + Subnet: cfg.Network.Subnet, + MTU: cfg.Network.MTU, + CNIBinDir: cm.Dir(), + ControlPorts: controlPorts, + L2BridgeEgress: cfg.Network.L2BridgeEgress, + HostNIC: cfg.Network.HostNIC, + IPPool: cfg.Network.IPPool, + Gateway: cfg.Network.Gateway, + PublicDNS: cfg.Network.PublicDNS, + ExtraAllowedCIDRs: cfg.Network.ExtraAllowedDestinations, + AllowHostAccess: needsHostAccess(cfg), + Log: log, }) if err != nil { return fmt.Errorf("initializing networking: %w", err) @@ -427,12 +434,19 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP // Initialize container networking net, err := networking.New(networking.Config{ - DataDir: configDir, - Subnet: cfg.Network.Subnet, - MTU: cfg.Network.MTU, - CNIBinDir: cm.Dir(), - GatewayPorts: gatewayPorts, - Log: log, + DataDir: configDir, + Subnet: cfg.Network.Subnet, + MTU: cfg.Network.MTU, + CNIBinDir: cm.Dir(), + GatewayPorts: gatewayPorts, + L2BridgeEgress: cfg.Network.L2BridgeEgress, + HostNIC: cfg.Network.HostNIC, + IPPool: cfg.Network.IPPool, + Gateway: cfg.Network.Gateway, + PublicDNS: cfg.Network.PublicDNS, + ExtraAllowedCIDRs: cfg.Network.ExtraAllowedDestinations, + AllowHostAccess: needsHostAccess(cfg), + Log: log, }) if err != nil { return fmt.Errorf("initializing networking: %w", err) @@ -1137,6 +1151,21 @@ func crictlCmd() *cli.Command { } } +// needsHostAccess reports whether ephemerd runs anything that job containers +// must be able to reach over the network on the host address: +// +// - dind: the per-job Docker API listener, which on Windows is a TCP listener +// on the host address handed to the job as DOCKER_HOST. +// - the Go module proxy: bound to the same address and injected as GOPROXY. +// +// It only affects the Windows L2Bridge egress path, where the ACL ladder +// otherwise blocks the host along with the rest of RFC1918 — with neither +// feature enabled the strictest posture (host unreachable) applies. On NAT and +// on Linux the gateway is already reachable and this changes nothing. +func needsHostAccess(cfg *config.Config) bool { + return cfg.Dind.Enabled || cfg.ModuleProxy.Enabled +} + func joinPath(parts ...string) string { result := parts[0] for _, p := range parts[1:] { diff --git a/cmd/ephemerd/run.go b/cmd/ephemerd/run.go index c5f60ddc..cf857470 100644 --- a/cmd/ephemerd/run.go +++ b/cmd/ephemerd/run.go @@ -157,34 +157,91 @@ func runWorkflow(ctx context.Context, workflowPath string, jobFilter string, ima socketPath = `\\.\pipe\ephemerd-run-` + filepath.Base(tmpDir) } - image := resolveRunImage(imageFlag, platform) + // Load the service config once. A local run must use the SAME container + // network the host is configured for — building its own would put the job + // on an unfiltered network on a host that was deliberately configured to + // filter, and on Windows would also install the NAT-era netsh rules that + // block RFC1918 host-wide. A missing or unreadable config is not fatal: + // the run falls back to the built-in default network. + cfg := loadRunConfig() runner := &workflow.Runner{ DataDir: tmpDir, SocketPath: socketPath, - Image: image, + Image: resolveRunImage(imageFlag, platform, cfg), + Network: runNetworkOptions(cfg, log), Log: log, } return runner.RunJob(ctx, jobName, job, repoDir) } +// loadRunConfig reads the service config for a local run. A missing or +// malformed config.toml is not an error here — the run simply falls back to the +// built-in defaults for both image and network — so it returns nil rather than +// failing the command. +func loadRunConfig() *config.Config { + cfg, err := config.Load(filepath.Join(configDir, "config.toml")) + if err != nil { + return nil + } + return cfg +} + +// runNetworkOptions maps the host's [network] config onto a local run and warns +// when the job will land on the default, unfilterable container network. +// +// On Windows the default is an HNS NAT network on the Hyper-V vSwitch. Container +// egress there CANNOT be filtered in software — WFP, the Hyper-V firewall, VFP +// on a NAT switch and netsh have all been ruled out on metal, because WinNAT's +// translation path never presents the packet to an inspectable filtering layer. +// A job on that network can reach anything the host can, including the LAN and +// any management planes on it. network.l2bridge_egress is the only path that +// actually enforces. See docs/guides/security.md ("Network Firewall"). +func runNetworkOptions(cfg *config.Config, log *slog.Logger) workflow.NetworkOptions { + if cfg == nil { + warnUnfilteredRunNetwork(log, "no readable config.toml") + return workflow.NetworkOptions{} + } + + opts := workflow.NetworkOptions{ + Subnet: cfg.Network.Subnet, + MTU: cfg.Network.MTU, + L2BridgeEgress: cfg.Network.L2BridgeEgress, + HostNIC: cfg.Network.HostNIC, + IPPool: cfg.Network.IPPool, + Gateway: cfg.Network.Gateway, + PublicDNS: cfg.Network.PublicDNS, + ExtraAllowedCIDRs: cfg.Network.ExtraAllowedDestinations, + AllowHostAccess: needsHostAccess(cfg), + } + + if runtime.GOOS == "windows" && !opts.L2BridgeEgress { + warnUnfilteredRunNetwork(log, "network.l2bridge_egress is not enabled") + } + return opts +} + +func warnUnfilteredRunNetwork(log *slog.Logger, reason string) { + if runtime.GOOS != "windows" { + return + } + log.Warn("this job's container egress is NOT filtered: it runs on the default Hyper-V vSwitch (HNS NAT), "+ + "where container egress cannot be filtered in software — the job can reach your LAN and anything on it. "+ + "Set network.l2bridge_egress (requires a WIRED adapter and a reserved network.ip_pool) to enforce egress", + "reason", reason) +} + // resolveRunImage determines the container image for a run job. // Priority: --image flag → service config.toml → empty (caller applies the // built-in default — see workflow.Runner.RunJob, which substitutes // defaultImage when this returns ""). -func resolveRunImage(flagValue string, platform workflow.TargetPlatform) string { +func resolveRunImage(flagValue string, platform workflow.TargetPlatform, cfg *config.Config) string { if flagValue != "" { return flagValue } - - osName := platform.String() - cfgPath := filepath.Join(configDir, "config.toml") - if cfg, err := config.Load(cfgPath); err == nil { - if img := cfg.GitHub.DefaultImageFor(osName); img != "" { - return img - } + if cfg == nil { + return "" } - - return "" + return cfg.GitHub.DefaultImageFor(platform.String()) } diff --git a/cmd/ephemerd/run_test.go b/cmd/ephemerd/run_test.go index 6bb5bc16..91fd3d6a 100644 --- a/cmd/ephemerd/run_test.go +++ b/cmd/ephemerd/run_test.go @@ -41,7 +41,7 @@ default_image_windows = "ghcr.io/from-config:windows" configDir = dir t.Setenv("GITHUB_TOKEN", "ghp_test") - if got := resolveRunImage("ghcr.io/explicit:v1", workflow.PlatformLinux); got != "ghcr.io/explicit:v1" { + if got := resolveRunImage("ghcr.io/explicit:v1", workflow.PlatformLinux, loadRunConfig()); got != "ghcr.io/explicit:v1" { t.Errorf("flag-wins: got %q, want explicit override", got) } } @@ -58,7 +58,7 @@ default_image_windows = "ghcr.io/from-config:windows" configDir = dir t.Setenv("GITHUB_TOKEN", "ghp_test") - got := resolveRunImage("", workflow.PlatformLinux) + got := resolveRunImage("", workflow.PlatformLinux, loadRunConfig()) if got != "ghcr.io/from-config:linux" { t.Errorf("config-wins linux: got %q, want %q", got, "ghcr.io/from-config:linux") } @@ -76,7 +76,7 @@ default_image_windows = "ghcr.io/from-config:windows" configDir = dir t.Setenv("GITHUB_TOKEN", "ghp_test") - got := resolveRunImage("", workflow.PlatformWindows) + got := resolveRunImage("", workflow.PlatformWindows, loadRunConfig()) if got != "ghcr.io/from-config:windows" { t.Errorf("config-wins windows: got %q, want %q", got, "ghcr.io/from-config:windows") } @@ -88,7 +88,7 @@ func TestResolveRunImage_NoConfigFile(t *testing.T) { defer configDirGuard(t)() configDir = t.TempDir() - if got := resolveRunImage("", workflow.PlatformLinux); got != "" { + if got := resolveRunImage("", workflow.PlatformLinux, loadRunConfig()); got != "" { t.Errorf("no-config: got %q, want empty (caller defaults)", got) } } @@ -103,7 +103,7 @@ func TestResolveRunImage_ConfigParseError(t *testing.T) { writeConfig(t, dir, "this is not valid TOML [\n") configDir = dir - if got := resolveRunImage("", workflow.PlatformLinux); got != "" { + if got := resolveRunImage("", workflow.PlatformLinux, loadRunConfig()); got != "" { t.Errorf("config-parse-error: got %q, want empty fallback", got) } } @@ -123,7 +123,109 @@ owner = "testorg" // Windows has no built-in default image (the runtime picks one from // the host build number), so DefaultImageFor("windows") returns "" — // resolver must propagate the empty string. - if got := resolveRunImage("", workflow.PlatformWindows); got != "" { + if got := resolveRunImage("", workflow.PlatformWindows, loadRunConfig()); got != "" { t.Errorf("no-windows-override: got %q, want empty (caller defaults)", got) } } + +// A local run must inherit the host's L2Bridge settings. Before this, `ephemerd +// run` built its own default network: on Windows an HNS NAT network plus the +// NAT-era netsh rules that block RFC1918 host-wide — which severs the host's own +// DNS when its resolver is a LAN address — and put the job on an UNFILTERED +// network on a host deliberately configured to filter. +func TestRunNetworkOptions_CarriesL2BridgeConfig(t *testing.T) { + defer configDirGuard(t)() + dir := t.TempDir() + writeConfig(t, dir, ` +[github] +owner = "testorg" + +[network] +l2bridge_egress = true +host_nic = "Ethernet" +ip_pool = "192.0.2.192/27" +public_dns = ["9.9.9.9"] +extra_allowed_destinations = ["198.51.100.0/24"] +`) + configDir = dir + t.Setenv("GITHUB_TOKEN", "ghp_test") + + opts := runNetworkOptions(loadRunConfig(), quietLog()) + + if !opts.L2BridgeEgress { + t.Error("L2BridgeEgress not carried into the run; the job would land on the unfiltered NAT network") + } + if opts.HostNIC != "Ethernet" { + t.Errorf("HostNIC: got %q, want %q", opts.HostNIC, "Ethernet") + } + if opts.IPPool != "192.0.2.192/27" { + t.Errorf("IPPool: got %q, want %q", opts.IPPool, "192.0.2.192/27") + } + if len(opts.PublicDNS) != 1 || opts.PublicDNS[0] != "9.9.9.9" { + t.Errorf("PublicDNS: got %v, want [9.9.9.9]", opts.PublicDNS) + } + if len(opts.ExtraAllowedCIDRs) != 1 || opts.ExtraAllowedCIDRs[0] != "198.51.100.0/24" { + t.Errorf("ExtraAllowedCIDRs: got %v, want [198.51.100.0/24]", opts.ExtraAllowedCIDRs) + } +} + +// AllowHostAccess must follow the same rule serve uses (needsHostAccess): the +// host /32 allow exists only because dind and the module proxy serve job +// containers over the network. With neither enabled the strict posture applies. +func TestRunNetworkOptions_AllowHostAccessFollowsDind(t *testing.T) { + defer configDirGuard(t)() + dir := t.TempDir() + writeConfig(t, dir, ` +[github] +owner = "testorg" + +[network] +l2bridge_egress = true +host_nic = "Ethernet" +ip_pool = "192.0.2.192/27" + +[dind] +enabled = true +`) + configDir = dir + t.Setenv("GITHUB_TOKEN", "ghp_test") + + if opts := runNetworkOptions(loadRunConfig(), quietLog()); !opts.AllowHostAccess { + t.Error("AllowHostAccess must be set when dind is enabled, or dind cannot reach the host and jobs fail to provision") + } +} + +func TestRunNetworkOptions_StrictWhenNothingServesContainers(t *testing.T) { + defer configDirGuard(t)() + dir := t.TempDir() + writeConfig(t, dir, ` +[github] +owner = "testorg" + +[network] +l2bridge_egress = true +host_nic = "Ethernet" +ip_pool = "192.0.2.192/27" + +[dind] +enabled = false +`) + configDir = dir + t.Setenv("GITHUB_TOKEN", "ghp_test") + + if opts := runNetworkOptions(loadRunConfig(), quietLog()); opts.AllowHostAccess { + t.Error("AllowHostAccess must stay false when nothing serves containers — that is the strictest posture") + } +} + +// No config at all is not fatal: the run falls back to the built-in default +// network (and warns on Windows that egress is unfiltered). +func TestRunNetworkOptions_NoConfigFallsBack(t *testing.T) { + defer configDirGuard(t)() + configDir = t.TempDir() + + opts := runNetworkOptions(loadRunConfig(), quietLog()) + if opts.L2BridgeEgress || opts.HostNIC != "" || opts.IPPool != "" { + t.Errorf("no-config: expected zero-value options, got %+v", opts) + } +} diff --git a/config.example.toml b/config.example.toml index f47fcd01..36b80759 100644 --- a/config.example.toml +++ b/config.example.toml @@ -81,12 +81,88 @@ owner = "your-org" [network] # Container subnet. Ephemerd auto-picks a free subnet if this conflicts with # an existing network (e.g. Podman, Docker). You don't need to change this. +# On the Windows L2Bridge path below it means something different — see there. # subnet = "10.88.0.0/16" # Bridge MTU. Auto-detected from the host's default interface. # Override if containers have connectivity issues on unusual networks. # mtu = 1500 +# --- Windows L2Bridge egress (opt-in; Windows hosts only) ------------------- +# By default Windows containers use an HNS NAT network. NAT cannot +# software-filter Windows container egress (VFP does not engage on a NAT +# network), so egress enforcement there leans on the Hyper-V firewall. Setting +# l2bridge_egress = true instead puts containers on an L2Bridge with a +# VFP-managed vSwitch port, where per-endpoint ACLs actually enforce: all of +# RFC1918 (10/8, 172.16/12, 192.168/16, 169.254/16) — including the LAN router +# — is blocked, and only the internet is reachable. Leave false to keep NAT. +# +# TRADE-OFF: on L2Bridge your job containers are no longer behind NAT. They are +# peers on this host's LAN with real LAN addresses, which is exactly why the +# ACLs can enforce — and why you must reserve addresses for them (ip_pool). +# l2bridge_egress = false +# +# Host NIC the L2Bridge binds onto. REQUIRED when l2bridge_egress = true; there +# is no default (the correct adapter name is host-specific). Use the name shown +# by `Get-NetAdapter`. Ignored when l2bridge_egress is false. +# +# STRONGLY RECOMMENDED: bind a DEDICATED NIC, not the host's management NIC. +# Creating the L2Bridge builds an external Hyper-V vSwitch on this adapter and +# migrates its IP onto a "vEthernet ()" adapter. On a host reached over +# that same NIC there is a brief connectivity blip during creation, and any +# failure mid-creation can leave the box unreachable — a hazard for remote +# nodes. A second NIC dedicated to container traffic keeps the management path +# untouched. (ephemerd tolerates the vSwitch rename either way, so restarts are +# safe once the network exists.) +# host_nic = "Ethernet 2" +# +# Addresses ephemerd may assign to job containers. REQUIRED when +# l2bridge_egress = true, with NO default. +# +# Why you have to set this: an L2Bridge HNS network must declare a subnet (HNS +# refuses to create one without), and left to itself HNS then picks endpoint +# addresses from anywhere inside that subnet — i.e. straight into your DHCP +# server's scope. ephemerd instead allocates from this pool and pins each +# container to the address it picked. Reserve the range on your DHCP server (or +# place it outside the DHCP scope) BEFORE enabling this, or you will get +# duplicate-address conflicts on your LAN. +# +# Either form works. Size it for at least runner.max_concurrent containers. +# The examples below use RFC 5737 documentation addresses — replace them. +# ip_pool = "192.0.2.192/27" # CIDR: .193-.222 (network/broadcast excluded) +# ip_pool = "192.0.2.200-192.0.2.230" # or an inclusive range +# +# The LAN itself. Both are auto-derived from host_nic at startup — subnet from +# the address configured on the adapter, gateway from its IPv4 default route — +# and ephemerd logs what it inferred. Set them only if the adapter has no +# default route of its own, or carries a prefix other than the LAN you want +# declared. ip_pool must lie inside subnet and must not contain either the +# host's own address or the gateway; startup refuses the config if it does. +# subnet = "192.0.2.0/24" +# gateway = "192.0.2.1" +# +# Public DNS resolvers handed to L2Bridge containers so name resolution goes to +# the internet rather than the (blocked) LAN router. Default 1.1.1.1, 8.8.8.8. +# public_dns = ["1.1.1.1", "8.8.8.8"] +# +# Extra destination CIDRs permitted through the L2Bridge egress ACLs, ABOVE the +# RFC1918 block (reserved for future use; default none — the strict posture). +# extra_allowed_destinations = [] +# +# NOTE — host reachability. When dind or the Go module proxy is enabled, +# ephemerd adds a single /32 allow for its own host address, because both serve +# job containers over the network (DOCKER_HOST / GOPROXY) and would otherwise be +# blocked with the rest of RFC1918. HNS Switch ACLs cannot be port-scoped +# without blackholing the whole port, so the control-plane ports are fenced off +# at the host firewall instead. With neither feature enabled, this host is +# unreachable from job containers like every other RFC1918 address. +# +# NOTE — migration. Switching an existing Windows node from NAT to L2Bridge +# needs a REBOOT, not just a daemon restart: creating an L2Bridge network beside +# a live HNS NAT network has been observed to wedge HNS (containers start with +# no egress, runners register then go offline, service stop hangs on drain). +# Drain the node, set the keys, reboot. + [runner] # Max concurrent jobs diff --git a/docs/arch/windows-egress-wfp-investigation.md b/docs/arch/windows-egress-wfp-investigation.md index b790e289..f52d6165 100644 --- a/docs/arch/windows-egress-wfp-investigation.md +++ b/docs/arch/windows-egress-wfp-investigation.md @@ -1,5 +1,15 @@ # Windows container egress: WFP IPFORWARD investigation (negative result) +> **Addendum (2026-08, resolved).** A host-side software path was subsequently +> found and shipped: HNS **L2Bridge** networks with per-endpoint **VFP Switch +> ACLs** (`network.l2bridge_egress`), proven on metal. The container is a peer on +> the host LAN (no NAT), so the VFP dataplane filters its egress at the vSwitch +> port. The "network-level isolation only" conclusion below therefore applies +> specifically to the default **NAT** stack this investigation used — where it +> still holds, and where ephemerd now installs nothing and logs that egress is +> unfiltered — **not universally**. See `docs/guides/security.md`. The body below +> is preserved unchanged as the historical record for the NAT stack. + ## TL;DR On Windows Server 2025 (build 26100) with a Hyper-V-isolated job container on an diff --git a/docs/guides/security.md b/docs/guides/security.md index 7211e18b..41f1e05e 100644 --- a/docs/guides/security.md +++ b/docs/guides/security.md @@ -27,18 +27,65 @@ macOS jobs run in full virtual machines via Apple's Virtualization.framework. Ea ## Network Firewall -By default, containers are blocked from reaching private network ranges: +The intent is that containers cannot reach private network ranges: - `10.0.0.0/8` (RFC 1918) - `172.16.0.0/12` (RFC 1918) - `192.168.0.0/16` (RFC 1918) - `169.254.0.0/16` (link-local) -This prevents jobs from scanning or accessing other machines on your local network, cloud metadata services (169.254.169.254), or other containers. Outbound internet access is allowed so jobs can fetch dependencies, push artifacts, and interact with external APIs. +This keeps jobs from scanning or reaching other machines on your LAN, cloud metadata services (169.254.169.254), or other containers, while outbound internet access stays open so jobs can fetch dependencies and talk to external APIs. -On Linux, these rules are enforced via iptables in the CNI bridge configuration. On Windows, per-endpoint HCN ACL policies block the same ranges. +**How well that intent holds depends on the platform. Read the Windows section — the default there does not enforce.** -The container's own subnet (default `10.88.0.0/16`) is excluded from the block list so containers can communicate with their gateway for outbound NAT. +### Linux and macOS — enforced + +On Linux this is enforced with iptables rules in the CNI bridge configuration. The container's own subnet (default `10.88.0.0/16`) is excluded so containers can reach their gateway for DNS and outbound NAT. + +macOS jobs run inside a Linux VM sidecar and get the identical in-VM iptables stack, and the sidecar is itself NAT-hidden behind the host. + +### Windows default (HNS NAT) — NOT enforced + +> **Job containers on the default Windows network can reach your entire LAN, including any management interfaces on it.** Treat a Windows runner on the default network as if it were an unfiltered host on your network. + +By default, Windows job containers attach to an HNS **NAT** network on the Hyper-V vSwitch. ephemerd still applies per-endpoint HCN ACL policies there, but **they do not take effect**, and no software mechanism on that stack does. This was established by exhaustive testing on real hardware, not inferred: + +- **Host WFP filters** (`netsh`, the WFP API, every layer including IPFORWARD and OUTBOUND) never see container egress. WinNAT's translation path does not present the packet to an inspectable filtering layer, even though the packet does traverse `tcpip.sys`. +- **HNS Switch ACLs** are a VFP construct, and VFP is not engaged on a NAT switch. They apply successfully and do nothing. +- **Enabling VFP** on the NAT switch default-denies everything — HNS only programs selective VFP policy for L2Bridge and Overlay. +- **The Hyper-V firewall** (`New-NetFirewallHyperVRule`) has nothing to bind to: Hyper-V-isolated containers on the NAT switch never register a VM creator, so `Get-NetFirewallHyperVVMCreator` returns nothing. +- **`netsh` host rules** are post-NAT. Container traffic is indistinguishable from the host's own by then, so any rule broad enough to catch a container also blackholes the host — including the host's own DNS if your resolver is a LAN address. + +There is no configuration that makes the NAT path filterable. If you need enforced egress on Windows, use L2Bridge. + +### Windows L2Bridge — the enforcing path + +Setting `network.l2bridge_egress = true` moves job containers onto an L2Bridge network, where VFP *is* engaged and the ACL ladder genuinely enforces. This is the same stack Kubernetes Windows CNIs (Calico, Antrea) use in production. + +```toml +[network] +l2bridge_egress = true +host_nic = "Ethernet 2" # dedicated NIC to bridge onto (see below) +ip_pool = "192.0.2.192/27" # reserved range, see below +``` + +**A reserved `ip_pool` your DHCP server will never lease is mandatory.** On L2Bridge, containers are addressed on your LAN rather than behind NAT, so ephemerd must be told which addresses it may hand out. There is deliberately **no default** — any built-in guess would collide with live DHCP leases. Size it for at least `runner.max_concurrent` containers, and add the matching exclusion on your DHCP server *before* enabling. Container egress ACLs rewrite each container's source MAC to the host NIC's, so the containers appear on the segment as the host itself. + +**A dedicated NIC is strongly recommended.** Creating the L2Bridge builds an external Hyper-V vSwitch on `host_nic` and migrates its IP onto a `vEthernet ()` adapter. If that is the host's only NIC — the one you administer it over — there is a connectivity blip during creation, and a failure mid-creation can leave a remote node unreachable. A second NIC dedicated to container traffic keeps the management path untouched. ephemerd tolerates the vSwitch rename, so once the network exists, daemon restarts are safe. Wi-Fi adapters are untested and not recommended: Hyper-V external switches on 802.11 historically require bridging workarounds. + +`subnet`, `gateway`, and DNS are derived from `host_nic` at startup; set them explicitly only to override. Startup fails fast with a message naming the offending key rather than guessing. + +**Understand the trade-offs before enabling:** + +- Containers hold **real LAN addresses** and are routable L2 peers of your network. The ACLs are load-bearing — a mis-scoped pool or block list means full LAN access, a worse blast radius than NAT. +- The block list covers the container's own subnet and its default gateway. Containers still *route* through the gateway but cannot *address* it. +- If `dind` or the Go module proxy is enabled, containers must be able to reach the host (that is how they reach the Docker API and `GOPROXY`), so a `/32` allow for the host is added automatically. Because a port-scoped Switch ACL disables the whole VFP port, that allow covers **all** host ports — so do not run anything on a Windows runner host that you would not expose to job containers. With both features disabled, the host stays blocked. +- **Migrating an existing node requires a reboot**, not just a service restart: creating an L2Bridge network beside a live NAT network leaves HNS in a broken state. Reserve the pool, drain the node, set the keys, then reboot. +- Anti-spoofing is not currently enforced — a container can forge a source address on the segment. + +### Local runs + +`ephemerd run` uses whatever network the host's `config.toml` specifies, so a local run on an L2Bridge host is filtered the same way a real job is. With no config, or on a Windows host without `l2bridge_egress`, it falls back to the default NAT network and logs a warning that the job's egress is unfiltered. ## Capability Restrictions diff --git a/pkg/config/config.go b/pkg/config/config.go index bd294531..18be3e82 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -213,8 +213,111 @@ func (w *WebhookConfig) ResolvedReconcileInterval() time.Duration { // NetworkConfig configures container networking. type NetworkConfig struct { - Subnet string `toml:"subnet"` // container subnet (auto-selected if empty) - MTU int `toml:"mtu"` // bridge MTU (auto-detected from host if 0) + // Subnet is the container subnet. + // + // Linux (CNI bridge): auto-selected when empty, avoiding ranges already in + // use on the host. + // + // Windows L2Bridge (l2bridge_egress = true): the CIDR of the LAN the bridge + // is attached to — containers are peers on it, not behind NAT — declared as + // the HNS network's Ipam subnet. Auto-derived from the address configured on + // host_nic when empty, which is the expected setting. Pin it only when the + // adapter carries a prefix that differs from the LAN you want declared. + // + // Windows NAT (the default): not consulted; the HNS NAT network always uses + // the built-in 10.88.0.0/16. + Subnet string `toml:"subnet"` + + MTU int `toml:"mtu"` // bridge MTU (auto-detected from host if 0) + + // L2BridgeEgress opts a Windows pool into L2Bridge container networking + // with VFP-enforced egress filtering, instead of the default HNS NAT. + // NAT cannot software-filter Windows container egress (VFP does not engage + // on a NAT network); L2Bridge puts the container on a VFP-managed vSwitch + // port so per-endpoint ACLs actually enforce. Windows only; ignored on + // Linux/macOS. Default false — NAT stays the default and this flag flips + // nothing until an operator opts a pool in. + L2BridgeEgress bool `toml:"l2bridge_egress"` + + // HostNIC is the host network adapter name the L2Bridge binds onto + // (e.g. "Ethernet"). REQUIRED when L2BridgeEgress is true — the bridge + // has no uplink without it. There is no default: the correct NIC name is + // host-specific (do not assume "Ethernet 2", which was a spike's + // hot-added test NIC). Ignored when L2BridgeEgress is false. + HostNIC string `toml:"host_nic"` + + // IPPool is the range of LAN addresses ephemerd may assign to job + // containers on the L2Bridge path. REQUIRED when L2BridgeEgress is true, + // with no default, and validated at load time. + // + // Why it cannot be inferred: an L2Bridge network must declare a subnet (HNS + // rejects a subnet-less one outright), and once it has one HNS will assign + // endpoint addresses from anywhere inside it — on a real LAN, straight into + // the site DHCP server's scope. ephemerd therefore allocates addresses + // itself, and only the operator knows which slice of their LAN the DHCP + // server is configured never to lease. + // + // Accepts a CIDR ("192.0.2.192/27" — network and broadcast excluded) or an + // inclusive range ("192.0.2.200-192.0.2.230"). Must lie inside Subnet and + // must not contain the host's own address or the LAN gateway. Size it for + // at least runner.max_concurrent addresses. Ignored when L2BridgeEgress is + // false. + IPPool string `toml:"ip_pool"` + + // Gateway is the LAN router the L2Bridge default route points at (the HNS + // Ipam route next hop). Auto-derived from the default route on HostNIC when + // empty, which is the expected setting; pin it only when the adapter has no + // default route of its own or carries more than one. + // + // Containers route THROUGH this address while the egress ACLs stop them + // ADDRESSING it — the gateway gets no allow rule. Windows L2Bridge only. + Gateway string `toml:"gateway"` + + // PublicDNS is the DNS resolver list handed to L2Bridge containers. Public + // resolvers keep container DNS off the LAN router (which the egress ACLs + // block along with the rest of RFC1918). Empty falls back to a built-in + // public default (1.1.1.1, 8.8.8.8). Only consulted on the L2Bridge path. + PublicDNS []string `toml:"public_dns"` + + // ExtraAllowedDestinations are additional CIDRs permitted through the + // L2Bridge egress ACLs at a precedence ABOVE the RFC1918 block (so a + // listed destination wins over the block). Reserved for future use — + // default empty, which reproduces the strict Linux end-state (no RFC1918 + // carve-outs at all). Only consulted on the L2Bridge path. + ExtraAllowedDestinations []string `toml:"extra_allowed_destinations"` +} + +// validate rejects an L2Bridge configuration that cannot be completed safely at +// runtime. It runs at config load, on every platform, so a node whose config was +// rendered wrong dies at startup with a specific message instead of coming up +// and quietly mis-addressing containers. +// +// Only the two settings that cannot be inferred are required. subnet, gateway, +// and public_dns are all derived from the host's real primary adapter when +// unset, so nothing here assumes any particular network. ip_pool deliberately +// has NO default: on L2Bridge, containers take addresses on the operator's own +// LAN, and a built-in guess would hand out addresses the site's DHCP server is +// also leasing. Semantic checks that need the host (does the pool fit inside the +// adapter's subnet, does it swallow the gateway) happen in pkg/networking once +// the adapter has been read. +func (n *NetworkConfig) validate() error { + if !n.L2BridgeEgress { + return nil + } + if strings.TrimSpace(n.HostNIC) == "" { + return fmt.Errorf(`network.host_nic is required when network.l2bridge_egress = true: ` + + `set it to the host adapter the bridge attaches to — the name shown by ` + "`Get-NetAdapter`" + `, ` + + `e.g. host_nic = "Ethernet". There is no default; the correct adapter is host-specific`) + } + if strings.TrimSpace(n.IPPool) == "" { + return fmt.Errorf(`network.ip_pool is required when network.l2bridge_egress = true: ` + + `on L2Bridge, job containers are addressed on this host's own LAN rather than behind NAT, ` + + `so ephemerd must be told which addresses it may hand out. Set it to a range your DHCP server ` + + `is configured never to lease — either a CIDR (ip_pool = "192.0.2.192/27") or an inclusive ` + + `range (ip_pool = "192.0.2.200-192.0.2.230") — sized for at least runner.max_concurrent ` + + `containers. There is no default: any built-in guess would collide with live DHCP leases`) + } + return nil } type ContainerdConfig struct { @@ -1334,6 +1437,10 @@ func (c *Config) validate() error { } } + if err := c.Network.validate(); err != nil { + return err + } + // Webhook secret handling depends on who owns the tunnel: // - "external": ingress and the GitHub webhook are configured elsewhere, // so the secret must match that external config — we cannot invent one. diff --git a/pkg/config/network_test.go b/pkg/config/network_test.go new file mode 100644 index 00000000..054e49fe --- /dev/null +++ b/pkg/config/network_test.go @@ -0,0 +1,129 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// The addresses here are RFC 5737 documentation ranges. Neither these tests nor +// the shipped defaults encode any real site's LAN — network.ip_pool has no +// default at all, which is the point of most of this file. + +// loadNetworkTOML writes a minimal valid config with the given [network] block +// and runs it through Load, returning the validation error (if any). +func loadNetworkTOML(t *testing.T, networkBlock string) (*Config, error) { + t.Helper() + t.Setenv("GITHUB_TOKEN", "ghp_test123") + + path := filepath.Join(t.TempDir(), "config.toml") + body := "[github]\nowner = \"testorg\"\n\n[network]\n" + networkBlock + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return Load(path) +} + +// TestNetworkValidate_L2BridgeRequiresIPPool is the fail-fast the whole L2Bridge +// IPAM design rests on. On L2Bridge, containers take addresses on the operator's +// own LAN; ephemerd must never guess a range, because a guess collides with live +// DHCP leases. Opting in without ip_pool has to stop the daemon at startup with +// a message naming the key. +func TestNetworkValidate_L2BridgeRequiresIPPool(t *testing.T) { + _, err := loadNetworkTOML(t, "l2bridge_egress = true\nhost_nic = \"Ethernet\"\n") + if err == nil { + t.Fatal("Load accepted l2bridge_egress = true with no ip_pool; want a startup failure") + } + msg := err.Error() + for _, want := range []string{"network.ip_pool", "l2bridge_egress", "DHCP"} { + if !strings.Contains(msg, want) { + t.Errorf("error %q does not mention %q", msg, want) + } + } + // The message must show the operator what a value looks like, in + // documentation address space — never a real LAN. + if !strings.Contains(msg, "192.0.2.") { + t.Errorf("error %q gives no example value", msg) + } +} + +// TestNetworkValidate_L2BridgeRequiresHostNIC covers the other setting that +// cannot be inferred: which adapter to bridge onto. +func TestNetworkValidate_L2BridgeRequiresHostNIC(t *testing.T) { + _, err := loadNetworkTOML(t, "l2bridge_egress = true\nip_pool = \"192.0.2.192/27\"\n") + if err == nil { + t.Fatal("Load accepted l2bridge_egress = true with no host_nic") + } + if !strings.Contains(err.Error(), "network.host_nic") { + t.Errorf("error %q does not name network.host_nic", err) + } +} + +// TestNetworkValidate_L2BridgeMinimalConfig verifies the intended surface: two +// required keys, everything else derived from the host at runtime. +func TestNetworkValidate_L2BridgeMinimalConfig(t *testing.T) { + cfg, err := loadNetworkTOML(t, "l2bridge_egress = true\nhost_nic = \"Ethernet\"\nip_pool = \"192.0.2.192/27\"\n") + if err != nil { + t.Fatalf("Load rejected a minimal valid L2Bridge config: %v", err) + } + if !cfg.Network.L2BridgeEgress || cfg.Network.HostNIC != "Ethernet" || cfg.Network.IPPool != "192.0.2.192/27" { + t.Errorf("parsed [network] = %+v, want the three keys round-tripped", cfg.Network) + } + // Everything optional stays empty so the runtime derives it from the host's + // real adapter. A non-empty default here would be a hardcoded network. + if cfg.Network.Subnet != "" || cfg.Network.Gateway != "" || len(cfg.Network.PublicDNS) != 0 { + t.Errorf("optional keys defaulted to %q / %q / %v, want empty (derived at runtime)", + cfg.Network.Subnet, cfg.Network.Gateway, cfg.Network.PublicDNS) + } +} + +// TestNetworkValidate_L2BridgeOptionalOverrides verifies the escape hatches +// parse when an operator needs to pin them. +func TestNetworkValidate_L2BridgeOptionalOverrides(t *testing.T) { + cfg, err := loadNetworkTOML(t, `l2bridge_egress = true +host_nic = "Ethernet" +ip_pool = "192.0.2.200-192.0.2.230" +subnet = "192.0.2.0/24" +gateway = "192.0.2.1" +public_dns = ["9.9.9.9"] +extra_allowed_destinations = ["203.0.113.0/24"] +`) + if err != nil { + t.Fatalf("Load: %v", err) + } + n := cfg.Network + if n.IPPool != "192.0.2.200-192.0.2.230" || n.Subnet != "192.0.2.0/24" || n.Gateway != "192.0.2.1" { + t.Errorf("address keys = %+v, want the configured values", n) + } + if len(n.PublicDNS) != 1 || n.PublicDNS[0] != "9.9.9.9" { + t.Errorf("public_dns = %v, want [9.9.9.9]", n.PublicDNS) + } + if len(n.ExtraAllowedDestinations) != 1 || n.ExtraAllowedDestinations[0] != "203.0.113.0/24" { + t.Errorf("extra_allowed_destinations = %v", n.ExtraAllowedDestinations) + } +} + +// TestNetworkValidate_NotRequiredWhenOptedOut pins that the default path — NAT, +// and every Linux/macOS node — is untouched by any of this. +func TestNetworkValidate_NotRequiredWhenOptedOut(t *testing.T) { + if _, err := loadNetworkTOML(t, "mtu = 1400\n"); err != nil { + t.Fatalf("Load rejected a config that never opts into L2Bridge: %v", err) + } + if _, err := loadNetworkTOML(t, "l2bridge_egress = false\n"); err != nil { + t.Fatalf("Load rejected l2bridge_egress = false: %v", err) + } +} + +// TestNetworkValidate_BlankStringsAreNotSet guards against a rendered config +// that emits the keys with empty values passing validation. +func TestNetworkValidate_BlankStringsAreNotSet(t *testing.T) { + _, err := loadNetworkTOML(t, "l2bridge_egress = true\nhost_nic = \" \"\nip_pool = \"192.0.2.192/27\"\n") + if err == nil { + t.Error("whitespace-only host_nic accepted") + } + _, err = loadNetworkTOML(t, "l2bridge_egress = true\nhost_nic = \"Ethernet\"\nip_pool = \" \"\n") + if err == nil { + t.Error("whitespace-only ip_pool accepted") + } +} diff --git a/pkg/dind/dind.go b/pkg/dind/dind.go index e86e9855..a0ea8cd6 100644 --- a/pkg/dind/dind.go +++ b/pkg/dind/dind.go @@ -53,6 +53,9 @@ type Server struct { server *http.Server client *client.Client network *networking.Manager + // hostPort is the TCP port the Windows listener opened in the host + // firewall for the container pool (L2Bridge only); 0 means none open. + hostPort int buildkit *buildkit.Server // shared embedded BuildKit solver (nil → fall back to platform default) runnerNetNS string // path to runner container's net namespace; used to install DNAT rules for port bindings allowPrivileged bool // gate for docker run --privileged / --cap-add; see config.DindConfig.AllowPrivileged @@ -348,6 +351,11 @@ func (s *Server) Stop() { s.log.Debug("closing listener", "error", err) } } + // Remove the L2Bridge host-firewall allow opened for this job's listener. + if s.hostPort != 0 && s.network != nil { + s.network.CloseHostPort(s.hostPort) + s.hostPort = 0 + } // Clean up the socket and job docker directory dockerDir := filepath.Dir(s.sockPath) diff --git a/pkg/dind/listen_windows.go b/pkg/dind/listen_windows.go index 34d8f50f..68e1a992 100644 --- a/pkg/dind/listen_windows.go +++ b/pkg/dind/listen_windows.go @@ -37,6 +37,21 @@ func (s *Server) listen() (net.Listener, error) { _ = ln.Close() return nil, fmt.Errorf("unexpected listener address type: %T", ln.Addr()) } + + // On the L2Bridge path the container reaches this listener at the host's + // LAN address, where the host's own inbound Windows Firewall default-denies + // it (the VFP host /32 allow only governs the container's egress port). Open + // a scoped inbound allow for exactly this port from the container pool. No-op + // on NAT. Failure is not fatal: log and continue — dind then simply is not + // reachable, which the job surfaces, rather than taking provisioning down. + if s.network != nil { + if err := s.network.OpenHostPort(tcpAddr.Port); err != nil { + s.log.Warn("failed to open dind host port for the container pool; docker may be unreachable on L2Bridge", "port", tcpAddr.Port, "error", err) + } else { + s.hostPort = tcpAddr.Port + } + } + s.endpoint = fmt.Sprintf("tcp://%s:%d", host, tcpAddr.Port) return ln, nil } diff --git a/pkg/networking/firewall_hyperv_windows_test.go b/pkg/networking/firewall_hyperv_windows_test.go deleted file mode 100644 index ed4e740f..00000000 --- a/pkg/networking/firewall_hyperv_windows_test.go +++ /dev/null @@ -1,245 +0,0 @@ -//go:build windows - -package networking - -import ( - "slices" - "strings" - "testing" -) - -// A concrete container VMCreatorId for the tests. The real value is discovered -// at runtime from Get-NetFirewallHyperVVMCreator (excluding WSL); the rule -// construction is identical whatever GUID it resolves to. -const testContainerVMCreatorID = "{9E9E4CB2-1B2C-4D3E-8F90-ABCDEF012345}" - -// TestHyperVEgressRules_BlocksEveryDeniedRange verifies each RFC1918 + -// link-local range produces exactly one outbound Block rule scoped to the -// container VMCreatorId; a missing range would let a job reach that slice of -// the LAN. -func TestHyperVEgressRules_BlocksEveryDeniedRange(t *testing.T) { - rules, err := hyperVEgressRules(testContainerVMCreatorID, DefaultSubnet, defaultGateway, nil) - if err != nil { - t.Fatalf("hyperVEgressRules: %v", err) - } - if len(rules) != len(egressBlockedCIDRs) { - t.Fatalf("got %d rules, want %d (one outbound block per denied range)", len(rules), len(egressBlockedCIDRs)) - } - for i, cidr := range egressBlockedCIDRs { - r := rules[i] - if r.direction != "Outbound" || r.action != "Block" { - t.Errorf("rule %s is not an outbound block: dir=%s action=%s", r.name, r.direction, r.action) - } - if r.vmCreatorID != testContainerVMCreatorID { - t.Errorf("rule %s not scoped to the container creator: %q", r.name, r.vmCreatorID) - } - if len(r.remoteAddrs) == 0 { - t.Errorf("rule %s has no RemoteAddresses scope", r.name) - } - wantName := firewallRulePrefix + "-" + creatorTag(testContainerVMCreatorID) + "-block-" + strings.ReplaceAll(cidr, "/", "_") - if r.name != wantName { - t.Errorf("rule[%d].name = %q, want %q", i, r.name, wantName) - } - } -} - -// TestHyperVEgressRules_GatewayAndSubnetNeverBlocked pins the safety property: -// the container subnet — which contains the NAT gateway (DNS, default route, -// GatewayPorts) and the other containers — must never appear inside a block -// rule's RemoteAddresses. Blocking the gateway would brick all container -// networking. The 10/8 block must be split exactly around the subnet. -func TestHyperVEgressRules_GatewayAndSubnetNeverBlocked(t *testing.T) { - rules, err := hyperVEgressRules(testContainerVMCreatorID, "10.88.0.0/16", "10.88.0.1", nil) - if err != nil { - t.Fatalf("hyperVEgressRules: %v", err) - } - - for _, r := range rules { - for _, addr := range r.remoteAddrs { - if strings.Contains(addr, "10.88.") { - t.Errorf("rule %s blocks the container subnet: RemoteAddresses contains %q", r.name, addr) - } - } - } - - tag := creatorTag(testContainerVMCreatorID) - wantName := firewallRulePrefix + "-" + tag + "-block-10.0.0.0_8" - want := []string{"10.0.0.0-10.87.255.255", "10.89.0.0-10.255.255.255"} - for _, r := range rules { - if r.name == wantName { - if !slices.Equal(r.remoteAddrs, want) { - t.Errorf("10/8 block RemoteAddresses = %v, want %v", r.remoteAddrs, want) - } - return - } - } - t.Errorf("no block rule found for 10.0.0.0/8 (name %q)", wantName) -} - -// TestHyperVEgressRules_ControlPortRules confirms the container->gateway -// control-plane blocks mirror the Linux drops: outbound, TCP, one specific port -// each, RemoteAddresses = gateway only — never a blanket gateway block and -// never port 53 (DNS must survive). -func TestHyperVEgressRules_ControlPortRules(t *testing.T) { - ports := []int{10000, 10001, 10002} // containerd, dispatch, debug exec - rules, err := hyperVEgressRules(testContainerVMCreatorID, DefaultSubnet, defaultGateway, ports) - if err != nil { - t.Fatalf("hyperVEgressRules: %v", err) - } - - var control []hyperVRule - for _, r := range rules { - if len(r.remotePorts) > 0 { - control = append(control, r) - } - } - if len(control) != len(ports) { - t.Fatalf("got %d control rules, want %d (one per control port)", len(control), len(ports)) - } - - for i, port := range []string{"10000", "10001", "10002"} { - r := control[i] - if r.direction != "Outbound" || r.action != "Block" || r.protocol != "TCP" { - t.Errorf("rule %s is not an outbound TCP block: dir=%s action=%s proto=%s", r.name, r.direction, r.action, r.protocol) - } - if !slices.Equal(r.remotePorts, []string{port}) { - t.Errorf("rule %s RemotePorts = %v, want [%s]", r.name, r.remotePorts, port) - } - if !slices.Equal(r.remoteAddrs, []string{defaultGateway}) { - t.Errorf("rule %s RemoteAddresses = %v, want [%s]", r.name, r.remoteAddrs, defaultGateway) - } - if port == "53" { - t.Errorf("rule %s blocks DNS (port 53) — must not", r.name) - } - } -} - -// TestHyperVRuleCommand_Rendering pins the exact PowerShell the install path -// runs: New-NetFirewallHyperVRule with the VMCreatorId scoping, Outbound/Block, -// and RemoteAddresses/RemotePorts rendered as quoted PowerShell arrays (so a -// string[] parameter receives distinct elements, not one comma-joined string). -func TestHyperVRuleCommand_Rendering(t *testing.T) { - block := hyperVRule{ - name: "ephemerd-egress-9e9e4cb2-block-10.0.0.0_8", - displayName: "ephemerd egress block 10.0.0.0/8", - direction: "Outbound", - action: "Block", - vmCreatorID: testContainerVMCreatorID, - remoteAddrs: []string{"10.0.0.0-10.87.255.255", "10.89.0.0-10.255.255.255"}, - } - got := block.command() - for _, want := range []string{ - "New-NetFirewallHyperVRule", - "-Name 'ephemerd-egress-9e9e4cb2-block-10.0.0.0_8'", - "-DisplayName 'ephemerd egress block 10.0.0.0/8'", - "-Direction Outbound", - "-Action Block", - "-VMCreatorId '{9E9E4CB2-1B2C-4D3E-8F90-ABCDEF012345}'", - "-RemoteAddresses '10.0.0.0-10.87.255.255','10.89.0.0-10.255.255.255'", - } { - if !strings.Contains(got, want) { - t.Errorf("command() = %q\n missing %q", got, want) - } - } - // An all-protocol block must not emit -Protocol (default Any) and must not - // emit -RemotePorts. - if strings.Contains(got, "-Protocol") { - t.Errorf("all-protocol block should omit -Protocol: %q", got) - } - if strings.Contains(got, "-RemotePorts") { - t.Errorf("block without ports should omit -RemotePorts: %q", got) - } - - control := hyperVRule{ - name: "ephemerd-egress-9e9e4cb2-control-10000", - displayName: "ephemerd egress block control tcp/10000", - direction: "Outbound", - action: "Block", - vmCreatorID: testContainerVMCreatorID, - protocol: "TCP", - remoteAddrs: []string{"10.88.0.1"}, - remotePorts: []string{"10000"}, - } - gotC := control.command() - for _, want := range []string{ - "-Protocol TCP", - "-RemoteAddresses '10.88.0.1'", - "-RemotePorts '10000'", - } { - if !strings.Contains(gotC, want) { - t.Errorf("control command() = %q\n missing %q", gotC, want) - } - } -} - -// TestHyperVRuleRemoveCommand verifies removal targets exactly the name add -// created (that is what makes remove-before-add idempotent) and stays quiet on -// a fresh host. -func TestHyperVRuleRemoveCommand(t *testing.T) { - r := hyperVRule{name: "ephemerd-egress-9e9e4cb2-block-192.168.0.0_16"} - got := r.removeCommand() - want := "Remove-NetFirewallHyperVRule -Name 'ephemerd-egress-9e9e4cb2-block-192.168.0.0_16' -ErrorAction SilentlyContinue" - if got != want { - t.Errorf("removeCommand() = %q, want %q", got, want) - } -} - -// TestHyperVRuleNames pins the naming contract: every rule carries the ephemerd -// prefix (so the set is findable and removable by removeByPrefixScript) and is -// scoped to the creator via a short tag so multiple creators do not collide. -func TestHyperVRuleNames(t *testing.T) { - rules, err := hyperVEgressRules(testContainerVMCreatorID, DefaultSubnet, defaultGateway, []int{10000}) - if err != nil { - t.Fatalf("hyperVEgressRules: %v", err) - } - tag := creatorTag(testContainerVMCreatorID) - if tag != "9e9e4cb2" { - t.Errorf("creatorTag = %q, want %q", tag, "9e9e4cb2") - } - for _, r := range rules { - if !strings.HasPrefix(r.name, firewallRulePrefix+"-") { - t.Errorf("rule name %q missing %q prefix", r.name, firewallRulePrefix) - } - if !strings.Contains(r.name, tag) { - t.Errorf("rule name %q missing creator tag %q", r.name, tag) - } - if r.displayName == "" { - t.Errorf("rule %q has empty DisplayName (mandatory for New-NetFirewallHyperVRule)", r.name) - } - } - - // removeByPrefixScript must match those names. - if !strings.Contains(removeByPrefixScript(), firewallRulePrefix+"-*") { - t.Errorf("removeByPrefixScript does not match the rule-name prefix: %q", removeByPrefixScript()) - } -} - -// TestCreatorTag covers normalization: hex-only, lowercased, first 8, with a -// safe fallback for a GUID that yields no hex. -func TestCreatorTag(t *testing.T) { - tests := []struct { - in, want string - }{ - {"{9E9E4CB2-1B2C-4D3E-8F90-ABCDEF012345}", "9e9e4cb2"}, - {"{40E0AC32-46A5-438A-A0B2-2B479E8F2E90}", "40e0ac32"}, - {"{GGGG}", "any"}, - {"", "any"}, - } - for _, tt := range tests { - if got := creatorTag(tt.in); got != tt.want { - t.Errorf("creatorTag(%q) = %q, want %q", tt.in, got, tt.want) - } - } -} - -// TestPSArrayAndQuote covers the PowerShell rendering helpers, including the -// embedded-quote escape that keeps a crafted value from breaking out of the -// argument. -func TestPSArrayAndQuote(t *testing.T) { - if got := psQuote("a'b"); got != "'a''b'" { - t.Errorf("psQuote = %q, want %q", got, "'a''b'") - } - if got := psArray([]string{"x", "y"}); got != "'x','y'" { - t.Errorf("psArray = %q, want %q", got, "'x','y'") - } -} diff --git a/pkg/networking/firewall_windows.go b/pkg/networking/firewall_windows.go index 5a5f3db4..8d7a554b 100644 --- a/pkg/networking/firewall_windows.go +++ b/pkg/networking/firewall_windows.go @@ -3,9 +3,7 @@ package networking import ( - "encoding/binary" "fmt" - "net" "os/exec" "strconv" "strings" @@ -13,216 +11,47 @@ import ( // Egress firewall for Windows job containers. // -// Two layers restrict what a Windows job can reach: +// The ONLY working host-side egress enforcement on Windows is the L2Bridge VFP +// Switch-ACL path (opt-in via network.l2bridge_egress): per-endpoint ACLs on +// the container's vSwitch port, built by buildL2BridgeEgressACLPolicies in +// network_windows.go and applied in setup(), plus the small host-firewall +// backstop below that fences the control-plane ports back off. This was proven +// on metal. // -// 1. Per-endpoint HNS ACL policies (applyACLPolicies in network_windows.go) — -// VFP rules on the container's vSwitch port, applied in setup(). -// 2. The Hyper-V firewall rules installed here, programmed with -// New-NetFirewallHyperVRule and scoped to the container VMCreatorId. +// The default NAT network CANNOT be egress-filtered by any host-side mechanism, +// and ephemerd does not try. Every candidate was disproven on real hardware: // -// Layer 2 exists because layer 1 alone left the fleet reachable in practice: -// the containment suite (.github/workflows/containment.yml, "Fleet management -// planes must be unreachable") reached the Incus daemon and Grafana from a -// Hyper-V-isolated job (#135), so per-endpoint vSwitch ACLs cannot be the only -// line of defense. +// - Host Windows Defender Firewall (netsh/MPSSVC): evaluates the forwarded +// container->LAN traffic POST-NAT at the host endpoint, where the source is +// the host's own LAN address, so a rule scoped to the container subnet +// matches nothing (verified on a live Windows node — every management-plane +// probe still succeeded). +// - The Hyper-V firewall (New-NetFirewallHyperVRule, -VMCreatorId scoped): +// runhcs NAT containers register NO Hyper-V VM creator, so there is no +// creator to scope a rule to; the rules install but filter nothing. +// - A host-side WFP filter at IPFORWARD_V4 (the Linux FORWARD analogue): the +// container's forwarded + SNAT'd egress is never classified at ANY host +// IPv4 WFP layer — the packets travel the Hyper-V vSwitch/HNS datapath, +// out-of-band of the host tcpip.sys WFP hooks. Full evidence: +// docs/arch/windows-egress-wfp-investigation.md. // -// Why NOT the host Windows Defender Firewall (the netsh approach of #136). -// #136 installed host MPSSVC rules scoped localip=, on the -// theory that Windows Firewall would see the container's source IP on the -// WinNAT-forwarded flow. On real hardware it does not: the host firewall -// (WFP/MPSSVC) evaluates the forwarded container->LAN traffic POST-NAT at the -// host endpoint, where the source is the host's own LAN address, so a rule -// scoped to the 10.88/16 container source matches nothing. Verified against a -// live v0.1.6 Windows node: every management-plane probe still succeeded. The -// host firewall is the wrong enforcement point for NATed container egress. -// -// The Hyper-V firewall filters at the container's vNIC boundary, BEFORE NAT, -// where the packet still carries the container's own address — the correct -// enforcement point for Hyper-V-isolated Windows containers. Rules are scoped -// by -VMCreatorId so they apply to container ports, not to the host or to -// unrelated Hyper-V workloads (regular Hyper-V VMs are not filtered by the -// Hyper-V firewall at all; it governs container-class workloads — WSL, Windows -// Sandbox, and Windows containers). -// -// Why NOT a host-side WFP filter at IPFORWARD_V4 (the Linux FORWARD analogue). -// Tempting, since firewall_linux.go enforces in FORWARD. It was built and driven -// on a live Server 2025 node (a pure-Go tailscale/wf spike, no callout driver) -// and does not work: the container's forwarded + SNAT'd egress is never -// classified at ANY host IPv4 WFP layer — not IPFORWARD_V4, not -// INBOUND_IPPACKET_V4 (arrival-interface scoped or not), not OUTBOUND_IPPACKET_V4 -// (dest-only, post-NAT). Positive controls prove WFP itself is enforced on the -// host (blocking the host's own 1.1.1.1 at ALE and OUTBOUND both worked), so the -// packets simply travel the Hyper-V vSwitch/HNS datapath, out-of-band of the -// host tcpip.sys WFP hooks (Get-NetNat is empty and per-interface Forwarding is -// Disabled). Full evidence: docs/arch/windows-egress-wfp-investigation.md. -// The definitive fix for this stack is network-level: an isolated VLAN whose -// uplink denies RFC1918, not a host-side software filter. -// -// Gateway safety (identical reasoning to #136, and to firewall_linux.go). -// The container subnet contains the NAT gateway (DNS, the default route, -// module-proxy GatewayPorts) and the other containers. Blocking the gateway -// would brick all container networking. Rather than rely on Hyper-V firewall -// rule-priority ordering (allow-above-deny) to rescue the gateway — an -// unverified semantic, and unverified assumptions are exactly what sank #136 — -// the container subnet is subtracted from every blocked range up front -// (subtractCIDR), so the gateway and the container-to-container range never -// appear inside any Block rule in the first place. Everything outside the -// blocked ranges — the internet, including GitHub/Docker Hub/registries — is -// untouched: the creator's default outbound action stays Allow and only -// RFC1918 + link-local is denied. +// So on the NAT path ephemerd installs nothing and logs that container egress +// is unfiltered, pointing the operator at network.l2bridge_egress. Enforcing +// NAT egress requires a network-level control (an isolated VLAN whose uplink +// denies RFC1918), not a host-side software filter. // // IPv4 only, deliberately: the HCN NAT network is IPv4-only (no v6 IPAM), so // containers have no IPv6 path. -// firewallRulePrefix names every rule ephemerd installs — Hyper-V firewall -// rules and the netsh fallback rules alike — so the set is findable and -// removable on Cleanup. +// firewallRulePrefix names every rule ephemerd installs (the L2Bridge +// host-firewall backstop) so the set is findable and removable on Cleanup. const firewallRulePrefix = "ephemerd-egress" -// wslVMCreatorID is the well-known Hyper-V firewall VMCreatorId for the Windows -// Subsystem for Linux (documented by Microsoft). WSL is not an ephemerd job -// container, so it is excluded from egress filtering — narrowing the blast -// radius to the container runtime's own creator(s). -const wslVMCreatorID = "{40E0AC32-46A5-438A-A0B2-2B479E8F2E90}" - -// ------------------------------------------------------------------------- -// Hyper-V firewall (primary path) -// ------------------------------------------------------------------------- - -// hyperVRule is one New-NetFirewallHyperVRule invocation. Kept as structured -// fields (rather than a flat argv) so both the rendered PowerShell command and -// the unit tests derive from the same source, and so array-valued parameters -// (-RemoteAddresses, -RemotePorts) render as real PowerShell arrays. -type hyperVRule struct { - name string // -Name; unique, ephemerd-prefixed; used for idempotent remove-before-add and Cleanup - displayName string // -DisplayName; mandatory, human-facing - direction string // -Direction (Outbound) - action string // -Action (Block) - vmCreatorID string // -VMCreatorId '{GUID}' - protocol string // -Protocol; "" omits it (matches Any) - remoteAddrs []string // -RemoteAddresses; addresses/CIDRs/ranges; empty omits it (matches Any) - remotePorts []string // -RemotePorts; empty omits it (matches Any) -} - // psQuote single-quotes a value for PowerShell, doubling embedded quotes. func psQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") + "'" } -// psArray renders a slice as a PowerShell array literal of quoted strings -// ('a','b'), so a string[] parameter receives distinct elements rather than one -// comma-joined string. -func psArray(vals []string) string { - q := make([]string, len(vals)) - for i, v := range vals { - q[i] = psQuote(v) - } - return strings.Join(q, ",") -} - -// command renders the New-NetFirewallHyperVRule command that creates the rule. -func (r hyperVRule) command() string { - var b strings.Builder - b.WriteString("New-NetFirewallHyperVRule") - b.WriteString(" -Name " + psQuote(r.name)) - b.WriteString(" -DisplayName " + psQuote(r.displayName)) - b.WriteString(" -Direction " + r.direction) - b.WriteString(" -Action " + r.action) - b.WriteString(" -VMCreatorId " + psQuote(r.vmCreatorID)) - if r.protocol != "" { - b.WriteString(" -Protocol " + r.protocol) - } - if len(r.remoteAddrs) > 0 { - b.WriteString(" -RemoteAddresses " + psArray(r.remoteAddrs)) - } - if len(r.remotePorts) > 0 { - b.WriteString(" -RemotePorts " + psArray(r.remotePorts)) - } - return b.String() -} - -// removeCommand renders the idempotent delete for this rule by name. -func (r hyperVRule) removeCommand() string { - return "Remove-NetFirewallHyperVRule -Name " + psQuote(r.name) + " -ErrorAction SilentlyContinue" -} - -// creatorTag reduces a VMCreatorId GUID to a short, filesystem-safe token -// (first 8 hex digits, lowercased) used to keep rule names unique per creator -// when more than one container creator is present. -func creatorTag(vmCreatorID string) string { - var b strings.Builder - for _, r := range vmCreatorID { - switch { - case r >= '0' && r <= '9', r >= 'a' && r <= 'f': - b.WriteRune(r) - case r >= 'A' && r <= 'F': - b.WriteRune(r + ('a' - 'A')) - } - if b.Len() >= 8 { - break - } - } - if b.Len() == 0 { - return "any" - } - return b.String() -} - -// hyperVEgressRules returns the Hyper-V firewall rule set for one container -// VMCreatorId: an outbound Block for every denied range (with the container -// subnet — and thus the gateway — carved out) plus one outbound Block per -// control port for container->gateway control-plane traffic. -// -// Exposed as a pure function (no side effects) so tests can assert the exact -// rule set without invoking PowerShell. -func hyperVEgressRules(vmCreatorID, subnet, gateway string, controlPorts []int) ([]hyperVRule, error) { - tag := creatorTag(vmCreatorID) - var rules []hyperVRule - - // Outbound RFC1918 + link-local blocks. subtractCIDR removes the container - // subnet from any overlapping range so the gateway (DNS/NAT/default route) - // and the container-to-container range never appear inside a Block. The - // internet is never in these ranges, so it is left fully open. - for _, cidr := range egressBlockedCIDRs { - remote, err := subtractCIDR(cidr, subnet) - if err != nil { - return nil, fmt.Errorf("computing blocked ranges for %s: %w", cidr, err) - } - if len(remote) == 0 { - continue // fully covered by the container subnet - } - rules = append(rules, hyperVRule{ - name: fmt.Sprintf("%s-%s-block-%s", firewallRulePrefix, tag, strings.ReplaceAll(cidr, "/", "_")), - displayName: "ephemerd egress block " + cidr, - direction: "Outbound", - action: "Block", - vmCreatorID: vmCreatorID, - remoteAddrs: remote, - }) - } - - // Control-plane blocks: container -> gateway on the ephemerd control ports - // (containerd, dispatch gRPC, debug exec). Intentionally narrow — the - // gateway address, one TCP port each — so DNS (53), NAT, and the other - // gateway services stay reachable. Mirrors controlPlaneInputRules on Linux; - // evaluated Outbound here because the Hyper-V firewall filters at the - // container vNIC, where container->gateway is egress. - for _, port := range controlPorts { - rules = append(rules, hyperVRule{ - name: fmt.Sprintf("%s-%s-control-%d", firewallRulePrefix, tag, port), - displayName: fmt.Sprintf("ephemerd egress block control tcp/%d", port), - direction: "Outbound", - action: "Block", - vmCreatorID: vmCreatorID, - protocol: "TCP", - remoteAddrs: []string{gateway}, - remotePorts: []string{strconv.Itoa(port)}, - }) - } - - return rules, nil -} - // powershellArgs wraps a script for non-interactive execution. func powershellArgs(script string) []string { return []string{"-NonInteractive", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", script} @@ -246,142 +75,202 @@ func powershellOutput(script string) (string, error) { return string(out), nil } -// hyperVFirewallAvailable reports whether the Hyper-V firewall cmdlets exist on -// this host. They ship with Windows 11 22H2 / Windows Server 2025 and are -// absent on older builds, where the netsh fallback is used instead. -func hyperVFirewallAvailable() bool { - return powershell("if (Get-Command New-NetFirewallHyperVRule -ErrorAction SilentlyContinue) { exit 0 } else { exit 1 }") == nil +// ------------------------------------------------------------------------- +// L2Bridge host-firewall backstop +// ------------------------------------------------------------------------- +// +// On the L2Bridge path the primary egress enforcement is the per-endpoint VFP +// ACL ladder (buildL2BridgeEgressACLPolicies), which was proven on metal. +// +// What the host firewall CAN do here, and could not on NAT, is match on the +// container's source address. On NAT the host sees container egress post-NAT +// with its own address as the source, so a host rule scoped to the container +// source matches nothing. L2Bridge does not NAT: a container's packet reaches +// the host carrying the container's own pool address, as ordinary inbound LAN +// traffic. So an INBOUND rule scoped remoteip= matches exactly the job +// containers and nothing else. +// +// That is used for one job: closing the control-plane ports back off after the +// VFP ladder's host /32 allow opens the host (AllowHostAccess). The allow has to +// be address-scoped — a port-scoped Switch ACL blackholes the entire VFP port — +// so the port precision lives here instead. + +// l2BridgeControlPlaneRules returns the inbound host-firewall blocks for +// container -> ephemerd control-plane traffic on the L2Bridge path: one rule per +// control port, scoped to the host address on the bridged LAN and to the source +// range containers are allocated from. +// +// Pure (no side effects) so the exact rule set is unit-testable without netsh. +func l2BridgeControlPlaneRules(hostIP, ipPool string, controlPorts []int) []winFirewallRule { + if hostIP == "" || ipPool == "" { + return nil + } + rules := make([]winFirewallRule, 0, len(controlPorts)) + for _, port := range controlPorts { + rules = append(rules, winFirewallRule{ + name: fmt.Sprintf("%s-l2b-control-%d", firewallRulePrefix, port), + spec: []string{ + "dir=in", + "action=block", + "protocol=TCP", + "localip=" + hostIP, + "localport=" + strconv.Itoa(port), + "remoteip=" + ipPool, + "profile=any", + "enable=yes", + }, + }) + } + return rules } -// discoverContainerVMCreators returns the VMCreatorIds of the Hyper-V firewall -// VM creators present on the host, excluding WSL. On a Windows container host -// the remaining creator(s) are the container runtime's, which is what job -// containers run under. Returns an empty slice (not an error) when no creators -// are registered yet. -func discoverContainerVMCreators() ([]string, error) { - out, err := powershellOutput("Get-NetFirewallHyperVVMCreator | Select-Object -ExpandProperty VMCreatorId") - if err != nil { - return nil, err +// installL2BridgeFirewallRules programs the L2Bridge backstop. Best-effort like +// the rest of installFirewallRules: the VFP ladder is the enforcement point and +// a host that cannot program netsh must not fail daemon startup. +func (w *windowsNetworking) installL2BridgeFirewallRules() error { + if w.plan == nil { + return nil } - var creators []string - for _, line := range strings.Split(out, "\n") { - id := strings.TrimSpace(line) - if id == "" { - continue - } - if strings.EqualFold(id, wslVMCreatorID) { - continue // not an ephemerd job container + rules := l2BridgeControlPlaneRules(w.plan.HostIP, w.plan.PoolSpec, w.cfg.ControlPorts) + if len(rules) == 0 { + w.cfg.Log.Info("L2Bridge egress: no control ports to fence off at the host firewall") + return nil + } + for _, r := range rules { + _ = netsh(r.deleteArgs()...) // idempotent: clear any rule of this name first + w.cfg.Log.Info("adding L2Bridge control-plane firewall rule", "rule", r.name) + if err := netsh(r.addArgs()...); err != nil { + w.cfg.Log.Warn("failed to add L2Bridge control-plane firewall rule", "rule", r.name, "error", err) } - creators = append(creators, id) } - return creators, nil + w.cfg.Log.Info("L2Bridge control-plane firewall rules installed", "rules", len(rules)) + return nil } -// enableHyperVFirewallScript ensures the Hyper-V firewall is enabled for a -// creator with permissive defaults: internet, the gateway (DNS/NAT), and -// host<->container control traffic all stay open, and only our explicit Block -// rules restrict RFC1918. Without this the rules would exist but not enforce on -// a creator whose firewall was never enabled. Best-effort — logged, not fatal. -func enableHyperVFirewallScript(creator string) string { - return fmt.Sprintf( - "Set-NetFirewallHyperVVMSetting -Name %s -Enabled True -DefaultInboundAction Allow -DefaultOutboundAction Allow -ErrorAction Stop", - psQuote(creator), - ) +// hostPortAllowRule is the scoped inbound allow that makes one host TCP port +// reachable from the container pool on the L2Bridge path (see openHostPort). +func hostPortAllowRule(hostIP, ipPool string, port int) winFirewallRule { + return winFirewallRule{ + name: fmt.Sprintf("%s-l2b-hostport-%d", firewallRulePrefix, port), + spec: []string{ + "dir=in", + "action=allow", + "protocol=TCP", + "localip=" + hostIP, + "localport=" + strconv.Itoa(port), + "remoteip=" + ipPool, + "profile=any", + "enable=yes", + }, + } } -// removeByPrefixScript deletes every Hyper-V firewall rule ephemerd installed, -// across all creators, matched by the ephemerd- name prefix. Catches stale -// rules leaked by a crashed run as well as the current set. -func removeByPrefixScript() string { - return fmt.Sprintf( - "Get-NetFirewallHyperVRule | Where-Object { $_.Name -like '%s-*' } | Remove-NetFirewallHyperVRule -ErrorAction SilentlyContinue", - firewallRulePrefix, - ) +// openHostPort adds a scoped inbound allow so job containers can reach one host +// TCP port (a per-job dind Docker API listener, or the module proxy). Without +// it the host's default inbound deny drops the connection even though the VFP +// host /32 allow permits the container to send. Scoped to remoteip= and +// localport=, so nothing else on the host opens. No-op unless the +// L2Bridge egress path is active with a resolved plan. +func (w *windowsNetworking) openHostPort(port int) error { + if !w.cfg.L2BridgeEgress || w.plan == nil || w.plan.HostIP == "" || w.plan.PoolSpec == "" { + return nil + } + r := hostPortAllowRule(w.plan.HostIP, w.plan.PoolSpec, port) + _ = netsh(r.deleteArgs()...) // idempotent + if err := netsh(r.addArgs()...); err != nil { + return fmt.Errorf("opening host port %d for the container pool: %w", port, err) + } + w.cfg.Log.Info("opened L2Bridge host port for container pool", "port", port, "pool", w.plan.PoolSpec) + return nil } -func (w *windowsNetworking) installFirewallRules() error { - // Degrade gracefully at every step: a host that cannot program the - // Hyper-V firewall must never fail daemon startup. It falls back to the - // netsh host-firewall rules (weaker, but better than nothing on builds - // without the Hyper-V firewall) or, failing that, to the per-endpoint HNS - // ACLs already applied in setup(). - if !hyperVFirewallAvailable() { - w.cfg.Log.Warn("New-NetFirewallHyperVRule unavailable on this host; falling back to netsh host-firewall rules") - return w.installNetshFirewallRules() +// closeHostPort removes the allow added by openHostPort. +func (w *windowsNetworking) closeHostPort(port int) { + if !w.cfg.L2BridgeEgress || w.plan == nil || w.plan.HostIP == "" || w.plan.PoolSpec == "" { + return + } + r := hostPortAllowRule(w.plan.HostIP, w.plan.PoolSpec, port) + if err := netsh(r.deleteArgs()...); err != nil { + w.cfg.Log.Debug("failed to remove L2Bridge host-port allow", "port", port, "error", err) } +} - creators, err := discoverContainerVMCreators() - if err != nil { - w.cfg.Log.Warn("failed to enumerate Hyper-V firewall VM creators; falling back to netsh host-firewall rules", "error", err) - return w.installNetshFirewallRules() +// hostPortRulePrefix is the DisplayName prefix of every per-job host-port allow +// (see hostPortAllowRule). Swept by prefix on shutdown so a hard kill — which +// skips dind's per-job CloseHostPort — cannot leak stale inbound allows. +const hostPortRulePrefix = firewallRulePrefix + "-l2b-hostport-" + +// removeL2BridgeFirewallRules deletes the backstop rules by name, and sweeps any +// leaked per-job host-port allows by prefix. +func (w *windowsNetworking) removeL2BridgeFirewallRules() { + if w.plan != nil { + for _, r := range l2BridgeControlPlaneRules(w.plan.HostIP, w.plan.PoolSpec, w.cfg.ControlPorts) { + if err := netsh(r.deleteArgs()...); err != nil { + w.cfg.Log.Debug("failed to remove L2Bridge control-plane firewall rule", "rule", r.name, "error", err) + } + } } - if len(creators) == 0 { - w.cfg.Log.Warn("no container Hyper-V firewall VM creator registered yet; falling back to netsh host-firewall rules") - return w.installNetshFirewallRules() + // Prefix-sweep the per-job host-port allows. netsh delete-by-name has no + // wildcard, so go through the firewall cmdlets. Runs regardless of w.plan — + // a hard kill can leave these behind for a pool that no longer resolves, and + // startup Cleanup must still reclaim them. + if err := powershell("Get-NetFirewallRule -DisplayName '" + hostPortRulePrefix + "*' -ErrorAction SilentlyContinue | Remove-NetFirewallRule -ErrorAction SilentlyContinue"); err != nil { + w.cfg.Log.Debug("failed to prefix-sweep L2Bridge host-port allows", "error", err) } +} - // init() always creates the HCN network on DefaultSubnet with - // defaultGateway (cfg.Subnet is not consulted on Windows), so the firewall - // must match those. - installed := 0 - for _, creator := range creators { - rules, err := hyperVEgressRules(creator, DefaultSubnet, defaultGateway, w.cfg.ControlPorts) +// defaultGatewayForAdapter returns the IPv4 default-route next hop reachable via +// the named adapter, used to fill in network.gateway when the operator has not +// pinned it. The lowest-metric route wins, matching what Windows itself would +// pick. Returns an empty string (no error) when the adapter has no default +// route, which the caller turns into a "set network.gateway" message. +func defaultGatewayForAdapter(name string) (string, error) { + // Try the configured name and its vEthernet-renamed form (creating the + // L2Bridge network renames the NIC — see adapterNameCandidates). + var lastErr error + for _, candidate := range adapterNameCandidates(name) { + out, err := powershellOutput(fmt.Sprintf( + "(Get-NetRoute -InterfaceAlias %s -DestinationPrefix '0.0.0.0/0' -ErrorAction SilentlyContinue "+ + "| Sort-Object -Property RouteMetric | Select-Object -First 1).NextHop", + psQuote(candidate), + )) if err != nil { - w.cfg.Log.Warn("failed to build Hyper-V firewall rules", "creator", creator, "error", err) + lastErr = err continue } - - if err := powershell(enableHyperVFirewallScript(creator)); err != nil { - w.cfg.Log.Warn("failed to enable Hyper-V firewall for creator (rules may not enforce)", "creator", creator, "error", err) - } - - for _, r := range rules { - // Idempotent: remove any rule carrying this name from a previous - // run before adding, so re-running install never duplicates. - _ = powershell(r.removeCommand()) - - w.cfg.Log.Info("adding Hyper-V firewall rule", "rule", r.name, "creator", creator) - if err := powershell(r.command()); err != nil { - // Non-fatal: one failed rule degrades to the remaining rules - // plus the per-endpoint ACLs, never to refusing to start. - w.cfg.Log.Warn("failed to add Hyper-V firewall rule", "rule", r.name, "error", err) - continue - } - installed++ + if hop := strings.TrimSpace(out); hop != "" { + return hop, nil } } + return "", lastErr +} - w.cfg.Log.Info("Hyper-V firewall rules installed", "rules", installed, "creators", len(creators)) +func (w *windowsNetworking) installFirewallRules() error { + // L2Bridge: the VFP ACL ladder (applied per-endpoint in setup()) is the + // enforcement point; this installs only the host-firewall backstop that + // fences the control-plane ports back off. + if w.cfg.L2BridgeEgress { + return w.installL2BridgeFirewallRules() + } + + // NAT path: there is no host-side mechanism that can filter container egress + // on this stack (WFP, the Hyper-V firewall, and netsh were all disproven on + // metal — see the file header). Installing rules here would be security + // theater: they match nothing. Log the gap and install nothing. + w.cfg.Log.Warn("Windows container egress is NOT filtered on the default NAT network — no host-side mechanism can filter it on this stack; set network.l2bridge_egress to enforce egress (see docs/guides/security.md)") return nil } func (w *windowsNetworking) removeFirewallRules() { - // Always attempt to remove the netsh fallback rules too — harmless if they - // were never installed — so a host that switched paths between runs does - // not leak the other path's rules. - w.removeNetshFirewallRules() - - if !hyperVFirewallAvailable() { - return - } - // Remove by prefix: catches every ephemerd rule across all creators, - // including stale ones, without needing to recompute per-creator names. - if err := powershell(removeByPrefixScript()); err != nil { - w.cfg.Log.Debug("failed to remove Hyper-V firewall rules", "error", err) - } + // removeL2BridgeFirewallRules is safe on either path — on NAT nothing + // matches — and also sweeps any leaked per-job host-port allows by prefix, + // so it must run unconditionally regardless of which path installed rules. + w.removeL2BridgeFirewallRules() } // ------------------------------------------------------------------------- -// netsh host firewall (fallback for hosts without the Hyper-V firewall) +// netsh host-firewall helpers (shared by the L2Bridge backstop above) // ------------------------------------------------------------------------- -// -// Retained only as a degraded fallback for Windows builds that lack -// New-NetFirewallHyperVRule (pre-Server 2025 / Windows 11 22H2). On such hosts -// these host-global rules are strictly better than nothing, even though — as -// #136 proved on Server 2025 — the host firewall evaluates NATed container -// egress post-NAT and cannot match on the container source. The outbound -// blocks are scoped localip= so a mis-scoped rule degrades to -// a no-op rather than cutting the host off its own management LAN. func netsh(args ...string) error { out, err := exec.Command("netsh", args...).CombinedOutput() @@ -408,144 +297,3 @@ func (r winFirewallRule) addArgs() []string { func (r winFirewallRule) deleteArgs() []string { return []string{"advfirewall", "firewall", "delete", "rule", "name=" + r.name} } - -// hostFirewallRules returns the netsh fallback rule set for the given container -// subnet, gateway, and control-plane ports: outbound blocks for every denied -// range (with the container subnet carved out) plus inbound blocks for -// container->gateway traffic on the control ports. -// -// Exposed as a pure function (no side effects) so tests can assert the exact -// rule set without invoking netsh. -func hostFirewallRules(subnet, gateway string, controlPorts []int) ([]winFirewallRule, error) { - var rules []winFirewallRule - - for _, cidr := range egressBlockedCIDRs { - remote, err := subtractCIDR(cidr, subnet) - if err != nil { - return nil, fmt.Errorf("computing blocked ranges for %s: %w", cidr, err) - } - if len(remote) == 0 { - continue // fully covered by the container subnet - } - rules = append(rules, winFirewallRule{ - name: firewallRulePrefix + "-block-" + strings.ReplaceAll(cidr, "/", "_"), - spec: []string{ - "dir=out", - "action=block", - "protocol=any", - "localip=" + subnet, - "remoteip=" + strings.Join(remote, ","), - "profile=any", - "enable=yes", - }, - }) - } - - for _, port := range controlPorts { - rules = append(rules, winFirewallRule{ - name: fmt.Sprintf("%s-control-%d", firewallRulePrefix, port), - spec: []string{ - "dir=in", - "action=block", - "protocol=TCP", - "localip=" + gateway, - "localport=" + strconv.Itoa(port), - "remoteip=" + subnet, - "profile=any", - "enable=yes", - }, - }) - } - - return rules, nil -} - -// subtractCIDR removes exclude from cidr and renders the remainder in -// address-range syntax: the original CIDR when the two do not overlap, -// otherwise up to two "start-end" ranges. Returns an empty slice when exclude -// covers cidr entirely. IPv4 only — the HCN NAT network has no IPv6 IPAM. -func subtractCIDR(cidr, exclude string) ([]string, error) { - clo, chi, err := v4Range(cidr) - if err != nil { - return nil, err - } - xlo, xhi, err := v4Range(exclude) - if err != nil { - return nil, err - } - - if xhi < clo || xlo > chi { - return []string{cidr}, nil // no overlap — keep the CIDR as-is - } - - var out []string - if xlo > clo { - out = append(out, u32ToIP(clo)+"-"+u32ToIP(xlo-1)) - } - if xhi < chi { - out = append(out, u32ToIP(xhi+1)+"-"+u32ToIP(chi)) - } - return out, nil -} - -// v4Range returns the first and last address of an IPv4 CIDR as uint32. -func v4Range(cidr string) (lo, hi uint32, err error) { - _, ipnet, err := net.ParseCIDR(cidr) - if err != nil { - return 0, 0, fmt.Errorf("parsing %s: %w", cidr, err) - } - ip4 := ipnet.IP.To4() - if ip4 == nil { - return 0, 0, fmt.Errorf("parsing %s: not an IPv4 CIDR", cidr) - } - ones, bits := ipnet.Mask.Size() - if bits != 32 { - return 0, 0, fmt.Errorf("parsing %s: not an IPv4 mask", cidr) - } - lo = binary.BigEndian.Uint32(ip4) - hi = lo | (1<<(32-ones) - 1) - return lo, hi, nil -} - -// u32ToIP renders a uint32 back to dotted-quad form. -func u32ToIP(v uint32) string { - return net.IPv4(byte(v>>24), byte(v>>16), byte(v>>8), byte(v)).String() -} - -func (w *windowsNetworking) installNetshFirewallRules() error { - rules, err := hostFirewallRules(DefaultSubnet, defaultGateway, w.cfg.ControlPorts) - if err != nil { - w.cfg.Log.Warn("building netsh host firewall rules", "error", err) - return nil - } - - for _, r := range rules { - // Idempotent: delete any rule carrying this name from a previous run - // before adding. netsh delete removes every rule matching the name; - // "no rules match" on a fresh host is expected and ignored. - _ = netsh(r.deleteArgs()...) - - w.cfg.Log.Info("adding netsh firewall rule", "rule", r.name) - if err := netsh(r.addArgs()...); err != nil { - // Non-fatal: degrade to the per-endpoint ACLs rather than refusing - // to start. - w.cfg.Log.Warn("failed to add netsh firewall rule", "rule", r.name, "error", err) - } - } - - w.cfg.Log.Info("netsh host firewall rules installed", "rules", len(rules)) - return nil -} - -func (w *windowsNetworking) removeNetshFirewallRules() { - rules, err := hostFirewallRules(DefaultSubnet, defaultGateway, w.cfg.ControlPorts) - if err != nil { - w.cfg.Log.Debug("failed to rebuild netsh firewall rule set for removal", "error", err) - return - } - for _, r := range rules { - if err := netsh(r.deleteArgs()...); err != nil { - w.cfg.Log.Debug("failed to remove netsh firewall rule", "rule", r.name, "error", err) - } - } -} diff --git a/pkg/networking/firewall_windows_test.go b/pkg/networking/firewall_windows_test.go index 4a861705..7542fc32 100644 --- a/pkg/networking/firewall_windows_test.go +++ b/pkg/networking/firewall_windows_test.go @@ -3,13 +3,11 @@ package networking import ( - "slices" "strings" - "testing" ) // specValue extracts the value of a key=value pair from a rule spec, or "" -// when the key is absent. +// when the key is absent. Shared by the L2Bridge host-firewall backstop tests. func specValue(r winFirewallRule, key string) string { for _, s := range r.spec { if v, ok := strings.CutPrefix(s, key+"="); ok { @@ -18,212 +16,3 @@ func specValue(r winFirewallRule, key string) string { } return "" } - -func TestSubtractCIDR(t *testing.T) { - tests := []struct { - name string - cidr string - exclude string - want []string - wantErr bool - }{ - { - name: "subnet inside range splits it", - cidr: "10.0.0.0/8", - exclude: "10.88.0.0/16", - want: []string{"10.0.0.0-10.87.255.255", "10.89.0.0-10.255.255.255"}, - }, - { - name: "no overlap keeps the CIDR", - cidr: "172.16.0.0/12", - exclude: "10.88.0.0/16", - want: []string{"172.16.0.0/12"}, - }, - { - name: "exclude equals the range", - cidr: "10.88.0.0/16", - exclude: "10.88.0.0/16", - want: nil, - }, - { - name: "exclude covers the range", - cidr: "10.88.0.0/16", - exclude: "10.0.0.0/8", - want: nil, - }, - { - name: "exclude at the start leaves one range", - cidr: "10.0.0.0/8", - exclude: "10.0.0.0/16", - want: []string{"10.1.0.0-10.255.255.255"}, - }, - { - name: "exclude at the end leaves one range", - cidr: "10.0.0.0/8", - exclude: "10.255.0.0/16", - want: []string{"10.0.0.0-10.254.255.255"}, - }, - { - name: "malformed CIDR errors", - cidr: "not-a-cidr", - exclude: "10.88.0.0/16", - wantErr: true, - }, - { - name: "IPv6 CIDR errors", - cidr: "fc00::/7", - exclude: "10.88.0.0/16", - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := subtractCIDR(tt.cidr, tt.exclude) - if tt.wantErr { - if err == nil { - t.Fatalf("subtractCIDR(%q, %q) = %v, want error", tt.cidr, tt.exclude, got) - } - return - } - if err != nil { - t.Fatalf("subtractCIDR(%q, %q): %v", tt.cidr, tt.exclude, err) - } - if !slices.Equal(got, tt.want) { - t.Errorf("subtractCIDR(%q, %q) = %v, want %v", tt.cidr, tt.exclude, got, tt.want) - } - }) - } -} - -// TestHostFirewallRules_BlocksEveryDeniedRange verifies each RFC1918 + -// link-local range produces an outbound block rule; a missing range would let -// a job reach that slice of the LAN. -func TestHostFirewallRules_BlocksEveryDeniedRange(t *testing.T) { - rules, err := hostFirewallRules(DefaultSubnet, defaultGateway, nil) - if err != nil { - t.Fatalf("hostFirewallRules: %v", err) - } - if len(rules) != len(egressBlockedCIDRs) { - t.Fatalf("got %d rules, want %d (one outbound block per denied range)", len(rules), len(egressBlockedCIDRs)) - } - for i, cidr := range egressBlockedCIDRs { - r := rules[i] - wantName := firewallRulePrefix + "-block-" + strings.ReplaceAll(cidr, "/", "_") - if r.name != wantName { - t.Errorf("rule[%d].name = %q, want %q", i, r.name, wantName) - } - if specValue(r, "dir") != "out" || specValue(r, "action") != "block" { - t.Errorf("rule %s is not an outbound block: %v", r.name, r.spec) - } - if specValue(r, "remoteip") == "" { - t.Errorf("rule %s has no remoteip scope", r.name) - } - } -} - -// TestHostFirewallRules_GatewayAndSubnetNeverBlocked pins the safety property: -// the container subnet — which contains the NAT gateway (DNS, default route, -// GatewayPorts) and the other containers — must never appear inside a blocked -// range. Windows Firewall has no rule ordering and Block beats Allow, so a -// blocked gateway could not be rescued by an allow rule: it would brick all -// container networking. -func TestHostFirewallRules_GatewayAndSubnetNeverBlocked(t *testing.T) { - rules, err := hostFirewallRules("10.88.0.0/16", "10.88.0.1", nil) - if err != nil { - t.Fatalf("hostFirewallRules: %v", err) - } - - for _, r := range rules { - remote := specValue(r, "remoteip") - if strings.Contains(remote, "10.88.") { - t.Errorf("rule %s blocks the container subnet: remoteip=%s", r.name, remote) - } - // Outbound blocks must be scoped to container-sourced traffic so the - // host's own LAN access can never match. - if specValue(r, "localip") != "10.88.0.0/16" { - t.Errorf("rule %s not scoped to the container subnet: localip=%s", r.name, specValue(r, "localip")) - } - } - - // The 10.0.0.0/8 block must be split exactly around the subnet. - want := "10.0.0.0-10.87.255.255,10.89.0.0-10.255.255.255" - for _, r := range rules { - if r.name == firewallRulePrefix+"-block-10.0.0.0_8" { - if got := specValue(r, "remoteip"); got != want { - t.Errorf("10/8 block remoteip = %q, want %q", got, want) - } - return - } - } - t.Error("no block rule found for 10.0.0.0/8") -} - -// TestHostFirewallRules_ControlPortRules confirms the container→gateway -// control-plane blocks mirror the Linux INPUT drops: inbound, TCP, one -// specific port each, source = container subnet, destination = gateway — -// never a blanket gateway block and never port 53. -func TestHostFirewallRules_ControlPortRules(t *testing.T) { - ports := []int{10000, 10001, 10002} // containerd, dispatch, debug exec - rules, err := hostFirewallRules(DefaultSubnet, defaultGateway, ports) - if err != nil { - t.Fatalf("hostFirewallRules: %v", err) - } - - var control []winFirewallRule - for _, r := range rules { - if specValue(r, "dir") == "in" { - control = append(control, r) - } - } - if len(control) != len(ports) { - t.Fatalf("got %d inbound rules, want %d (one per control port)", len(control), len(ports)) - } - - for i, port := range []string{"10000", "10001", "10002"} { - r := control[i] - if specValue(r, "action") != "block" || specValue(r, "protocol") != "TCP" { - t.Errorf("rule %s is not a TCP block: %v", r.name, r.spec) - } - if specValue(r, "localport") != port { - t.Errorf("rule %s localport = %s, want %s", r.name, specValue(r, "localport"), port) - } - if specValue(r, "remoteip") != DefaultSubnet { - t.Errorf("rule %s missing source-subnet scope: %v", r.name, r.spec) - } - if specValue(r, "localip") != defaultGateway { - t.Errorf("rule %s missing gateway-dest scope: %v", r.name, r.spec) - } - if specValue(r, "localport") == "53" { - t.Errorf("rule %s blocks DNS (port 53) — must not", r.name) - } - } -} - -// TestHostFirewallRules_NamesAndArgs pins the naming and argv contract: every -// rule carries the ephemerd prefix (so the set is findable and removable), -// and delete targets exactly the name add created (that is what makes -// delete-before-add idempotent). -func TestHostFirewallRules_NamesAndArgs(t *testing.T) { - rules, err := hostFirewallRules(DefaultSubnet, defaultGateway, []int{10000}) - if err != nil { - t.Fatalf("hostFirewallRules: %v", err) - } - for _, r := range rules { - if !strings.HasPrefix(r.name, firewallRulePrefix) { - t.Errorf("rule name %q missing %q prefix", r.name, firewallRulePrefix) - } - add := r.addArgs() - wantAdd := []string{"advfirewall", "firewall", "add", "rule", "name=" + r.name} - if !slices.Equal(add[:5], wantAdd) { - t.Errorf("addArgs()[:5] = %v, want %v", add[:5], wantAdd) - } - if !slices.Equal(add[5:], r.spec) { - t.Errorf("addArgs() spec = %v, want %v", add[5:], r.spec) - } - wantDel := []string{"advfirewall", "firewall", "delete", "rule", "name=" + r.name} - if !slices.Equal(r.deleteArgs(), wantDel) { - t.Errorf("deleteArgs() = %v, want %v", r.deleteArgs(), wantDel) - } - } -} diff --git a/pkg/networking/hostport_windows_test.go b/pkg/networking/hostport_windows_test.go new file mode 100644 index 00000000..180264a5 --- /dev/null +++ b/pkg/networking/hostport_windows_test.go @@ -0,0 +1,46 @@ +//go:build windows + +package networking + +import ( + "strings" + "testing" +) + +// TestHostPortAllowRule_ScopedToPoolAndPort proves the dind-over-L2Bridge fix +// opens exactly one host port to exactly the container pool. The VFP host /32 +// allow lets a container's packet leave toward the host, but the host's own +// inbound Windows Firewall default-denies it — so the per-job dind listener is +// unreachable without this rule. It must stay tightly scoped: a broad inbound +// allow would expose RDP/SMB/RPC on the host to job containers. +func TestHostPortAllowRule_ScopedToPoolAndPort(t *testing.T) { + const ( + hostIP = "192.0.2.10" + pool = "192.0.2.192/27" + port = 63933 + ) + r := hostPortAllowRule(hostIP, pool, port) + spec := strings.Join(r.spec, " ") + + for _, want := range []string{ + "dir=in", "action=allow", "protocol=TCP", + "localip=" + hostIP, "localport=63933", "remoteip=" + pool, + } { + if !strings.Contains(spec, want) { + t.Errorf("host-port allow rule missing %q; spec = %q", want, spec) + } + } + // Must be port-scoped, not a blanket host allow. + if strings.Contains(spec, "localport=any") || !strings.Contains(spec, "localport=63933") { + t.Errorf("rule is not scoped to the single dind port: %q", spec) + } + // The rule name must carry the port so concurrent jobs get distinct rules. + if !strings.Contains(r.name, "63933") { + t.Errorf("rule name %q must be port-specific so concurrent jobs do not collide", r.name) + } + // The name must fall under the prefix the shutdown sweep matches, or a + // hard-killed job leaks its allow forever. + if !strings.HasPrefix(r.name, hostPortRulePrefix) { + t.Errorf("rule name %q must start with the sweep prefix %q", r.name, hostPortRulePrefix) + } +} diff --git a/pkg/networking/l2bridge.go b/pkg/networking/l2bridge.go new file mode 100644 index 00000000..cdfa717d --- /dev/null +++ b/pkg/networking/l2bridge.go @@ -0,0 +1,384 @@ +package networking + +import ( + "encoding/binary" + "fmt" + "net" + "strings" + "sync" +) + +// Address planning for the Windows L2Bridge egress path. +// +// This file is deliberately platform-independent: it holds the pure address +// arithmetic (pool parsing, allocation, host-network derivation, validation) so +// the whole plan is unit-testable on any OS, while network_windows.go keeps the +// HNS calls. +// +// WHY AN EXPLICIT POOL IS REQUIRED +// +// On L2Bridge, containers are not behind NAT — they are peers on the host's own +// LAN. HNS refuses to create an L2Bridge network that declares no Ipam subnet: +// +// hcnCreateNetwork failed in Win32: The network does not have a subnet for +// this endpoint. (0x803b0005) +// +// so "just DHCP" is not an option; the network must carry a subnet. Given one, +// HNS will happily assign endpoint addresses itself — from anywhere in that +// subnet, which on a real LAN means colliding with the site DHCP server's +// scope. Measured on metal: four container runs landed on four unrelated +// addresses scattered across the declared prefix. +// +// So ephemerd allocates the addresses itself, out of an operator-declared range +// (network.ip_pool) that the site's DHCP server is configured not to hand out, +// and pins each endpoint with an explicit IpConfiguration. There is deliberately +// NO default pool: any built-in guess would either be wrong for the operator's +// LAN or would quietly hand out addresses that DHCP also leases. Missing config +// fails fast at startup instead. + +// ipRange is an inclusive IPv4 address range, held as host-order uint32 so pool +// arithmetic is exact and cheap. +type ipRange struct { + lo, hi uint32 +} + +func (r ipRange) size() uint64 { return uint64(r.hi) - uint64(r.lo) + 1 } + +func (r ipRange) contains(v uint32) bool { return v >= r.lo && v <= r.hi } + +// containsRange reports whether other lies entirely inside r. +func (r ipRange) containsRange(other ipRange) bool { + return other.lo >= r.lo && other.hi <= r.hi +} + +func (r ipRange) String() string { return u32ToIPv4(r.lo) + "-" + u32ToIPv4(r.hi) } + +// ipv4ToU32 converts a dotted-quad to host-order uint32. +func ipv4ToU32(ip net.IP) (uint32, bool) { + v4 := ip.To4() + if v4 == nil { + return 0, false + } + return binary.BigEndian.Uint32(v4), true +} + +// u32ToIPv4 renders a host-order uint32 back to dotted-quad form. +func u32ToIPv4(v uint32) string { + return net.IPv4(byte(v>>24), byte(v>>16), byte(v>>8), byte(v)).String() +} + +// parseIPPool parses an address pool in either supported form: +// +// "192.0.2.32-192.0.2.63" explicit inclusive start-end range +// "192.0.2.32/27" CIDR (network and broadcast addresses excluded) +// +// Both forms are IPv4 only — the Windows container stack ephemerd drives has no +// IPv6 path. A CIDR of /31 or /32 has no network/broadcast to reserve and is +// used whole. +func parseIPPool(spec string) (ipRange, error) { + s := strings.TrimSpace(spec) + if s == "" { + return ipRange{}, fmt.Errorf("empty address pool") + } + + if start, end, ok := strings.Cut(s, "-"); ok { + lo, err := parseIPv4(strings.TrimSpace(start)) + if err != nil { + return ipRange{}, fmt.Errorf("pool %q: start address: %w", spec, err) + } + hi, err := parseIPv4(strings.TrimSpace(end)) + if err != nil { + return ipRange{}, fmt.Errorf("pool %q: end address: %w", spec, err) + } + if hi < lo { + return ipRange{}, fmt.Errorf("pool %q: end address is below the start address", spec) + } + return ipRange{lo: lo, hi: hi}, nil + } + + if !strings.Contains(s, "/") { + return ipRange{}, fmt.Errorf("pool %q: want a CIDR (192.0.2.32/27) or a start-end range (192.0.2.32-192.0.2.63)", spec) + } + + _, ipnet, err := net.ParseCIDR(s) + if err != nil { + return ipRange{}, fmt.Errorf("pool %q: %w", spec, err) + } + base, ok := ipv4ToU32(ipnet.IP) + if !ok { + return ipRange{}, fmt.Errorf("pool %q: not an IPv4 CIDR", spec) + } + ones, bits := ipnet.Mask.Size() + if bits != 32 { + return ipRange{}, fmt.Errorf("pool %q: not an IPv4 mask", spec) + } + lo, hi := base, base|(1<<(32-ones)-1) + if ones <= 30 { + // Skip the network and broadcast addresses; handing either to a + // container would be a guaranteed-broken endpoint. + lo, hi = lo+1, hi-1 + } + return ipRange{lo: lo, hi: hi}, nil +} + +// parseIPv4 parses a bare dotted-quad address. +func parseIPv4(s string) (uint32, error) { + ip := net.ParseIP(s) + if ip == nil { + return 0, fmt.Errorf("%q is not an IP address", s) + } + v, ok := ipv4ToU32(ip) + if !ok { + return 0, fmt.Errorf("%q is not an IPv4 address", s) + } + return v, nil +} + +// ipAllocator hands out addresses from an ipRange, one per container endpoint. +// +// ephemerd owns allocation on the L2Bridge path (rather than letting HNS pick) +// precisely so every container address stays inside the operator's reserved +// range. Allocation is lowest-free-first and idempotent per id, so a retried +// setup for the same container reuses its address instead of leaking one. +type ipAllocator struct { + mu sync.Mutex + pool ipRange + reserved map[uint32]struct{} // never handed out (host, gateway, adopted endpoints) + byID map[string]uint32 + inUse map[uint32]string +} + +// newIPAllocator builds an allocator over pool. Addresses in reserved are +// permanently withheld; malformed entries are ignored (callers pass addresses +// they have already validated). +func newIPAllocator(pool ipRange, reserved ...string) *ipAllocator { + a := &ipAllocator{ + pool: pool, + reserved: map[uint32]struct{}{}, + byID: map[string]uint32{}, + inUse: map[uint32]string{}, + } + for _, r := range reserved { + a.reserve(r) + } + return a +} + +// reserve withholds an address from future allocations. Used for the host and +// gateway addresses, and for endpoints adopted from a previous daemon run. +// A non-IPv4 or out-of-pool address is a no-op. +func (a *ipAllocator) reserve(addr string) { + v, err := parseIPv4(strings.TrimSpace(addr)) + if err != nil { + return + } + a.mu.Lock() + defer a.mu.Unlock() + a.reserved[v] = struct{}{} +} + +// allocate returns the lowest free address in the pool for id, or an error when +// the pool is exhausted. Callers must treat exhaustion as fatal for the job: +// starting a container without an address of our choosing means either no +// network or an HNS-picked address that may collide with a DHCP lease. +func (a *ipAllocator) allocate(id string) (string, error) { + a.mu.Lock() + defer a.mu.Unlock() + + if v, ok := a.byID[id]; ok { + return u32ToIPv4(v), nil + } + + for v := a.pool.lo; ; v++ { + _, isReserved := a.reserved[v] + _, isUsed := a.inUse[v] + if !isReserved && !isUsed { + a.byID[id] = v + a.inUse[v] = id + return u32ToIPv4(v), nil + } + if v == a.pool.hi { + break // written this way so a pool ending at 255.255.255.255 cannot wrap + } + } + + return "", fmt.Errorf("address pool %s exhausted (%d in use): raise network.ip_pool or lower runner.max_concurrent", a.pool, len(a.inUse)) +} + +// release returns id's address to the pool. Safe to call for an unknown id. +func (a *ipAllocator) release(id string) { + a.mu.Lock() + defer a.mu.Unlock() + if v, ok := a.byID[id]; ok { + delete(a.byID, id) + delete(a.inUse, v) + } +} + +// l2BridgePlan is the fully resolved address plan for the L2Bridge network: +// everything HNS needs, plus the host address containers use to reach services +// ephemerd hosts (dind, the module proxy). +type l2BridgePlan struct { + // Subnet is the LAN CIDR the HNS network declares in its Ipam. Required — + // HNS rejects a subnet-less L2Bridge network outright (0x803b0005). + Subnet string + // PrefixLen is Subnet's prefix length, stamped onto each endpoint's + // IpConfiguration so the container's route table matches the real LAN. + PrefixLen int + // Gateway is the LAN router, used as the Ipam default-route next hop. + // Containers route THROUGH it while the egress ACLs stop them ADDRESSING + // it — that combination was verified on metal. + Gateway string + // HostIP is the ephemerd host's own address on HostNIC. + HostIP string + // Pool is the operator-declared range container addresses come from. + Pool ipRange + // PoolSpec is the pool exactly as configured, kept for firewall rules and + // log lines that should echo the operator's own words. + PoolSpec string + // DerivedSubnet / DerivedGateway record whether each value was read off the + // host rather than configured, so startup can log what it inferred. + DerivedSubnet bool + DerivedGateway bool +} + +// hostNetLookup abstracts the two host queries the plan needs, so +// resolveL2BridgePlan is testable without a Windows host or a real NIC. +type hostNetLookup struct { + // iface returns the host's IPv4 address and its network on the named + // adapter. + iface func(name string) (net.IP, *net.IPNet, error) + // gateway returns the default-route next hop reachable via the named + // adapter. + gateway func(name string) (string, error) +} + +// resolveL2BridgePlan turns the operator's [network] config plus the host's +// actual NIC state into a complete address plan, or a specific error naming the +// key that needs setting. +// +// Nothing here defaults to a particular network. subnet and gateway are read off +// the host's real primary adapter when the operator has not pinned them; ip_pool +// has no default at all, because only the operator knows which addresses their +// DHCP server is configured never to lease. +func resolveL2BridgePlan(cfg Config, look hostNetLookup) (*l2BridgePlan, error) { + nic := strings.TrimSpace(cfg.HostNIC) + if nic == "" { + return nil, fmt.Errorf("network.host_nic is required when network.l2bridge_egress = true: " + + "set it to the host adapter to bridge onto (the name shown by `Get-NetAdapter`, e.g. \"Ethernet\"). " + + "There is no default — the correct adapter is host-specific") + } + if strings.TrimSpace(cfg.IPPool) == "" { + return nil, errIPPoolRequired + } + + hostIP, hostNet, err := look.iface(nic) + if err != nil { + return nil, err + } + hostU32, ok := ipv4ToU32(hostIP) + if !ok { + return nil, fmt.Errorf("adapter %q: host address %s is not IPv4 (network.l2bridge_egress is IPv4 only)", nic, hostIP) + } + + plan := &l2BridgePlan{HostIP: u32ToIPv4(hostU32)} + + // Subnet: the operator's if pinned, otherwise the adapter's own prefix. + subnetSpec := strings.TrimSpace(cfg.Subnet) + if subnetSpec == "" { + subnetSpec = hostNet.String() + plan.DerivedSubnet = true + } + subnetRange, prefixLen, err := cidrRange(subnetSpec) + if err != nil { + return nil, fmt.Errorf("network.subnet %q: %w", subnetSpec, err) + } + if !subnetRange.contains(hostU32) { + return nil, fmt.Errorf("network.subnet %s does not contain the address %s configured on adapter %q: "+ + "either clear network.subnet to derive it from the adapter, or correct it to the LAN the adapter is on", + subnetSpec, plan.HostIP, nic) + } + plan.Subnet = subnetSpec + plan.PrefixLen = prefixLen + + // Gateway: the operator's if pinned, otherwise the adapter's default route. + gwSpec := strings.TrimSpace(cfg.Gateway) + if gwSpec == "" { + if look.gateway == nil { + return nil, errGatewayRequired(nic, fmt.Errorf("no default-route lookup available on this platform")) + } + gwSpec, err = look.gateway(nic) + if err != nil { + return nil, errGatewayRequired(nic, err) + } + gwSpec = strings.TrimSpace(gwSpec) + if gwSpec == "" { + return nil, errGatewayRequired(nic, fmt.Errorf("adapter has no IPv4 default route")) + } + plan.DerivedGateway = true + } + gwU32, err := parseIPv4(gwSpec) + if err != nil { + return nil, fmt.Errorf("network.gateway: %w", err) + } + if !subnetRange.contains(gwU32) { + return nil, fmt.Errorf("network.gateway %s is outside network.subnet %s: the L2Bridge default route "+ + "next hop must be an on-link address of the bridged LAN", gwSpec, plan.Subnet) + } + plan.Gateway = u32ToIPv4(gwU32) + + // Pool: must be inside the subnet and must not swallow the host or router. + pool, err := parseIPPool(cfg.IPPool) + if err != nil { + return nil, fmt.Errorf("network.ip_pool: %w", err) + } + if !subnetRange.containsRange(pool) { + return nil, fmt.Errorf("network.ip_pool %s (%s) is not inside network.subnet %s: containers are LAN peers "+ + "on L2Bridge, so the pool must be a reserved slice of the bridged LAN", cfg.IPPool, pool, plan.Subnet) + } + if pool.contains(hostU32) { + return nil, fmt.Errorf("network.ip_pool %s contains this host's own address %s: pick a range that excludes it", + cfg.IPPool, plan.HostIP) + } + if pool.contains(gwU32) { + return nil, fmt.Errorf("network.ip_pool %s contains the LAN gateway %s: pick a range that excludes it", + cfg.IPPool, plan.Gateway) + } + plan.Pool = pool + plan.PoolSpec = strings.TrimSpace(cfg.IPPool) + + return plan, nil +} + +// errIPPoolRequired is the backstop for the one setting that cannot be +// inferred. pkg/config rejects a missing ip_pool at load time, before any HNS +// object exists, with the full explanation; this catches a networking.Config +// assembled by any other caller. +var errIPPoolRequired = fmt.Errorf( + `network.ip_pool is required when network.l2bridge_egress = true: ` + + `set it to a slice of your LAN that DHCP never leases, as a CIDR ("192.0.2.192/27") ` + + `or an inclusive range ("192.0.2.200-192.0.2.230"). There is no default`) + +// errGatewayRequired reports that the LAN default route could not be derived and +// tells the operator which key pins it manually. +func errGatewayRequired(nic string, cause error) error { + return fmt.Errorf("could not derive the LAN default gateway for adapter %q (%w): "+ + "set network.gateway explicitly to the router address the L2Bridge should route through", nic, cause) +} + +// cidrRange returns a CIDR's inclusive address range and prefix length. +func cidrRange(cidr string) (ipRange, int, error) { + _, ipnet, err := net.ParseCIDR(strings.TrimSpace(cidr)) + if err != nil { + return ipRange{}, 0, err + } + base, ok := ipv4ToU32(ipnet.IP) + if !ok { + return ipRange{}, 0, fmt.Errorf("not an IPv4 CIDR") + } + ones, bits := ipnet.Mask.Size() + if bits != 32 { + return ipRange{}, 0, fmt.Errorf("not an IPv4 mask") + } + return ipRange{lo: base, hi: base | (1<<(32-ones) - 1)}, ones, nil +} diff --git a/pkg/networking/l2bridge_test.go b/pkg/networking/l2bridge_test.go new file mode 100644 index 00000000..cbd9d68b --- /dev/null +++ b/pkg/networking/l2bridge_test.go @@ -0,0 +1,342 @@ +package networking + +import ( + "fmt" + "net" + "strings" + "testing" +) + +// Addresses in these tests come from RFC 5737 documentation ranges +// (192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24). Nothing here — and nothing in +// the shipped defaults — encodes a real site's LAN. + +func mustPool(t *testing.T, spec string) ipRange { + t.Helper() + p, err := parseIPPool(spec) + if err != nil { + t.Fatalf("parseIPPool(%q): %v", spec, err) + } + return p +} + +func TestParseIPPool_Forms(t *testing.T) { + tests := []struct { + spec string + wantLo string + wantHi string + wantCount uint64 + }{ + // Inclusive start-end range, used verbatim. + {"192.0.2.200-192.0.2.230", "192.0.2.200", "192.0.2.230", 31}, + {" 192.0.2.10 - 192.0.2.10 ", "192.0.2.10", "192.0.2.10", 1}, + // CIDR: network and broadcast addresses are not handed to containers. + {"192.0.2.192/27", "192.0.2.193", "192.0.2.222", 30}, + {"192.0.2.0/24", "192.0.2.1", "192.0.2.254", 254}, + // /31 and /32 have no network/broadcast to reserve. + {"192.0.2.8/31", "192.0.2.8", "192.0.2.9", 2}, + {"192.0.2.8/32", "192.0.2.8", "192.0.2.8", 1}, + } + for _, tt := range tests { + got := mustPool(t, tt.spec) + if u32ToIPv4(got.lo) != tt.wantLo || u32ToIPv4(got.hi) != tt.wantHi { + t.Errorf("parseIPPool(%q) = %s-%s, want %s-%s", + tt.spec, u32ToIPv4(got.lo), u32ToIPv4(got.hi), tt.wantLo, tt.wantHi) + } + if got.size() != tt.wantCount { + t.Errorf("parseIPPool(%q).size() = %d, want %d", tt.spec, got.size(), tt.wantCount) + } + } +} + +func TestParseIPPool_Rejects(t *testing.T) { + for _, spec := range []string{ + "", // unset + "192.0.2.5", // bare address, neither form + "192.0.2.230-192.0.2.200", // reversed + "192.0.2.999-192.0.2.1000", // not addresses + "not-an-address", // garbage + "2001:db8::/64", // IPv6: no v6 path on this stack + "192.0.2.0/33", // impossible mask + "192.0.2.200-", // half a range + } { + if _, err := parseIPPool(spec); err == nil { + t.Errorf("parseIPPool(%q) accepted an invalid pool", spec) + } + } +} + +// TestIPAllocator_AllocatesWithinPool is the core guarantee: every address a +// container gets comes out of the operator's reserved range, so it can never +// collide with an address the site's DHCP server hands out. +func TestIPAllocator_AllocatesWithinPool(t *testing.T) { + pool := mustPool(t, "192.0.2.200-192.0.2.203") + a := newIPAllocator(pool) + + seen := map[string]bool{} + for i := range 4 { + got, err := a.allocate(fmt.Sprintf("job-%d", i)) + if err != nil { + t.Fatalf("allocate #%d: %v", i, err) + } + v, err := parseIPv4(got) + if err != nil { + t.Fatalf("allocate returned %q: %v", got, err) + } + if !pool.contains(v) { + t.Errorf("allocated %s outside the pool %s", got, pool) + } + if seen[got] { + t.Errorf("allocated %s twice", got) + } + seen[got] = true + } + + // Exhaustion must be an error, never a silent fallback to an + // HNS-chosen address that could collide with a DHCP lease. + if _, err := a.allocate("job-overflow"); err == nil { + t.Fatal("allocate succeeded on an exhausted pool; want an error so the job is refused") + } else if !strings.Contains(err.Error(), "network.ip_pool") { + t.Errorf("exhaustion error %q does not name the config key to raise", err) + } +} + +func TestIPAllocator_ReservesHostAndGateway(t *testing.T) { + pool := mustPool(t, "192.0.2.1-192.0.2.3") + a := newIPAllocator(pool, "192.0.2.1", "192.0.2.2") + + got, err := a.allocate("job") + if err != nil { + t.Fatalf("allocate: %v", err) + } + if got != "192.0.2.3" { + t.Errorf("allocate = %s, want 192.0.2.3 (the reserved host and gateway must be skipped)", got) + } +} + +func TestIPAllocator_ReleaseAndIdempotence(t *testing.T) { + a := newIPAllocator(mustPool(t, "192.0.2.10-192.0.2.11")) + + first, err := a.allocate("job-a") + if err != nil { + t.Fatalf("allocate: %v", err) + } + // Re-allocating for the same id must return the same address, so a retried + // setup does not burn a second one. + again, err := a.allocate("job-a") + if err != nil { + t.Fatalf("re-allocate: %v", err) + } + if again != first { + t.Errorf("re-allocate for the same id = %s, want the original %s", again, first) + } + + if _, err := a.allocate("job-b"); err != nil { + t.Fatalf("allocate second: %v", err) + } + if _, err := a.allocate("job-c"); err == nil { + t.Fatal("pool of 2 handed out a third address") + } + + // After release the address returns to the pool — otherwise a long-lived + // daemon leaks the pool one job at a time. + a.release("job-a") + reused, err := a.allocate("job-c") + if err != nil { + t.Fatalf("allocate after release: %v", err) + } + if reused != first { + t.Errorf("allocate after release = %s, want the freed %s", reused, first) + } + + a.release("never-allocated") // must not panic +} + +// fakeLookup builds a hostNetLookup that reports a fixed adapter address and +// default route, so the plan can be exercised without a Windows host. +func fakeLookup(hostCIDR, gateway string) hostNetLookup { + return hostNetLookup{ + iface: func(string) (net.IP, *net.IPNet, error) { + ip, ipnet, err := net.ParseCIDR(hostCIDR) + if err != nil { + return nil, nil, err + } + return ip.To4(), ipnet, nil + }, + gateway: func(string) (string, error) { return gateway, nil }, + } +} + +// TestResolveL2BridgePlan_DerivesFromHost verifies the auto-derivation contract: +// with only host_nic and ip_pool set, the subnet and gateway come off the host's +// real adapter. Nothing is defaulted to a fixed network. +func TestResolveL2BridgePlan_DerivesFromHost(t *testing.T) { + plan, err := resolveL2BridgePlan( + Config{HostNIC: "Ethernet", IPPool: "198.51.100.200-198.51.100.230"}, + fakeLookup("198.51.100.7/24", "198.51.100.1"), + ) + if err != nil { + t.Fatalf("resolveL2BridgePlan: %v", err) + } + if plan.Subnet != "198.51.100.0/24" || !plan.DerivedSubnet { + t.Errorf("subnet = %q derived=%v, want 198.51.100.0/24 derived", plan.Subnet, plan.DerivedSubnet) + } + if plan.PrefixLen != 24 { + t.Errorf("prefix len = %d, want 24", plan.PrefixLen) + } + if plan.Gateway != "198.51.100.1" || !plan.DerivedGateway { + t.Errorf("gateway = %q derived=%v, want 198.51.100.1 derived", plan.Gateway, plan.DerivedGateway) + } + if plan.HostIP != "198.51.100.7" { + t.Errorf("host ip = %q, want 198.51.100.7", plan.HostIP) + } + if plan.PoolSpec != "198.51.100.200-198.51.100.230" { + t.Errorf("pool spec = %q, want the operator's own string", plan.PoolSpec) + } +} + +// TestResolveL2BridgePlan_ExplicitOverridesWin verifies a pinned subnet/gateway +// is used as-is and reported as not derived. +func TestResolveL2BridgePlan_ExplicitOverridesWin(t *testing.T) { + plan, err := resolveL2BridgePlan( + Config{ + HostNIC: "Ethernet", + IPPool: "198.51.100.96/29", + Subnet: "198.51.100.0/25", + Gateway: "198.51.100.9", + }, + fakeLookup("198.51.100.7/24", "198.51.100.1"), + ) + if err != nil { + t.Fatalf("resolveL2BridgePlan: %v", err) + } + if plan.Subnet != "198.51.100.0/25" || plan.DerivedSubnet { + t.Errorf("subnet = %q derived=%v, want the configured 198.51.100.0/25", plan.Subnet, plan.DerivedSubnet) + } + if plan.Gateway != "198.51.100.9" || plan.DerivedGateway { + t.Errorf("gateway = %q derived=%v, want the configured 198.51.100.9", plan.Gateway, plan.DerivedGateway) + } +} + +// TestResolveL2BridgePlan_Failures walks every way the plan can be incomplete. +// Each must fail fast with a message naming the key to fix — never fall back to +// a guessed pool, which would hand containers addresses DHCP is also leasing. +func TestResolveL2BridgePlan_Failures(t *testing.T) { + tests := []struct { + name string + cfg Config + look hostNetLookup + wantText string + }{ + { + name: "no host_nic", + cfg: Config{IPPool: "198.51.100.200-198.51.100.230"}, + look: fakeLookup("198.51.100.7/24", "198.51.100.1"), + wantText: "network.host_nic", + }, + { + name: "no ip_pool", + cfg: Config{HostNIC: "Ethernet"}, + look: fakeLookup("198.51.100.7/24", "198.51.100.1"), + wantText: "network.ip_pool", + }, + { + name: "adapter not found", + cfg: Config{HostNIC: "Nope", IPPool: "198.51.100.200-198.51.100.230"}, + look: hostNetLookup{ + iface: func(string) (net.IP, *net.IPNet, error) { return nil, nil, fmt.Errorf("no such adapter") }, + gateway: func(string) (string, error) { return "198.51.100.1", nil }, + }, + wantText: "no such adapter", + }, + { + name: "no default route on the adapter", + cfg: Config{HostNIC: "Ethernet", IPPool: "198.51.100.200-198.51.100.230"}, + look: hostNetLookup{ + iface: fakeLookup("198.51.100.7/24", "").iface, + gateway: func(string) (string, error) { return "", nil }, + }, + wantText: "network.gateway", + }, + { + name: "pool outside the subnet", + cfg: Config{HostNIC: "Ethernet", IPPool: "203.0.113.200-203.0.113.230"}, + look: fakeLookup("198.51.100.7/24", "198.51.100.1"), + wantText: "not inside network.subnet", + }, + { + name: "pool swallows this host", + cfg: Config{HostNIC: "Ethernet", IPPool: "198.51.100.1-198.51.100.50"}, + look: fakeLookup("198.51.100.7/24", "198.51.100.1"), + wantText: "own address", + }, + { + name: "pool swallows the gateway", + cfg: Config{HostNIC: "Ethernet", IPPool: "198.51.100.200-198.51.100.254"}, + look: fakeLookup("198.51.100.7/24", "198.51.100.201"), + wantText: "LAN gateway", + }, + { + name: "gateway outside the subnet", + cfg: Config{HostNIC: "Ethernet", IPPool: "198.51.100.200-198.51.100.230", Gateway: "203.0.113.1"}, + look: fakeLookup("198.51.100.7/24", "198.51.100.1"), + wantText: "outside network.subnet", + }, + { + name: "pinned subnet does not contain the adapter address", + cfg: Config{HostNIC: "Ethernet", IPPool: "203.0.113.200-203.0.113.230", Subnet: "203.0.113.0/24"}, + look: fakeLookup("198.51.100.7/24", "198.51.100.1"), + wantText: "does not contain the address", + }, + { + name: "malformed pool", + cfg: Config{HostNIC: "Ethernet", IPPool: "banana"}, + look: fakeLookup("198.51.100.7/24", "198.51.100.1"), + wantText: "network.ip_pool", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + plan, err := resolveL2BridgePlan(tt.cfg, tt.look) + if err == nil { + t.Fatalf("resolveL2BridgePlan succeeded, want an error; got plan %+v", plan) + } + if !strings.Contains(err.Error(), tt.wantText) { + t.Errorf("error %q does not mention %q, so an operator cannot tell what to fix", err, tt.wantText) + } + }) + } +} + +// TestResolveL2BridgePlan_NoBuiltInNetwork is the guard against re-introducing a +// hardcoded LAN. With no adapter information available the plan must fail, not +// silently fall back to some built-in subnet, gateway, or pool. +func TestResolveL2BridgePlan_NoBuiltInNetwork(t *testing.T) { + _, err := resolveL2BridgePlan( + Config{HostNIC: "Ethernet", IPPool: "198.51.100.200-198.51.100.230"}, + hostNetLookup{ + iface: func(string) (net.IP, *net.IPNet, error) { return nil, nil, fmt.Errorf("adapter has no address") }, + gateway: func(string) (string, error) { return "", fmt.Errorf("no route") }, + }, + ) + if err == nil { + t.Fatal("resolveL2BridgePlan invented an address plan with no host information") + } +} + +// TestResolveL2BridgePlan_PoolFitsMaxConcurrent documents the sizing +// relationship an operator has to satisfy, using the shipped default +// runner.max_concurrent of 4. +func TestResolveL2BridgePlan_PoolFitsMaxConcurrent(t *testing.T) { + plan, err := resolveL2BridgePlan( + Config{HostNIC: "Ethernet", IPPool: "198.51.100.200-198.51.100.203"}, + fakeLookup("198.51.100.7/24", "198.51.100.1"), + ) + if err != nil { + t.Fatalf("resolveL2BridgePlan: %v", err) + } + if plan.Pool.size() < 4 { + t.Errorf("pool holds %d addresses, too few for the default max_concurrent of 4", plan.Pool.size()) + } +} diff --git a/pkg/networking/l2bridge_windows.go b/pkg/networking/l2bridge_windows.go new file mode 100644 index 00000000..fd9437df --- /dev/null +++ b/pkg/networking/l2bridge_windows.go @@ -0,0 +1,80 @@ +//go:build windows + +package networking + +import ( + "fmt" + "net" + "strings" +) + +// adapterNameCandidates returns the adapter names to try for a configured +// network.host_nic, in order. +// +// Creating the L2Bridge network CONSUMES the configured NIC: HNS binds an +// external vSwitch to it and the host's side of it re-enumerates as +// "vEthernet ()". So on every daemon start after the first, the raw +// configured name no longer exists — resolving only the literal name made the +// daemon crash-loop on restart ("no adapter with that name on this host", +// found on metal 2026-08-13). Deleting the network reverses the rename, so +// the unwrapped form must be tried too when the operator configured the +// vEthernet name. +func adapterNameCandidates(name string) []string { + candidates := []string{name} + if inner, ok := strings.CutPrefix(name, "vEthernet ("); ok { + if inner, ok := strings.CutSuffix(inner, ")"); ok { + candidates = append(candidates, inner) + } + } else { + candidates = append(candidates, "vEthernet ("+name+")") + } + return candidates +} + +// hostIPv4OnInterface returns the first routable IPv4 address configured on the +// named adapter, together with its network. On Windows the adapter name is the +// friendly name shown by Get-NetAdapter, which is also what the HNS +// NetAdapterName network policy expects — the same string in both places. +// The vEthernet-renamed form of the adapter is accepted too (see +// adapterNameCandidates). +// +// This lives in a Windows-only file because it reads the live host; the address +// arithmetic it feeds (resolveL2BridgePlan and friends) stays in l2bridge.go so +// it builds and is tested on every platform. +func hostIPv4OnInterface(name string) (net.IP, *net.IPNet, error) { + var iface *net.Interface + var err error + for _, candidate := range adapterNameCandidates(name) { + iface, err = net.InterfaceByName(candidate) + if err == nil { + break + } + } + if iface == nil { + return nil, nil, fmt.Errorf("network.host_nic %q: no adapter with that name on this host "+ + "(also tried %v; check `Get-NetAdapter`): %w", name, adapterNameCandidates(name)[1:], err) + } + addrs, err := iface.Addrs() + if err != nil { + return nil, nil, fmt.Errorf("network.host_nic %q: reading adapter addresses: %w", name, err) + } + for _, a := range addrs { + ipnet, ok := a.(*net.IPNet) + if !ok { + continue + } + v4 := ipnet.IP.To4() + if v4 == nil || v4.IsLoopback() || v4.IsLinkLocalUnicast() { + continue + } + // Normalize to a 4-byte IP and a /n IPv4 mask. + ones, bits := ipnet.Mask.Size() + if bits != 32 { + continue + } + return v4, &net.IPNet{IP: v4.Mask(ipnet.Mask), Mask: net.CIDRMask(ones, 32)}, nil + } + return nil, nil, fmt.Errorf("network.host_nic %q (resolved adapter %q): adapter has no routable IPv4 address "+ + "(an APIPA/link-local-only adapter cannot bridge); give the adapter a static or DHCP address, "+ + "or set network.host_nic to the adapter that carries the LAN", name, iface.Name) +} diff --git a/pkg/networking/network_darwin.go b/pkg/networking/network_darwin.go index 48fab791..b6291aa1 100644 --- a/pkg/networking/network_darwin.go +++ b/pkg/networking/network_darwin.go @@ -51,3 +51,11 @@ func (d *darwinNetworking) removeFirewallRules() {} func (d *darwinNetworking) cleanup() {} func cleanStaleBridge(_ *slog.Logger) {} // no-op on macOS + +// hostAddr: no L2Bridge on macOS — the generic subnet derivation applies. +func (d *darwinNetworking) hostAddr() string { return "" } + +// openHostPort/closeHostPort are Windows-L2Bridge-only; the macOS path +// delegates networking to the in-VM Linux stack. +func (d *darwinNetworking) openHostPort(int) error { return nil } +func (d *darwinNetworking) closeHostPort(int) {} diff --git a/pkg/networking/network_linux.go b/pkg/networking/network_linux.go index 6e13c45b..1eee219d 100644 --- a/pkg/networking/network_linux.go +++ b/pkg/networking/network_linux.go @@ -104,6 +104,14 @@ func (l *linuxNetworking) teardown(ctx context.Context, id string, netns string) return nil } +// hostAddr: no L2Bridge on Linux — the generic subnet derivation applies. +func (l *linuxNetworking) hostAddr() string { return "" } + +// openHostPort/closeHostPort are Windows-L2Bridge-only; Linux containers reach +// the bridge gateway without a host-firewall carve-out. +func (l *linuxNetworking) openHostPort(int) error { return nil } +func (l *linuxNetworking) closeHostPort(int) {} + func (l *linuxNetworking) cleanup() { log := l.cfg.Log diff --git a/pkg/networking/network_windows.go b/pkg/networking/network_windows.go index e963637c..36d5d6d0 100644 --- a/pkg/networking/network_windows.go +++ b/pkg/networking/network_windows.go @@ -15,12 +15,29 @@ import ( const ( networkName = "ephemerd" defaultGateway = "10.88.0.1" + + // l2BridgeNetworkName is the HNS network created on the L2Bridge egress + // path. It is deliberately distinct from networkName so a host that + // previously ran the NAT path (network "ephemerd") does not collide with, + // or get mistaken for, the L2Bridge network. + l2BridgeNetworkName = "ephemerd-l2bridge" ) +// defaultPublicDNS is the DNS resolver list handed to L2Bridge containers when +// Config.PublicDNS is empty. Public resolvers so container DNS never needs the +// LAN router — which the egress ACLs block along with the rest of RFC1918. +var defaultPublicDNS = []string{"1.1.1.1", "8.8.8.8"} + type windowsNetworking struct { cfg Config network *hcn.HostComputeNetwork mu sync.Mutex + + // plan and ipam are set only on the L2Bridge path: the resolved LAN + // address plan, and the allocator that hands each endpoint an address out + // of the operator's reserved pool. + plan *l2BridgePlan + ipam *ipAllocator } func newPlatformNetworking() platformNetworking { @@ -30,6 +47,12 @@ func newPlatformNetworking() platformNetworking { func (w *windowsNetworking) init(cfg Config) error { w.cfg = cfg + // L2Bridge egress path (opt-in). NAT stays the default: only a pool that + // explicitly sets L2BridgeEgress reaches VFP-enforced egress filtering. + if cfg.L2BridgeEgress { + return w.initL2Bridge(cfg) + } + // Check if network already exists (from previous run) existing, err := hcn.GetNetworkByName(networkName) if err == nil { @@ -77,16 +100,205 @@ func (w *windowsNetworking) init(cfg Config) error { return nil } +// publicDNS returns the configured public resolver list, or the built-in +// default when none is configured. +func (w *windowsNetworking) publicDNS() []string { + if len(w.cfg.PublicDNS) > 0 { + return w.cfg.PublicDNS + } + return defaultPublicDNS +} + +// initL2Bridge creates (or adopts) the L2Bridge HNS network bound to the +// configured host NIC. Unlike the NAT network, this puts containers on a +// VFP-managed vSwitch port so the per-endpoint Switch ACLs applied in setup() +// actually enforce. +// +// DNS is overridden to the public resolvers at the network and endpoint level +// so the container never needs the LAN router for name resolution — the egress +// ACLs block the router. +// +// IPAM — why the network declares a subnet and ephemerd owns allocation. +// DHCP IPAM is not available. Declaring no Ipams fails on metal (Server 2025 +// 26100) at network creation: +// +// hcnCreateNetwork failed in Win32: The network does not have a subnet for +// this endpoint. (0x803b0005) / ErrorCode 2151350277 +// +// so an L2Bridge network must carry a subnet plus a default route. Given one, +// HNS assigns endpoint addresses on its own — scattered anywhere across the +// declared prefix, which on a real LAN means straight into the site DHCP +// server's scope. ephemerd therefore pins each endpoint to an address it +// allocated out of the operator's reserved network.ip_pool (see setup and +// l2bridge.go). The subnet and the default-route next hop are read off the host +// adapter at runtime unless the operator pinned them; nothing here assumes any +// particular network. +func (w *windowsNetworking) initL2Bridge(cfg Config) error { + // Fail closed on an incomplete plan: silently falling back to NAT would + // defeat the egress guarantee the operator opted into, and guessing a pool + // would hand out addresses that collide with live DHCP leases. + plan, err := resolveL2BridgePlan(cfg, hostNetLookup{ + iface: hostIPv4OnInterface, + gateway: defaultGatewayForAdapter, + }) + if err != nil { + return fmt.Errorf("L2Bridge egress enabled but the address plan is incomplete: %w", err) + } + w.plan = plan + w.ipam = newIPAllocator(plan.Pool, plan.HostIP, plan.Gateway) + + cfg.Log.Info("L2Bridge address plan resolved", + "host_nic", cfg.HostNIC, + "subnet", plan.Subnet, "subnet_derived", plan.DerivedSubnet, + "gateway", plan.Gateway, "gateway_derived", plan.DerivedGateway, + "host_ip", plan.HostIP, + "ip_pool", plan.PoolSpec, "pool_size", plan.Pool.size()) + + if cfg.AllowHostAccess { + // Worth a warning on its own line: this is the one rule that lets a job + // container address the ephemerd host at all. + cfg.Log.Warn("L2Bridge egress permits containers to reach this host (required by dind / the module proxy)", + "host_ip", plan.HostIP, "control_ports_blocked", cfg.ControlPorts) + } + + if existing, err := hcn.GetNetworkByName(l2BridgeNetworkName); err == nil { + w.network = existing + w.adoptExistingEndpoints(existing) + cfg.Log.Info("HCN L2Bridge network found", "name", l2BridgeNetworkName, "id", existing.Id, "host_nic", cfg.HostNIC) + return nil + } + + adapterPol, err := json.Marshal(hcn.NetAdapterNameNetworkPolicySetting{ + NetworkAdapterName: cfg.HostNIC, + }) + if err != nil { + return fmt.Errorf("marshaling NetAdapterName policy for %q: %w", cfg.HostNIC, err) + } + + network := &hcn.HostComputeNetwork{ + Name: l2BridgeNetworkName, + Type: hcn.L2Bridge, + Ipams: []hcn.Ipam{ + { + // "Static" means HNS does not run a DHCP client for the + // endpoints; the addresses come from this subnet. ephemerd + // narrows that further by pinning each endpoint itself. + Type: "Static", + Subnets: []hcn.Subnet{ + { + IpAddressPrefix: plan.Subnet, + Routes: []hcn.Route{ + { + // The LAN router. Containers route THROUGH it + // while the egress ACLs stop them ADDRESSING + // it — verified working on metal. + NextHop: plan.Gateway, + DestinationPrefix: "0.0.0.0/0", + }, + }, + }, + }, + }, + }, + Policies: []hcn.NetworkPolicy{ + { + Type: hcn.NetAdapterName, // binds the L2Bridge to the physical NIC + Settings: adapterPol, + }, + }, + Dns: hcn.Dns{ + ServerList: w.publicDNS(), + }, + SchemaVersion: hcn.SchemaVersion{ + Major: 2, + Minor: 0, + }, + } + + created, err := network.Create() + if err != nil { + return fmt.Errorf("creating HCN L2Bridge network on %q (subnet %s, gateway %s): %w", + cfg.HostNIC, plan.Subnet, plan.Gateway, err) + } + w.network = created + + cfg.Log.Info("HCN L2Bridge network created", + "name", l2BridgeNetworkName, "id", created.Id, "host_nic", cfg.HostNIC, + "subnet", plan.Subnet, "gateway", plan.Gateway) + return nil +} + +// adoptExistingEndpoints marks the addresses of endpoints already present on an +// adopted network as in use, so a daemon restart that finds leftover endpoints +// does not hand the same address to a new container. +// +// These reservations are held for the life of the process: ephemerd did not +// allocate them, so it has no id to release them under. The addresses return to +// the pool at the next restart, once the stale endpoints are gone. Size the pool +// with a little headroom rather than exactly runner.max_concurrent. +// +// Best-effort: a failure to enumerate is logged, not fatal — HNS rejects a +// duplicate address at endpoint creation anyway, which fails the job closed. +func (w *windowsNetworking) adoptExistingEndpoints(network *hcn.HostComputeNetwork) { + eps, err := hcn.ListEndpointsOfNetwork(network.Id) + if err != nil { + w.cfg.Log.Warn("could not enumerate existing L2Bridge endpoints; pool may briefly double-allocate", "error", err) + return + } + for _, ep := range eps { + for _, ipc := range ep.IpConfigurations { + if ipc.IpAddress != "" { + w.ipam.reserve(ipc.IpAddress) + } + } + } + if len(eps) > 0 { + w.cfg.Log.Info("reserved addresses of pre-existing L2Bridge endpoints", "endpoints", len(eps)) + } +} + +// hostAddr reports the address containers use to reach services ephemerd hosts. +// On L2Bridge that is the host's own LAN address — there is no bridge gateway to +// bind to, and the NAT path's hard-coded 10.88.0.1 exists on no interface once +// the NAT network is out of the picture. Returning it here is what keeps the +// per-job dind listener (pkg/dind/listen_windows.go) and the Go module proxy +// bindable, and therefore jobs provisionable. +func (w *windowsNetworking) hostAddr() string { + if w.cfg.L2BridgeEgress && w.plan != nil { + return w.plan.HostIP + } + // NAT path: unchanged — the generic subnet derivation yields defaultGateway. + return "" +} + func (w *windowsNetworking) setup(ctx context.Context, id string, netns string) (*SetupResult, error) { w.mu.Lock() defer w.mu.Unlock() - // Create endpoint on the network + // DNS: on L2Bridge, hand the container the configured PUBLIC resolvers so + // name resolution goes to the internet (permitted by the low-precedence + // allow-any ACL) and never to the LAN router (blocked by the RFC1918 ACLs). + // On NAT, DNS is the gateway's forwarder as before. + dnsServers := []string{"8.8.8.8", "8.8.4.4"} + if w.cfg.L2BridgeEgress { + dnsServers = w.publicDNS() + } + + // Create endpoint on the network. + // + // On NAT, HNS assigns from the NAT subnet as before (no IpConfigurations). + // + // On L2Bridge the container is a peer on the host's LAN, so leaving the + // choice to HNS means an address anywhere in the declared subnet — which on + // a real LAN overlaps the site DHCP server's scope. ephemerd pins the + // endpoint to an address it allocated from the operator's reserved + // network.ip_pool instead, at the LAN's own prefix length so the container's + // route table matches its neighbours'. endpoint := &hcn.HostComputeEndpoint{ Name: id + "-ep", HostComputeNetwork: w.network.Id, Dns: hcn.Dns{ - ServerList: []string{"8.8.8.8", "8.8.4.4"}, + ServerList: dnsServers, }, SchemaVersion: hcn.SchemaVersion{ Major: 2, @@ -94,22 +306,55 @@ func (w *windowsNetworking) setup(ctx context.Context, id string, netns string) }, } + var allocatedIP string + if w.cfg.L2BridgeEgress { + if w.ipam == nil || w.plan == nil { + return nil, fmt.Errorf("L2Bridge egress enabled but no address plan was resolved for %s (refusing to let HNS pick an address that may collide with a DHCP lease)", id) + } + ip, err := w.ipam.allocate(id) + if err != nil { + return nil, fmt.Errorf("allocating an address for %s: %w", id, err) + } + allocatedIP = ip + // PrefixLength is deliberately NOT set: HNS derives it from the + // network's declared subnet. The 2026-08-12 hand proof pinned exactly + // this shape and worked; Microsoft's own sdnbridge CNI also omits it. + // Setting it was the delta in the dead deployment (gateway ARP replies + // dropped by the vSwitch as "Invalid Packet" before reaching the port). + endpoint.IpConfigurations = []hcn.IpConfig{ + { + IpAddress: allocatedIP, + }, + } + } + created, err := w.network.CreateEndpoint(endpoint) if err != nil { + if allocatedIP != "" { + w.ipam.release(id) + } return nil, fmt.Errorf("creating HCN endpoint for %s: %w", id, err) } - // Apply ACL policies to block private network access. This is the ONLY - // egress restriction on Windows (there is no global firewall backstop — - // installFirewallRules is a no-op), so a failure here means the container - // would otherwise run with unrestricted egress to the host LAN, other - // RFC1918 services, and link-local metadata endpoints. Fail CLOSED: tear - // down the endpoint we just created and refuse the job rather than start a - // container we cannot firewall. + // Apply ACL policies to block private network access. On L2Bridge the VFP + // Switch-ACL ladder applied here is the ONLY egress restriction that + // enforces — there is no host-side firewall mechanism that can filter this + // traffic (see the header in firewall_windows.go). A failure here therefore + // means the container would run with unrestricted egress to the host LAN, + // other RFC1918 services, and link-local metadata endpoints. Fail CLOSED: + // tear down the endpoint we just created and refuse the job rather than start + // a container we cannot firewall. + // + // The ACLs are applied to the endpoint BEFORE the container is started + // (setup runs ahead of task creation), and the L2Bridge rule set is STATIC + // — it does not depend on the gateway or DNS (DNS is public, the + // router gets no allow) — so there is no window in which the + // container has LAN connectivity but no filters. if err := w.applyACLPolicies(created); err != nil { if delErr := created.Delete(); delErr != nil { w.cfg.Log.Warn("failed to delete endpoint after ACL failure", "id", id, "error", delErr) } + w.releaseIP(id) return nil, fmt.Errorf("applying egress ACL policies for %s (refusing to start unfirewalled): %w", id, err) } @@ -121,23 +366,38 @@ func (w *windowsNetworking) setup(ctx context.Context, id string, netns string) ns, err = ns.Create() if err != nil { _ = created.Delete() + w.releaseIP(id) return nil, fmt.Errorf("creating HCN namespace for %s: %w", id, err) } if err := hcn.AddNamespaceEndpoint(ns.Id, created.Id); err != nil { _ = ns.Delete() _ = created.Delete() + w.releaseIP(id) return nil, fmt.Errorf("attaching endpoint to namespace for %s: %w", id, err) } - w.cfg.Log.Debug("HCN endpoint created", "id", id, "endpoint", created.Id, "namespace", ns.Id) - return &SetupResult{NetNS: ns.Id, EndpointID: created.Id}, nil + w.cfg.Log.Debug("HCN endpoint created", "id", id, "endpoint", created.Id, "namespace", ns.Id, "ip", allocatedIP) + return &SetupResult{NetNS: ns.Id, EndpointID: created.Id, IP: allocatedIP}, nil +} + +// releaseIP returns a container's pool address, if it holds one. Safe on the NAT +// path (no allocator) and for containers that never got an address. +func (w *windowsNetworking) releaseIP(id string) { + if w.ipam != nil { + w.ipam.release(id) + } } func (w *windowsNetworking) teardown(ctx context.Context, id string, netns string) error { w.mu.Lock() defer w.mu.Unlock() + // Return the pool address whatever happens below: an endpoint that no + // longer exists (or one we fail to delete) must not strand its address, or + // a long-lived daemon would leak the pool one job at a time. + defer w.releaseIP(id) + // Find endpoint by name endpoint, err := hcn.GetEndpointByName(id + "-ep") if err != nil { @@ -172,55 +432,207 @@ var egressBlockedCIDRs = []string{ "169.254.0.0/16", } -// buildEgressBlockPolicies constructs the per-endpoint block ACLs from -// egressBlockedCIDRs. It fails closed: a marshal error on any rule, or an empty -// resulting set, is an error rather than a silently weaker rule set. Split out -// from applyACLPolicies so the (pure) rule construction is unit-testable -// without a live HCN endpoint. -func buildEgressBlockPolicies() ([]hcn.EndpointPolicy, error) { +// VFP Switch-ACL precedence for the L2Bridge egress model. VFP evaluates ACLs +// by Priority, LOWER number = HIGHER precedence (evaluated first, first match +// wins). The ladder is: operator carve-outs (top) > RFC1918 block > allow-any +// (bottom). The two allow-any rules at the bottom are what stop the port from +// default-denying everything — VFP is default-DENY the moment any ACL is +// present. +// +// Every rule in this ladder is address-scoped. See the note in +// buildL2BridgeEgressACLPolicies: mixing in a port-scoped rule with no address +// scope (the removed UDP 67/68 DHCP allows) blackholes the entire port on +// metal. +// +// CRITICAL — priorities below 100 kill the port (proven on metal 2026-08-14, +// l2test bisect harness, Server 2025 build 26100). A Switch-rule ACL with +// Priority < 100 silently kills the endpoint's ENTIRE VFP dataplane: the port +// drops every inbound frame — including the gateway's ARP reply (pktmon shows +// "Invalid Packet" at the vSwitch before the port) — so the container can +// never resolve its next hop and has no connectivity at all, while HNS reports +// the endpoint healthy and applies the policy without error. This is +// independent of the rule's action, direction, or address: an irrelevant +// Block 203.0.113.1/32 at priority 90 reproduces it, and the identical ladder +// with every priority >= 100 works (internet up, RFC1918 blocked). This is +// what broke the 2026-08-13 production deployment (host-allow at 90), and the +// earlier "port-scoped DHCP rules blackhole the port" incident (rules at +// 90/95) was in the same band. NO RULE MAY EVER CARRY Priority < 100; the +// regression tests enforce it. +const ( + // aclPriorityMinimum is the lowest Switch-ACL priority that is safe on + // metal. Rules below this kill the port — see the block comment above. + aclPriorityMinimum uint16 = 100 + // aclPriorityHostAllow carves the ephemerd host's own /32 out ABOVE the + // RFC1918 block. Emitted only when Config.AllowHostAccess is set, which is + // what makes the per-job dind Docker API and the module proxy reachable. + aclPriorityHostAllow uint16 = 100 + // aclPriorityExtraAllow carves configured destinations out ABOVE the + // RFC1918 block. Unused by default (no carve-outs) — reserved for future + // operator-allowed destinations. + aclPriorityExtraAllow uint16 = 150 + // aclPriorityBlock denies the RFC1918 + link-local supernets, whole. No + // gateway/own-subnet carve-out: on L2Bridge the container is a LAN peer, + // so carving the subnet would expose the management plane and the router. + aclPriorityBlock uint16 = 200 + // aclPriorityAllowAny permits everything not blocked above (the internet) + // and, crucially, inbound return traffic. Lowest precedence. + aclPriorityAllowAny uint16 = 65500 +) + +// aclAnyProtocol matches any IP protocol in an HNS ACL ("256"). Every rule in +// the ladder uses it, per the proven metal run. +const aclAnyProtocol = "256" + +// marshalACL serializes one AclPolicySetting into an EndpointPolicy, failing +// closed on a marshal error (a rule we cannot serialize is a rule we cannot +// enforce; never skip it and continue with a weaker set). +func marshalACL(acl hcn.AclPolicySetting) (hcn.EndpointPolicy, error) { + settings, err := json.Marshal(acl) + if err != nil { + return hcn.EndpointPolicy{}, fmt.Errorf("marshaling ACL %+v: %w", acl, err) + } + return hcn.EndpointPolicy{Type: hcn.ACL, Settings: settings}, nil +} + +// buildL2BridgeEgressACLPolicies constructs the router-safe VFP Switch-ACL set +// for an L2Bridge endpoint. It is a pure function (no HCN calls) so the exact +// emitted policy set — actions, directions, remotes, protocols, priorities — is +// unit-testable. Fails closed on any marshal error. +// +// The model matches the Linux end-state (firewall_linux.go): block ALL of +// 10/8, 172.16/12, 192.168/16, 169.254/16 — including the LAN router and the +// container's own subnet — and permit only the internet. DNS is handled out of +// band (public resolvers on the endpoint), so unlike Linux there is no DNS or +// gateway carve-out here, and no container-to-container allow (that would be +// LAN access, which we block). extraAllowed carves additional CIDRs out above +// the block for future use; empty (the default) reproduces the strict posture. +// +// hostIP, when non-empty, adds one further carve-out: a /32 allow for the +// ephemerd host itself. Nothing ephemerd serves TO containers over the network +// works without it — the per-job dind Docker API and the Go module proxy both +// listen on the host address, and a container that cannot address the host +// cannot use either. It is deliberately an address-scoped /32 and not a +// port-scoped rule: see the port-scoping note above; scoping it to the dind +// port would blackhole the whole VFP port. The exposure that opens (every port +// the host has listening, not just ephemerd's) is closed back down at the host +// firewall by l2BridgeControlPlaneRules, which CAN match the container source +// on this path because L2Bridge does not NAT. +func buildL2BridgeEgressACLPolicies(extraAllowed []string, hostIP string) ([]hcn.EndpointPolicy, error) { var policies []hcn.EndpointPolicy + add := func(acl hcn.AclPolicySetting) error { + p, err := marshalACL(acl) + if err != nil { + return err + } + policies = append(policies, p) + return nil + } - for _, cidr := range egressBlockedCIDRs { - if cidr == DefaultSubnet { - continue + // NO DHCP tier. An earlier revision allowed UDP 67/68 (Out with + // RemotePorts, In with LocalPorts, no address scope) at a precedence above + // the block, so a lease/renew could survive the RFC1918 deny. On metal + // those two rules BLACKHOLE THE PORT: with them present nothing egresses at + // all — not the internet, not even a destination carved out by an explicit + // higher-precedence Allow. + // + // Measured on mfl-win-amd64-101 (Server 2025 26100), same endpoint, same + // code path, only these two rules varying: + // + // blocks + allow-any + DHCP rules -> ALL probes fail (1.1.1.1, 8.8.8.8, + // DNS, and every RFC1918 target) + // blocks + allow-any -> exactly the intended posture: + // Grafana/Incus/Proxmox/router/host + // blocked, 1.1.1.1 + 8.8.8.8 + DNS + + // public HTTPS all reachable + // + // HNS accepts the rules (ApplyPolicy returns success) but the resulting VFP + // rule set drops everything, so this fails CLOSED rather than open — it + // breaks jobs instead of leaking. Port-scoped ACLs carrying no address + // scope must not be mixed into this ladder. + // + // Nothing here needs DHCP: the endpoint is addressed by HNS IPAM, not by a + // DHCP client in the container. + + // Tier: the ephemerd host's own /32, when something it serves must be + // reachable (dind, the module proxy). Highest precedence in the ladder. + if hostIP != "" { + if err := add(hcn.AclPolicySetting{ + Protocols: aclAnyProtocol, + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeOut, + RemoteAddresses: hostIP + "/32", + RuleType: hcn.RuleTypeSwitch, + Priority: aclPriorityHostAllow, + }); err != nil { + return nil, err } + } - acl := hcn.AclPolicySetting{ - Protocols: "6,17", // TCP + UDP - Action: hcn.ActionTypeBlock, + // Tier: operator carve-outs (future use). Allowed ABOVE the block so a + // listed destination wins. Empty by default. + for _, cidr := range extraAllowed { + if err := add(hcn.AclPolicySetting{ + Protocols: aclAnyProtocol, + Action: hcn.ActionTypeAllow, Direction: hcn.DirectionTypeOut, RemoteAddresses: cidr, RuleType: hcn.RuleTypeSwitch, - Priority: 100, + Priority: aclPriorityExtraAllow, + }); err != nil { + return nil, err } + } - settings, err := json.Marshal(acl) - if err != nil { - // Fail closed: a rule we cannot serialize is a rule we cannot - // enforce. Do not skip it and continue with a weaker rule set. - return nil, fmt.Errorf("marshaling egress block ACL for %s: %w", cidr, err) + // Tier: block the RFC1918 + link-local supernets, whole. No carve-out. + for _, cidr := range egressBlockedCIDRs { + if err := add(hcn.AclPolicySetting{ + Protocols: aclAnyProtocol, + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeOut, + RemoteAddresses: cidr, + RuleType: hcn.RuleTypeSwitch, + Priority: aclPriorityBlock, + }); err != nil { + return nil, err } - - policies = append(policies, hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: settings, - }) } - if len(policies) == 0 { - // Nothing to block would mean no egress restriction at all — treat as - // an error so the caller refuses to start an unfirewalled container. - return nil, fmt.Errorf("no egress block ACLs constructed (would run unfirewalled)") + // Tier: allow-any Out AND In (lowest precedence). BOTH are mandatory: + // without them the port default-denies everything (internet included), and + // the INBOUND allow is required or TCP return traffic (SYN-ACK) is dropped + // and even permitted destinations fail. + for _, dir := range []hcn.DirectionType{hcn.DirectionTypeOut, hcn.DirectionTypeIn} { + if err := add(hcn.AclPolicySetting{ + Protocols: aclAnyProtocol, + Action: hcn.ActionTypeAllow, + Direction: dir, + RemoteAddresses: "0.0.0.0/0", + RuleType: hcn.RuleTypeSwitch, + Priority: aclPriorityAllowAny, + }); err != nil { + return nil, err + } } return policies, nil } -// applyACLPolicies blocks container traffic to RFC 1918 and link-local ranges. -// The full rule set is built up front and applied atomically; any failure is -// returned so the caller (setup) can treat it as fatal for the job. +// applyACLPolicies applies the per-endpoint egress ACLs. Only the L2Bridge path +// has a working enforcement point: it applies the router-safe VFP ladder +// (buildL2BridgeEgressACLPolicies), built up front and applied atomically, with +// any failure returned so the caller (setup) can treat it as fatal for the job. +// +// On the NAT path there is nothing to apply: block ACLs on the NAT vSwitch port +// were inert on metal (they never filtered the NAT'd egress), and no host-side +// mechanism can filter NAT container egress on this stack (see the header in +// firewall_windows.go). NAT egress is unfiltered by design — installFirewallRules +// logs that gap and points the operator at network.l2bridge_egress. func (w *windowsNetworking) applyACLPolicies(endpoint *hcn.HostComputeEndpoint) error { - policies, err := buildEgressBlockPolicies() + if !w.cfg.L2BridgeEgress { + return nil + } + + policies, err := buildL2BridgeEgressACLPolicies(w.cfg.ExtraAllowedCIDRs, w.hostAllowIP()) if err != nil { return err } @@ -230,6 +642,17 @@ func (w *windowsNetworking) applyACLPolicies(endpoint *hcn.HostComputeEndpoint) }) } +// hostAllowIP returns the host address to carve out of the egress block, or "" +// for the strict posture in which the host is unreachable like the rest of +// RFC1918. Non-empty only when the operator runs something containers must +// reach (dind, the module proxy) AND the address plan resolved. +func (w *windowsNetworking) hostAllowIP() string { + if w.cfg.AllowHostAccess && w.plan != nil { + return w.plan.HostIP + } + return "" +} + // installFirewallRules and removeFirewallRules live in firewall_windows.go // (mirroring firewall_linux.go): the host-global Windows Firewall backstop // that complements the per-endpoint ACLs applied above. diff --git a/pkg/networking/network_windows_test.go b/pkg/networking/network_windows_test.go index d6642fc2..bf26e627 100644 --- a/pkg/networking/network_windows_test.go +++ b/pkg/networking/network_windows_test.go @@ -4,52 +4,357 @@ package networking import ( "encoding/json" + "net" + "strings" "testing" "github.com/Microsoft/hcsshim/hcn" ) -// TestBuildEgressBlockPolicies verifies the fail-closed egress rule set (WIN-4). -// Every RFC1918 + link-local range must produce an outbound Block ACL; a -// partial or empty set would let a job reach the host LAN / other tenants / -// cloud metadata. -func TestBuildEgressBlockPolicies(t *testing.T) { - policies, err := buildEgressBlockPolicies() - if err != nil { - t.Fatalf("buildEgressBlockPolicies: %v", err) - } - - // Collect the CIDRs that actually became block rules. - got := map[string]bool{} - for _, p := range policies { +// decodeACLs unmarshals every EndpointPolicy back into an AclPolicySetting, +// failing the test on a non-ACL policy or a malformed setting. +func decodeACLs(t *testing.T, policies []hcn.EndpointPolicy) []hcn.AclPolicySetting { + t.Helper() + acls := make([]hcn.AclPolicySetting, 0, len(policies)) + for i, p := range policies { if p.Type != hcn.ACL { - t.Errorf("policy type = %v, want ACL", p.Type) + t.Errorf("policy[%d] type = %v, want ACL", i, p.Type) } var acl hcn.AclPolicySetting if err := json.Unmarshal(p.Settings, &acl); err != nil { - t.Fatalf("unmarshal ACL setting: %v", err) + t.Fatalf("unmarshal ACL[%d]: %v", i, err) } - if acl.Action != hcn.ActionTypeBlock { - t.Errorf("CIDR %s action = %v, want Block", acl.RemoteAddresses, acl.Action) - } - if acl.Direction != hcn.DirectionTypeOut { - t.Errorf("CIDR %s direction = %v, want Out", acl.RemoteAddresses, acl.Direction) + acls = append(acls, acl) + } + return acls +} + +// TestL2BridgeEgressACLPolicies_LadderShape pins the full router-safe VFP +// ladder: the two mandatory allow-any rules (Out+In) and a whole-supernet block +// for every RFC1918 + link-local range — with the exact actions, directions, +// protocols, and priorities that were proven on metal. +func TestL2BridgeEgressACLPolicies_LadderShape(t *testing.T) { + acls := decodeACLs(t, mustBuildL2Bridge(t, nil, "")) + + var ( + allowAnyOut, allowAnyIn bool + blockedSupernets = map[string]hcn.AclPolicySetting{} + ) + for _, a := range acls { + switch { + case a.Action == hcn.ActionTypeAllow && a.RemoteAddresses == "0.0.0.0/0" && a.Direction == hcn.DirectionTypeOut: + allowAnyOut = true + assertACL(t, "allow-any-out", a, aclAnyProtocol, aclPriorityAllowAny) + case a.Action == hcn.ActionTypeAllow && a.RemoteAddresses == "0.0.0.0/0" && a.Direction == hcn.DirectionTypeIn: + allowAnyIn = true + assertACL(t, "allow-any-in", a, aclAnyProtocol, aclPriorityAllowAny) + case a.Action == hcn.ActionTypeBlock: + blockedSupernets[a.RemoteAddresses] = a } - got[acl.RemoteAddresses] = true } - // Every configured range (minus the container's own subnet) must be blocked. + if !allowAnyOut || !allowAnyIn { + t.Errorf("missing mandatory allow-any rule: out=%v in=%v (both required or the port default-denies / drops SYN-ACK)", allowAnyOut, allowAnyIn) + } + + // Every RFC1918 + link-local supernet must be blocked, whole, Out, at the + // block priority. for _, cidr := range egressBlockedCIDRs { - if cidr == DefaultSubnet { + a, ok := blockedSupernets[cidr] + if !ok { + t.Errorf("supernet %s is not blocked", cidr) + continue + } + assertACL(t, "block "+cidr, a, aclAnyProtocol, aclPriorityBlock) + if a.Direction != hcn.DirectionTypeOut { + t.Errorf("block %s direction = %v, want Out", cidr, a.Direction) + } + } + if len(blockedSupernets) != len(egressBlockedCIDRs) { + t.Errorf("got %d block rules, want %d (one per supernet, no extras)", len(blockedSupernets), len(egressBlockedCIDRs)) + } +} + +// TestL2BridgeEgressACLPolicies_NoGatewayOrSubnetCarveOut pins the load-bearing +// safety property that differs from NAT: on L2Bridge the container is a LAN +// peer, so there must be NO carve-out — no allow for any RFC1918 address, and no +// block that excludes a subnet. The router and the management plane stay +// unreachable, matching the Linux end-state. +func TestL2BridgeEgressACLPolicies_NoGatewayOrSubnetCarveOut(t *testing.T) { + acls := decodeACLs(t, mustBuildL2Bridge(t, nil, "")) + + for _, a := range acls { + // The only allows permitted with default (empty) extraAllowed are the + // two allow-any (0.0.0.0/0) rules. + if a.Action != hcn.ActionTypeAllow { continue } - if !got[cidr] { - t.Errorf("egress range %s was not turned into a block rule", cidr) + if a.RemoteAddresses != "0.0.0.0/0" { + t.Errorf("unexpected allow carve-out to %q (only 0.0.0.0/0 is permitted with no extra-allowed configured)", a.RemoteAddresses) } } - // Link-local / metadata range must always be present. - if !got["169.254.0.0/16"] { - t.Error("link-local/metadata range 169.254.0.0/16 not blocked") + // Blocks must be the whole supernets, never a range/exclusion. + for _, a := range acls { + if a.Action == hcn.ActionTypeBlock && a.RemoteAddresses != "" { + if _, _, err := net.ParseCIDR(a.RemoteAddresses); err != nil { + t.Errorf("block RemoteAddresses %q is not a plain CIDR (a carve-out range leaked in): %v", a.RemoteAddresses, err) + } + } + } +} + +// TestL2BridgeEgressACLPolicies_Precedence pins the priority ladder: operator +// carve-outs must sit ABOVE the RFC1918 block (lower number = higher +// precedence), which must sit above the allow-any floor. +func TestL2BridgeEgressACLPolicies_Precedence(t *testing.T) { + if !(aclPriorityExtraAllow < aclPriorityBlock && + aclPriorityBlock < aclPriorityAllowAny) { + t.Fatalf("priority ladder broken: extra=%d block=%d allowany=%d (want extra < block < allowany)", + aclPriorityExtraAllow, aclPriorityBlock, aclPriorityAllowAny) + } + if aclPriorityHostAllow >= aclPriorityBlock { + t.Fatalf("host allow (%d) must outrank the RFC1918 block (%d) or dind/module-proxy traffic is denied", + aclPriorityHostAllow, aclPriorityBlock) + } +} + +// TestL2BridgeEgressACLPolicies_NoRuleBelowPriority100 is a regression guard +// for the metal finding that killed the 2026-08-13 production deployment: a +// Switch-rule ACL with Priority < 100 silently kills the endpoint's ENTIRE +// VFP dataplane on Server 2025 (build 26100). The port drops every inbound +// frame — including the gateway's ARP reply — so the container never resolves +// its next hop and has no connectivity, while HNS applies the policy without +// error and reports the endpoint healthy. Proven independent of the rule's +// action, direction, and address by the l2test bisect harness (an irrelevant +// Block 203.0.113.1/32 at priority 90 reproduces it; the identical ladder +// with every priority >= 100 works). NO rule may ever carry Priority < 100. +func TestL2BridgeEgressACLPolicies_NoRuleBelowPriority100(t *testing.T) { + for _, p := range []uint16{aclPriorityHostAllow, aclPriorityExtraAllow, aclPriorityBlock, aclPriorityAllowAny} { + if p < aclPriorityMinimum { + t.Fatalf("ladder constant %d is below the safe minimum %d (kills the VFP port on metal)", p, aclPriorityMinimum) + } + } + for _, extra := range [][]string{nil, {"203.0.113.0/24"}} { + for _, hostIP := range []string{"", "198.51.100.7"} { + for _, a := range decodeACLs(t, mustBuildL2Bridge(t, extra, hostIP)) { + if a.Priority < aclPriorityMinimum { + t.Errorf("emitted ACL with priority %d < %d (kills the VFP port on metal): %+v", + a.Priority, aclPriorityMinimum, a) + } + } + } + } +} + +// TestL2BridgeEgressACLPolicies_EveryRuleIsAddressScoped is a regression guard +// for the metal finding that motivated removing the DHCP allows: a Switch ACL +// carrying only a port scope (no RemoteAddresses/LocalAddresses) blackholes the +// entire VFP port. HNS accepts such a rule, then nothing egresses at all — not +// the internet, not even an explicitly allowed destination. Keep every rule in +// this ladder address-scoped. +func TestL2BridgeEgressACLPolicies_EveryRuleIsAddressScoped(t *testing.T) { + for _, extra := range [][]string{nil, {"203.0.113.0/24"}} { + for _, a := range decodeACLs(t, mustBuildL2Bridge(t, extra, "198.51.100.7")) { + if a.RemoteAddresses == "" && a.LocalAddresses == "" { + t.Errorf("ACL with no address scope: %+v (blackholes the whole port on metal)", a) + } + if a.RemotePorts != "" || a.LocalPorts != "" { + t.Errorf("port-scoped ACL in the L2Bridge ladder: %+v", a) + } + } + } +} + +// TestL2BridgeEgressACLPolicies_ExtraAllowed verifies configured carve-outs are +// emitted as Out allows ABOVE the block so they win over the RFC1918 deny. +func TestL2BridgeEgressACLPolicies_ExtraAllowed(t *testing.T) { + extra := []string{"192.168.50.10/32", "172.20.0.0/16"} + acls := decodeACLs(t, mustBuildL2Bridge(t, extra, "")) + + found := map[string]hcn.AclPolicySetting{} + for _, a := range acls { + if a.Action == hcn.ActionTypeAllow && a.RemoteAddresses != "" && a.RemoteAddresses != "0.0.0.0/0" { + found[a.RemoteAddresses] = a + } + } + for _, cidr := range extra { + a, ok := found[cidr] + if !ok { + t.Errorf("extra-allowed %s not emitted", cidr) + continue + } + if a.Direction != hcn.DirectionTypeOut || a.Priority != aclPriorityExtraAllow { + t.Errorf("extra-allowed %s = %+v, want Out priority %d", cidr, a, aclPriorityExtraAllow) + } + if a.Priority >= aclPriorityBlock { + t.Errorf("extra-allowed %s priority %d not above block %d (would not win)", cidr, a.Priority, aclPriorityBlock) + } + } +} + +func mustBuildL2Bridge(t *testing.T, extra []string, hostIP string) []hcn.EndpointPolicy { + t.Helper() + policies, err := buildL2BridgeEgressACLPolicies(extra, hostIP) + if err != nil { + t.Fatalf("buildL2BridgeEgressACLPolicies: %v", err) + } + return policies +} + +func assertACL(t *testing.T, name string, a hcn.AclPolicySetting, wantProto string, wantPrio uint16) { + t.Helper() + if a.Protocols != wantProto { + t.Errorf("%s protocol = %q, want %q", name, a.Protocols, wantProto) + } + if a.Priority != wantPrio { + t.Errorf("%s priority = %d, want %d", name, a.Priority, wantPrio) + } + if a.RuleType != hcn.RuleTypeSwitch { + t.Errorf("%s ruletype = %q, want Switch", name, a.RuleType) + } +} + +// TestL2BridgeEgressACLPolicies_HostAllow verifies the one carve-out that keeps +// jobs provisionable: with a host address supplied, the ladder emits an +// address-scoped /32 Allow ABOVE the RFC1918 block. Without it the container +// cannot reach the per-job dind Docker API (DOCKER_HOST) or the module proxy, +// both of which listen on the host address — the host is inside 192.168/16 or +// 10/8 like every other LAN peer. +func TestL2BridgeEgressACLPolicies_HostAllow(t *testing.T) { + const hostIP = "198.51.100.7" + acls := decodeACLs(t, mustBuildL2Bridge(t, nil, hostIP)) + + var found *hcn.AclPolicySetting + for i, a := range acls { + if a.Action == hcn.ActionTypeAllow && a.RemoteAddresses == hostIP+"/32" { + found = &acls[i] + } + } + if found == nil { + t.Fatalf("no /32 allow for the host address %s (dind and the module proxy would be unreachable)", hostIP) + } + assertACL(t, "host-allow", *found, aclAnyProtocol, aclPriorityHostAllow) + if found.Direction != hcn.DirectionTypeOut { + t.Errorf("host allow direction = %v, want Out", found.Direction) + } + if found.Priority >= aclPriorityBlock { + t.Errorf("host allow priority %d is not above the block priority %d (it would never win)", found.Priority, aclPriorityBlock) + } + // It must stay address-scoped. Narrowing it to the dind port would look + // safer and would in fact blackhole the whole VFP port (see the metal note + // on the removed DHCP rules). + if found.RemotePorts != "" || found.LocalPorts != "" { + t.Errorf("host allow is port-scoped (%+v); port-scoped Switch ACLs blackhole the VFP port", *found) + } +} + +// TestL2BridgeEgressACLPolicies_NoHostAllowByDefault pins the strict default: +// with no host address supplied (nothing ephemerd serves to containers), the +// ephemerd host is blocked along with the rest of RFC1918, exactly as verified +// on metal. +func TestL2BridgeEgressACLPolicies_NoHostAllowByDefault(t *testing.T) { + for _, a := range decodeACLs(t, mustBuildL2Bridge(t, nil, "")) { + if a.Action == hcn.ActionTypeAllow && a.RemoteAddresses != "0.0.0.0/0" { + t.Errorf("unexpected allow %q with no host access configured (want only 0.0.0.0/0)", a.RemoteAddresses) + } + } +} + +// TestL2BridgeControlPlaneRules verifies the host-firewall backstop that closes +// the control-plane ports back off after the host /32 allow opens the host. +// These are INBOUND rules scoped to the container pool as the remote — which +// only matches because L2Bridge does not NAT (on NAT the host sees its own +// address as the source; that is what sank #136). +func TestL2BridgeControlPlaneRules(t *testing.T) { + const ( + hostIP = "198.51.100.7" + pool = "198.51.100.200-198.51.100.230" + ) + rules := l2BridgeControlPlaneRules(hostIP, pool, []int{10000, 10001, 10002}) + if len(rules) != 3 { + t.Fatalf("got %d rules, want 3 (one per control port)", len(rules)) + } + for _, r := range rules { + if !strings.HasPrefix(r.name, firewallRulePrefix) { + t.Errorf("rule %q lacks the %q prefix, so Cleanup would not find it", r.name, firewallRulePrefix) + } + if specValue(r, "dir") != "in" { + t.Errorf("rule %s dir = %q, want in (container->host is inbound at the host)", r.name, specValue(r, "dir")) + } + if specValue(r, "action") != "block" { + t.Errorf("rule %s action = %q, want block", r.name, specValue(r, "action")) + } + if specValue(r, "localip") != hostIP { + t.Errorf("rule %s localip = %q, want %s", r.name, specValue(r, "localip"), hostIP) + } + if specValue(r, "remoteip") != pool { + t.Errorf("rule %s remoteip = %q, want the container pool %s", r.name, specValue(r, "remoteip"), pool) + } + if specValue(r, "localport") == "" { + t.Errorf("rule %s has no localport; a portless block would cut the host off the container entirely", r.name) + } + } +} + +// TestL2BridgeControlPlaneRules_NoPlanNoRules verifies the backstop stays silent +// when there is nothing to scope it to, rather than emitting a rule with an +// empty address that would match everything. +func TestL2BridgeControlPlaneRules_NoPlanNoRules(t *testing.T) { + if rules := l2BridgeControlPlaneRules("", "198.51.100.0/24", []int{10000}); rules != nil { + t.Errorf("rules emitted with no host address: %+v", rules) + } + if rules := l2BridgeControlPlaneRules("198.51.100.7", "", []int{10000}); rules != nil { + t.Errorf("rules emitted with no pool: %+v", rules) + } + if rules := l2BridgeControlPlaneRules("198.51.100.7", "198.51.100.0/24", nil); len(rules) != 0 { + t.Errorf("rules emitted with no control ports: %+v", rules) + } +} + +// TestWindowsHostAddr_L2BridgeReportsHostIP is the regression guard for the +// binding that took down provisioning on the ICS experiment: on L2Bridge there +// is no 10.88.0.1 on any interface, so anything that binds GatewayIP() — the +// per-job dind listener and the Go module proxy — must be handed the host's own +// LAN address instead. +func TestWindowsHostAddr_L2BridgeReportsHostIP(t *testing.T) { + w := &windowsNetworking{ + cfg: Config{L2BridgeEgress: true}, + plan: &l2BridgePlan{HostIP: "198.51.100.7"}, + } + if got := w.hostAddr(); got != "198.51.100.7" { + t.Errorf("hostAddr() = %q, want the host's L2Bridge address", got) + } + m := &Manager{cfg: Config{}, platform: w} + if got := m.GatewayIP(); got != "198.51.100.7" { + t.Errorf("GatewayIP() = %q, want the host's L2Bridge address (10.88.0.1 binds to nothing on L2Bridge)", got) + } + + // NAT path unchanged: no plan, no override, the old gateway still applies. + nat := &windowsNetworking{cfg: Config{}} + if got := nat.hostAddr(); got != "" { + t.Errorf("NAT hostAddr() = %q, want empty (generic derivation)", got) + } + if got := (&Manager{cfg: Config{}, platform: nat}).GatewayIP(); got != defaultGateway { + t.Errorf("NAT GatewayIP() = %q, want %q", got, defaultGateway) + } +} + +// TestWindowsHostAllowIP_GatedOnAllowHostAccess pins that the host carve-out is +// opt-in: it appears only when ephemerd actually serves something to containers. +func TestWindowsHostAllowIP_GatedOnAllowHostAccess(t *testing.T) { + plan := &l2BridgePlan{HostIP: "198.51.100.7"} + off := &windowsNetworking{cfg: Config{L2BridgeEgress: true}, plan: plan} + if got := off.hostAllowIP(); got != "" { + t.Errorf("hostAllowIP() = %q with AllowHostAccess=false, want empty (strict posture)", got) + } + on := &windowsNetworking{cfg: Config{L2BridgeEgress: true, AllowHostAccess: true}, plan: plan} + if got := on.hostAllowIP(); got != "198.51.100.7" { + t.Errorf("hostAllowIP() = %q with AllowHostAccess=true, want the host address", got) + } + // No plan (init failed or NAT) must never produce a carve-out. + none := &windowsNetworking{cfg: Config{L2BridgeEgress: true, AllowHostAccess: true}} + if got := none.hostAllowIP(); got != "" { + t.Errorf("hostAllowIP() = %q with no resolved plan, want empty", got) } } diff --git a/pkg/networking/networking.go b/pkg/networking/networking.go index 1874cc12..73974134 100644 --- a/pkg/networking/networking.go +++ b/pkg/networking/networking.go @@ -29,6 +29,37 @@ type Config struct { // dispatch server listening on the bridge). ControlPorts []int + // The fields below configure the Windows L2Bridge egress path (see + // network_windows.go and l2bridge.go). They are ignored on Linux/macOS. + // When L2BridgeEgress is false (the default), Windows uses the HNS NAT + // network and none of them are consulted. + // + // L2BridgeEgress is the opt-in. HostNIC (the host adapter to bridge onto) + // and IPPool (the reserved LAN range container addresses come from) are + // REQUIRED when it is set; Subnet and Gateway are derived from HostNIC when + // empty. PublicDNS defaults to public resolvers so container DNS never needs + // the blocked LAN router. ExtraAllowedCIDRs carves destinations out above + // the RFC1918 block. + L2BridgeEgress bool + HostNIC string + IPPool string + Gateway string + PublicDNS []string + ExtraAllowedCIDRs []string + + // AllowHostAccess permits job containers to address the ephemerd host + // itself on the L2Bridge path. Required by anything ephemerd serves TO + // containers over the network — the per-job dind Docker API and the Go + // module proxy both listen on the host address — because the egress ACLs + // otherwise block the host along with the rest of RFC1918. + // + // It is an address-scoped /32 allow, so it opens every port the host has + // listening, not just ephemerd's. The control-plane ports are blocked back + // off at the host firewall (see l2BridgeControlPlaneRules), which CAN match + // on the container source here because L2Bridge does not NAT. Left false + // when nothing needs to be reachable, which is the strictest posture. + AllowHostAccess bool + Log *slog.Logger } @@ -134,6 +165,26 @@ type platformNetworking interface { installFirewallRules() error removeFirewallRules() cleanup() + + // hostAddr returns the host address containers reach ephemerd's own + // services on, when the platform knows it better than GatewayIP's + // subnet arithmetic does. Empty means "use the generic derivation". + // + // This exists for the Windows L2Bridge path, where containers are LAN + // peers and the reachable host address is the host's own adapter address + // — not the .1 of any container subnet. + hostAddr() string + + // openHostPort / closeHostPort open and close a scoped host-firewall + // inbound allow for one TCP port, from the container pool to the host. + // Needed only on the Windows L2Bridge path: the VFP host /32 allow lets a + // container's packet leave its port toward the host, but the host's own + // inbound Windows Firewall default-denies it, so per-job services ephemerd + // binds on the host (the dind Docker API, the module proxy) are otherwise + // unreachable. Scoped to remoteip=, localport= so ONLY that + // service opens — RDP/SMB/RPC stay blocked. No-op on NAT and non-Windows. + openHostPort(port int) error + closeHostPort(port int) } // New creates and initializes the networking manager for the current platform. @@ -159,11 +210,23 @@ func (m *Manager) Teardown(ctx context.Context, id string, netns string) error { return m.platform.teardown(ctx, id, netns) } -// GatewayIP returns the bridge gateway IP address (e.g., "10.88.0.1"). -// This is the first usable IP in the container subnet, reachable from -// inside containers. Used by services that need to be accessible to jobs -// (e.g., Go module proxy, DNS). +// GatewayIP returns the host address that services ephemerd runs for jobs must +// bind to in order to be reachable from inside containers — the Go module proxy +// and, on Windows, the per-job dind Docker API listener. +// +// Normally that is the bridge gateway (e.g. "10.88.0.1"), the first usable +// address of the container subnet. On the Windows L2Bridge path there is no +// such bridge gateway: containers are peers on the host's LAN, so the platform +// reports the host's own adapter address instead. Binding to the old hard-coded +// 10.88.0.1 there would fail outright — no interface holds that address once the +// NAT network is out of the picture — and take dind provisioning down with it. func (m *Manager) GatewayIP() string { + if m.platform != nil { + if addr := m.platform.hostAddr(); addr != "" { + return addr + } + } + subnet := m.cfg.Subnet if subnet == "" { subnet = DefaultSubnet @@ -185,6 +248,25 @@ func (m *Manager) InstallFirewallRules() error { return m.platform.installFirewallRules() } +// OpenHostPort opens a scoped host-firewall inbound allow for one TCP port from +// the container pool to the host, so a per-job service ephemerd binds on the +// host (dind's Docker API, the module proxy) is reachable from job containers. +// Only the Windows L2Bridge path does anything; elsewhere it is a no-op. Pair +// with CloseHostPort on teardown. +func (m *Manager) OpenHostPort(port int) error { + if m.platform == nil { + return nil + } + return m.platform.openHostPort(port) +} + +// CloseHostPort removes an allow previously added by OpenHostPort. +func (m *Manager) CloseHostPort(port int) { + if m.platform != nil { + m.platform.closeHostPort(port) + } +} + // Cleanup removes all networking state: firewall rules, bridge interface, // CNI config, IP allocations, and DNS files. Called on shutdown. func (m *Manager) Cleanup() { diff --git a/pkg/runtime/provisioning_test.go b/pkg/runtime/provisioning_test.go new file mode 100644 index 00000000..5c333665 --- /dev/null +++ b/pkg/runtime/provisioning_test.go @@ -0,0 +1,54 @@ +package runtime + +import "testing" + +// TestProvisioning_InFlightIDsSurviveSweep is the regression guard for the +// 2026-08-14 metal race: the orphan sweep decides "orphan" by the absence of a +// containerd container, but a job's runner-dir copy and workdir exist for the +// whole provisioning window (copyDirForJob → NewContainer), which on a cold +// Windows image pull is minutes long. A sweep firing in that window must NOT +// treat the in-flight job as an orphan, or it deletes a live job's runner dir +// and corrupts it into a self-update loop. +func TestProvisioning_InFlightIDsSurviveSweep(t *testing.T) { + r := &Runtime{} + const id = "ephemerd-github-ephpm-live_shannon" + + done := r.beginProvisioning(id) + + // The sweep builds its keep set from live containerd containers (none here, + // mirroring the window before NewContainer runs) and then unions in-flight + // provisioning IDs. + keep := map[string]struct{}{} + r.addProvisioning(keep) + if _, ok := keep[id]; !ok { + t.Fatalf("in-flight provisioning id %q missing from the sweep keep set — its runner dir would be deleted mid-provision", id) + } + + // After provisioning completes the container exists in containerd and the + // in-flight guard is released; the ID no longer needs the provisioning set. + done() + keep2 := map[string]struct{}{} + r.addProvisioning(keep2) + if _, ok := keep2[id]; ok { + t.Errorf("id %q still marked in-flight after done() — leak", id) + } +} + +// TestProvisioning_ConcurrentJobsIndependent ensures one job finishing +// provisioning does not unguard another still in flight. +func TestProvisioning_ConcurrentJobsIndependent(t *testing.T) { + r := &Runtime{} + doneA := r.beginProvisioning("job-a") + _ = r.beginProvisioning("job-b") // still in flight + + doneA() + + keep := map[string]struct{}{} + r.addProvisioning(keep) + if _, ok := keep["job-a"]; ok { + t.Errorf("job-a should be released") + } + if _, ok := keep["job-b"]; !ok { + t.Errorf("job-b must still be guarded while in flight") + } +} diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 7f540fb0..a37369d9 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -138,6 +138,45 @@ type Runtime struct { cfg Config client *client.Client pullMu sync.Mutex // serializes image pulls to avoid content store contention + + // provisioning holds the IDs of jobs whose on-disk state (runner-dir copy, + // job workdir, snapshot) exists but whose container is not yet registered + // in containerd — i.e. jobs somewhere between the copyDirForJob and + // NewContainer calls in Create. The orphan sweep decides "orphan" by the + // absence of a containerd container, so without this a sweep that fires + // during the provisioning window (a Windows image pull can take minutes) + // deletes a live job's runner dir out from under it, corrupting it into a + // self-update loop (observed on metal 2026-08-14). SweepOrphans unions + // these IDs into its keep set. + provMu sync.Mutex + provisioning map[string]struct{} +} + +// beginProvisioning marks id as in-flight so the orphan sweep will not reclaim +// its on-disk state before its container exists. The returned func clears it +// and must be deferred by the caller. +func (r *Runtime) beginProvisioning(id string) func() { + r.provMu.Lock() + if r.provisioning == nil { + r.provisioning = make(map[string]struct{}) + } + r.provisioning[id] = struct{}{} + r.provMu.Unlock() + return func() { + r.provMu.Lock() + delete(r.provisioning, id) + r.provMu.Unlock() + } +} + +// addProvisioning inserts the in-flight provisioning IDs into keep so a sweep +// preserves their on-disk state. +func (r *Runtime) addProvisioning(keep map[string]struct{}) { + r.provMu.Lock() + defer r.provMu.Unlock() + for id := range r.provisioning { + keep[id] = struct{}{} + } } // Client returns the underlying containerd client. Used by the in-VM @@ -277,6 +316,8 @@ func (r *Runtime) SweepOrphans(ctx context.Context) error { for _, c := range containers { live[c.ID()] = struct{}{} } + // Jobs mid-provision have on-disk state but no container yet: keep them. + r.addProvisioning(live) return r.sweepOrphanState(ctx, live) } @@ -671,6 +712,14 @@ func (r *Runtime) Create(ctx context.Context, cfg CreateConfig) (*RunnerEnv, err jitConfig := cfg.JITConfig ctx = namespaces.WithNamespace(ctx, namespace) + // Protect this job's on-disk state (runner-dir copy, job workdir, snapshot) + // from the orphan sweep for the whole provisioning window — from here until + // Create returns. Until NewContainer runs there is no containerd container + // for the sweep to key off, so without this a sweep firing mid-provision + // (an image pull can take minutes) would delete a live job's runner dir. + doneProvisioning := r.beginProvisioning(id) + defer doneProvisioning() + // Use a default image when no custom image is specified. // If runner.default_image is set in config, use that. // Otherwise: Linux uses the official GHA runner image, diff --git a/pkg/workflow/runner.go b/pkg/workflow/runner.go index 23a36b0a..911a798c 100644 --- a/pkg/workflow/runner.go +++ b/pkg/workflow/runner.go @@ -24,11 +24,32 @@ const ( defaultImage = "ghcr.io/actions/actions-runner:latest" ) +// NetworkOptions carries the host's [network] configuration into a local run. +// +// Without it a local run silently built its own default network — on Windows an +// HNS NAT network plus the NAT-era netsh host-firewall rules — even on a host +// configured for L2Bridge egress. That was wrong twice over: the container came +// up unfiltered on NAT rather than on the enforcing L2Bridge path, and the netsh +// rules block RFC1918 host-wide, which severs the host's own DNS when its +// resolver is a LAN address. Mirrors the fields serve passes to networking.New. +type NetworkOptions struct { + Subnet string + MTU int + L2BridgeEgress bool + HostNIC string + IPPool string + Gateway string + PublicDNS []string + ExtraAllowedCIDRs []string + AllowHostAccess bool +} + // Runner executes workflow jobs locally using embedded containerd. type Runner struct { DataDir string SocketPath string // optional: containerd socket override for isolation from the service Image string // container image; empty falls back to defaultImage + Network NetworkOptions Log *slog.Logger } @@ -67,11 +88,23 @@ func (r *Runner) RunJob(ctx context.Context, jobName string, job Job, repoDir st return fmt.Errorf("extracting CNI plugins: %w", err) } - // Initialize networking + // Initialize networking. A local run joins whatever network the host is + // configured for rather than inventing its own — on the L2Bridge path it + // adopts the existing network and reserves the addresses of endpoints + // already on it, so it cannot hand a job an address the service is using. net, err := networking.New(networking.Config{ - DataDir: r.DataDir, - CNIBinDir: cm.Dir(), - Log: r.Log, + DataDir: r.DataDir, + CNIBinDir: cm.Dir(), + Subnet: r.Network.Subnet, + MTU: r.Network.MTU, + L2BridgeEgress: r.Network.L2BridgeEgress, + HostNIC: r.Network.HostNIC, + IPPool: r.Network.IPPool, + Gateway: r.Network.Gateway, + PublicDNS: r.Network.PublicDNS, + ExtraAllowedCIDRs: r.Network.ExtraAllowedCIDRs, + AllowHostAccess: r.Network.AllowHostAccess, + Log: r.Log, }) if err != nil { return fmt.Errorf("initializing networking: %w", err)