diff --git a/Documentation/guides/fork_vfork_migration.rst b/Documentation/guides/fork_vfork_migration.rst index fa9a305d3c8e6..d859e1ab59333 100644 --- a/Documentation/guides/fork_vfork_migration.rst +++ b/Documentation/guides/fork_vfork_migration.rst @@ -58,8 +58,9 @@ Two things break, and they break loudly rather than quietly: environment no longer builds.** ``fork()`` is not declared in ``unistd.h`` there, so you get a compile error naming the function. That is the intended outcome: a build error is strictly better than the silent wrongness it -replaces. Today that is every in-tree architecture, so every caller of -``fork()`` has to be looked at. +replaces. Today that is every configuration without a per-process address +environment -- every flat and every protected build -- so most callers of +``fork()`` have to be looked at. **Code calling** ``fork()`` **on a target that does have real** ``fork()`` **changes behaviour** -- from sharing to copying. Code that (perhaps @@ -211,7 +212,7 @@ Known gaps complete -- ``addrenv_fork()``, the ``up_addrenv_fork()`` hook, the syscall, the libc wrapper and the ``ostest`` case -- so an architecture provides ``fork()`` by implementing ``up_addrenv_fork()`` and selecting ``CONFIG_ARCH_HAVE_FORK``, -with no further generic work. +with no further generic work. RISC-V selects it today. **A windowed ABI needs its stack rebased, not just copied.** On Xtensa, giving a child a relocated copy of the parent's stack takes more than the copy: diff --git a/arch/Kconfig b/arch/Kconfig index 60039f1991e4e..72d47c8d4d39b 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -83,6 +83,7 @@ config ARCH_RENESAS config ARCH_RISCV bool "RISC-V" + select ARCH_HAVE_FORK if BUILD_KERNEL && ARCH_ADDRENV && !ARCH_STACK_DYNAMIC select ARCH_HAVE_BACKTRACE select ARCH_HAVE_CPUINFO select ARCH_HAVE_INTERRUPTSTACK @@ -493,7 +494,7 @@ config ARCH_HAVE_VFORK config ARCH_HAVE_FORK bool default n - depends on ARCH_ADDRENV + depends on ARCH_ADDRENV && !ARCH_STACK_DYNAMIC ---help--- The architecture can implement POSIX fork(): the child receives its own copy of the parent's memory at the same virtual addresses, may @@ -505,13 +506,19 @@ config ARCH_HAVE_FORK allocated pages holding a copy of the parent's contents, mapped at the same virtual addresses. - No architecture selects this yet. Two things are needed. First, - up_addrenv_fork() itself. Second, the architecture must build the - child's register context from the *user's* saved system call frame: - in a kernel build fork() is reached through a system call, so the - return address and stack pointer the architecture's fork entry point - can see for itself are the kernel's, not the caller's, and a child - built from those resumes at a kernel address. + It also requires the architecture to build the child's register + context from the *user's* saved system call frame: in a kernel build + fork() is reached through a system call, so the return address and + stack pointer the architecture's fork entry point can see for itself + are the kernel's, not the caller's, and a child built from those + resumes at a kernel address. + + The protected configurations are excluded, MPU and MMU alike. A + protected build has one address space protected by a fixed set of + regions, and its up_addrenv_*() are stubs; there is no per-process + mapping to duplicate at the same virtual addresses, so POSIX fork() + semantics cannot be provided. vfork(), which shares the parent's + memory, works there as everywhere else. Where this is not selected fork() is not provided at all, and code that calls it fails to build. diff --git a/arch/risc-v/src/common/riscv_addrenv.c b/arch/risc-v/src/common/riscv_addrenv.c index 022abfeb7b095..d1132fa2a5cc8 100644 --- a/arch/risc-v/src/common/riscv_addrenv.c +++ b/arch/risc-v/src/common/riscv_addrenv.c @@ -341,6 +341,33 @@ static inline bool vaddr_is_shm(uintptr_t vaddr) #endif } +#ifdef CONFIG_ARCH_HAVE_FORK +/**************************************************************************** + * Name: vaddr_is_text + * + * Description: + * Check if a vaddr is part of the .text area, which is mapped read/execute + * while everything else is mapped read/write. The two arms mirror exactly + * the two layouts up_addrenv_create() builds. + * + ****************************************************************************/ + +static inline bool vaddr_is_text(const arch_addrenv_t *addrenv, + uintptr_t vaddr) +{ +#if (CONFIG_ARCH_TEXT_VBASE != 0x0) && (CONFIG_ARCH_HEAP_VBASE != 0x0) + UNUSED(addrenv); + return vaddr >= CONFIG_ARCH_TEXT_VBASE && vaddr < ARCH_TEXT_VEND; +#else + /* Contiguous layout: the reserve sits below .text, and .data begins where + * .text ends. + */ + + return vaddr >= addrenv->textvbase && vaddr < addrenv->datavbase; +#endif +} +#endif /* CONFIG_ARCH_HAVE_FORK */ + /**************************************************************************** * Public Functions ****************************************************************************/ @@ -496,6 +523,172 @@ int up_addrenv_create(size_t textsize, size_t datasize, size_t heapsize, return ret; } +#ifdef CONFIG_ARCH_HAVE_FORK +/**************************************************************************** + * Name: up_addrenv_fork + * + * Description: + * Duplicate an address environment for POSIX fork(). The destination gets + * its own page tables and its own physical pages, holding a copy of the + * source's contents and mapped at the same virtual addresses. + * + * The walk mirrors up_addrenv_destroy(): every final level page table the + * source has under its static tables is visited, and every page it maps is + * duplicated. So the cost is the size of the process, not the size of its + * address space -- but it is an eager copy, with no copy-on-write, so + * forking needs as much free page memory as the parent occupies. + * + * Input Parameters: + * src - The address environment to be duplicated. + * dest - The location to receive the duplicate. + * + * Returned Value: + * Zero (OK) on success; a negated errno value on failure. + * + ****************************************************************************/ + +int up_addrenv_fork(const arch_addrenv_t *src, arch_addrenv_t *dest) +{ + uintptr_t *sptprev; + uintptr_t *sptlast; + uintptr_t *dptprev; + uintptr_t *dptlast; + uintptr_t paddr; + uintptr_t vaddr; + uintptr_t pgvaddr; + size_t pgsize; + int i; + int j; + int ret; + + DEBUGASSERT(src && dest); + + memset(dest, 0, sizeof(arch_addrenv_t)); + + /* Give the child its own static page tables and kernel mappings */ + + ret = create_spgtables(dest); + if (ret < 0) + { + serr("ERROR: Failed to create static page tables\n"); + goto errout; + } + + ret = copy_kernel_mappings(dest); + if (ret < 0) + { + serr("ERROR: Failed to copy kernel mappings to new environment\n"); + goto errout; + } + + /* The duplicate lives at the same virtual addresses as the original -- + * that is what makes it a copy rather than merely a similar process -- so + * the bases are carried over unchanged. + */ + + dest->textvbase = src->textvbase; + dest->datavbase = src->datavbase; + dest->heapvbase = src->heapvbase; + dest->heapsize = src->heapsize; + dest->satp = mmu_satp_reg(dest->spgtables[0], 0); + + /* Make sure the source's page tables are visible before walking them */ + + UP_ISB(); + UP_DMB(); + + vaddr = ARCH_ADDRENV_VBASE; + pgsize = mmu_get_region_size(ARCH_SPGTS); + sptprev = (uintptr_t *)riscv_pgvaddr(src->spgtables[ARCH_SPGTS - 1]); + dptprev = (uintptr_t *)riscv_pgvaddr(dest->spgtables[ARCH_SPGTS - 1]); + + if (sptprev == NULL || dptprev == NULL) + { + ret = -EINVAL; + goto errout; + } + + i = (ARCH_SPGTS < 2) ? vaddr / pgsize : 0; + for (; i < ENTRIES_PER_PGT; i++, vaddr += pgsize) + { + sptlast = (uintptr_t *)riscv_pgvaddr(mmu_pte_to_paddr(sptprev[i])); + if (sptlast == NULL) + { + continue; + } + + /* Hook the static tables up for this address, then give the child its + * own final level page table here. + */ + + map_spgtables(dest, vaddr); + + paddr = mm_pgalloc(1); + if (!paddr) + { + ret = -ENOMEM; + goto errout; + } + + riscv_pgwipe(paddr); + mmu_ln_setentry(ARCH_SPGTS, (uintptr_t)dptprev, paddr, vaddr, + MMU_UPGT_FLAGS); + dptlast = (uintptr_t *)riscv_pgvaddr(paddr); + + for (j = 0; j < ENTRIES_PER_PGT; j++) + { + uintptr_t pgvaddr_src; + uintptr_t srcpage; + uintptr_t destpage; + + srcpage = mmu_pte_to_paddr(sptlast[j]); + if (!srcpage) + { + continue; + } + + pgvaddr = vaddr + ((uintptr_t)j << MM_PGSHIFT); + + if (vaddr_is_shm(pgvaddr)) + { + /* Shared memory stays shared across fork(). Map the very same + * page; up_addrenv_destroy() knows not to free SHM pages, so + * this does not hand one page to the allocator twice. + */ + + dptlast[j] = sptlast[j]; + continue; + } + + destpage = mm_pgalloc(1); + if (!destpage) + { + ret = -ENOMEM; + goto errout; + } + + pgvaddr_src = riscv_pgvaddr(srcpage); + memcpy((void *)riscv_pgvaddr(destpage), (const void *)pgvaddr_src, + MM_PGSIZE); + + mmu_ln_setentry(ARCH_SPGTS + 1, (uintptr_t)dptlast, destpage, + pgvaddr, + vaddr_is_text(src, pgvaddr) ? MMU_UTEXT_FLAGS + : MMU_UDATA_FLAGS); + } + } + + UP_ISB(); + UP_DMB(); + + return OK; + +errout: + up_addrenv_destroy(dest); + return ret; +} +#endif /* CONFIG_ARCH_HAVE_FORK */ + /**************************************************************************** * Name: up_addrenv_destroy * diff --git a/arch/risc-v/src/common/riscv_fork.c b/arch/risc-v/src/common/riscv_fork.c index 107576297857c..9272d94b291f5 100644 --- a/arch/risc-v/src/common/riscv_fork.c +++ b/arch/risc-v/src/common/riscv_fork.c @@ -121,24 +121,25 @@ pid_t riscv_fork(bool vfork, const struct fork_s *context) return (pid_t)ERROR; } - /* Copy parent user stack to child */ - - stacktop = (uintptr_t)parent->stack_base_ptr + parent->adj_stack_size; - DEBUGASSERT(stacktop > parent->xcp.sregs[REG_SP]); - stackutil = stacktop - parent->xcp.sregs[REG_SP]; - if (child->stack_base_ptr == parent->stack_base_ptr) { - /* The child is running at the parent's stack addresses, inside its - * own duplicated address environment. There is nothing to relocate: - * every stack address the child inherits is still the address it - * names. + /* A fork() child inherits the parent's stack address: its copy of the + * parent's stack is already in place, with its contents, at the same + * virtual address, so there is nothing to copy and nothing to + * relocate. The child simply resumes on the stack pointer the parent + * called with. */ newsp = parent->xcp.sregs[REG_SP]; } else { + /* Copy parent user stack to child */ + + stacktop = (uintptr_t)parent->stack_base_ptr + parent->adj_stack_size; + DEBUGASSERT(stacktop > parent->xcp.sregs[REG_SP]); + stackutil = stacktop - parent->xcp.sregs[REG_SP]; + /* Copy goes to child's user stack top */ newtop = (uintptr_t)child->stack_base_ptr + child->adj_stack_size;