arch/armv7-a: Implement up_addrenv_fork() and provide POSIX fork()

Documentation/guides/fork_vfork_migration.rst

@@ -211,7 +211,7 @@ Known gaps
 complete -- ``addrenv_fork()``, the ``up_addrenv_fork()`` hook, the syscall, the
 libc wrapper and the ``ostest`` case -- so an architecture provides ``fork()``
 by implementing ``up_addrenv_fork()`` and selecting ``CONFIG_ARCH_HAVE_FORK``,
-with no further generic work.
+with no further generic work.  armv7-a selects it today.
 
 **A windowed ABI needs its stack rebased, not just copied.**  On Xtensa,
 giving a child a relocated copy of the parent's stack takes more than the copy:

arch/arm/Kconfig

@@ -1115,6 +1115,7 @@ config ARCH_CORTEXM7
 config ARCH_ARMV7A
 	bool
 	default n
+	select ARCH_HAVE_FORK if BUILD_KERNEL && ARCH_ADDRENV
 	select ARCH_HAVE_CPUINFO
 	select ARCH_HAVE_DEBUG
 	select ARCH_HAVE_PERF_EVENTS

arch/arm/src/armv7-a/addrenv.h

@@ -85,6 +85,26 @@ int arm_addrenv_create_region(uintptr_t *l1table, unsigned int listlen,
                               uintptr_t vaddr, size_t regionsize,
                               uint32_t mmuflags);
 
+/****************************************************************************
+ * Name: arm_addrenv_fork_region
+ *
+ * Description:
+ *   Duplicate one memory region of an address environment for fork().  The
+ *   destination gets its own page tables and, unless `share' is requested,
+ *   its own copy of the source's pages, mapped at the same virtual
+ *   addresses.
+ *
+ * Returned Value:
+ *   Zero (OK) on success; a negated errno value on failure.
+ *
+ ****************************************************************************/
+
+#ifdef CONFIG_ARCH_HAVE_FORK
+int arm_addrenv_fork_region(uintptr_t *srcl1, uintptr_t *destl1,
+                            unsigned int listlen, uintptr_t vaddr,
+                            uint32_t mmuflags, bool share);
+#endif
+
 /****************************************************************************
  * Name: arm_addrenv_destroy_region
  *

arch/arm/src/armv7-a/arm_addrenv.c

@@ -255,6 +255,125 @@ int up_addrenv_create(size_t textsize, size_t datasize, size_t heapsize,
   return ret;
 }
 
+#ifdef CONFIG_ARCH_HAVE_FORK
+/****************************************************************************
+ * Name: up_addrenv_fork
+ *
+ * Description:
+ *   Duplicate an address environment for POSIX fork().  The destination is
+ *   backed by fresh pages holding a copy of the source's contents, mapped at
+ *   the same virtual addresses.
+ *
+ *   Only the pages the source actually has mapped are duplicated, so the
+ *   cost is the size of the process, not the size of its address space.
+ *   There is no copy-on-write, so this needs as much free page memory as
+ *   the parent occupies and returns -ENOMEM when that is not available.
+ *
+ * Input Parameters:
+ *   src  - The address environment to be duplicated.
+ *   dest - The location to receive the duplicate.
+ *
+ * Returned Value:
+ *   Zero (OK) on success; a negated errno value on failure.
+ *
+ ****************************************************************************/
+
+int up_addrenv_fork(FAR const arch_addrenv_t *src, FAR arch_addrenv_t *dest)
+{
+  size_t npage = MM_NPAGES(PGTABLE_SIZE);
+  int ret;
+
+  binfo("src=%p dest=%p\n", src, dest);
+
+  DEBUGASSERT(src && dest && src->l1table);
+
+  memset(dest, 0, sizeof(arch_addrenv_t));
+
+  /* Give the child its own L1 page table, seeded from the kernel's, exactly
+   * as up_addrenv_create() does.
+   */
+
+  dest->l1table = (uintptr_t *)mm_pgalloc_align(npage, npage);
+  if (dest->l1table == NULL)
+    {
+      ret = -ENOMEM;
+      goto errout;
+    }
+
+  memcpy(dest->l1table, (void *)PGTABLE_BASE_VADDR, PGTABLE_SIZE);
+
+  /* The duplicate lives at the same virtual addresses as the original --
+   * that is the whole point -- so the bases are simply carried over.
+   */
+
+  dest->textvbase = src->textvbase;
+  dest->datavbase = src->datavbase;
+  dest->heapvbase = src->heapvbase;
+  dest->heapsize  = src->heapsize;
+#ifdef CONFIG_ARCH_VMA_MAPPING
+  dest->shmvbase  = src->shmvbase;
+#endif
+
+  ret = arm_addrenv_fork_region(src->l1table, dest->l1table,
+                                ARCH_TEXT_NSECTS, CONFIG_ARCH_TEXT_VBASE,
+                                MMU_L2_UTEXTFLAGS, false);
+  if (ret < 0)
+    {
+      berr("ERROR: Failed to duplicate .text region: %d\n", ret);
+      goto errout;
+    }
+
+  ret = arm_addrenv_fork_region(src->l1table, dest->l1table,
+                                ARCH_DATA_NSECTS, CONFIG_ARCH_DATA_VBASE,
+                                MMU_L2_UDATAFLAGS, false);
+  if (ret < 0)
+    {
+      berr("ERROR: Failed to duplicate .bss/.data region: %d\n", ret);
+      goto errout;
+    }
+
+#ifdef CONFIG_BUILD_KERNEL
+  ret = arm_addrenv_fork_region(src->l1table, dest->l1table,
+                                ARCH_HEAP_NSECTS, CONFIG_ARCH_HEAP_VBASE,
+                                MMU_L2_UDATAFLAGS, false);
+  if (ret < 0)
+    {
+      berr("ERROR: Failed to duplicate heap region: %d\n", ret);
+      goto errout;
+    }
+
+  /* Note that the stack region is deliberately not duplicated here.  The
+   * child's stack is allocated separately by nxtask_setup_fork() and filled
+   * in by arm_fork(), and up_addrenv_destroy() does not own the stack region
+   * either -- up_addrenv_ustackfree() does.  Duplicating it here would leak
+   * it.  CONFIG_ARCH_HAVE_FORK therefore depends on
+   * !CONFIG_ARCH_STACK_DYNAMIC.
+   */
+
+#ifdef CONFIG_ARCH_VMA_MAPPING
+  /* Shared memory stays shared across fork(), so the child is given the
+   * parent's own pages rather than copies of them.
+   */
+
+  ret = arm_addrenv_fork_region(src->l1table, dest->l1table,
+                                ARCH_SHM_NSECTS, CONFIG_ARCH_SHM_VBASE,
+                                MMU_L2_UDATAFLAGS, true);
+  if (ret < 0)
+    {
+      berr("ERROR: Failed to share SHM region: %d\n", ret);
+      goto errout;
+    }
+#endif
+#endif /* CONFIG_BUILD_KERNEL */
+
+  return OK;
+
+errout:
+  up_addrenv_destroy(dest);
+  return ret;
+}
+#endif /* CONFIG_ARCH_HAVE_FORK */
+
 /****************************************************************************
  * Name: up_addrenv_destroy
  *

arch/arm/src/armv7-a/arm_addrenv_utils.c

@@ -149,6 +149,132 @@ int arm_addrenv_create_region(uintptr_t *l1table, unsigned int listlen,
   return npages;
 }
 
+#ifdef CONFIG_ARCH_HAVE_FORK
+/****************************************************************************
+ * Name: arm_addrenv_fork_region
+ *
+ * Description:
+ *   Duplicate one memory region of an address environment for fork().
+ *
+ *   The destination gets its own L2 page tables and, unless `share' is
+ *   requested, its own physical pages holding a copy of the source's
+ *   contents.  Both are mapped at the same virtual addresses as the source,
+ *   which is what makes the duplicate an exact copy:  a pointer the parent
+ *   held into its own memory means the same thing in the child.
+ *
+ *   Only the sections the source actually has mapped are duplicated, so the
+ *   copy is the size of the parent, not the size of the region.
+ *
+ *   `share' maps the source's own pages into the destination rather than
+ *   copying them.  It is used for the shared memory region, which POSIX says
+ *   remains shared across fork().
+ *
+ * Input Parameters:
+ *   srcl1    - The L1 page table of the address environment to duplicate
+ *   destl1   - The L1 page table receiving the duplicate
+ *   listlen  - Number of sections in the region
+ *   vaddr    - Virtual base address of the region
+ *   mmuflags - L2 flags for the duplicated pages
+ *   share    - Map the source's pages rather than copying them
+ *
+ * Returned Value:
+ *   Zero (OK) on success; a negated errno value on failure.  On failure the
+ *   caller is expected to tear the destination down with
+ *   up_addrenv_destroy(), which copes with a partially built environment.
+ *
+ ****************************************************************************/
+
+int arm_addrenv_fork_region(uintptr_t *srcl1, uintptr_t *destl1,
+                            unsigned int listlen, uintptr_t vaddr,
+                            uint32_t mmuflags, bool share)
+{
+  uintptr_t l1entry;
+  uintptr_t *srcl2;
+  uintptr_t *destl2;
+  uintptr_t paddr;
+  unsigned int i;
+  unsigned int j;
+
+  binfo("listlen=%d vaddr=%08lx share=%d\n", listlen,
+        (unsigned long)vaddr, share);
+
+  for (i = 0; i < listlen; i++, vaddr += SECTION_SIZE)
+    {
+      /* Skip sections the source never populated */
+
+      l1entry = mmu_l1table_getentry(srcl1, vaddr);
+      if (l1entry == 0)
+        {
+          continue;
+        }
+
+      srcl2 = (uintptr_t *)arm_pgvaddr(l1entry & PTE_SMALL_PADDR_MASK);
+
+      /* Allocate one physical page for the destination L2 page table */
+
+      paddr = mm_pgalloc(1);
+      if (!paddr)
+        {
+          return -ENOMEM;
+        }
+
+      DEBUGASSERT(MM_ISALIGNED(paddr));
+
+      mmu_l1table_setentry(destl1, paddr, vaddr, MMU_L1_PGTABFLAGS);
+
+      destl2 = (uintptr_t *)arm_pgvaddr(paddr);
+      memset(destl2, 0, ENTRIES_PER_L2TABLE * sizeof(uintptr_t));
+
+      for (j = 0; j < ENTRIES_PER_L2TABLE; j++)
+        {
+          uintptr_t srcpage = srcl2[j];
+          uintptr_t destpage;
+
+          if (srcpage == 0)
+            {
+              continue;
+            }
+
+          if (share)
+            {
+              /* Map the very same page.  Note that up_addrenv_destroy()
+               * tears a shared region down with keep=true, so this does not
+               * hand the same page to the page allocator twice.
+               */
+
+              destl2[j] = srcpage;
+              continue;
+            }
+
+          srcpage &= PTE_SMALL_PADDR_MASK;
+
+          destpage = mm_pgalloc(1);
+          if (!destpage)
+            {
+              return -ENOMEM;
+            }
+
+          memcpy((void *)arm_pgvaddr(destpage),
+                 (const void *)arm_pgvaddr(srcpage), MM_PGSIZE);
+
+          up_flush_dcache(arm_pgvaddr(destpage),
+                          arm_pgvaddr(destpage) + MM_PGSIZE);
+
+          set_l2_entry(destl2, destpage, vaddr + (j << MM_PGSHIFT),
+                       mmuflags);
+        }
+
+      /* Make sure the initialized L2 table is visible to the MMU */
+
+      up_flush_dcache((uintptr_t)destl2,
+                      (uintptr_t)destl2 +
+                      ENTRIES_PER_L2TABLE * sizeof(uintptr_t));
+    }
+
+  return OK;
+}
+#endif /* CONFIG_ARCH_HAVE_FORK */
+
 /****************************************************************************
  * Name: arm_addrenv_destroy_region
  *

arch/arm/src/armv7-a/arm_initialstate.c

@@ -55,6 +55,7 @@
 void up_initial_state(struct tcb_s *tcb)
 {
   struct xcptcontext *xcp = &tcb->xcp;
+  uintptr_t topstack;
   uint32_t cpsr;
 #ifdef CONFIG_ARCH_KERNEL_STACK
   uint32_t *kstack = xcp->kstack;
@@ -85,21 +86,47 @@ void up_initial_state(struct tcb_s *tcb)
       return;
     }
 
+  topstack = (uintptr_t)tcb->stack_base_ptr + tcb->adj_stack_size;
+
 #ifdef CONFIG_ARCH_KERNEL_STACK
   xcp->kstack = kstack;
+
+  /* A user process in a kernel build keeps its register save area on its own
+   * kernel stack, not at the top of its user stack.  Two reasons:
+   *
+   *   - The area must not be user-writable.  It holds the CPSR the thread is
+   *     resumed with, so a task that could scribble on it could resume
+   *     itself in a privileged mode.
+   *   - The kernel must be able to write it without regard to which address
+   *     environment is current, and without regard to what the user stack
+   *     already contains.  A fork() child inherits the parent's stack
+   *     address, so the top of "its" stack is occupied by the parent's live
+   *     frames -- including the exception frame the child is built from,
+   *     which sits just below the caller's stack pointer.  Zeroing
+   *     XCPTCONTEXT_SIZE bytes there would destroy both.
+   *
+   * This is what risc-v and arm64 already do; see riscv_initialstate.c.
+   */
+
+  if (kstack != NULL)
+    {
+      topstack = (uintptr_t)kstack + ARCH_KERNEL_STACKSIZE;
+    }
 #endif
 
-  /* Initialize the context registers to stack top */
+  /* Initialize the context registers to the top of whichever stack holds the
+   * save area
+   */
 
-  xcp->regs = (void *)((uint32_t)tcb->stack_base_ptr +
-                                 tcb->adj_stack_size -
-                                 XCPTCONTEXT_SIZE);
+  xcp->regs = (void *)(topstack - XCPTCONTEXT_SIZE);
 
   /* Initialize the xcp registers */
 
   memset(xcp->regs, 0, XCPTCONTEXT_SIZE);
 
-  /* Save the initial stack pointer */
+  /* Save the initial stack pointer.  The thread itself always starts on its
+   * user stack, whichever stack the save area came off.
+   */
 
   xcp->regs[REG_SP] = (uint32_t)tcb->stack_base_ptr +
                                 tcb->adj_stack_size;

arch/arm/src/common/arm_fork.c

@@ -277,6 +277,11 @@ static pid_t arm_fork_direct(bool vfork, struct tcb_s *parent,
  *   in a system call at all, so it inherits none of the parent's nesting
  *   state.
  *
+ *   A process in a kernel build keeps its register save area on its kernel
+ *   stack, so building the child's context writes nothing to any user stack
+ *   -- see up_initial_state().  That is what lets a child which shares the
+ *   parent's stack addresses be left exactly as it is.
+ *
  * Input Parameters:
  *   vfork  - true for vfork(), false for fork()
  *   parent - The calling task's TCB
@@ -345,14 +350,6 @@ static pid_t arm_fork_syscall(bool vfork, struct tcb_s *parent)
                          child->adj_stack_size;
       newsp  = newtop - stackutil;
 
-      /* Put the child's register save area where the parent's is:  just
-       * below the stack the caller was using.  It cannot be left at the top
-       * of the child's stack, which is where up_initial_state() put it,
-       * because the copy of the parent's stack below is about to land there.
-       */
-
-      child->xcp.regs = (uint32_t *)(newsp - XCPTCONTEXT_SIZE);
-
       memcpy((void *)newsp, (const void *)oldsp, stackutil);
 
       /* Was there a frame pointer in place before? */