Respect VmallocTotal from /proc/meminfo as virtual address space limit

src/coreclr/gc/unix/cgroup.cpp

@@ -21,10 +21,10 @@ Module Name:
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
-#include <sys/resource.h>
 #include <sys/vfs.h>
 #include <errno.h>
 #include <limits>
+#include <minipal/vmlimit.h>
 
 #include "config.gc.h"
 
@@ -579,14 +579,9 @@ size_t GetRestrictedPhysicalMemoryLimit()
         return 0;
     }
 
-    struct rlimit curr_rlimit;
-    size_t rlimit_soft_limit = (size_t)RLIM_INFINITY;
-    if (getrlimit(RLIMIT_AS, &curr_rlimit) == 0)
-    {
-        rlimit_soft_limit = curr_rlimit.rlim_cur;
-    }
-    physical_memory_limit = (physical_memory_limit < rlimit_soft_limit) ?
-                            physical_memory_limit : rlimit_soft_limit;
+    size_t vm_limit = minipal_get_virtual_address_space_limit();
+    physical_memory_limit = (physical_memory_limit < vm_limit) ?
+                            physical_memory_limit : vm_limit;
 
     // Ensure that limit is not greater than real memory size
     long pages = sysconf(_SC_PHYS_PAGES);

src/coreclr/gc/unix/gcenv.unix.cpp

@@ -26,6 +26,7 @@
 #include <minipal/memorybarrierprocesswide.h>
 #include <minipal/thread.h>
 #include <minipal/time.h>
+#include <minipal/vmlimit.h>
 #include <minipal/cpucount.h>
 
 #if HAVE_SWAPCTL
@@ -1007,13 +1008,11 @@ static size_t GetCurrentVirtualMemorySize()
 //  non zero if it has succeeded, GetVirtualMemoryMaxAddress() if not available
 size_t GCToOSInterface::GetVirtualMemoryLimit()
 {
-#ifdef RLIMIT_AS
-    rlimit addressSpaceLimit;
-    if ((getrlimit(RLIMIT_AS, &addressSpaceLimit) == 0) && (addressSpaceLimit.rlim_cur != RLIM_INFINITY))
+    size_t addressSpaceLimit = minipal_get_virtual_address_space_limit();
+    if (addressSpaceLimit != SIZE_MAX)
     {
-        return addressSpaceLimit.rlim_cur;
+        return addressSpaceLimit;
     }
-#endif // RLIMIT_AS
 
     // No virtual memory limit
     return GetVirtualMemoryMaxAddress();
@@ -1239,15 +1238,15 @@ void GCToOSInterface::GetMemoryStatus(uint64_t restricted_limit, uint32_t* memor
             }
 
 #if HAVE_PROCFS_STATM
-            rlimit addressSpaceLimit;
-            if ((getrlimit(RLIMIT_AS, &addressSpaceLimit) == 0) && (addressSpaceLimit.rlim_cur != RLIM_INFINITY))
+            size_t virtualAddressSpaceLimit = minipal_get_virtual_address_space_limit();
+            if (virtualAddressSpaceLimit != SIZE_MAX)
             {
                 // If there is virtual address space limit set, compute virtual memory load and change
                 // the load to this one in case it is higher than the physical memory load
                 size_t used_virtual = GetCurrentVirtualMemorySize();
                 if (used_virtual != (size_t)-1)
                 {
-                    uint32_t load_virtual = (uint32_t)(((float)used_virtual * 100) / (float)addressSpaceLimit.rlim_cur);
+                    uint32_t load_virtual = (uint32_t)(((float)used_virtual * 100) / (float)virtualAddressSpaceLimit);
                     if (load_virtual > load)
                     {
                         load = load_virtual;

src/coreclr/minipal/Unix/doublemapping.cpp

@@ -25,6 +25,7 @@
 #endif // TARGET_LINUX && !MFD_CLOEXEC
 #include "minipal.h"
 #include "minipal/cpufeatures.h"
+#include <minipal/vmlimit.h>
 
 #ifndef TARGET_APPLE
 #if !defined(TARGET_WASI)
@@ -109,19 +110,15 @@ bool VMToOSInterface::CreateDoubleMemoryMapper(void** pHandle, size_t *pMaxExecu
     // Clip the maximum double mapped memory size to 1/4 of the virtual address space limit.
     // When such a limit is set, GC reserves 1/2 of it, so we need to leave something
     // for the rest of the process.
-#ifdef RLIMIT_AS
-    // OpenBSD has no address-space rlimit (RLIMIT_AS), so this clipping is skipped there.
-    // WASI also has no RLIMIT_AS.
-    struct rlimit virtualAddressSpaceLimit;
-    if ((getrlimit(RLIMIT_AS, &virtualAddressSpaceLimit) == 0) && (virtualAddressSpaceLimit.rlim_cur != RLIM_INFINITY))
-    {
-        virtualAddressSpaceLimit.rlim_cur /= 4;
-        if (maxDoubleMappedMemorySize > virtualAddressSpaceLimit.rlim_cur)
+    size_t virtualAddressSpaceLimit = minipal_get_virtual_address_space_limit();
+    if (virtualAddressSpaceLimit != SIZE_MAX)
+    {
+        virtualAddressSpaceLimit /= 4;
+        if (maxDoubleMappedMemorySize > virtualAddressSpaceLimit)
         {
-            maxDoubleMappedMemorySize = virtualAddressSpaceLimit.rlim_cur;
+            maxDoubleMappedMemorySize = virtualAddressSpaceLimit;
         }
     }
-#endif // RLIMIT_AS
 
     // Clip the maximum double mapped memory size to the file size limit
 #ifdef RLIMIT_FSIZE

src/coreclr/pal/src/map/virtual.cpp

@@ -31,7 +31,6 @@ SET_DEFAULT_DEBUG_CHANNEL(VIRTUAL); // some headers have code with asserts, so d
 #include "common.h"
 #include <clrconfignocache.h>
 
-#include <sys/resource.h>
 #include <sys/types.h>
 #include <sys/mman.h>
 #include <errno.h>
@@ -41,6 +40,7 @@ SET_DEFAULT_DEBUG_CHANNEL(VIRTUAL); // some headers have code with asserts, so d
 #include <dlfcn.h>
 #include <minipal/utils.h>
 #include <minipal/ospagesize.h>
+#include <minipal/vmlimit.h>
 
 #if HAVE_VM_ALLOCATE
 #include <mach/vm_map.h>
@@ -1616,20 +1616,15 @@ void ExecutableMemoryAllocator::TryReserveInitialMemory()
     int32_t sizeOfAllocation = MaxExecutableMemorySizeNearCoreClr;
     int32_t initialReserveLimit = -1;
 #if defined(TARGET_WASI)
-    // WASI has neither RLIMIT_AS nor RLIMIT_DATA nor RLIM_INFINITY; wasm32 has a
-    // hard 4 GiB limit and no separate VM accounting. Skip the rlimit-based
-    // sizing and let sizeOfAllocation stay at MaxExecutableMemorySizeNearCoreClr.
+    // WASI has a hard 4 GiB limit and no separate VM accounting.
+    // Skip the virtual address space limit check and let sizeOfAllocation stay
+    // at MaxExecutableMemorySizeNearCoreClr.
 #else
-#ifdef RLIMIT_AS
-    int addressSpace = RLIMIT_AS;
-#else
-    int addressSpace = RLIMIT_DATA;
-#endif
-    rlimit addressSpaceLimit;
-    if ((getrlimit(addressSpace, &addressSpaceLimit) == 0) && (addressSpaceLimit.rlim_cur != RLIM_INFINITY))
+    size_t virtualAddressSpaceLimit = minipal_get_virtual_address_space_limit();
+    if (virtualAddressSpaceLimit != SIZE_MAX)
     {
         // By default reserve max 20% of the available virtual address space
-        rlim_t initialExecMemoryPerc = 20;
+        size_t initialExecMemoryPerc = 20;
         CLRConfigNoCache defInitialExecMemoryPerc = CLRConfigNoCache::Get("InitialExecMemoryPercent", /*noprefix*/ false, &getenv);
         if (defInitialExecMemoryPerc.IsSet())
         {
@@ -1640,7 +1635,12 @@ void ExecutableMemoryAllocator::TryReserveInitialMemory()
             }
         }
 
-        initialReserveLimit = addressSpaceLimit.rlim_cur * initialExecMemoryPerc / 100;
+        size_t computedLimit = virtualAddressSpaceLimit * initialExecMemoryPerc / 100;
+        if (computedLimit > INT32_MAX)
+        {
+            computedLimit = INT32_MAX;
+        }
+        initialReserveLimit = (int32_t)computedLimit;
         if (initialReserveLimit < sizeOfAllocation)
         {
             sizeOfAllocation = initialReserveLimit;

src/native/minipal/CMakeLists.txt

@@ -29,7 +29,7 @@ if(NOT WIN32 AND NOT HOST_WASM AND NOT (CLR_CMAKE_TARGET_ARCH_WASM AND NOT CLR_C
 endif()
 
 if(CLR_CMAKE_HOST_UNIX)
-    list(APPEND SOURCES cpucount.c)
+    list(APPEND SOURCES cpucount.c vmlimit.c)
 endif()
 
 # Provide an object library for scenarios where we ship static libraries

src/native/minipal/vmlimit.c

@@ -0,0 +1,87 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#include "vmlimit.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+
+#include <sys/resource.h>
+
+#if !defined(__APPLE__) && !defined(__HAIKU__)
+static size_t get_vmalloc_total(void)
+{
+    FILE* memInfoFile = fopen("/proc/meminfo", "r");
+    if (memInfoFile == NULL)
+    {
+        return SIZE_MAX;
+    }
+
+    size_t result = SIZE_MAX;
+    char* line = NULL;
+    size_t lineLen = 0;
+
+    while (getline(&line, &lineLen, memInfoFile) != -1)
+    {
+        uint64_t value;
+        char units = '\0';
+        if (sscanf(line, "VmallocTotal: %" SCNu64 " %cB", &value, &units) >= 1)
+        {
+            uint64_t multiplier = 1;
+            switch (units)
+            {
+                case 'g':
+                case 'G': multiplier = 1024 * 1024 * 1024; break;
+                case 'm':
+                case 'M': multiplier = 1024 * 1024; break;
+                case 'k':
+                case 'K': multiplier = 1024; break;
+            }
+
+            uint64_t total = value * multiplier;
+            result = (total < (uint64_t)SIZE_MAX) ? (size_t)total : SIZE_MAX;
+            break;
+        }
+    }
+
+    free(line);
+    fclose(memInfoFile);
+
+    return result;
+}
+#endif // !defined(__APPLE__) && !defined(__HAIKU__)
+
+size_t minipal_get_virtual_address_space_limit(void)
+{
+    // Cache the result since the values don't change during process lifetime.
+    // Use 0 as the "not yet computed" sentinel since a zero limit is not a valid result.
+    static volatile size_t cached_limit = 0;
+
+    if (cached_limit != 0)
+    {
+        return cached_limit;
+    }
+
+    size_t limit = SIZE_MAX;
+
+#ifdef RLIMIT_AS
+    struct rlimit addressSpaceLimit;
+    if ((getrlimit(RLIMIT_AS, &addressSpaceLimit) == 0) && (addressSpaceLimit.rlim_cur != RLIM_INFINITY))
+    {
+        limit = addressSpaceLimit.rlim_cur;
+    }
+#endif
+
+#if !defined(__APPLE__) && !defined(__HAIKU__)
+    size_t vmallocTotal = get_vmalloc_total();
+    if (vmallocTotal < limit)
+    {
+        limit = vmallocTotal;
+    }
+#endif
+
+    cached_limit = limit;
+
+    return limit;
+}

src/native/minipal/vmlimit.h

@@ -0,0 +1,25 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#ifndef HAVE_MINIPAL_VMLIMIT_H
+#define HAVE_MINIPAL_VMLIMIT_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif // __cplusplus
+
+// Returns the effective virtual address space limit for the current process
+// by taking the minimum of RLIMIT_AS (if set and finite) and the VmallocTotal
+// value from /proc/meminfo (on Linux).
+// Returns SIZE_MAX if neither limit is available or applicable.
+size_t minipal_get_virtual_address_space_limit(void);
+
+#ifdef __cplusplus
+}
+#endif // __cplusplus
+
+#endif // HAVE_MINIPAL_VMLIMIT_H