diff --git a/runtime-common/core/allocator/details/malloc-interface.h b/runtime-common/core/allocator/details/malloc-interface.h new file mode 100644 index 0000000000..ea34faa96e --- /dev/null +++ b/runtime-common/core/allocator/details/malloc-interface.h @@ -0,0 +1,165 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/wrappers/likely.h" +#include "runtime-common/core/utils/kphp-assert-core.h" + +namespace kphp::memory::details { + +struct control_block { +private: + static constexpr auto SIZE_FIELD_BITSIZE{48}; + static constexpr auto BASE_OFFSET_FIELD_BITSIZE{16}; + static constexpr uint64_t BLOCK_SIZE_MASK{(1UL << SIZE_FIELD_BITSIZE) - 1}; + static constexpr uint64_t BASE_OFFSET_MASK{(1UL << BASE_OFFSET_FIELD_BITSIZE) - 1}; + + static_assert(SIZE_FIELD_BITSIZE + BASE_OFFSET_FIELD_BITSIZE == std::numeric_limits::digits); + +public: + static constexpr uint64_t max_size() noexcept { + return 1UL << SIZE_FIELD_BITSIZE; + } + + static constexpr uint64_t max_alignment() noexcept { + return 1UL << BASE_OFFSET_FIELD_BITSIZE; + } + + uint64_t raw() const noexcept { + return (static_cast(base_offset) << SIZE_FIELD_BITSIZE) | (static_cast(size) & BLOCK_SIZE_MASK); + } + + static control_block from_raw(uint64_t raw) noexcept { + return control_block{.size = raw & BLOCK_SIZE_MASK, .base_offset = static_cast((raw >> SIZE_FIELD_BITSIZE) & BASE_OFFSET_MASK)}; + } + + uint64_t size : SIZE_FIELD_BITSIZE; + uint16_t base_offset : BASE_OFFSET_FIELD_BITSIZE; +}; + +inline bool is_power_of_2(uint64_t v) noexcept { + return v && !(v & (v - 1)); +} + +static_assert(sizeof(control_block) == sizeof(uint64_t), "Control block's size must be equal to uint64"); + +constexpr uint64_t MALLOC_REPLACER_MAX_ALLOC = 0xFFFFFF00; // 4GiB + +template +struct malloc_interface { + static auto alloc(size_t size) noexcept -> void* { + constexpr size_t cb_size{sizeof(kphp::memory::details::control_block)}; + if (unlikely(size > std::min(kphp::memory::details::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - cb_size)) { + php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); + return nullptr; + } + const size_t total_size{size + cb_size}; + void* base{get_allocator_func().alloc_script_memory(total_size)}; + if (unlikely(base == nullptr)) { + php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size); + return base; + } + *(static_cast(base)) = kphp::memory::details::control_block{.size = total_size, .base_offset = cb_size}.raw(); + return static_cast(static_cast(base) + cb_size); + } + + static auto alloc_aligned(size_t size, std::align_val_t alignment) noexcept -> void* { + // Check that provided alignment is power of two + const size_t align{static_cast(alignment)}; + if (unlikely(align == 0 || !kphp::memory::details::is_power_of_2(align) || align >= kphp::memory::details::control_block::max_alignment())) { + php_warning("allocation alignment have to be non-zero power of two and not greater than %" PRIu64 ", got : %lu", + kphp::memory::details::control_block::max_alignment(), align); + return nullptr; + } + + // Check that memory is enough + constexpr size_t cb_size{sizeof(kphp::memory::details::control_block)}; + if (unlikely(size > std::min(kphp::memory::details::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - (align - 1) - cb_size)) { + php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); + return nullptr; + } + + // Request mem from underlying memory manager + const size_t total_size{size + (align - 1) + cb_size}; + void* base{get_allocator_func().alloc_script_memory(total_size)}; + if (unlikely(base == nullptr)) { + php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size); + return base; + } + + const uint64_t base_u{reinterpret_cast(base)}; + // The smallest multiple of `align` greater than or equal to requested memory + const uint64_t aligned_u{((base_u + cb_size) + (align - 1)) & ~(align - 1)}; + const uint64_t base_offset_u{aligned_u - base_u}; + + // Save control block + *(reinterpret_cast(aligned_u - cb_size)) = // NOLINT + kphp::memory::details::control_block{.size = total_size, .base_offset = static_cast(base_offset_u)}.raw(); + + return reinterpret_cast(aligned_u); // NOLINT + } + + static auto calloc(size_t num, size_t size) noexcept -> void* { + void* ptr{alloc(num * size)}; + if (unlikely(ptr == nullptr)) { + return nullptr; + } + return std::memset(ptr, 0, num * size); + } + + static auto free(void* ptr) noexcept -> void { + if (unlikely(ptr == nullptr)) { + return; + } + + constexpr size_t cb_size{sizeof(kphp::memory::details::control_block)}; + const auto mem{reinterpret_cast(ptr)}; + + const auto cb{kphp::memory::details::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT + void* base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT + + get_allocator_func().free_script_memory(base, cb.size); + } + + static auto realloc(void* ptr, size_t new_size) noexcept -> void* { + if (unlikely(ptr == nullptr)) { + return alloc(new_size); + } + + if (unlikely(new_size == 0)) { + free(ptr); + return nullptr; + } + + constexpr size_t cb_size{sizeof(kphp::memory::details::control_block)}; + const auto mem{reinterpret_cast(ptr)}; + + const auto cb{kphp::memory::details::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT + + void* old_base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT + const size_t old_size{cb.size}; + + void* new_ptr{alloc(new_size)}; + if (likely(new_ptr != nullptr)) { + std::memcpy(new_ptr, ptr, std::min(new_size, old_size)); + get_allocator_func().free_script_memory(old_base, old_size); + } + return new_ptr; + } + + static auto strdup(const char* str1) noexcept -> char* { + auto* str2{static_cast(alloc(std::strlen(str1) + 1))}; + return std::strcpy(str2, str1); + } +}; + +} // namespace kphp::memory::details diff --git a/runtime-common/core/allocator/platform-allocator.h b/runtime-common/core/allocator/platform-allocator.h index 6ae1134683..9d6fec27ef 100644 --- a/runtime-common/core/allocator/platform-allocator.h +++ b/runtime-common/core/allocator/platform-allocator.h @@ -6,7 +6,7 @@ #include -#include "runtime-common/core/allocator/runtime-allocator.h" +#include "runtime-common/core/allocator/platform-malloc-interface.h" namespace kphp::memory { @@ -25,11 +25,11 @@ struct platform_allocator { }; constexpr value_type* allocate(size_t n) noexcept { - return static_cast(RuntimeAllocator::get().alloc_global_memory(n * sizeof(T))); + return static_cast(kphp::memory::platform::alloc(n * sizeof(T))); } - constexpr void deallocate(T* p, size_t n) noexcept { - RuntimeAllocator::get().free_global_memory(p, n * sizeof(T)); + constexpr void deallocate(T* p, size_t /*unused*/) noexcept { + kphp::memory::platform::free(p); } }; diff --git a/runtime-common/core/allocator/platform-malloc-interface.h b/runtime-common/core/allocator/platform-malloc-interface.h index 5afb94558a..ffe34ef5e1 100644 --- a/runtime-common/core/allocator/platform-malloc-interface.h +++ b/runtime-common/core/allocator/platform-malloc-interface.h @@ -10,46 +10,26 @@ #include #include "common/wrappers/likely.h" -#include "runtime-common/core/allocator/runtime-allocator.h" -#include "runtime-common/core/utils/kphp-assert-core.h" namespace kphp::memory::platform { constexpr int64_t MALLOC_REPLACER_SIZE_OFFSET = sizeof(size_t); constexpr uint64_t MALLOC_REPLACER_MAX_ALLOC = 0xFFFFFF00; -inline void* alloc(size_t size) noexcept { - if (unlikely(size > MALLOC_REPLACER_MAX_ALLOC - MALLOC_REPLACER_SIZE_OFFSET)) { - php_warning("attempt to allocate too much memory by malloc replacer : %lu", size); - return nullptr; - } - const size_t real_size{size + MALLOC_REPLACER_SIZE_OFFSET}; - void* ptr{RuntimeAllocator::get().alloc_global_memory(real_size)}; - - if (unlikely(ptr == nullptr)) { - php_warning("not enough platform memory to allocate: %lu", size); - return ptr; - } - *static_cast(ptr) = real_size; - return static_cast(ptr) + MALLOC_REPLACER_SIZE_OFFSET; -} +auto alloc(size_t size) noexcept -> void*; -inline void* calloc(size_t num, size_t size) noexcept { +inline auto calloc(size_t num, size_t size) noexcept -> void* { void* ptr{kphp::memory::platform::alloc(num * size)}; if (unlikely(ptr == nullptr)) { return nullptr; } + return std::memset(ptr, 0, num * size); } -inline void free(void* ptr) noexcept { - if (likely(ptr != nullptr)) { - void* real_ptr{static_cast(ptr) - MALLOC_REPLACER_SIZE_OFFSET}; - RuntimeAllocator::get().free_global_memory(real_ptr, *static_cast(real_ptr)); - } -} +auto free(void* ptr) noexcept -> void; -inline void* realloc(void* ptr, size_t new_size) noexcept { +inline auto realloc(void* ptr, size_t new_size) noexcept -> void* { if (unlikely(ptr == nullptr)) { return kphp::memory::platform::alloc(new_size); } @@ -59,14 +39,15 @@ inline void* realloc(void* ptr, size_t new_size) noexcept { return nullptr; } - void* real_ptr{static_cast(ptr) - sizeof(size_t)}; - const size_t old_size{*static_cast(real_ptr)}; + void* real_ptr{static_cast(ptr) - MALLOC_REPLACER_SIZE_OFFSET}; + const size_t old_size{*static_cast(real_ptr) - MALLOC_REPLACER_SIZE_OFFSET}; void* new_ptr{kphp::memory::platform::alloc(new_size)}; if (likely(new_ptr != nullptr)) { std::memcpy(new_ptr, ptr, std::min(new_size, old_size)); - RuntimeAllocator::get().free_global_memory(real_ptr, old_size); + kphp::memory::platform::free(ptr); } + return new_ptr; } diff --git a/runtime-common/core/allocator/pool-allocator.h b/runtime-common/core/allocator/pool-allocator.h new file mode 100644 index 0000000000..38b928e16f --- /dev/null +++ b/runtime-common/core/allocator/pool-allocator.h @@ -0,0 +1,38 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "common/mixin/not_copyable.h" +#include "runtime-common/core/memory-resource/unsynchronized_pool_resource.h" + +namespace kphp::memory { + +struct pool_allocator : private vk::not_copyable { +private: + memory_resource::unsynchronized_pool_resource memory_resource; + size_t m_min_extra_mem_size{0}; + + auto request_extra_memory(size_t requested_size) noexcept -> void; + +public: + pool_allocator() = default; + pool_allocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept; + + auto init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void; + auto free() noexcept -> void; + + auto alloc(size_t size) noexcept -> void*; + auto calloc(size_t size) noexcept -> void*; + auto realloc(void* mem, size_t new_size, size_t old_size) noexcept -> void*; + auto free(void* mem, size_t size) noexcept -> void; + + auto get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource& { + return memory_resource; + } +}; + +} // namespace kphp::memory diff --git a/runtime-common/core/allocator/runtime-allocator.h b/runtime-common/core/allocator/runtime-allocator.h index aac9d859d8..46b948b130 100644 --- a/runtime-common/core/allocator/runtime-allocator.h +++ b/runtime-common/core/allocator/runtime-allocator.h @@ -6,34 +6,27 @@ #include -#include "common/mixin/not_copyable.h" -#include "runtime-common/core/memory-resource/unsynchronized_pool_resource.h" +#include "runtime-common/core/allocator/pool-allocator.h" -struct RuntimeAllocator final : vk::not_copyable { - static RuntimeAllocator& get() noexcept; - - RuntimeAllocator() = default; - RuntimeAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size); - - void init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size); - void free(); +struct RuntimeAllocator final { +private: + kphp::memory::pool_allocator m_allocator; - void* alloc_script_memory(size_t size) noexcept; - void* alloc0_script_memory(size_t size) noexcept; - void* realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept; - void free_script_memory(void* mem, size_t size) noexcept; +public: + static auto get() noexcept -> RuntimeAllocator&; - void* alloc_global_memory(size_t size) noexcept; - void* alloc0_global_memory(size_t size) noexcept; - void* realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept; - void free_global_memory(void* mem, size_t size) noexcept; + RuntimeAllocator() = default; + RuntimeAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept; -private: - void request_extra_memory(size_t requested_size) noexcept; + auto init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void; + auto free() noexcept -> void; -public: - memory_resource::unsynchronized_pool_resource memory_resource; + auto alloc_script_memory(size_t size) noexcept -> void*; + auto calloc_script_memory(size_t size) noexcept -> void*; + auto realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void*; + auto free_script_memory(void* mem, size_t size) noexcept -> void; -private: - size_t m_min_extra_mem_size{0}; + auto get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource& { + return m_allocator.get_memory_resource(); + } }; diff --git a/runtime-common/core/allocator/script-malloc-interface.h b/runtime-common/core/allocator/script-malloc-interface.h index 6af4350f87..de6b3f1775 100644 --- a/runtime-common/core/allocator/script-malloc-interface.h +++ b/runtime-common/core/allocator/script-malloc-interface.h @@ -4,170 +4,36 @@ #pragma once -#include #include -#include #include -#include -#include "common/wrappers/likely.h" +#include "runtime-common/core/allocator/details/malloc-interface.h" #include "runtime-common/core/allocator/runtime-allocator.h" -#include "runtime-common/core/utils/kphp-assert-core.h" -namespace kphp { +namespace kphp::memory::script { -namespace memory { - -namespace script { - -constexpr uint64_t MALLOC_REPLACER_MAX_ALLOC = 0xFFFFFF00; // 4GiB - -namespace details { -struct control_block { -private: - static constexpr auto SIZE_FIELD_BITSIZE{48}; - static constexpr auto BASE_OFFSET_FIELD_BITSIZE{16}; - static constexpr uint64_t BLOCK_SIZE_MASK{(1UL << SIZE_FIELD_BITSIZE) - 1}; - static constexpr uint64_t BASE_OFFSET_MASK{(1UL << BASE_OFFSET_FIELD_BITSIZE) - 1}; - - static_assert(SIZE_FIELD_BITSIZE + BASE_OFFSET_FIELD_BITSIZE == std::numeric_limits::digits); - -public: - static constexpr uint64_t max_size() noexcept { - return 1UL << SIZE_FIELD_BITSIZE; - } - - static constexpr uint64_t max_alignment() noexcept { - return 1UL << BASE_OFFSET_FIELD_BITSIZE; - } - - uint64_t raw() const noexcept { - return (static_cast(base_offset) << SIZE_FIELD_BITSIZE) | (static_cast(size) & BLOCK_SIZE_MASK); - } - - static control_block from_raw(uint64_t raw) noexcept { - return control_block{.size = raw & BLOCK_SIZE_MASK, .base_offset = static_cast((raw >> SIZE_FIELD_BITSIZE) & BASE_OFFSET_MASK)}; - } - - uint64_t size : SIZE_FIELD_BITSIZE; - uint16_t base_offset : BASE_OFFSET_FIELD_BITSIZE; -}; - -inline bool is_power_of_2(uint64_t v) noexcept { - return v && !(v & (v - 1)); +inline auto alloc(size_t size) noexcept -> void* { + return kphp::memory::details::malloc_interface::alloc(size); } -static_assert(sizeof(control_block) == sizeof(uint64_t), "Control block's size must be equal to uint64"); - -} // namespace details - -inline void* alloc(size_t size) noexcept { - constexpr size_t cb_size{sizeof(details::control_block)}; - if (unlikely(size > std::min(details::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - cb_size)) { - php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); - return nullptr; - } - const size_t total_size{size + cb_size}; - void* base{RuntimeAllocator::get().alloc_script_memory(total_size)}; - if (unlikely(base == nullptr)) { - php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size); - return base; - } - *(static_cast(base)) = details::control_block{.size = total_size, .base_offset = cb_size}.raw(); - return static_cast(static_cast(base) + cb_size); +inline auto alloc_aligned(size_t size, std::align_val_t alignment) noexcept -> void* { + return kphp::memory::details::malloc_interface::alloc_aligned(size, alignment); } -inline void* alloc_aligned(size_t size, std::align_val_t alignment) noexcept { - // Check that provided alignment is power of two - const size_t align{static_cast(alignment)}; - if (unlikely(align == 0 || !details::is_power_of_2(align) || align >= details::control_block::max_alignment())) { - php_warning("allocation alignment have to be non-zero power of two and not greater than %" PRIu64 ", got : %lu", details::control_block::max_alignment(), - align); - return nullptr; - } - - // Check that memory is enough - constexpr size_t cb_size{sizeof(details::control_block)}; - if (unlikely(size > std::min(details::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - (align - 1) - cb_size)) { - php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); - return nullptr; - } - - // Request mem from underlying memory manager - const size_t total_size{size + (align - 1) + cb_size}; - void* base{RuntimeAllocator::get().alloc_script_memory(total_size)}; - if (unlikely(base == nullptr)) { - php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size); - return base; - } - - const uint64_t base_u{reinterpret_cast(base)}; - // The smallest multiple of `align` greater than or equal to requested memory - const uint64_t aligned_u{((base_u + cb_size) + (align - 1)) & ~(align - 1)}; - const uint64_t base_offset_u{aligned_u - base_u}; - - // Save control block - *(reinterpret_cast(aligned_u - cb_size)) = // NOLINT - details::control_block{.size = total_size, .base_offset = static_cast(base_offset_u)}.raw(); - - return reinterpret_cast(aligned_u); // NOLINT -} - -inline void* calloc(size_t num, size_t size) noexcept { - void* ptr{kphp::memory::script::alloc(num * size)}; - if (unlikely(ptr == nullptr)) { - return nullptr; - } - return std::memset(ptr, 0, num * size); +inline auto calloc(size_t num, size_t size) noexcept -> void* { + return kphp::memory::details::malloc_interface::calloc(num, size); } -inline void free(void* ptr) noexcept { - if (unlikely(ptr == nullptr)) { - return; - } - - constexpr size_t cb_size{sizeof(details::control_block)}; - const auto mem{reinterpret_cast(ptr)}; - - const auto cb{details::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT - void* base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT - - RuntimeAllocator::get().free_script_memory(base, cb.size); +inline auto free(void* ptr) noexcept -> void { + kphp::memory::details::malloc_interface::free(ptr); } -inline void* realloc(void* ptr, size_t new_size) noexcept { - if (unlikely(ptr == nullptr)) { - return kphp::memory::script::alloc(new_size); - } - - if (unlikely(new_size == 0)) { - kphp::memory::script::free(ptr); - return nullptr; - } - - constexpr size_t cb_size{sizeof(details::control_block)}; - const auto mem{reinterpret_cast(ptr)}; - - const auto cb{details::control_block::from_raw(*reinterpret_cast(mem - cb_size))}; // NOLINT - - void* old_base{reinterpret_cast(mem - cb.base_offset)}; // NOLINT - const size_t old_size{cb.size}; - - void* new_ptr{kphp::memory::script::alloc(new_size)}; - if (likely(new_ptr != nullptr)) { - std::memcpy(new_ptr, ptr, std::min(new_size, old_size)); - RuntimeAllocator::get().free_script_memory(old_base, old_size); - } - return new_ptr; +inline auto realloc(void* ptr, size_t new_size) noexcept -> void* { + return kphp::memory::details::malloc_interface::realloc(ptr, new_size); } -inline char* strdup(const char* str1) noexcept { - auto* str2{static_cast(kphp::memory::script::alloc(std::strlen(str1) + 1))}; - return std::strcpy(str2, str1); +inline auto strdup(const char* str1) noexcept -> char* { + return kphp::memory::details::malloc_interface::strdup(str1); } -} // namespace script - -} // namespace memory - -} // namespace kphp +} // namespace kphp::memory::script diff --git a/runtime-common/core/core-types/definition/array.inl b/runtime-common/core/core-types/definition/array.inl index ce1270adae..da86d14d41 100644 --- a/runtime-common/core/core-types/definition/array.inl +++ b/runtime-common/core/core-types/definition/array.inl @@ -261,7 +261,7 @@ typename array::array_inner* array::array_inner::create(int64_t new_int_si auto shift_pointer_to_array_inner = [](void* mem) { return reinterpret_cast(static_cast(mem) + sizeof(array_inner_fields_for_map)); }; - array_inner* p = shift_pointer_to_array_inner(RuntimeAllocator::get().alloc0_script_memory(mem_size)); + array_inner* p = shift_pointer_to_array_inner(RuntimeAllocator::get().calloc_script_memory(mem_size)); p->is_vector_internal = false; p->ref_cnt = 0; p->max_key = -1; diff --git a/runtime-common/core/core-types/definition/string_buffer.cpp b/runtime-common/core/core-types/definition/string_buffer.cpp index f88f6c21c9..8d810194a6 100644 --- a/runtime-common/core/core-types/definition/string_buffer.cpp +++ b/runtime-common/core/core-types/definition/string_buffer.cpp @@ -5,11 +5,11 @@ #include #include -#include "runtime-common/core/allocator/runtime-allocator.h" +#include "runtime-common/core/allocator/platform-malloc-interface.h" #include "runtime-common/core/runtime-core.h" string_buffer::string_buffer(string::size_type buffer_len) noexcept - : buffer_end(static_cast(RuntimeAllocator::get().alloc_global_memory(buffer_len))), + : buffer_end(static_cast(kphp::memory::platform::alloc(buffer_len))), buffer_begin(buffer_end), buffer_len(buffer_len) {} @@ -20,14 +20,15 @@ string_buffer::string_buffer(string_buffer&& other) noexcept string_buffer& string_buffer::operator=(string_buffer&& other) noexcept { if (this != std::addressof(other)) { - RuntimeAllocator::get().free_global_memory(buffer_begin, buffer_len); + kphp::memory::platform::free(buffer_begin); buffer_end = std::exchange(other.buffer_end, nullptr); buffer_begin = std::exchange(other.buffer_begin, nullptr); buffer_len = std::exchange(other.buffer_len, 0); } + return *this; } string_buffer::~string_buffer() noexcept { - RuntimeAllocator::get().free_global_memory(buffer_begin, buffer_len); + kphp::memory::platform::free(buffer_begin); } diff --git a/runtime-common/core/core-types/definition/string_buffer.inl b/runtime-common/core/core-types/definition/string_buffer.inl index f3497a8676..d59e9ce50b 100644 --- a/runtime-common/core/core-types/definition/string_buffer.inl +++ b/runtime-common/core/core-types/definition/string_buffer.inl @@ -1,6 +1,7 @@ #pragma once #include "common/algorithms/simd-int-to-string.h" +#include "runtime-common/core/allocator/platform-malloc-interface.h" #ifndef INCLUDED_FROM_KPHP_CORE #error "this file must be included only from runtime-core.h" @@ -26,7 +27,7 @@ inline void string_buffer::resize(string::size_type new_buffer_len) noexcept { } string::size_type current_len = size(); - if (void* new_mem = RuntimeAllocator::get().realloc_global_memory(buffer_begin, new_buffer_len, buffer_len)) { + if (void* new_mem = kphp::memory::platform::realloc(buffer_begin, new_buffer_len)) { buffer_begin = static_cast(new_mem); buffer_len = new_buffer_len; buffer_end = buffer_begin + current_len; diff --git a/runtime-common/stdlib/math/bcmath-functions.cpp b/runtime-common/stdlib/math/bcmath-functions.cpp index ae91774d57..9acd78e92a 100644 --- a/runtime-common/stdlib/math/bcmath-functions.cpp +++ b/runtime-common/stdlib/math/bcmath-functions.cpp @@ -266,7 +266,7 @@ string bc_mul_positive_impl(const char* lhs, int lint, int ldot, int lfrac, int int result_size = result_len + result_scale + 3; string result(static_cast(result_size), false); - int* res = (int*)RuntimeAllocator::get().alloc0_script_memory(static_cast(sizeof(int) * result_size)); + int* res = (int*)RuntimeAllocator::get().calloc_script_memory(static_cast(sizeof(int) * result_size)); for (int i = -lscale; i < llen; i++) { int x = (i < 0 ? lhs[lfrac - i - 1] : lhs[ldot - i - 1]) - '0'; for (int j = -rscale; j < rlen; j++) { @@ -319,7 +319,7 @@ string bc_div_positive_impl(const char* lhs, int lint, int ldot, int lfrac, int int dividend_len = llen + lscale; int divider_len = rlen + rscale; - int* dividend = (int*)RuntimeAllocator::get().alloc0_script_memory(static_cast(sizeof(int) * (result_size + dividend_len + divider_len))); + int* dividend = (int*)RuntimeAllocator::get().calloc_script_memory(static_cast(sizeof(int) * (result_size + dividend_len + divider_len))); int* divider = (int*)RuntimeAllocator::get().alloc_script_memory(static_cast(sizeof(int) * divider_len)); for (int i = -lscale; i < llen; i++) { diff --git a/runtime-light/allocator/allocator-state.h b/runtime-light/allocator/allocator-state.h index a5cc2e79e7..e5dde50264 100644 --- a/runtime-light/allocator/allocator-state.h +++ b/runtime-light/allocator/allocator-state.h @@ -11,8 +11,6 @@ #include "runtime-common/core/allocator/runtime-allocator.h" #include "runtime-light/stdlib/diagnostics/logs.h" -inline constexpr auto DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE{static_cast(1 * 1024U * 1024U)}; // 1Mib - class AllocatorState final : private vk::not_copyable { uint32_t m_libc_alloc_allowed{}; diff --git a/runtime-light/allocator/allocator.cmake b/runtime-light/allocator/allocator.cmake index 85880d002e..68a7f446f0 100644 --- a/runtime-light/allocator/allocator.cmake +++ b/runtime-light/allocator/allocator.cmake @@ -1 +1,4 @@ -set(RUNTIME_LIGHT_ALLOCATOR_SRC allocator/runtime-light-allocator.cpp) +set(RUNTIME_LIGHT_ALLOCATOR_SRC + allocator/runtime-light-allocator.cpp + allocator/pool-allocator.cpp + allocator/platform-malloc-interface.cpp) diff --git a/runtime-light/allocator/platform-malloc-interface.cpp b/runtime-light/allocator/platform-malloc-interface.cpp new file mode 100644 index 0000000000..af0cceaf8f --- /dev/null +++ b/runtime-light/allocator/platform-malloc-interface.cpp @@ -0,0 +1,40 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include + +#include "common/wrappers/likely.h" +#include "runtime-common/core/allocator/platform-malloc-interface.h" +#include "runtime-common/core/utils/kphp-assert-core.h" +#include "runtime-light/k2-platform/k2-api.h" + +namespace kphp::memory::platform { + +auto alloc(size_t size) noexcept -> void* { + if (unlikely(size > MALLOC_REPLACER_MAX_ALLOC - MALLOC_REPLACER_SIZE_OFFSET)) { + php_warning("attempt to allocate too much memory by malloc replacer : %lu", size); + return nullptr; + } + + const size_t real_size{size + MALLOC_REPLACER_SIZE_OFFSET}; + void* ptr{k2::alloc(real_size)}; + + if (unlikely(ptr == nullptr)) { + php_warning("not enough platform memory to allocate: %lu", size); + return ptr; + } + + *static_cast(ptr) = real_size; + + return static_cast(ptr) + MALLOC_REPLACER_SIZE_OFFSET; +} + +void free(void* ptr) noexcept { + if (likely(ptr != nullptr)) { + void* real_ptr{static_cast(ptr) - MALLOC_REPLACER_SIZE_OFFSET}; + k2::free(real_ptr); + } +} + +} // namespace kphp::memory::platform diff --git a/runtime-light/allocator/pool-allocator.cpp b/runtime-light/allocator/pool-allocator.cpp new file mode 100644 index 0000000000..cb24f9228d --- /dev/null +++ b/runtime-light/allocator/pool-allocator.cpp @@ -0,0 +1,114 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include +#include +#include +#include + +#include "runtime-common/core/allocator/platform-malloc-interface.h" +#include "runtime-common/core/allocator/pool-allocator.h" +#include "runtime-light/stdlib/diagnostics/logs.h" + +namespace kphp::memory { + +pool_allocator::pool_allocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept + : m_min_extra_mem_size(min_extra_mem_size) { + // kphp::log::debug("create pool allocator -> {:p}: script memory -> {}, oom handling size -> {}", reinterpret_cast(this), script_mem_size, + // oom_handling_mem_size); + void* buffer{kphp::memory::platform::alloc(script_mem_size)}; + + kphp::log::assertion(buffer != nullptr); + + memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); +} + +auto pool_allocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void { + kphp::log::assertion(buffer != nullptr); + + // kphp::log::debug("init pool allocator -> {:p}: buffer -> {:p}, script memory -> {}, oom handling size -> {}", reinterpret_cast(this), buffer, + // script_mem_size, oom_handling_mem_size); + memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); +} + +auto pool_allocator::free() noexcept -> void { + // kphp::log::debug("free pool allocator -> {:p}", reinterpret_cast(this)); + auto* extra_memory{memory_resource.get_extra_memory_head()}; + while (extra_memory->get_pool_payload_size() != 0) { + auto* extra_memory_to_release{extra_memory}; + extra_memory = extra_memory->next_in_chain; + kphp::memory::platform::free(extra_memory_to_release); + } + + kphp::memory::platform::free(memory_resource.memory_begin()); +} + +auto pool_allocator::alloc(size_t size) noexcept -> void* { + kphp::log::assertion(size != 0); + + void* mem{memory_resource.allocate(size)}; + if (mem == nullptr) [[unlikely]] { + request_extra_memory(size); + mem = memory_resource.allocate(size); + + kphp::log::assertion(mem != nullptr); + } + + return mem; +} + +auto pool_allocator::calloc(size_t size) noexcept -> void* { + kphp::log::assertion(size != 0); + + void* mem{memory_resource.allocate0(size)}; + if (mem == nullptr) [[unlikely]] { + request_extra_memory(size); + mem = memory_resource.allocate0(size); + + kphp::log::assertion(mem != nullptr); + } + + return mem; +} + +auto pool_allocator::realloc(void* old_mem, size_t new_size, size_t old_size) noexcept -> void* { + kphp::log::assertion(new_size > old_size); + + void* new_mem{memory_resource.reallocate(old_mem, new_size, old_size)}; + if (new_mem == nullptr) [[unlikely]] { + request_extra_memory(new_size * 2); + new_mem = memory_resource.reallocate(old_mem, new_size, old_size); + + kphp::log::assertion(new_mem != nullptr); + } + + return new_mem; +} + +auto pool_allocator::free(void* mem, size_t size) noexcept -> void { + kphp::log::assertion(size != 0); + + memory_resource.deallocate(mem, size); +} + +auto pool_allocator::request_extra_memory(size_t requested_size) noexcept -> void { + // Extra mem size have to be greater than max chunk block + const auto min_size{std::max(m_min_extra_mem_size, memory_resource::unsynchronized_pool_resource::MAX_CHUNK_BLOCK_SIZE)}; + + size_t extra_mem_size{std::max(min_size, requested_size)}; + // Take into account internal layout of `memory_resource::extra_memory_pool` + extra_mem_size += sizeof(memory_resource::extra_memory_pool); + // The smallest power of two that is not smaller than `extra_mem_size` + extra_mem_size = std::bit_ceil(extra_mem_size); + + // kphp::log::debug("requested extra memory pool with size {} bytes, will be allocated {} bytes", requested_size, extra_mem_size); + + auto* extra_mem{kphp::memory::platform::alloc(extra_mem_size)}; + + kphp::log::assertion(extra_mem != nullptr); + + memory_resource.add_extra_memory(new (extra_mem) memory_resource::extra_memory_pool{extra_mem_size}); +} + +} // namespace kphp::memory diff --git a/runtime-light/allocator/runtime-light-allocator.cpp b/runtime-light/allocator/runtime-light-allocator.cpp index 8234c5ebb7..84e5a8f0b8 100644 --- a/runtime-light/allocator/runtime-light-allocator.cpp +++ b/runtime-light/allocator/runtime-light-allocator.cpp @@ -2,118 +2,36 @@ // Copyright (c) 2024 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt -#include -#include -#include -#include - +#include "runtime-common/core/allocator/runtime-allocator.h" #include "runtime-light/allocator/allocator-state.h" -#include "runtime-light/k2-platform/k2-api.h" -#include "runtime-light/stdlib/diagnostics/logs.h" -RuntimeAllocator& RuntimeAllocator::get() noexcept { +auto RuntimeAllocator::get() noexcept -> RuntimeAllocator& { return AllocatorState::get_mutable().allocator; } -RuntimeAllocator::RuntimeAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) - : m_min_extra_mem_size(min_extra_mem_size) { - // kphp::log::debug("create runtime allocator -> {:p}: script memory -> {}, oom handling size -> {}", reinterpret_cast(this), script_mem_size, - // oom_handling_mem_size); - void* buffer{alloc_global_memory(script_mem_size)}; - memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); -} - -void RuntimeAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) { - kphp::log::assertion(buffer != nullptr); - // kphp::log::debug("init runtime allocator -> {:p}: buffer -> {:p}, script memory -> {}, oom handling size -> {}", reinterpret_cast(this), buffer, - // script_mem_size, oom_handling_mem_size); - memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); -} - -void RuntimeAllocator::free() { - // kphp::log::debug("free runtime allocator -> {:p}", reinterpret_cast(this)); - auto* extra_memory{memory_resource.get_extra_memory_head()}; - while (extra_memory->get_pool_payload_size() != 0) { - auto* extra_memory_to_release{extra_memory}; - extra_memory = extra_memory->next_in_chain; - k2::free(extra_memory_to_release); - } - k2::free(memory_resource.memory_begin()); -} - -void* RuntimeAllocator::alloc_script_memory(size_t size) noexcept { - kphp::log::assertion(size != 0); - void* mem{memory_resource.allocate(size)}; - if (mem == nullptr) [[unlikely]] { - request_extra_memory(size); - mem = memory_resource.allocate(size); - kphp::log::assertion(mem != nullptr); - } - return mem; -} +RuntimeAllocator::RuntimeAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept + : m_allocator{script_mem_size, min_extra_mem_size, oom_handling_mem_size} {} -void* RuntimeAllocator::alloc0_script_memory(size_t size) noexcept { - kphp::log::assertion(size != 0); - void* mem{memory_resource.allocate0(size)}; - if (mem == nullptr) [[unlikely]] { - request_extra_memory(size); - mem = memory_resource.allocate0(size); - kphp::log::assertion(mem != nullptr); - } - return mem; +auto RuntimeAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void { + m_allocator.init(buffer, script_mem_size, oom_handling_mem_size); } -void* RuntimeAllocator::realloc_script_memory(void* old_mem, size_t new_size, size_t old_size) noexcept { - kphp::log::assertion(new_size > old_size); - void* new_mem{memory_resource.reallocate(old_mem, new_size, old_size)}; - if (new_mem == nullptr) [[unlikely]] { - request_extra_memory(new_size * 2); - new_mem = memory_resource.reallocate(old_mem, new_size, old_size); - kphp::log::assertion(new_mem != nullptr); - } - return new_mem; +auto RuntimeAllocator::free() noexcept -> void { + m_allocator.free(); } -void RuntimeAllocator::free_script_memory(void* mem, size_t size) noexcept { - kphp::log::assertion(size != 0); - memory_resource.deallocate(mem, size); +auto RuntimeAllocator::alloc_script_memory(size_t size) noexcept -> void* { + return m_allocator.alloc(size); } -void* RuntimeAllocator::alloc_global_memory(size_t size) noexcept { - void* mem{k2::alloc(size)}; - kphp::log::assertion(mem != nullptr); - return mem; +auto RuntimeAllocator::calloc_script_memory(size_t size) noexcept -> void* { + return m_allocator.calloc(size); } -void* RuntimeAllocator::alloc0_global_memory(size_t size) noexcept { - void* mem{k2::alloc(size)}; - kphp::log::assertion(mem != nullptr); - std::memset(mem, 0, size); - return mem; +auto RuntimeAllocator::realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void* { + return m_allocator.realloc(mem, new_size, old_size); } -void* RuntimeAllocator::realloc_global_memory(void* old_mem, size_t new_size, size_t /*unused*/) noexcept { - void* mem{k2::realloc(old_mem, new_size)}; - kphp::log::assertion(mem != nullptr); - return mem; -} - -void RuntimeAllocator::free_global_memory(void* mem, size_t /*unused*/) noexcept { - k2::free(mem); -} - -void RuntimeAllocator::request_extra_memory(size_t requested_size) noexcept { - // Extra mem size have to be greater than max chunk block - const auto min_size{std::max(m_min_extra_mem_size, memory_resource::unsynchronized_pool_resource::MAX_CHUNK_BLOCK_SIZE)}; - - size_t extra_mem_size{std::max(min_size, requested_size)}; - // Take into account internal layout of `memory_resource::extra_memory_pool` - extra_mem_size += sizeof(memory_resource::extra_memory_pool); - // The smallest power of two that is not smaller than `extra_mem_size` - extra_mem_size = std::bit_ceil(extra_mem_size); - - // kphp::log::debug("requested extra memory pool with size {} bytes, will be allocated {} bytes", requested_size, extra_mem_size); - - auto* extra_mem{alloc_global_memory(extra_mem_size)}; - memory_resource.add_extra_memory(new (extra_mem) memory_resource::extra_memory_pool{extra_mem_size}); +auto RuntimeAllocator::free_script_memory(void* mem, size_t size) noexcept -> void { + m_allocator.free(mem, size); } diff --git a/runtime-light/components/confdata/state/component-state.h b/runtime-light/components/confdata/state/component-state.h index 9e39724264..c111f8ee8d 100644 --- a/runtime-light/components/confdata/state/component-state.h +++ b/runtime-light/components/confdata/state/component-state.h @@ -15,7 +15,7 @@ #include "runtime-light/stdlib/diagnostics/logs.h" struct ComponentState final : private vk::not_copyable { - AllocatorState m_allocator_state{INIT_COMPONENT_ALLOCATOR_SIZE, DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE, 0}; + AllocatorState m_allocator_state{INIT_COMPONENT_ALLOCATOR_SIZE, DEFAULT_MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE, 0}; kphp::stl::string m_confdata_proxy_actor_name; private: @@ -31,7 +31,8 @@ struct ComponentState final : private vk::not_copyable { auto parse_args() noexcept -> void; static constexpr std::string_view CONFDATA_PROXY_ACTOR_NAME_ARG{"confdata-proxy-actor-name"}; - static constexpr auto INIT_COMPONENT_ALLOCATOR_SIZE{static_cast(1024U * 1024U)}; // 1MiB + static constexpr auto INIT_COMPONENT_ALLOCATOR_SIZE{static_cast(1024U * 1024U)}; // 1MiB + static constexpr auto DEFAULT_MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB }; inline ComponentState::ComponentState() noexcept { diff --git a/runtime-light/components/confdata/state/instance-state.h b/runtime-light/components/confdata/state/instance-state.h index 8bd4c66434..1f1c4373fa 100644 --- a/runtime-light/components/confdata/state/instance-state.h +++ b/runtime-light/components/confdata/state/instance-state.h @@ -19,14 +19,14 @@ struct InstanceState final : vk::not_copyable { enum class warmup_status : uint8_t { pending, done }; - AllocatorState m_allocator_state{INIT_INSTANCE_ALLOCATOR_SIZE, DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE, 0}; + AllocatorState m_allocator_state{INIT_INSTANCE_ALLOCATOR_SIZE, DEFAULT_MIN_INSTANCE_EXTRA_MEMORY_POOL_SIZE, 0}; warmup_status m_warmup_status{warmup_status::pending}; kphp::confdata::pagination m_pagination{}; kphp::log::contextual_tags m_instance_tags; - kphp::coro::instance_state m_coroutine_instance_state; + kphp::coro::instance_state m_coroutine_instance_state{INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE, DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE, 0}; kphp::coro::io_scheduler m_io_scheduler{m_coroutine_instance_state}; InstanceState() noexcept = default; @@ -35,7 +35,10 @@ struct InstanceState final : vk::not_copyable { auto init() noexcept -> void; private: - static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(16U * 1024U * 1024U); // 16MiB + static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(16U * 1024U * 1024U); // 16MiB + static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(2U * 1024U * 1024U); // 2MiB + static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE = static_cast(512U * 1024U); // 0.5MiB auto run() noexcept -> kphp::coro::task<>; auto accept_loop() noexcept -> kphp::coro::task<>; diff --git a/runtime-light/components/kphp/state/component-state.cpp b/runtime-light/components/kphp/state/component-state.cpp index 9cd5b6e9da..251f73d35d 100644 --- a/runtime-light/components/kphp/state/component-state.cpp +++ b/runtime-light/components/kphp/state/component-state.cpp @@ -132,6 +132,24 @@ void ComponentState::parse_min_instance_extra_memory_size_arg(std::string_view v kphp::log::info("set min instance extra memory size to {} bytes", min_instance_extra_memory_size); } +void ComponentState::parse_initial_instance_coroutine_memory_size_arg(std::string_view value_view) noexcept { + const auto parsed{parse_uint64(value_view)}; + if (!parsed) { + kphp::log::error("couldn't parse initial instance coroutine memory size, got {}", value_view); + } + initial_instance_coroutine_memory_size = *parsed; + kphp::log::info("set initial instance coroutine memory size to {} bytes", initial_instance_coroutine_memory_size); +} + +void ComponentState::parse_min_instance_extra_coroutine_memory_size_arg(std::string_view value_view) noexcept { + const auto parsed{parse_uint64(value_view)}; + if (!parsed) { + kphp::log::error("couldn't parse min instance extra coroutine memory size, got {}", value_view); + } + min_instance_extra_coroutine_memory_size = *parsed; + kphp::log::info("set min instance extra coroutine memory size to {} bytes", min_instance_extra_coroutine_memory_size); +} + void ComponentState::parse_args() noexcept { for (auto i = 0; i < argc; ++i) { const auto [arg_key, arg_value]{k2::arg_fetch(i)}; @@ -152,6 +170,10 @@ void ComponentState::parse_args() noexcept { parse_initial_instance_memory_size_arg(value_view); } else if (key_view == MIN_INSTANCE_EXTRA_MEMORY_SIZE_ARG) { parse_min_instance_extra_memory_size_arg(value_view); + } else if (key_view == INITIAL_INSTANCE_COROUTINE_MEMORY_SIZE_ARG) { + parse_initial_instance_coroutine_memory_size_arg(value_view); + } else if (key_view == MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_SIZE_ARG) { + parse_min_instance_extra_coroutine_memory_size_arg(value_view); } else { kphp::log::warning("unexpected argument format: {}", key_view); } diff --git a/runtime-light/components/kphp/state/component-state.h b/runtime-light/components/kphp/state/component-state.h index 9182b62107..f737cd5c62 100644 --- a/runtime-light/components/kphp/state/component-state.h +++ b/runtime-light/components/kphp/state/component-state.h @@ -19,7 +19,7 @@ #include "runtime-light/stdlib/kml/kml-state.h" struct ComponentState final : private vk::not_copyable { - AllocatorState component_allocator_state{INIT_COMPONENT_ALLOCATOR_SIZE, DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE, 0}; + AllocatorState component_allocator_state{INIT_COMPONENT_ALLOCATOR_SIZE, DEFAULT_MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE, 0}; KmlComponentState kml_component_state; // This member does not hold any KPHP types, so setting a reference counter is unnecessary. const uint32_t argc{k2::args_count()}; @@ -30,7 +30,9 @@ struct ComponentState final : private vk::not_copyable { string cluster_name{DEFAULT_CLUSTER_NAME.data(), DEFAULT_CLUSTER_NAME.size()}; bool exit_after_response{}; uint64_t initial_instance_memory_size{INIT_INSTANCE_ALLOCATOR_SIZE}; - uint64_t min_instance_extra_memory_size{DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE}; + uint64_t min_instance_extra_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_MEMORY_POOL_SIZE}; + uint64_t initial_instance_coroutine_memory_size{INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE}; + uint64_t min_instance_extra_coroutine_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE}; ComponentState() noexcept { parse_env(); @@ -63,8 +65,14 @@ struct ComponentState final : private vk::not_copyable { static constexpr std::string_view EXIT_AFTER_RESPONSE_ARG = "exit-after-response"; static constexpr std::string_view INITIAL_INSTANCE_MEMORY_SIZE_ARG = "initial-instance-memory-size"; static constexpr std::string_view MIN_INSTANCE_EXTRA_MEMORY_SIZE_ARG = "min-instance-extra-memory-size"; - static constexpr auto INIT_COMPONENT_ALLOCATOR_SIZE = static_cast(1024U * 1024U); // 1MiB - static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(64U * 1024U * 1024U); // 64MiB + static constexpr std::string_view INITIAL_INSTANCE_COROUTINE_MEMORY_SIZE_ARG = "initial-instance-coroutine-memory-size"; + static constexpr std::string_view MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_SIZE_ARG = "min-instance-extra-coroutine-memory-size"; + static constexpr auto INIT_COMPONENT_ALLOCATOR_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(64U * 1024U * 1024U); // 64MiB + static constexpr auto DEFAULT_MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(8U * 1024U * 1024U); // 8MiB + static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE = static_cast(4U * 1024U * 1024U); // 4MiB void parse_env() noexcept; @@ -83,4 +91,8 @@ struct ComponentState final : private vk::not_copyable { void parse_initial_instance_memory_size_arg(std::string_view) noexcept; void parse_min_instance_extra_memory_size_arg(std::string_view) noexcept; + + void parse_initial_instance_coroutine_memory_size_arg(std::string_view) noexcept; + + void parse_min_instance_extra_coroutine_memory_size_arg(std::string_view) noexcept; }; diff --git a/runtime-light/components/kphp/state/image-state.h b/runtime-light/components/kphp/state/image-state.h index 7d3fabae02..ae0207fba9 100644 --- a/runtime-light/components/kphp/state/image-state.h +++ b/runtime-light/components/kphp/state/image-state.h @@ -30,7 +30,7 @@ #include "runtime-light/stdlib/visitors/shape-visitors.h" struct ImageState final : private vk::not_copyable { - AllocatorState image_allocator_state{INIT_IMAGE_ALLOCATOR_SIZE, DEFAULT_MIN_EXTRA_MEMORY_POOL_SIZE, 0}; + AllocatorState image_allocator_state{INIT_IMAGE_ALLOCATOR_SIZE, MIN_EXTRA_MEMORY_POOL_SIZE, 0}; uint32_t pid{k2::getpid()}; uid_t uid{k2::getuid()}; @@ -102,5 +102,6 @@ struct ImageState final : private vk::not_copyable { } private: - static constexpr auto INIT_IMAGE_ALLOCATOR_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto INIT_IMAGE_ALLOCATOR_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto MIN_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB }; diff --git a/runtime-light/components/kphp/state/instance-state.h b/runtime-light/components/kphp/state/instance-state.h index 65ba170383..88396086cb 100644 --- a/runtime-light/components/kphp/state/instance-state.h +++ b/runtime-light/components/kphp/state/instance-state.h @@ -8,6 +8,7 @@ #include #include "common/mixin/not_copyable.h" +#include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/runtime-core.h" #include "runtime-common/core/std/containers.h" #include "runtime-light/allocator/allocator-state.h" @@ -60,9 +61,6 @@ struct InstanceState final : vk::not_copyable { template using deque = kphp::stl::deque; - template - using list = kphp::stl::list; - // It's important to use `{}` instead of `= default` here. // In the second case clang++ zeroes the whole structure. // It drastically ruins performance. Be careful! @@ -92,7 +90,8 @@ struct InstanceState final : vk::not_copyable { kphp::log::contextual_tags instance_tags; - kphp::coro::instance_state coroutine_instance_state; + kphp::coro::instance_state coroutine_instance_state{ComponentState::get().initial_instance_coroutine_memory_size, + ComponentState::get().min_instance_extra_coroutine_memory_size, 0}; kphp::coro::io_scheduler io_scheduler{coroutine_instance_state}; ForkInstanceState fork_instance_state; WaitQueueInstanceState wait_queue_instance_state; @@ -122,7 +121,7 @@ struct InstanceState final : vk::not_copyable { ErrorHandlingState error_handling_instance_state; KmlInstanceState kml_instance_state; - list> shutdown_functions; + kphp::stl::list, kphp::memory::script_allocator> shutdown_functions; private: kphp::coro::task<> init_cli_instance() noexcept; diff --git a/runtime-light/core/globals/php-script-globals.cpp b/runtime-light/core/globals/php-script-globals.cpp index 298b304de9..9d5a049c1d 100644 --- a/runtime-light/core/globals/php-script-globals.cpp +++ b/runtime-light/core/globals/php-script-globals.cpp @@ -5,18 +5,18 @@ #include "php-script-globals.h" #include "common/php-functions.h" -#include "runtime-common/core/allocator/runtime-allocator.h" +#include "runtime-common/core/allocator/platform-malloc-interface.h" #include "runtime-light/stdlib/diagnostics/logs.h" void PhpScriptMutableGlobals::once_alloc_linear_mem(unsigned int n_bytes) { kphp::log::assertion(g_linear_mem == nullptr); - g_linear_mem = static_cast(RuntimeAllocator::get().alloc0_global_memory(n_bytes)); + g_linear_mem = static_cast(kphp::memory::platform::calloc(1, n_bytes)); } void PhpScriptMutableGlobals::once_alloc_linear_mem(const char* lib_name, unsigned int n_bytes) { int64_t key_lib_name{string_hash(lib_name, strlen(lib_name))}; kphp::log::assertion(libs_linear_mem.find(key_lib_name) == libs_linear_mem.end()); - libs_linear_mem[key_lib_name] = static_cast(RuntimeAllocator::get().alloc0_global_memory(n_bytes)); + libs_linear_mem[key_lib_name] = static_cast(kphp::memory::platform::calloc(1, n_bytes)); } char* PhpScriptMutableGlobals::get_linear_mem(const char* lib_name) const { diff --git a/runtime-light/coroutine/coroutine-state.h b/runtime-light/coroutine/coroutine-state.h index 169ff00993..9c5f5a2462 100644 --- a/runtime-light/coroutine/coroutine-state.h +++ b/runtime-light/coroutine/coroutine-state.h @@ -7,15 +7,18 @@ #include "common/mixin/not_copyable.h" #include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h" namespace kphp::coro { struct instance_state final : private vk::not_copyable { - instance_state() noexcept = default; + instance_state(size_t coroutine_mem_size, size_t min_extra_coroutine_mem_size, size_t oom_handling_coroutine_mem_size) noexcept + : coroutine_allocator{coroutine_mem_size, min_extra_coroutine_mem_size, oom_handling_coroutine_mem_size} {} static instance_state& get() noexcept; + kphp::coro::detail::memory::RuntimeCoroutineAllocator coroutine_allocator; kphp::coro::async_stack_root coroutine_stack_root; }; diff --git a/runtime-light/coroutine/coroutine.cmake b/runtime-light/coroutine/coroutine.cmake new file mode 100644 index 0000000000..923f777a9b --- /dev/null +++ b/runtime-light/coroutine/coroutine.cmake @@ -0,0 +1,2 @@ +set(RUNTIME_LIGHT_COROUTINE_SRC + coroutine/detail/allocator/runtime-coroutine-allocator.cpp) diff --git a/runtime-light/coroutine/detail/allocator/coroutine-allocator.h b/runtime-light/coroutine/detail/allocator/coroutine-allocator.h new file mode 100644 index 0000000000..ea3d6b82bb --- /dev/null +++ b/runtime-light/coroutine/detail/allocator/coroutine-allocator.h @@ -0,0 +1,41 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h" + +namespace kphp::coro::detail::memory { + +template +struct coroutine_allocator { + using value_type = T; + + coroutine_allocator() noexcept = default; + + template + coroutine_allocator(const coroutine_allocator& /*unused*/) noexcept {} + + constexpr value_type* allocate(size_t n) noexcept { + return static_cast(kphp::coro::detail::memory::RuntimeCoroutineAllocator::get().alloc_script_memory(n * sizeof(T))); + } + + constexpr void deallocate(T* p, size_t n) noexcept { + kphp::coro::detail::memory::RuntimeCoroutineAllocator::get().free_script_memory(p, n * sizeof(T)); + } +}; + +template +constexpr bool operator==(const coroutine_allocator& /*unused*/, const coroutine_allocator& /*unused*/) { + return true; +} + +template +constexpr bool operator!=(const coroutine_allocator& /*unused*/, const coroutine_allocator& /*unused*/) { + return false; +} + +} // namespace kphp::coro::detail::memory diff --git a/runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h b/runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h new file mode 100644 index 0000000000..d62bdecc79 --- /dev/null +++ b/runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h @@ -0,0 +1,39 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include + +#include "runtime-common/core/allocator/details/malloc-interface.h" +#include "runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h" + +namespace kphp::coro::detail::memory { + +inline auto alloc(size_t size) noexcept -> void* { + return kphp::memory::details::malloc_interface::alloc(size); +} + +inline auto alloc_aligned(size_t size, std::align_val_t alignment) noexcept -> void* { + return kphp::memory::details::malloc_interface::alloc_aligned(size, alignment); +} + +inline auto calloc(size_t num, size_t size) noexcept -> void* { + return kphp::memory::details::malloc_interface::calloc(num, size); +} + +inline auto free(void* ptr) noexcept -> void { + kphp::memory::details::malloc_interface::free(ptr); +} + +inline auto realloc(void* ptr, size_t new_size) noexcept -> void* { + return kphp::memory::details::malloc_interface::realloc(ptr, new_size); +} + +inline auto strdup(const char* str1) noexcept -> char* { + return kphp::memory::details::malloc_interface::strdup(str1); +} + +} // namespace kphp::coro::detail::memory diff --git a/runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.cpp b/runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.cpp new file mode 100644 index 0000000000..413d9984a4 --- /dev/null +++ b/runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.cpp @@ -0,0 +1,14 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h" +#include "runtime-light/coroutine/coroutine-state.h" + +namespace kphp::coro::detail::memory { + +auto RuntimeCoroutineAllocator::get() noexcept -> RuntimeCoroutineAllocator& { + return kphp::coro::instance_state::get().coroutine_allocator; +} + +} // namespace kphp::coro::detail::memory diff --git a/runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h b/runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h new file mode 100644 index 0000000000..eb1b1fd65b --- /dev/null +++ b/runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h @@ -0,0 +1,54 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "runtime-common/core/allocator/pool-allocator.h" + +namespace kphp::coro::detail::memory { + +struct RuntimeCoroutineAllocator final { +private: + kphp::memory::pool_allocator m_allocator; + +public: + static auto get() noexcept -> RuntimeCoroutineAllocator&; + + RuntimeCoroutineAllocator() = default; + + RuntimeCoroutineAllocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept + : m_allocator{script_mem_size, min_extra_mem_size, oom_handling_mem_size} {} + + auto init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept -> void { + m_allocator.init(buffer, script_mem_size, oom_handling_mem_size); + } + + auto free() noexcept -> void { + m_allocator.free(); + } + + auto alloc_script_memory(size_t size) noexcept -> void* { + return m_allocator.alloc(size); + } + + auto calloc_script_memory(size_t size) noexcept -> void* { + return m_allocator.calloc(size); + } + + auto realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void* { + return m_allocator.realloc(mem, new_size, old_size); + } + + auto free_script_memory(void* mem, size_t size) noexcept -> void { + m_allocator.free(mem, size); + } + + auto get_memory_resource() noexcept -> memory_resource::unsynchronized_pool_resource& { + return m_allocator.get_memory_resource(); + } +}; + +} // namespace kphp::coro::detail::memory diff --git a/runtime-light/coroutine/detail/await-set.h b/runtime-light/coroutine/detail/await-set.h index 81fb98665e..b45f177559 100644 --- a/runtime-light/coroutine/detail/await-set.h +++ b/runtime-light/coroutine/detail/await-set.h @@ -11,8 +11,8 @@ #include #include "common/containers/intrusive-list.h" -#include "runtime-common/core/allocator/script-malloc-interface.h" #include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/void-value.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -53,16 +53,16 @@ class await_broker { template void* operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept { - return kphp::memory::script::alloc(n); + return kphp::coro::detail::memory::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::coro::detail::memory::alloc_aligned(n, al); } void operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept { - kphp::memory::script::free(ptr); + kphp::coro::detail::memory::free(ptr); } void start_task(await_set_task&& task, kphp::coro::async_stack_root& coroutine_stack_root, void* return_address) noexcept { @@ -175,16 +175,16 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { template void* operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept { - return kphp::memory::script::alloc(n); + return kphp::coro::detail::memory::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::coro::detail::memory::alloc_aligned(n, al); } void operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept { - kphp::memory::script::free(ptr); + kphp::coro::detail::memory::free(ptr); } std::suspend_always initial_suspend() const noexcept { diff --git a/runtime-light/coroutine/detail/task-self-deleting.h b/runtime-light/coroutine/detail/task-self-deleting.h index e6908f575b..ec8ddea722 100644 --- a/runtime-light/coroutine/detail/task-self-deleting.h +++ b/runtime-light/coroutine/detail/task-self-deleting.h @@ -9,10 +9,10 @@ #include #include "common/containers/intrusive-list.h" -#include "runtime-common/core/allocator/script-malloc-interface.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" #include "runtime-light/coroutine/coroutine-state.h" +#include "runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h" #include "runtime-light/stdlib/diagnostics/logs.h" namespace kphp::coro::detail { @@ -35,16 +35,16 @@ struct promise_self_deleting : kphp::coro::async_stack_element { template auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc(n); + return kphp::coro::detail::memory::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::coro::detail::memory::alloc_aligned(n, al); } auto operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept -> void { - kphp::memory::script::free(ptr); + kphp::coro::detail::memory::free(ptr); } auto get_return_object() noexcept -> task_self_deleting; diff --git a/runtime-light/coroutine/detail/when-all.h b/runtime-light/coroutine/detail/when-all.h index 2386e4fbb6..186ba2793c 100644 --- a/runtime-light/coroutine/detail/when-all.h +++ b/runtime-light/coroutine/detail/when-all.h @@ -16,6 +16,7 @@ #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" +#include "runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/void-value.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -152,16 +153,16 @@ class when_all_task_promise_base : public kphp::coro::async_stack_element { template auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc(n); + return kphp::coro::detail::memory::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::coro::detail::memory::alloc_aligned(n, al); } auto operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept -> void { - kphp::memory::script::free(ptr); + kphp::coro::detail::memory::free(ptr); } auto initial_suspend() const noexcept -> std::suspend_always { diff --git a/runtime-light/coroutine/detail/when-any.h b/runtime-light/coroutine/detail/when-any.h index f5ee01549e..8a90b1961f 100644 --- a/runtime-light/coroutine/detail/when-any.h +++ b/runtime-light/coroutine/detail/when-any.h @@ -14,6 +14,7 @@ #include #include "runtime-light/coroutine/concepts.h" +#include "runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/void-value.h" #include "runtime-light/metaprogramming/type-functions.h" @@ -162,16 +163,16 @@ class when_any_task_promise_base : public kphp::coro::async_stack_element { template auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc(n); + return kphp::coro::detail::memory::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::coro::detail::memory::alloc_aligned(n, al); } auto operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept -> void { - kphp::memory::script::free(ptr); + kphp::coro::detail::memory::free(ptr); } auto initial_suspend() const noexcept -> std::suspend_always { diff --git a/runtime-light/coroutine/event.h b/runtime-light/coroutine/event.h index 4be3984ac8..f73d9d246e 100644 --- a/runtime-light/coroutine/event.h +++ b/runtime-light/coroutine/event.h @@ -21,7 +21,7 @@ namespace kphp::coro { class event { - struct event_controller : kphp::memory::script_allocator_managed, vk::not_copyable { + struct event_controller : public memory::script_allocator_managed, private vk::not_copyable { // 1) std::monostate => not set and no coroutines are waiting // 2) non empty list => linked list of coroutines waiting for the event to trigger // 3) empty list => the event is triggered and all coroutines are resumed diff --git a/runtime-light/coroutine/shared-task.h b/runtime-light/coroutine/shared-task.h index af6fd6b591..e6148b6217 100644 --- a/runtime-light/coroutine/shared-task.h +++ b/runtime-light/coroutine/shared-task.h @@ -16,8 +16,8 @@ #include #include "common/containers/intrusive-list.h" -#include "runtime-common/core/allocator/script-malloc-interface.h" #include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/void-value.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -125,16 +125,16 @@ struct promise_base : kphp::coro::async_stack_element { template auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc(n); + return kphp::coro::detail::memory::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::coro::detail::memory::alloc_aligned(n, al); } auto operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept -> void { - kphp::memory::script::free(ptr); + kphp::coro::detail::memory::free(ptr); } private: diff --git a/runtime-light/coroutine/task.h b/runtime-light/coroutine/task.h index d5c064720b..8e874c480c 100644 --- a/runtime-light/coroutine/task.h +++ b/runtime-light/coroutine/task.h @@ -11,8 +11,8 @@ #include #include "common/containers/final_action.h" -#include "runtime-common/core/allocator/script-malloc-interface.h" #include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h" #include "runtime-light/stdlib/diagnostics/logs.h" namespace kphp::coro { @@ -66,16 +66,16 @@ struct promise_base : kphp::coro::async_stack_element { template auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc(n); + return kphp::coro::detail::memory::alloc(n); } template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::memory::script::alloc_aligned(n, al); + return kphp::coro::detail::memory::alloc_aligned(n, al); } auto operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept -> void { - kphp::memory::script::free(ptr); + kphp::coro::detail::memory::free(ptr); } void* m_next{}; diff --git a/runtime-light/runtime-light.cmake b/runtime-light/runtime-light.cmake index a0fe34d878..68eadb85ac 100644 --- a/runtime-light/runtime-light.cmake +++ b/runtime-light/runtime-light.cmake @@ -18,6 +18,7 @@ set(RUNTIME_LIGHT_LINK_FLAGS -stdlib=libc++ -static-libstdc++ -static-libgcc ${R include(${RUNTIME_LIGHT_DIR}/allocator/allocator.cmake) include(${RUNTIME_LIGHT_DIR}/components/kphp/kphp.cmake) include(${RUNTIME_LIGHT_DIR}/core/core.cmake) +include(${RUNTIME_LIGHT_DIR}/coroutine/coroutine.cmake) include(${RUNTIME_LIGHT_DIR}/server/server.cmake) include(${RUNTIME_LIGHT_DIR}/stdlib/stdlib.cmake) include(${RUNTIME_LIGHT_DIR}/tl/tl.cmake) @@ -25,6 +26,7 @@ include(${RUNTIME_LIGHT_DIR}/memory-resource-impl/memory-resource-impl.cmake) set(RUNTIME_LIGHT_SRC ${RUNTIME_LIGHT_CORE_SRC} + ${RUNTIME_LIGHT_COROUTINE_SRC} ${RUNTIME_LIGHT_STDLIB_SRC} ${RUNTIME_LIGHT_SCHEDULER_SRC} ${RUNTIME_LIGHT_SERVER_SRC} diff --git a/runtime-light/stdlib/memory/memory-usage.h b/runtime-light/stdlib/memory/memory-usage.h index 53cd3b158f..2391c8fc10 100644 --- a/runtime-light/stdlib/memory/memory-usage.h +++ b/runtime-light/stdlib/memory/memory-usage.h @@ -12,22 +12,22 @@ inline int64_t f$memory_get_peak_usage(bool real_usage = false) noexcept { if (real_usage) { - return static_cast(RuntimeAllocator::get().memory_resource.get_memory_stats().max_real_memory_used); + return static_cast(RuntimeAllocator::get().get_memory_resource().get_memory_stats().max_real_memory_used); } else { - return static_cast(RuntimeAllocator::get().memory_resource.get_memory_stats().max_memory_used); + return static_cast(RuntimeAllocator::get().get_memory_resource().get_memory_stats().max_memory_used); } } inline int64_t f$memory_get_usage([[maybe_unused]] bool real_usage = false) noexcept { - return static_cast(RuntimeAllocator::get().memory_resource.get_memory_stats().memory_used); + return static_cast(RuntimeAllocator::get().get_memory_resource().get_memory_stats().memory_used); } inline int64_t f$memory_get_total_usage() noexcept { - return static_cast(RuntimeAllocator::get().memory_resource.get_memory_stats().real_memory_used); + return static_cast(RuntimeAllocator::get().get_memory_resource().get_memory_stats().real_memory_used); } inline array f$memory_get_detailed_stats() noexcept { - const auto& stats{RuntimeAllocator::get().memory_resource.get_memory_stats()}; + const auto& stats{RuntimeAllocator::get().get_memory_resource().get_memory_stats()}; return array({std::make_pair(string{"memory_limit"}, static_cast(stats.memory_limit)), std::make_pair(string{"real_memory_used"}, static_cast(stats.real_memory_used)), std::make_pair(string{"memory_used"}, static_cast(stats.memory_used)), diff --git a/runtime/context/runtime-core-allocator.cpp b/runtime/context/runtime-core-allocator.cpp index 1d98c592a0..3d0f7f6b6b 100644 --- a/runtime/context/runtime-core-allocator.cpp +++ b/runtime/context/runtime-core-allocator.cpp @@ -5,11 +5,11 @@ #include "runtime/allocator.h" #include "runtime/context/runtime-context.h" -void RuntimeAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) { +void RuntimeAllocator::init(void* buffer, size_t script_mem_size, size_t oom_handling_mem_size) noexcept { dl::init_script_allocator(buffer, script_mem_size, oom_handling_mem_size); } -void RuntimeAllocator::free() { +void RuntimeAllocator::free() noexcept { dl::free_script_allocator(); } @@ -21,7 +21,7 @@ void* RuntimeAllocator::alloc_script_memory(size_t size) noexcept { return dl::allocate(size); } -void* RuntimeAllocator::alloc0_script_memory(size_t size) noexcept { +void* RuntimeAllocator::calloc_script_memory(size_t size) noexcept { return dl::allocate0(size); } @@ -32,23 +32,3 @@ void* RuntimeAllocator::realloc_script_memory(void* mem, size_t new_size, size_t void RuntimeAllocator::free_script_memory(void* mem, size_t size) noexcept { dl::deallocate(mem, size); } - -void* RuntimeAllocator::alloc_global_memory(size_t size) noexcept { - return dl::heap_allocate(size); -} - -void* RuntimeAllocator::alloc0_global_memory(size_t size) noexcept { - void* ptr = dl::heap_allocate(size); - if (ptr != nullptr) { - memset(ptr, 0, size); - } - return ptr; -} - -void* RuntimeAllocator::realloc_global_memory(void* mem, size_t new_size, size_t old_size) noexcept { - return dl::heap_reallocate(mem, new_size, old_size); -} - -void RuntimeAllocator::free_global_memory(void* mem, size_t size) noexcept { - dl::heap_deallocate(mem, size); -} diff --git a/runtime/platform-malloc-interface.cpp b/runtime/platform-malloc-interface.cpp new file mode 100644 index 0000000000..29c3dde0f8 --- /dev/null +++ b/runtime/platform-malloc-interface.cpp @@ -0,0 +1,40 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include + +#include "common/wrappers/likely.h" +#include "runtime-common/core/allocator/platform-malloc-interface.h" +#include "runtime-common/core/utils/kphp-assert-core.h" +#include "runtime/allocator.h" + +namespace kphp::memory::platform { + +auto alloc(size_t size) noexcept -> void* { + if (unlikely(size > MALLOC_REPLACER_MAX_ALLOC - MALLOC_REPLACER_SIZE_OFFSET)) { + php_warning("attempt to allocate too much memory by malloc replacer : %lu", size); + return nullptr; + } + + const size_t real_size{size + MALLOC_REPLACER_SIZE_OFFSET}; + void* ptr{dl::heap_allocate(real_size)}; + + if (unlikely(ptr == nullptr)) { + php_warning("not enough platform memory to allocate: %lu", size); + return ptr; + } + + *static_cast(ptr) = real_size; + + return static_cast(ptr) + MALLOC_REPLACER_SIZE_OFFSET; +} + +void free(void* ptr) noexcept { + if (likely(ptr != nullptr)) { + void* real_ptr{static_cast(ptr) - MALLOC_REPLACER_SIZE_OFFSET}; + dl::heap_deallocate(real_ptr, *static_cast(real_ptr)); + } +} + +} // namespace kphp::memory::platform diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index 9615c1dcd4..9867553854 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -58,6 +58,7 @@ prepend(KPHP_RUNTIME_SOURCES ${BASE_DIR}/runtime/ ${KPHP_RUNTIME_PDO_MYSQL_SOURCES} ${KPHP_RUNTIME_PDO_PGSQL_SOURCES} allocator.cpp + platform-malloc-interface.cpp context/runtime-core-allocator.cpp context/runtime-context.cpp array_functions.cpp