diff --git a/tcmalloc/BUILD b/tcmalloc/BUILD index a674b06da..1bae80185 100644 --- a/tcmalloc/BUILD +++ b/tcmalloc/BUILD @@ -111,6 +111,7 @@ cc_library( ":common_8k_pages", ":malloc_hook", "//tcmalloc/internal:allocation_guard", + "//tcmalloc/internal:is_aligned_to", "//tcmalloc/internal:overflow", "//tcmalloc/internal:page_size", "@com_google_absl//absl/time", @@ -138,6 +139,7 @@ cc_library( ":common_8k_pages", ":malloc_hook", "//tcmalloc/internal:allocation_guard", + "//tcmalloc/internal:is_aligned_to", "//tcmalloc/internal:overflow", "//tcmalloc/internal:page_size", "@com_google_absl//absl/time", @@ -164,6 +166,7 @@ cc_library( ":common_deprecated_perthread", ":malloc_hook", "//tcmalloc/internal:allocation_guard", + "//tcmalloc/internal:is_aligned_to", "//tcmalloc/internal:overflow", "//tcmalloc/internal:page_size", "@com_google_absl//absl/time", @@ -307,6 +310,7 @@ create_tcmalloc_libraries( "//tcmalloc/internal:exponential_biased", "//tcmalloc/internal:gwp_asan_state", "//tcmalloc/internal:hook_list", + "//tcmalloc/internal:is_aligned_to", "//tcmalloc/internal:linked_list", "//tcmalloc/internal:logging", "//tcmalloc/internal:memory_stats", @@ -428,6 +432,7 @@ cc_library( ":common_large_pages", ":malloc_hook", "//tcmalloc/internal:allocation_guard", + "//tcmalloc/internal:is_aligned_to", "//tcmalloc/internal:overflow", "//tcmalloc/internal:page_size", "@com_google_absl//absl/time", @@ -453,6 +458,7 @@ cc_library( ":common_256k_pages", ":malloc_hook", "//tcmalloc/internal:allocation_guard", + "//tcmalloc/internal:is_aligned_to", "//tcmalloc/internal:overflow", "//tcmalloc/internal:page_size", "@com_google_absl//absl/time", @@ -481,6 +487,7 @@ cc_library( ":common_256k_pages_numa_aware", ":malloc_hook", "//tcmalloc/internal:allocation_guard", + "//tcmalloc/internal:is_aligned_to", "//tcmalloc/internal:overflow", "//tcmalloc/internal:page_size", "@com_google_absl//absl/time", @@ -510,6 +517,7 @@ cc_library( ":common_small_but_slow", ":malloc_hook", "//tcmalloc/internal:allocation_guard", + "//tcmalloc/internal:is_aligned_to", "//tcmalloc/internal:overflow", "//tcmalloc/internal:page_size", "@com_google_absl//absl/time", @@ -537,6 +545,7 @@ cc_library( ":common_numa_aware", ":malloc_hook", "//tcmalloc/internal:allocation_guard", + "//tcmalloc/internal:is_aligned_to", "//tcmalloc/internal:overflow", "//tcmalloc/internal:page_size", "@com_google_absl//absl/time", @@ -563,6 +572,7 @@ cc_library( ":common_legacy_locking", ":malloc_hook", "//tcmalloc/internal:allocation_guard", + "//tcmalloc/internal:is_aligned_to", "//tcmalloc/internal:overflow", "//tcmalloc/internal:page_size", "@com_google_absl//absl/time", @@ -590,6 +600,7 @@ cc_library( ":malloc_hook", "//tcmalloc/internal:allocation_guard", "//tcmalloc/internal:delay_injection", + "//tcmalloc/internal:is_aligned_to", "//tcmalloc/internal:overflow", "//tcmalloc/internal:page_size", "@com_google_absl//absl/time", @@ -1435,6 +1446,7 @@ create_tcmalloc_testsuite( copts = TCMALLOC_DEFAULT_COPTS, deps = [ ":malloc_extension", + "//tcmalloc/internal:is_aligned_to", "//tcmalloc/internal:page_size", "//tcmalloc/testing:testutil", "@com_github_google_benchmark//:benchmark", diff --git a/tcmalloc/cpu_cache.h b/tcmalloc/cpu_cache.h index 2015a21d4..d17aa3930 100644 --- a/tcmalloc/cpu_cache.h +++ b/tcmalloc/cpu_cache.h @@ -158,6 +158,10 @@ class StaticForwarder { return Parameters::per_cpu_caches_dynamic_slab_shrink_threshold(); } + static bool release_drained_slab_metadata() { + return Parameters::release_drained_slab_metadata(); + } + bool reuse_size_classes() const { return state_.size_class_configuration() == SizeClassConfiguration::kReuse || @@ -464,6 +468,14 @@ class CpuCache { // Reports total number of times any CPU has been reclaimed. uint64_t GetNumReclaims() const; + // Reports number of times the has been unpopulated + // (which happens when its metadata gets released, after all per-CPU + // metadata slabs on the same hugepage ave been reclaimed). + uint64_t GetNumUnpopulates(int cpu) const; + + // Reports total number of times any CPU has been unpopulated. + uint64_t GetNumUnpopulates() const; + // Reports number of cpus that have touched set to true. int CountTouchedCpus() const; @@ -668,6 +680,9 @@ class CpuCache { std::atomic reclaim_used_bytes; // Tracks number of times this CPU has been reclaimed. std::atomic num_reclaims; + // Tracks number of times this CPU has been unpopulated + // (see GetNumUnpopulates()). + std::atomic num_unpopulates; }; // Determines how we distribute memory in the per-cpu cache to the various @@ -1407,6 +1422,8 @@ template inline void CpuCache::TryReclaimingCaches() { const int num_cpus = NumCPUs(); + bool any_drained = false; + for (int cpu = 0; cpu < num_cpus; ++cpu) { // Nothing to reclaim if the cpu is not populated. if (!HasPopulated(cpu)) { @@ -1428,6 +1445,7 @@ inline void CpuCache::TryReclaimingCaches() { // stayed constant since the last interval. if (used_bytes != 0 && used_bytes == prev_used_bytes && misses == 0) { Reclaim(cpu); + any_drained = true; } // Takes a snapshot of used bytes in the cache at the end of this interval @@ -1438,6 +1456,26 @@ inline void CpuCache::TryReclaimingCaches() { resize_[cpu].reclaim_used_bytes.store(used_bytes, std::memory_order_relaxed); } + + if (any_drained && forwarder_.release_drained_slab_metadata()) { + freelist_.ReleaseSlabMetadataForDrainedCpus( + [this](int cpu) { return HasPopulated(cpu); }, + [this](int cpu) + ABSL_NO_THREAD_SAFETY_ANALYSIS { resize_[cpu].lock.lock(); }, + [this](int cpu) + ABSL_NO_THREAD_SAFETY_ANALYSIS { resize_[cpu].lock.unlock(); }, + [this](int cpu) { + TC_CHECK_EQ( + resize_[cpu].available, resize_[cpu].capacity, + "CPU %u was not actually drained, or available is out of sync", + cpu); + resize_[cpu].populated.store(false, std::memory_order_release); + resize_[cpu].num_unpopulates.fetch_add(1, std::memory_order_relaxed); + }, + [this](void* slab_addr, size_t slab_size) { + return MadviseAwaySlabs(slab_addr, slab_size); + }); + } } template @@ -2271,6 +2309,20 @@ inline uint64_t CpuCache::GetNumReclaims() const { return reclaims; } +template +inline uint64_t CpuCache::GetNumUnpopulates(int cpu) const { + return resize_[cpu].num_unpopulates.load(std::memory_order_relaxed); +} + +template +inline uint64_t CpuCache::GetNumUnpopulates() const { + uint64_t reclaims = 0; + const int num_cpus = NumCPUs(); + for (int cpu = 0; cpu < num_cpus; ++cpu) + reclaims += resize_[cpu].num_unpopulates.load(std::memory_order_relaxed); + return reclaims; +} + template inline int CpuCache::CountTouchedCpus() const { if (resize_ == nullptr) return 0; diff --git a/tcmalloc/cpu_cache_test.cc b/tcmalloc/cpu_cache_test.cc index 0a9290026..fb6edf936 100644 --- a/tcmalloc/cpu_cache_test.cc +++ b/tcmalloc/cpu_cache_test.cc @@ -337,6 +337,10 @@ class TestStaticForwarder { return false; } + bool release_drained_slab_metadata() const { + return release_drained_slab_metadata_; + } + size_t arena_reported_nonresident_bytes_ = 0; int64_t arena_reported_impending_bytes_ = 0; size_t shrink_to_usage_limit_calls_ = 0; @@ -345,6 +349,7 @@ class TestStaticForwarder { double dynamic_slab_shrink_threshold_ = -1; DynamicSlab dynamic_slab_ = DynamicSlab::kNoop; std::optional size_map_; + bool release_drained_slab_metadata_ = false; private: NumaTopology numa_topology_; @@ -1267,7 +1272,7 @@ static void ColdCacheOperations(CpuCache& cache, int cpu_id, // Runs multiple allocate and deallocate operation on the cpu cache to collect // misses. Once we collect enough misses on this cache, we can shuffle cpu // caches to steal capacity from colder caches to the hot cache. -static void HotCacheOperations(CpuCache& cache, int cpu_id) { +static void HotCacheOperations(CpuCache& cache, int cpu_id, bool reclaim) { constexpr size_t kPtrs = 4096; std::vector ptrs; ptrs.resize(kPtrs); @@ -1288,10 +1293,13 @@ static void HotCacheOperations(CpuCache& cache, int cpu_id) { } } - // We reclaim the cache to reset it so that we record underflows/overflows the - // next time we allocate and deallocate objects. Without reclaim, the cache - // would stay warmed up and it would take more time to drain the colder cache. - cache.Reclaim(cpu_id); + if (reclaim) { + // We reclaim the cache to reset it so that we record underflows/overflows + // the next time we allocate and deallocate objects. Without reclaim, the + // cache would stay warmed up and it would take more time to drain the + // colder cache. + cache.Reclaim(cpu_id); + } } class DynamicWideSlabTest : public testing::Test {}; @@ -1319,7 +1327,7 @@ TEST_F(DynamicWideSlabTest, DynamicSlabThreshold) { constexpr int kCpuId1 = 1; // Accumulate overflows and underflows for kCpuId0. - HotCacheOperations(cache, kCpuId0); + HotCacheOperations(cache, kCpuId0, true); CpuCache::CpuCacheMissStats interval_misses = cache.GetIntervalCacheMissStats(kCpuId0, MissCount::kSlabResize); // Make sure that overflows/underflows ratio is greater than the threshold @@ -1539,7 +1547,7 @@ TEST(CpuCacheTest, ColdHotCacheShuffleTest) { CpuCache::kCacheCapacityThreshold * max_cpu_cache_size; ++num_tries) { ColdCacheOperations(cache, cold_cpu_id, size_class); - HotCacheOperations(cache, hot_cpu_id); + HotCacheOperations(cache, hot_cpu_id, true); cache.ShuffleCpuCaches(); // Check that the capacity is preserved. @@ -1568,7 +1576,7 @@ TEST(CpuCacheTest, ColdHotCacheShuffleTest) { // change the capacity of either of the caches. for (int i = 0; i < 100; ++i) { ColdCacheOperations(cache, cold_cpu_id, size_class); - HotCacheOperations(cache, hot_cpu_id); + HotCacheOperations(cache, hot_cpu_id, true); cache.ShuffleCpuCaches(); // Check that the capacity is preserved. @@ -1611,6 +1619,7 @@ TEST(CpuCacheTest, ReclaimCpuCache) { // None of the caches should have been reclaimed yet. EXPECT_EQ(cache.GetNumReclaims(cpu), 0); + EXPECT_EQ(cache.GetNumUnpopulates(cpu), 0); // Check that caches are empty. uint64_t used_bytes = cache.UsedBytes(cpu); @@ -1709,6 +1718,102 @@ TEST(CpuCacheTest, ReclaimCpuCache) { cache.Deactivate(); } +TEST(CpuCacheTest, ReclaimCpuCacheAndUnpopulate) { + if (!subtle::percpu::IsFast()) { + return; + } + + for (bool enabled : {false, true}) { + SCOPED_TRACE(absl::StrFormat("Feature enabled: %d", enabled)); + + CpuCache cache; + cache.forwarder().release_drained_slab_metadata_ = enabled; + cache.Activate(); + + const size_t kSizeClass = 2; + + const int num_cpus = NumCPUs(); + + // Verify that we fill at least three hugepages; one (or more) + // to be unpopulated, one not to be, and one account for misalignment + // before or after. + uint8_t per_cpu_shift = CpuCachePeer::GetSlabShift(cache); + const auto shift = subtle::percpu::ToShiftType(per_cpu_shift); + const size_t slabs_size = + subtle::percpu::GetSlabsAllocSize(shift, num_cpus); + if (slabs_size < 3 * kHugePageSize) { + TC_LOG("Not enough CPUs to run test; skipping."); + return; + } + + for (int cpu = 0; cpu < num_cpus; ++cpu) { + SCOPED_TRACE(absl::StrFormat("Failed CPU: %d", cpu)); + ColdCacheOperations(cache, cpu, kSizeClass); + EXPECT_TRUE(cache.HasPopulated(cpu)); + EXPECT_EQ(cache.GetNumUnpopulates(cpu), 0); + } + + // None of the caches are stable, so nothing should be reclaimed + // and nothing should be unpopulated. + cache.TryReclaimingCaches(); + EXPECT_EQ(cache.GetNumReclaims(), 0); + EXPECT_EQ(cache.GetNumUnpopulates(), 0); + + // Do some work on every other CPUs. This should block all unpopulates, + // as no hugepage will contain all-reclaimed caches. The other ones + // should be reclaimed, though. + int num_idle_cpus = 0; + for (int cpu = 0; cpu < num_cpus; ++cpu) { + if (cpu % 2 == 0) { + HotCacheOperations(cache, cpu, false); + } else { + ++num_idle_cpus; + } + } + cache.TryReclaimingCaches(); + EXPECT_EQ(cache.GetNumReclaims(), num_idle_cpus); + EXPECT_EQ(cache.GetNumUnpopulates(), 0); + + // Now do work on only one CPU, to record some misses on that, + // but let the others stay idle. (We do an extra reclaim first, + // or HotCacheOperations() wouldn't actually cause misses. + // This reclaim gets included in GetNumReclaims() below.) + // We should have unpopulates after another round of reclaim, + // but not everything. + // + // The “arbitrary” CPU must already be touched (so even), + // and we'd like it to be so far in that we know that it + // would actually get unpopulated if untouched. + constexpr int kArbitraryCpu = 80; // Must already be touched. + if (kArbitraryCpu >= num_cpus) { + TC_LOG("Not enough CPUs to run test; skipping."); + return; + } + HotCacheOperations(cache, kArbitraryCpu, false); + cache.TryReclaimingCaches(); + + EXPECT_EQ(cache.GetNumReclaims(kArbitraryCpu), 0); + EXPECT_EQ(cache.GetNumReclaims(), num_cpus - 1); + + if (enabled) { + // The touched CPU cannot be unpopulated, and since it shares hugepage + // with at least one of its neighbors, at least one of those (probably + // both) must remain, too. + EXPECT_EQ(cache.GetNumUnpopulates(kArbitraryCpu), 0); + EXPECT_LT(cache.GetNumUnpopulates(kArbitraryCpu - 1) + + cache.GetNumUnpopulates(kArbitraryCpu + 1), + 2); + + EXPECT_GT(cache.GetNumUnpopulates(), 0); + EXPECT_LT(cache.GetNumUnpopulates(), num_cpus); + } else { + EXPECT_EQ(cache.GetNumUnpopulates(), 0); + } + + cache.Deactivate(); + } +} + TEST(CpuCacheTest, SizeClassCapacityTest) { if (!subtle::percpu::IsFast()) { return; diff --git a/tcmalloc/huge_page_aware_allocator.h b/tcmalloc/huge_page_aware_allocator.h index abfdaac0e..d3dc7475b 100644 --- a/tcmalloc/huge_page_aware_allocator.h +++ b/tcmalloc/huge_page_aware_allocator.h @@ -100,6 +100,10 @@ class StaticForwarder { return Parameters::madvise_cold_regions_nohugepage(); } + static bool release_drained_slab_metadata() { + return Parameters::release_drained_slab_metadata(); + } + // Arena state. static Arena& arena(); diff --git a/tcmalloc/internal/BUILD b/tcmalloc/internal/BUILD index ad651056b..108c5c367 100644 --- a/tcmalloc/internal/BUILD +++ b/tcmalloc/internal/BUILD @@ -533,6 +533,7 @@ cc_library( ], deps = [ ":config", + ":is_aligned_to", ":logging", ":page_size", ":range_tracker", @@ -617,6 +618,7 @@ cc_test( copts = TCMALLOC_DEFAULT_COPTS, linkstatic = 1, deps = [ + ":is_aligned_to", ":mincore", ":page_size", "@com_github_google_benchmark//:benchmark", @@ -632,6 +634,7 @@ cc_test( deps = [ ":allocation_guard", ":config", + ":is_aligned_to", ":page_size", ":range_tracker", ":residency", @@ -900,6 +903,7 @@ cc_library( ], deps = [ ":delay_injection", + ":is_aligned_to", ":logging", ":mincore", ":optimization", @@ -909,6 +913,7 @@ cc_library( "@com_google_absl//absl/base", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:dynamic_annotations", + "@com_google_absl//absl/container:fixed_array", "@com_google_absl//absl/functional:function_ref", "@com_google_absl//absl/numeric:bits", ], @@ -1335,6 +1340,7 @@ cc_library( ":config", ":environment", ":exponential_biased", + ":is_aligned_to", ":logging", ":memory_tag", ":numa", @@ -1575,3 +1581,18 @@ cc_library( "@com_google_absl//absl/base:core_headers", ], ) + +cc_library( + name = "is_aligned_to", + hdrs = ["is_aligned_to.h"], + copts = TCMALLOC_DEFAULT_COPTS, + visibility = [ + "//tcmalloc:__pkg__", + "//tcmalloc:__subpackages__", + ], + deps = [ + ":config", + ":logging", + "@com_google_absl//absl/numeric:bits", + ], +) diff --git a/tcmalloc/internal/is_aligned_to.h b/tcmalloc/internal/is_aligned_to.h new file mode 100644 index 000000000..31ad2367e --- /dev/null +++ b/tcmalloc/internal/is_aligned_to.h @@ -0,0 +1,51 @@ +// Copyright 2026 The TCMalloc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TCMALLOC_IS_ALIGNED_TO_H_ +#define TCMALLOC_IS_ALIGNED_TO_H_ + +#include +#include +#include + +#include "absl/numeric/bits.h" +#include "tcmalloc/internal/config.h" +#include "tcmalloc/internal/logging.h" + +GOOGLE_MALLOC_SECTION_BEGIN +namespace tcmalloc::tcmalloc_internal { + +constexpr bool IsAlignedTo(uintptr_t val, size_t alignment) { + TC_ASSERT(absl::has_single_bit(alignment), + "Alignment %zu needs to be a power of two", alignment); + return (val & (alignment - 1)) == 0; +} + +constexpr bool IsAlignedTo(uintptr_t val, std::align_val_t alignment) { + return IsAlignedTo(val, static_cast(alignment)); +} + +inline bool IsAlignedTo(const void* addr, size_t alignment) { + return IsAlignedTo(reinterpret_cast(addr), alignment); +} + +inline bool IsAlignedTo(const void* addr, std::align_val_t alignment) { + return IsAlignedTo(reinterpret_cast(addr), + static_cast(alignment)); +} + +} // namespace tcmalloc::tcmalloc_internal +GOOGLE_MALLOC_SECTION_END + +#endif diff --git a/tcmalloc/internal/mincore_test.cc b/tcmalloc/internal/mincore_test.cc index adc77f6c1..5c10165e5 100644 --- a/tcmalloc/internal/mincore_test.cc +++ b/tcmalloc/internal/mincore_test.cc @@ -25,6 +25,7 @@ #include "benchmark/benchmark.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "tcmalloc/internal/is_aligned_to.h" #include "tcmalloc/internal/page_size.h" namespace tcmalloc { @@ -44,7 +45,7 @@ class MInCoreMock : public MInCoreInterface { const size_t kHardwarePageSize = GetPageSize(); uintptr_t uAddress = reinterpret_cast(addr); // Check that we only pass page aligned addresses into mincore(). - EXPECT_THAT(uAddress & (kHardwarePageSize - 1), Eq(0)); + EXPECT_TRUE(IsAlignedTo(uAddress, kHardwarePageSize)); uintptr_t uEndAddress = uAddress + length; int index = 0; diff --git a/tcmalloc/internal/pageflags.cc b/tcmalloc/internal/pageflags.cc index 4e074b7e1..e34274928 100644 --- a/tcmalloc/internal/pageflags.cc +++ b/tcmalloc/internal/pageflags.cc @@ -30,6 +30,7 @@ #include "absl/status/status.h" #include "absl/strings/numbers.h" #include "tcmalloc/internal/config.h" +#include "tcmalloc/internal/is_aligned_to.h" #include "tcmalloc/internal/logging.h" #include "tcmalloc/internal/util.h" @@ -343,7 +344,7 @@ PageFlagsBase::PageFlagsBitmaps PageFlags::GetSinglePageBitmaps( const void* addr) { PageFlagsBitmaps ret; uintptr_t currPage = reinterpret_cast(addr); - if ((currPage & (kHugePageSize - 1)) != 0) { + if (!IsAlignedTo(currPage, kHugePageSize)) { TC_LOG("Address is not hugepage aligned"); ret.status = absl::StatusCode::kFailedPrecondition; return ret; diff --git a/tcmalloc/internal/parameter_accessors.h b/tcmalloc/internal/parameter_accessors.h index 8d93888cd..1e9bc8466 100644 --- a/tcmalloc/internal/parameter_accessors.h +++ b/tcmalloc/internal/parameter_accessors.h @@ -120,6 +120,9 @@ ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_GetSizeClasses( std::vector* absl_nonnull size_classes); ABSL_ATTRIBUTE_WEAK size_t TCMalloc_Internal_GetPageSize(); + +ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetReleaseDrainedSlabMetadata( + bool v); } #endif // TCMALLOC_INTERNAL_PARAMETER_ACCESSORS_H_ diff --git a/tcmalloc/internal/percpu.cc b/tcmalloc/internal/percpu.cc index 083f6ba6e..fadc2ffa1 100644 --- a/tcmalloc/internal/percpu.cc +++ b/tcmalloc/internal/percpu.cc @@ -402,6 +402,17 @@ void FenceCpu(int vcpu) { void FenceAllCpus() { #if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ + // An effect of fencing all CPUs is that the cached slabs pointer is reset + // because our rseq machinery resets it on every thread schedule. This is + // desirable if e.g. because we don't want to hit the fast path the next time + // the CPU allocates (i.e., we changed something under the thread). + // This also happens to our own thread due to the syscall (depending a bit + // on the kernel version). However, when fake CPUs are enabled in tests, + // we've unsubscribed from rseq and thus the syscall doesn't reset the + // slabs pointer, so uncache it explicitly here so that all CPUs are + // handled equal in this respect. + tcmalloc_slabs = 0; + if (using_upstream_fence.load(std::memory_order_relaxed)) { UpstreamRseqFenceCpu(-1); return; diff --git a/tcmalloc/internal/percpu_tcmalloc.h b/tcmalloc/internal/percpu_tcmalloc.h index 989ef4fa8..b27fc169f 100644 --- a/tcmalloc/internal/percpu_tcmalloc.h +++ b/tcmalloc/internal/percpu_tcmalloc.h @@ -35,9 +35,11 @@ #include "absl/base/casts.h" #include "absl/base/dynamic_annotations.h" #include "absl/base/optimization.h" +#include "absl/container/fixed_array.h" #include "absl/functional/function_ref.h" #include "absl/numeric/bits.h" #include "tcmalloc/internal/delay_injection.h" +#include "tcmalloc/internal/is_aligned_to.h" #include "tcmalloc/internal/logging.h" #include "tcmalloc/internal/mincore.h" #include "tcmalloc/internal/optimization.h" @@ -293,6 +295,21 @@ class TcmallocSlab { // Push/Pop/Grow/Shrink concurrently (even on the same CPU) is safe. void Drain(int cpu, DrainHandler drain_handler); + enum HugePageStatus : uint8_t { kNotTouched = 0, kCannotFree, kShouldFree }; + + void ReleaseSlabMetadataForDrainedCpus( + absl::FunctionRef populated, + absl::FunctionRef lock_cpu, + absl::FunctionRef unlock_cpu, + absl::FunctionRef unpopulate, + absl::FunctionRef madvise_away_slabs); + + void ReleaseSlabMetadataForDrainedAndStoppedCpus( + absl::FixedArray& hugepage_status, + absl::FunctionRef populated, + absl::FunctionRef unpopulate, + absl::FunctionRef madvise_away_slabs); + PerCPUMetadataState MetadataMemoryUsage() const; // Gets the current shift of the slabs. Intended for use by the thread that @@ -378,6 +395,7 @@ class TcmallocSlab { static Header LoadHeader(AtomicHeader* hdrp); static void StoreHeader(AtomicHeader* hdrp, Header hdr); void DrainCpu(void* slabs, Shift shift, int cpu, DrainHandler drain_handler); + bool CpuIsDrained(void* slabs, Shift shift, int cpu); void DrainOldSlabs(void* slabs, Shift shift, int cpu, const std::array& old_begins, DrainHandler drain_handler); @@ -1262,6 +1280,19 @@ void TcmallocSlab::DrainCpu(void* slabs, Shift shift, int cpu, } } +template +bool TcmallocSlab::CpuIsDrained(void* slabs, Shift shift, int cpu) { + for (size_t size_class = 1; size_class < NumClasses; ++size_class) { + uint16_t begin = begins_[size_class].load(std::memory_order_relaxed); + auto* hdrp = GetHeader(slabs, shift, cpu, size_class); + Header hdr = LoadHeader(hdrp); + if (hdr.end != 0 && hdr.end != begin) { + return false; + } + } + return true; +} + template void TcmallocSlab::DrainOldSlabs( void* slabs, Shift shift, int cpu, @@ -1463,6 +1494,135 @@ void TcmallocSlab::Drain(int cpu, DrainHandler drain_handler) { DrainCpu(slabs, shift, cpu, drain_handler); } +template +void TcmallocSlab::ReleaseSlabMetadataForDrainedCpus( + absl::FunctionRef populated, + absl::FunctionRef lock_cpu, + absl::FunctionRef unlock_cpu, + absl::FunctionRef unpopulate, + absl::FunctionRef madvise_away_slabs) { + const int n_cpus = num_cpus(); + + // For each hugepage touched by our slabs, track whether there is something + // there that needs to be freed (because all CPUs belonging to that hugepage + // are drained). + // + // The max per-CPU metadata size is smaller than a hugepage (asserted below) + // and we are aligned to it (also checked below), so it's fine to allocate + // tracking for as many hugepages as we have CPUs. + absl::FixedArray hugepage_status(n_cpus, kNotTouched); + + // We can't allocate while holding the per-cpu spinlocks. + AllocationGuard enforce_no_alloc; + + // Stop all CPUs. We also need to lock them since we are touching the + // populated bit later (but we need to wait locking them until we have + // allocated hugepage_status[]). + for (int cpu = 0; cpu < n_cpus; ++cpu) lock_cpu(cpu); + for (auto& state : state_) { + TC_CHECK(!state.stopped.load(std::memory_order_relaxed)); + state.stopped.store(true, std::memory_order_relaxed); + } + FenceAllCpus(); + + // See which ones are actually drained, and which hugepages we can free. + ReleaseSlabMetadataForDrainedAndStoppedCpus(hugepage_status, populated, + unpopulate, madvise_away_slabs); + + // Restart the CPUs again. + for (auto& state : state_) { + state.stopped.store(false, std::memory_order_release); + } + for (int cpu = 0; cpu < n_cpus; ++cpu) unlock_cpu(cpu); +} + +template +void TcmallocSlab::ReleaseSlabMetadataForDrainedAndStoppedCpus( + absl::FixedArray& hugepage_status, + absl::FunctionRef populated, + absl::FunctionRef unpopulate, + absl::FunctionRef madvise_away_slabs) { + const int n_cpus = num_cpus(); + + const auto [slabs, shift] = GetSlabsAndShift(std::memory_order_relaxed); + const size_t slab_size_bytes = 1ULL << static_cast(shift); + TC_ASSERT(slab_size_bytes <= kHugePageSize); + + // If the slab is not aligned to its own size, freeing any hugepage + // would tear through a CPU's data, and we can do nothing. + if (!IsAlignedTo(slabs, slab_size_bytes)) { + TC_BUG("Slabs are not properly aligned"); + return; + } + + auto address_to_hugepage_number = [](const void* addr) { + return reinterpret_cast(addr) / kHugePageSize; + }; + size_t base_hugepage_nr = address_to_hugepage_number(slabs); + + void* slabs_start = CpuMemoryStart(slabs, shift, 0); + if (!IsAlignedTo(slabs_start, kHugePageSize)) { + // If our slab doesn't doesn't start hugepage-aligned, + // we cannot free the first hugepage. + hugepage_status[0] = kCannotFree; + } + + // We cannot free the last page page either, if the slabs doesn't + // end perfectly on a hugepage boundary. (At the very least, + // we'd risk tearing a hugepage.) + void* slabs_end = CpuMemoryStart(slabs, shift, n_cpus + 1); + hugepage_status[address_to_hugepage_number(slabs_end) - base_hugepage_nr] = + kCannotFree; + + // Go through all the CPUs and figure out which hugepage its slab + // lives in. (Because we've already tested that slabs are slab-aligned + // and not larger than a hugepage, and they are also powers of two, + // it can never cross hugepages.) + for (size_t cpu = 0; cpu < n_cpus; ++cpu) { + if (!populated(cpu)) { + continue; + } + + size_t slab_hugepage = + address_to_hugepage_number(CpuMemoryStart(slabs, shift, cpu)); + HugePageStatus& status = hugepage_status[slab_hugepage - base_hugepage_nr]; + + if (status == kCannotFree) { + // No need to check, don't do anything. + } else if (CpuIsDrained(slabs, shift, cpu)) { + status = kShouldFree; + } else { + status = kCannotFree; + } + } + + for (size_t hugepage_idx = 0; hugepage_idx < hugepage_status.size(); + ++hugepage_idx) { + if (hugepage_status[hugepage_idx] != kShouldFree) { + continue; + } + + void* hugepage_start = reinterpret_cast( + (base_hugepage_nr + hugepage_idx) * kHugePageSize); + + // Coalesce neighboring madvises. + size_t bytes_to_free = kHugePageSize; + while (hugepage_idx + 1 < hugepage_status.size() && + hugepage_status[hugepage_idx + 1] == kShouldFree) { + bytes_to_free += kHugePageSize; + ++hugepage_idx; + } + + madvise_away_slabs(hugepage_start, bytes_to_free); + size_t first_cpu = (reinterpret_cast(hugepage_start) - + reinterpret_cast(slabs)) / + slab_size_bytes; + for (unsigned i = 0; i < bytes_to_free / slab_size_bytes; ++i) { + unpopulate(first_cpu + i); + } + } +} + template void TcmallocSlab::StopCpu(int cpu) { TC_ASSERT(cpu >= 0 && cpu < num_cpus(), "cpu=%d", cpu); diff --git a/tcmalloc/internal/percpu_tcmalloc_test.cc b/tcmalloc/internal/percpu_tcmalloc_test.cc index 4a461349a..27f896322 100644 --- a/tcmalloc/internal/percpu_tcmalloc_test.cc +++ b/tcmalloc/internal/percpu_tcmalloc_test.cc @@ -490,6 +490,14 @@ TEST_F(TcmallocSlabTest, ResizeMaxCapacities) { }, new_max_capacity, /*classes_to_resize=*/2); + + // UpdateMaxCapacities() zeroes out our thread's slabs pointer, + // which Grow() expects to be there. Normally, any other caller + // of Grow() would be from deallocation, which updates the slab + // pointer before doing anything, so explicitly put it back here. + auto [cpu, cached] = slab_.CacheCpuSlab(); + EXPECT_TRUE(cached); + ASSERT_NE(old_slabs, nullptr); mprotect(old_slabs, old_slabs_size, PROT_READ | PROT_WRITE); sized_aligned_delete(old_slabs, old_slabs_size, diff --git a/tcmalloc/internal/residency_test.cc b/tcmalloc/internal/residency_test.cc index 2263d49e6..5c08287ab 100644 --- a/tcmalloc/internal/residency_test.cc +++ b/tcmalloc/internal/residency_test.cc @@ -37,6 +37,7 @@ #include "absl/strings/string_view.h" #include "tcmalloc/internal/allocation_guard.h" #include "tcmalloc/internal/config.h" +#include "tcmalloc/internal/is_aligned_to.h" #include "tcmalloc/internal/page_size.h" #include "tcmalloc/internal/range_tracker.h" #include "tcmalloc/internal/util.h" @@ -350,7 +351,7 @@ TEST(PageMapIntegrationTest, WorksOnActualData) { MAP_ANONYMOUS | MAP_POPULATE | MAP_PRIVATE, -1, 0); ASSERT_NE(addr, MAP_FAILED) << errno; auto position = reinterpret_cast(addr); - if ((position & (kHugePageSize - 1)) != 0) { + if (!IsAlignedTo(position, kHugePageSize)) { position |= kHugePageSize - 1; position++; addr = reinterpret_cast(position); diff --git a/tcmalloc/internal/system_allocator.h b/tcmalloc/internal/system_allocator.h index bd023cd92..3a44ee921 100644 --- a/tcmalloc/internal/system_allocator.h +++ b/tcmalloc/internal/system_allocator.h @@ -25,6 +25,7 @@ #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" +#include "tcmalloc/internal/is_aligned_to.h" #include "tcmalloc/internal/logging.h" #ifdef __linux__ #include @@ -557,7 +558,7 @@ void* SystemAllocator::MmapAlignedLocked( }(); bool first = !next_addr; - if (!next_addr || next_addr & (alignment - 1) || + if (!next_addr || !IsAlignedTo(next_addr, alignment) || GetMemoryTag(reinterpret_cast(next_addr)) != tag || GetMemoryTag(reinterpret_cast(next_addr + size - 1)) != tag) { next_addr = RandomMmapHint(size, alignment, tag); diff --git a/tcmalloc/new_extension_test.cc b/tcmalloc/new_extension_test.cc index e8a8d470e..bf56429df 100644 --- a/tcmalloc/new_extension_test.cc +++ b/tcmalloc/new_extension_test.cc @@ -26,10 +26,13 @@ #include "absl/base/optimization.h" #include "absl/numeric/bits.h" #include "absl/random/random.h" +#include "tcmalloc/internal/is_aligned_to.h" #include "tcmalloc/internal/page_size.h" #include "tcmalloc/malloc_extension.h" #include "tcmalloc/testing/testutil.h" +using tcmalloc::tcmalloc_internal::IsAlignedTo; + namespace tcmalloc { namespace { @@ -285,9 +288,7 @@ TEST(HotColdNew, OperatorNewAligned) { void* ptr = ::operator new(size, alignment, static_cast(label)); ASSERT_NE(ptr, nullptr); - EXPECT_EQ(reinterpret_cast(ptr) & - (static_cast(alignment) - 1u), - 0); + EXPECT_TRUE(IsAlignedTo(ptr, alignment)); benchmark::DoNotOptimize(memset(ptr, 0xBF, size)); ptrs.emplace_back(SizedAlignedPtr{ptr, size, alignment}); } @@ -298,7 +299,7 @@ TEST(HotColdNew, OperatorNewAligned) { ::operator new(kSmall, static_cast(kSmallAlignment), static_cast<__hot_cold_t>(0)); ASSERT_NE(ptr, nullptr); - EXPECT_EQ(reinterpret_cast(ptr) & (kSmallAlignment - 1u), 0); + EXPECT_TRUE(IsAlignedTo(ptr, kSmallAlignment)); benchmark::DoNotOptimize(memset(ptr, 0xBF, kSmall)); ptrs.emplace_back(SizedAlignedPtr{ ptr, kSmall, static_cast(kSmallAlignment)}); @@ -341,9 +342,7 @@ TEST(HotColdNew, OperatorNewAlignedNothrow) { void* ptr = ::operator new(size, alignment, std::nothrow, static_cast(label)); ASSERT_NE(ptr, nullptr); - EXPECT_EQ(reinterpret_cast(ptr) & - (static_cast(alignment) - 1u), - 0); + EXPECT_TRUE(IsAlignedTo(ptr, alignment)); benchmark::DoNotOptimize(memset(ptr, 0xBF, size)); ptrs.emplace_back(SizedAlignedPtr{ptr, size, alignment}); } @@ -354,7 +353,7 @@ TEST(HotColdNew, OperatorNewAlignedNothrow) { ::operator new(kSmall, static_cast(kSmallAlignment), std::nothrow, static_cast<__hot_cold_t>(0)); ASSERT_NE(ptr, nullptr); - EXPECT_EQ(reinterpret_cast(ptr) & (kSmallAlignment - 1u), 0); + EXPECT_TRUE(IsAlignedTo(ptr, kSmallAlignment)); benchmark::DoNotOptimize(memset(ptr, 0xBF, kSmall)); ptrs.emplace_back(SizedAlignedPtr{ ptr, kSmall, static_cast(kSmallAlignment)}); @@ -397,9 +396,7 @@ TEST(HotColdNew, OperatorNewArrayAligned) { void* ptr = ::operator new[](size, alignment, static_cast(label)); ASSERT_NE(ptr, nullptr); - EXPECT_EQ(reinterpret_cast(ptr) & - (static_cast(alignment) - 1u), - 0); + EXPECT_TRUE(IsAlignedTo(ptr, alignment)); benchmark::DoNotOptimize(memset(ptr, 0xBF, size)); ptrs.emplace_back(SizedAlignedPtr{ptr, size, alignment}); } @@ -410,7 +407,7 @@ TEST(HotColdNew, OperatorNewArrayAligned) { ::operator new[](kSmall, static_cast(kSmallAlignment), static_cast<__hot_cold_t>(0)); ASSERT_NE(ptr, nullptr); - EXPECT_EQ(reinterpret_cast(ptr) & (kSmallAlignment - 1u), 0); + EXPECT_TRUE(IsAlignedTo(ptr, kSmallAlignment)); benchmark::DoNotOptimize(memset(ptr, 0xBF, kSmall)); ptrs.emplace_back(SizedAlignedPtr{ ptr, kSmall, static_cast(kSmallAlignment)}); @@ -453,9 +450,7 @@ TEST(HotColdNew, OperatorNewArrayAlignedNothrow) { void* ptr = ::operator new[](size, alignment, std::nothrow, static_cast(label)); ASSERT_NE(ptr, nullptr); - EXPECT_EQ(reinterpret_cast(ptr) & - (static_cast(alignment) - 1u), - 0); + EXPECT_TRUE(IsAlignedTo(ptr, alignment)); benchmark::DoNotOptimize(memset(ptr, 0xBF, size)); ptrs.emplace_back(SizedAlignedPtr{ptr, size, alignment}); } @@ -466,7 +461,7 @@ TEST(HotColdNew, OperatorNewArrayAlignedNothrow) { ::operator new[](kSmall, static_cast(kSmallAlignment), std::nothrow, static_cast<__hot_cold_t>(0)); ASSERT_NE(ptr, nullptr); - EXPECT_EQ(reinterpret_cast(ptr) & (kSmallAlignment - 1u), 0); + EXPECT_TRUE(IsAlignedTo(ptr, kSmallAlignment)); benchmark::DoNotOptimize(memset(ptr, 0xBF, kSmall)); ptrs.emplace_back(SizedAlignedPtr{ ptr, kSmall, static_cast(kSmallAlignment)}); diff --git a/tcmalloc/pages.h b/tcmalloc/pages.h index e9cfd96f1..f34276c95 100644 --- a/tcmalloc/pages.h +++ b/tcmalloc/pages.h @@ -28,6 +28,7 @@ #include "tcmalloc/common.h" #include "tcmalloc/internal/bytes.h" #include "tcmalloc/internal/config.h" +#include "tcmalloc/internal/is_aligned_to.h" #include "tcmalloc/internal/logging.h" #include "tcmalloc/internal/optimization.h" @@ -195,7 +196,7 @@ inline constexpr Length LengthFromBytes(size_t bytes) { TCMALLOC_ATTRIBUTE_CONST inline constexpr Length BytesToLengthCeil(size_t bytes) { return Length((bytes >> kPageShift) + - ((bytes & (kPageSize - 1)) > 0 ? 1 : 0)); + (IsAlignedTo(bytes, kPageSize) ? 0 : 1)); } TCMALLOC_ATTRIBUTE_CONST diff --git a/tcmalloc/parameters.cc b/tcmalloc/parameters.cc index bc5145e64..6919db4ae 100644 --- a/tcmalloc/parameters.cc +++ b/tcmalloc/parameters.cc @@ -231,6 +231,8 @@ ABSL_CONST_INIT std::atomic Parameters::back_size_threshold_bytes_( ABSL_CONST_INIT std::atomic Parameters::enable_unfiltered_collapse_( false); ABSL_CONST_INIT std::atomic Parameters::release_max_cold_pages_(false); +ABSL_CONST_INIT std::atomic Parameters::release_drained_slab_metadata_( + false); static std::atomic& madvise_cold_regions_nohugepage_enabled() { ABSL_CONST_INIT static absl::once_flag flag; @@ -343,7 +345,6 @@ ReleaseStalePages Parameters::release_stale_pages() { return v.load(std::memory_order_relaxed); } - int32_t Parameters::max_per_cpu_cache_size() { return tc_globals.cpu_cache().CacheLimit(); } @@ -604,7 +605,6 @@ void TCMalloc_Internal_SetPerCpuCachesDynamicSlabEnabled(bool v) { Parameters::per_cpu_caches_dynamic_slab_.store(v, std::memory_order_relaxed); } - uint8_t TCMalloc_Internal_GetMinHotAccessHint() { return static_cast(Parameters::min_hot_access_hint()); } @@ -660,6 +660,11 @@ void TCMalloc_Internal_SetMadviseColdRegionsNoHugepage(bool v) { std::memory_order_relaxed); } +void TCMalloc_Internal_SetReleaseDrainedSlabMetadata(bool v) { + Parameters::release_drained_slab_metadata_.store(v, + std::memory_order_relaxed); +} + } // extern "C" GOOGLE_MALLOC_SECTION_END diff --git a/tcmalloc/parameters.h b/tcmalloc/parameters.h index 6ccb0f989..21d6cd8b2 100644 --- a/tcmalloc/parameters.h +++ b/tcmalloc/parameters.h @@ -201,8 +201,15 @@ class Parameters { std::memory_order_relaxed); } - static HeapPartitioningMode heap_partitioning_mode(); + static bool release_drained_slab_metadata() { + return release_drained_slab_metadata_.load(std::memory_order_relaxed); + } + static void set_release_drained_slab_metadata(bool value) { + TCMalloc_Internal_SetReleaseDrainedSlabMetadata(value); + } + + static HeapPartitioningMode heap_partitioning_mode(); // TODO: b/527473378 - Remove this function once the experiment is cleaned up. static ReleaseStalePages release_stale_pages(); @@ -241,6 +248,7 @@ class Parameters { friend void ::TCMalloc_Internal_SetEnableUnfilteredCollapse(bool v); friend void ::TCMalloc_Internal_SetHugeRegionAdaptiveReleaseEnabled(bool v); friend void ::TCMalloc_Internal_SetReleaseMaxColdPages(bool v); + friend void ::TCMalloc_Internal_SetReleaseDrainedSlabMetadata(bool v); static std::atomic guarded_sampling_interval_; static std::atomic max_per_cpu_cache_size_; @@ -261,6 +269,7 @@ class Parameters { static std::atomic back_size_threshold_bytes_; static std::atomic enable_unfiltered_collapse_; static std::atomic release_max_cold_pages_; + static std::atomic release_drained_slab_metadata_; }; } // namespace tcmalloc_internal diff --git a/tcmalloc/sizemap.cc b/tcmalloc/sizemap.cc index b2c0ca1b6..fa61360ed 100644 --- a/tcmalloc/sizemap.cc +++ b/tcmalloc/sizemap.cc @@ -27,6 +27,7 @@ #include "tcmalloc/common.h" #include "tcmalloc/huge_page_aware_allocator.h" #include "tcmalloc/internal/config.h" +#include "tcmalloc/internal/is_aligned_to.h" #include "tcmalloc/internal/logging.h" #include "tcmalloc/internal/parameter_accessors.h" #include "tcmalloc/internal/sampled_allocation.h" @@ -109,7 +110,7 @@ bool SizeMap::IsValidSizeClass(size_t size, Length pages, const size_t alignment = size > SizeMap::kLargeSize ? kLargeSizeAlignment : static_cast(kAlignment); - if ((size & (alignment - 1)) != 0) { + if (!IsAlignedTo(size, alignment)) { TC_LOG("%v not aligned properly %v", size, alignment); return false; } diff --git a/tcmalloc/sizemap.h b/tcmalloc/sizemap.h index 69101c532..8397ffd13 100644 --- a/tcmalloc/sizemap.h +++ b/tcmalloc/sizemap.h @@ -27,6 +27,7 @@ #include "absl/types/span.h" #include "tcmalloc/common.h" #include "tcmalloc/internal/config.h" +#include "tcmalloc/internal/is_aligned_to.h" #include "tcmalloc/internal/logging.h" #include "tcmalloc/internal/optimization.h" #include "tcmalloc/internal/size_class_info.h" @@ -263,10 +264,11 @@ class SizeMap { static_assert((kMaxSize % kPageSize) == 0, "the loop below won't work"); // Profiles say we usually get the right class based on the size, // so avoid the loop overhead on the fast path. - if (ABSL_PREDICT_FALSE(class_to_size(size_class) & (align - 1))) { + if (ABSL_PREDICT_FALSE(!IsAlignedTo(class_to_size(size_class), align))) { do { ++size_class; - } while (ABSL_PREDICT_FALSE(class_to_size(size_class) & (align - 1))); + } while ( + ABSL_PREDICT_FALSE(!IsAlignedTo(class_to_size(size_class), align))); } return {true, size_class}; } diff --git a/tcmalloc/tcmalloc.cc b/tcmalloc/tcmalloc.cc index ce82d0b37..f392b9d6d 100644 --- a/tcmalloc/tcmalloc.cc +++ b/tcmalloc/tcmalloc.cc @@ -98,6 +98,7 @@ #include "tcmalloc/guarded_page_allocator.h" #include "tcmalloc/internal/allocation_guard.h" #include "tcmalloc/internal/config.h" +#include "tcmalloc/internal/is_aligned_to.h" #include "tcmalloc/internal/logging.h" #include "tcmalloc/internal/memory_tag.h" #include "tcmalloc/internal/optimization.h" @@ -1091,8 +1092,7 @@ bool CorrectAlignment(void* ptr, std::align_val_t alignment) { align = std::max(align, kPageSize); } - if (ABSL_PREDICT_FALSE((reinterpret_cast(ptr) & (align - 1)) != - 0)) { + if (ABSL_PREDICT_FALSE(!IsAlignedTo(ptr, align))) { ReportCorruptedFree(tc_globals, static_cast(align), ptr); return false; } diff --git a/tcmalloc/testing/BUILD b/tcmalloc/testing/BUILD index 7e34f467f..3ac274849 100644 --- a/tcmalloc/testing/BUILD +++ b/tcmalloc/testing/BUILD @@ -81,6 +81,7 @@ create_tcmalloc_testsuite( "//tcmalloc:malloc_extension", "//tcmalloc/internal:config", "//tcmalloc/internal:declarations", + "//tcmalloc/internal:is_aligned_to", "//tcmalloc/internal:logging", "//tcmalloc/internal:memory_tag", "//tcmalloc/internal:parameter_accessors", @@ -517,6 +518,7 @@ create_tcmalloc_testsuite( ":testutil", "//tcmalloc:malloc_extension", "//tcmalloc/internal:config", + "//tcmalloc/internal:is_aligned_to", "@com_github_google_benchmark//:benchmark", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_map", diff --git a/tcmalloc/testing/aligned_new_test.cc b/tcmalloc/testing/aligned_new_test.cc index 303c8e616..50a26a2b5 100644 --- a/tcmalloc/testing/aligned_new_test.cc +++ b/tcmalloc/testing/aligned_new_test.cc @@ -27,9 +27,12 @@ #include "absl/container/flat_hash_map.h" #include "absl/debugging/leak_check.h" #include "tcmalloc/internal/config.h" +#include "tcmalloc/internal/is_aligned_to.h" #include "tcmalloc/malloc_extension.h" #include "tcmalloc/testing/testutil.h" +using tcmalloc::tcmalloc_internal::IsAlignedTo; + namespace tcmalloc { namespace { @@ -87,7 +90,7 @@ TYPED_TEST_P(AlignedNew, AlignedTest) { for (int i = 0; i < kAllocations; i++) { TypeParam* p = new TypeParam(); benchmark::DoNotOptimize(p); - ASSERT_EQ(0, reinterpret_cast(p) & (alignof(TypeParam) - 1)); + ASSERT_TRUE(IsAlignedTo(p, alignof(TypeParam))); this->ptrs.emplace_back(p); } diff --git a/tcmalloc/testing/tcmalloc_test.cc b/tcmalloc/testing/tcmalloc_test.cc index 9e4d6d256..0bd6da5e7 100644 --- a/tcmalloc/testing/tcmalloc_test.cc +++ b/tcmalloc/testing/tcmalloc_test.cc @@ -74,6 +74,7 @@ #include "tcmalloc/huge_pages.h" #include "tcmalloc/internal/config.h" #include "tcmalloc/internal/declarations.h" +#include "tcmalloc/internal/is_aligned_to.h" #include "tcmalloc/internal/logging.h" #include "tcmalloc/internal/memory_tag.h" #include "tcmalloc/internal/parameter_accessors.h" @@ -903,7 +904,7 @@ TEST(TCMallocTest, FreeAlignedSized) { void* ptr = aligned_alloc(alignment, size); if (!kSanitizerPresent) { ASSERT_NE(ptr, nullptr) << alignment << " " << size; - ASSERT_EQ(reinterpret_cast(ptr) & (alignment - 1), 0); + ASSERT_TRUE(IsAlignedTo(ptr, alignment)); memset(ptr, 0, size); benchmark::DoNotOptimize(ptr); } @@ -938,7 +939,7 @@ TEST(TCMallocTest, sdallocx_alignment) { void* ptr; int err = posix_memalign(&ptr, alignment, size); ASSERT_EQ(err, 0) << alignment << " " << size; - ASSERT_EQ(reinterpret_cast(ptr) & (alignment - 1), 0); + ASSERT_TRUE(IsAlignedTo(ptr, alignment)); memset(ptr, 0, size); benchmark::DoNotOptimize(ptr); sdallocx(ptr, size, MALLOCX_LG_ALIGN(align)); @@ -975,7 +976,7 @@ TEST(TCMallocTest, aligned_alloc_at_least) { auto result = aligned_alloc_at_least(alignment, size); if (!kSanitizerPresent) { ASSERT_NE(result.ptr, nullptr) << alignment << " " << size; - ASSERT_EQ(reinterpret_cast(result.ptr) & (alignment - 1), 0); + ASSERT_TRUE(IsAlignedTo(result.ptr, alignment)); ASSERT_GE(result.size, size); memset(result.ptr, 0, result.size); benchmark::DoNotOptimize(result);