diff --git a/tcmalloc/central_freelist.h b/tcmalloc/central_freelist.h index e93f2b28f..39a8a1ca4 100644 --- a/tcmalloc/central_freelist.h +++ b/tcmalloc/central_freelist.h @@ -517,8 +517,12 @@ inline Span* CentralFreeList::ReleaseToSpans( nonempty_.Add(span, cur_index); span->set_nonempty_index(cur_index); } else if (cur_index != prev_index) { +#ifdef TCMALLOC_INTERNAL_LEGACY_LOCKING nonempty_.Remove(span, prev_index); nonempty_.Add(span, cur_index); +#else + nonempty_.Move(span, prev_index, cur_index); +#endif span->set_nonempty_index(cur_index); } return nullptr; @@ -774,8 +778,12 @@ inline int CentralFreeList::RemoveRange(absl::Span batch) { const uint8_t cur_index = IndexFor(span->is_long_lived_span(), cur_allocated, cur_bitwidth); if (cur_index != prev_index) { +#ifdef TCMALLOC_INTERNAL_LEGACY_LOCKING nonempty_.Remove(span, prev_index); nonempty_.Add(span, cur_index); +#else + nonempty_.Move(span, prev_index, cur_index); +#endif span->set_nonempty_index(cur_index); } } diff --git a/tcmalloc/hinted_tracker_lists.h b/tcmalloc/hinted_tracker_lists.h index 5f0951bfb..93afd68c8 100644 --- a/tcmalloc/hinted_tracker_lists.h +++ b/tcmalloc/hinted_tracker_lists.h @@ -95,6 +95,26 @@ class HintedTrackerLists { } --size_; } + + // Moves pointer from list `from` to list `to`. + // REQUIRES: from < N && to < N && pt != nullptr + void Move(TrackerType* absl_nonnull pt TCMALLOC_CAPTURED_BY_THIS, + const size_t from, const size_t to) { + TC_ASSERT_LT(from, N); + TC_ASSERT_LT(to, N); + // This check is not strictly required, but we should avoid the linked list + // manipulations if we can. + TC_ASSERT_NE(from, to); + TC_ASSERT_NE(pt, nullptr); + + if (lists_[from].remove(pt)) { + TC_ASSERT(nonempty_.GetBit(from)); + nonempty_.ClearBit(from); + } + lists_[to].prepend(pt); + nonempty_.SetBit(to); + } + const TrackerList& operator[](const size_t n) const { TC_ASSERT_LT(n, N); return lists_[n];