Skip to content

[libc++] Optimize heap operations #121480

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[libc++] Optimize heap operations #121480

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions libcxx/include/__algorithm/make_heap.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,10 @@ __make_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compar
__comp_ref_type<_Compare> __comp_ref = __comp;

using difference_type = typename iterator_traits<_RandomAccessIterator>::difference_type;
difference_type __n = __last - __first;
if (__n > 1) {
difference_type __n = __last - __first, __start = __n / 2;
while (__start != 0) {
// start from the first parent, there is no need to consider children
for (difference_type __start = (__n - 2) / 2; __start >= 0; --__start) {
std::__sift_down<_AlgPolicy>(__first, __comp_ref, __n, __first + __start);
}
std::__sift_down<_AlgPolicy>(__first, __comp_ref, __n, --__start);
}
}

Expand Down
2 changes: 1 addition & 1 deletion libcxx/include/__algorithm/partial_sort.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandomAccessIterator __part
for (; __i != __last; ++__i) {
if (__comp(*__i, *__first)) {
_IterOps<_AlgPolicy>::iter_swap(__i, __first);
std::__sift_down<_AlgPolicy>(__first, __comp, __len, __first);
std::__sift_down<_AlgPolicy>(__first, __comp, __len, 0);
}
}
std::__sort_heap<_AlgPolicy>(std::move(__first), std::move(__middle), __comp);
Expand Down
7 changes: 4 additions & 3 deletions libcxx/include/__algorithm/partial_sort_copy.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,16 +52,17 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator, _Random
_RandomAccessIterator __r = __result_first;
auto&& __projected_comp = std::__make_projected(__comp, __proj2);

if (__r != __result_last) {
if (__result_first != __result_last) {
for (; __first != __last && __r != __result_last; ++__first, (void)++__r)
*__r = *__first;
std::__make_heap<_AlgPolicy>(__result_first, __r, __projected_comp);
typename iterator_traits<_RandomAccessIterator>::difference_type __len = __r - __result_first;
for (; __first != __last; ++__first)
for (; __first != __last; ++__first) {
if (std::__invoke(__comp, std::__invoke(__proj1, *__first), std::__invoke(__proj2, *__result_first))) {
*__result_first = *__first;
std::__sift_down<_AlgPolicy>(__result_first, __projected_comp, __len, __result_first);
std::__sift_down<_AlgPolicy>(__result_first, __projected_comp, __len, 0);
}
}
std::__sort_heap<_AlgPolicy>(__result_first, __r, __projected_comp);
}

Expand Down
27 changes: 13 additions & 14 deletions libcxx/include/__algorithm/pop_heap.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,27 +33,21 @@ _LIBCPP_BEGIN_NAMESPACE_STD
template <class _AlgPolicy, class _Compare, class _RandomAccessIterator>
inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
__pop_heap(_RandomAccessIterator __first,
_RandomAccessIterator __last,
_RandomAccessIterator __bottom,
_Compare& __comp,
typename iterator_traits<_RandomAccessIterator>::difference_type __len) {
// Calling `pop_heap` on an empty range is undefined behavior, but in practice it will be a no-op.
_LIBCPP_ASSERT_PEDANTIC(__len > 0, "The heap given to pop_heap must be non-empty");

__comp_ref_type<_Compare> __comp_ref = __comp;

using value_type = typename iterator_traits<_RandomAccessIterator>::value_type;

if (__len > 1) {
value_type __top = _IterOps<_AlgPolicy>::__iter_move(__first); // create a hole at __first
_RandomAccessIterator __hole = std::__floyd_sift_down<_AlgPolicy>(__first, __comp_ref, __len);
--__last;
_RandomAccessIterator __hole = std::__floyd_sift_down<_AlgPolicy>(__first, __comp, __len);

if (__hole == __last) {
if (__hole == __bottom) {
*__hole = std::move(__top);
} else {
*__hole = _IterOps<_AlgPolicy>::__iter_move(__last);
++__hole;
*__last = std::move(__top);
std::__sift_up<_AlgPolicy>(__first, __hole, __comp_ref, __hole - __first);
*__hole = _IterOps<_AlgPolicy>::__iter_move(__bottom);
*__bottom = std::move(__top);
std::__sift_up<_AlgPolicy>(__first, __hole, __comp);
}
}
}
Expand All @@ -64,8 +58,13 @@ pop_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare _
static_assert(std::is_copy_constructible<_RandomAccessIterator>::value, "Iterators must be copy constructible.");
static_assert(std::is_copy_assignable<_RandomAccessIterator>::value, "Iterators must be copy assignable.");

// Calling `pop_heap` on an empty range is undefined behavior, but in practice it will be a no-op.
_LIBCPP_ASSERT_PEDANTIC(__len > 0, "The heap given to pop_heap must be non-empty");

__comp_ref_type<_Compare> __comp_ref = __comp;

typename iterator_traits<_RandomAccessIterator>::difference_type __len = __last - __first;
std::__pop_heap<_ClassicAlgPolicy>(std::move(__first), std::move(__last), __comp, __len);
std::__pop_heap<_ClassicAlgPolicy>(std::move(__first), std::move(--__last), __comp_ref, __len);
}

template <class _RandomAccessIterator>
Expand Down
48 changes: 23 additions & 25 deletions libcxx/include/__algorithm/push_heap.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,37 +29,35 @@ _LIBCPP_BEGIN_NAMESPACE_STD

template <class _AlgPolicy, class _Compare, class _RandomAccessIterator>
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
__sift_up(_RandomAccessIterator __first,
_RandomAccessIterator __last,
_Compare&& __comp,
typename iterator_traits<_RandomAccessIterator>::difference_type __len) {
using value_type = typename iterator_traits<_RandomAccessIterator>::value_type;

if (__len > 1) {
__len = (__len - 2) / 2;
_RandomAccessIterator __ptr = __first + __len;

if (__comp(*__ptr, *--__last)) {
value_type __t(_IterOps<_AlgPolicy>::__iter_move(__last));
do {
*__last = _IterOps<_AlgPolicy>::__iter_move(__ptr);
__last = __ptr;
if (__len == 0)
break;
__len = (__len - 1) / 2;
__ptr = __first + __len;
} while (__comp(*__ptr, __t));

*__last = std::move(__t);
}
__sift_up(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare&& __comp) {
using difference_type = typename iterator_traits<_RandomAccessIterator>::difference_type;
using value_type = typename iterator_traits<_RandomAccessIterator>::value_type;

difference_type __parent = __last - __first;
_LIBCPP_ASSERT_INTERNAL(__parent > 0, "shouldn't be called unless __last - __first > 0");
__parent = (__parent - 1) / 2;
_RandomAccessIterator __parent_i = __first + __parent;

if (__comp(*__parent_i, *__last)) {
value_type __t(_IterOps<_AlgPolicy>::__iter_move(__last));
do {
*__last = _IterOps<_AlgPolicy>::__iter_move(__parent_i);
__last = __parent_i;
if (__parent == 0)
break;
__parent = (__parent - 1) / 2;
__parent_i = __first + __parent;
} while (__comp(*__parent_i, __t));

*__last = std::move(__t);
}
}

template <class _AlgPolicy, class _RandomAccessIterator, class _Compare>
inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
__push_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare& __comp) {
typename iterator_traits<_RandomAccessIterator>::difference_type __len = __last - __first;
std::__sift_up<_AlgPolicy, __comp_ref_type<_Compare> >(std::move(__first), std::move(__last), __comp, __len);
if (__first != __last)
std::__sift_up<_AlgPolicy, __comp_ref_type<_Compare> >(std::move(__first), std::move(--__last), __comp);
}

template <class _RandomAccessIterator, class _Compare>
Expand Down
80 changes: 45 additions & 35 deletions libcxx/include/__algorithm/sift_down.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,84 +29,94 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
__sift_down(_RandomAccessIterator __first,
_Compare&& __comp,
typename iterator_traits<_RandomAccessIterator>::difference_type __len,
_RandomAccessIterator __start) {
typename iterator_traits<_RandomAccessIterator>::difference_type __start) {
using _Ops = _IterOps<_AlgPolicy>;

typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type;
// left-child of __start is at 2 * __start + 1
// right-child of __start is at 2 * __start + 2
difference_type __child = __start - __first;

if (__len < 2 || (__len - 2) / 2 < __child)
if (__len < 2)
return;

__child = 2 * __child + 1;
_RandomAccessIterator __child_i = __first + __child;
// left-child of __start is at 2 * __start + 1
// right-child of __start is at 2 * __start + 2
difference_type __child = 2 * __start + 1;
_RandomAccessIterator __child_i = __first + __child, __start_i = __first + __start;

if ((__child + 1) < __len && __comp(*__child_i, *(__child_i + difference_type(1)))) {
// right-child exists and is greater than left-child
++__child_i;
++__child;
if (__child < __len - 1) {
_RandomAccessIterator __right_i = _Ops::next(__child_i);
if (__comp(*__child_i, *__right_i)) {
// right-child exists and is greater than left-child
__child_i = __right_i;
++__child;
}
}

// check if we are in heap-order
if (__comp(*__child_i, *__start))
if (__comp(*__child_i, *__start_i))
// we are, __start is larger than its largest child
return;

value_type __top(_Ops::__iter_move(__start));
value_type __top(_Ops::__iter_move(__start_i));
do {
// we are not in heap-order, swap the parent with its largest child
*__start = _Ops::__iter_move(__child_i);
__start = __child_i;
*__start_i = _Ops::__iter_move(__child_i);
__start_i = __child_i;

if ((__len - 2) / 2 < __child)
if (__len / 2 - 1 < __child)
break;

// recompute the child based off of the updated parent
__child = 2 * __child + 1;
__child_i = __first + __child;

if ((__child + 1) < __len && __comp(*__child_i, *(__child_i + difference_type(1)))) {
// right-child exists and is greater than left-child
++__child_i;
++__child;
if (__child < __len - 1) {
_RandomAccessIterator __right_i = _Ops::next(__child_i);
if (__comp(*__child_i, *__right_i)) {
// right-child exists and is greater than left-child
__child_i = __right_i;
++__child;
}
}

// check if we are in heap-order
} while (!__comp(*__child_i, __top));
*__start = std::move(__top);
*__start_i = std::move(__top);
}

template <class _AlgPolicy, class _Compare, class _RandomAccessIterator>
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _RandomAccessIterator __floyd_sift_down(
_RandomAccessIterator __first,
_Compare&& __comp,
typename iterator_traits<_RandomAccessIterator>::difference_type __len) {
using difference_type = typename iterator_traits<_RandomAccessIterator>::difference_type;
_LIBCPP_ASSERT_INTERNAL(__len >= 2, "shouldn't be called unless __len >= 2");
_LIBCPP_ASSERT_INTERNAL(__len > 1, "shouldn't be called unless __len > 1");

_RandomAccessIterator __hole = __first;
_RandomAccessIterator __child_i = __first;
difference_type __child = 0;
using _Ops = _IterOps<_AlgPolicy>;

while (true) {
__child_i += difference_type(__child + 1);
__child = 2 * __child + 1;
typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;

if ((__child + 1) < __len && __comp(*__child_i, *(__child_i + difference_type(1)))) {
// right-child exists and is greater than left-child
++__child_i;
++__child;
difference_type __child = 1;
_RandomAccessIterator __hole = __first, __child_i = __first;

while (true) {
__child_i += __child;
__child *= 2;

if (__child < __len) {
_RandomAccessIterator __right_i = _Ops::next(__child_i);
if (__comp(*__child_i, *__right_i)) {
// right-child exists and is greater than left-child
__child_i = __right_i;
++__child;
}
}

// swap __hole with its largest child
*__hole = _IterOps<_AlgPolicy>::__iter_move(__child_i);
*__hole = _Ops::__iter_move(__child_i);
__hole = __child_i;

// if __hole is now a leaf, we're done
if (__child > (__len - 2) / 2)
if (__child > __len / 2)
return __hole;
}
}
Expand Down
4 changes: 2 additions & 2 deletions libcxx/include/__algorithm/sort_heap.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ __sort_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compar
__comp_ref_type<_Compare> __comp_ref = __comp;

using difference_type = typename iterator_traits<_RandomAccessIterator>::difference_type;
for (difference_type __n = __last - __first; __n > 1; --__last, (void)--__n)
std::__pop_heap<_AlgPolicy>(__first, __last, __comp_ref, __n);
for (difference_type __n = __last - __first; __n > 1; --__n)
std::__pop_heap<_AlgPolicy>(__first, --__last, __comp_ref, __n);
std::__check_strict_weak_ordering_sorted(__first, __saved_last, __comp_ref);
}

Expand Down