diff --git a/examples/6_resource_scope.cpp b/examples/6_resource_scope.cpp index 5d267117..4f56f286 100644 --- a/examples/6_resource_scope.cpp +++ b/examples/6_resource_scope.cpp @@ -17,7 +17,7 @@ int main() rg::init(1); rg::IOResource a; // scope-level=0 - rg::emplace_task( + rg::emplace_continuable_task( [](auto a) { std::cout << "scope = " << rg::scope_depth() << std::endl; @@ -34,8 +34,8 @@ int main() std::cout << "scope = " << rg::scope_depth() << std::endl; }, - a.read()) - .enable_stack_switching(); + a.read() + ); rg::finalize(); } diff --git a/examples/mpi.cpp b/examples/mpi.cpp index 94867e2c..68d8129a 100644 --- a/examples/mpi.cpp +++ b/examples/mpi.cpp @@ -37,7 +37,7 @@ struct MPIConfig int main() { spdlog::set_pattern("[thread %t] %^[%l]%$ %v"); - spdlog::set_level(spdlog::level::trace); + spdlog::set_level( spdlog::level::trace ); /* int prov; @@ -45,53 +45,51 @@ int main() assert( prov == MPI_THREAD_MULTIPLE ); */ - MPI_Init(nullptr, nullptr); + MPI_Init( nullptr, nullptr ); auto default_scheduler = std::make_shared(); auto mpi_request_pool = std::make_shared(); - hwloc_obj_t obj = hwloc_get_obj_by_type(redGrapes::SingletonContext::get().hwloc_ctx.topology, HWLOC_OBJ_PU, 1); - rg::memory::ChunkedBumpAlloc mpi_alloc( - rg::memory::HwlocAlloc(redGrapes::SingletonContext::get().hwloc_ctx, obj)); - auto mpi_worker = std::make_shared( - mpi_alloc, - redGrapes::SingletonContext::get().hwloc_ctx, - obj, - 4); + hwloc_obj_t obj = hwloc_get_obj_by_type( redGrapes::SingletonContext::get().hwloc_ctx.topology, HWLOC_OBJ_PU, 1 ); + rg::memory::ChunkedBumpAlloc< rg::memory::HwlocAlloc > mpi_alloc( rg::memory::HwlocAlloc( redGrapes::SingletonContext::get().hwloc_ctx, obj ) ); + auto mpi_worker = std::make_shared( mpi_alloc, redGrapes::SingletonContext::get().hwloc_ctx, obj, 4 ); // initialize main thread to execute tasks from the mpi-queue and poll - rg::SingletonContext::get().idle = [mpi_worker, mpi_request_pool] - { - mpi_request_pool->poll(); - - redGrapes::Task* task; - - if(task = mpi_worker->ready_queue.pop()) - redGrapes::SingletonContext::get().execute_task(*task); - - while(mpi_worker->init_dependencies(task, true)) - if(task) - { - redGrapes::SingletonContext::get().execute_task(*task); - break; - } - }; - - rg::init(4, rg::scheduler::make_tag_match_scheduler().add({}, default_scheduler).add({SCHED_MPI}, mpi_worker)); - + rg::SingletonContext::get().idle = + [mpi_worker, mpi_request_pool] + { + mpi_request_pool->poll(); + + redGrapes::Task * task; + + if( (task = mpi_worker->ready_queue.pop()) ) + redGrapes::SingletonContext::get().execute_task( *task ); + + while( mpi_worker->init_dependencies( task, true ) ) + if( task ) + { + redGrapes::SingletonContext::get().execute_task( *task ); + break; + } + }; + + rg::init(4, + rg::scheduler::make_tag_match_scheduler() + .add({}, default_scheduler) + .add({ SCHED_MPI }, mpi_worker)); + // initialize MPI config - rg::IOResource mpi_config; + rg::IOResource< MPIConfig > mpi_config; rg::emplace_task( - [](auto config) - { + []( auto config ) { MPI_Comm_rank(MPI_COMM_WORLD, &config->world_rank); MPI_Comm_size(MPI_COMM_WORLD, &config->world_size); }, - mpi_config.write()) - .scheduling_tags(std::bitset<64>().set(SCHED_MPI)); + mpi_config.write() + ).scheduling_tags( std::bitset<64>().set(SCHED_MPI) ); // main loop - rg::FieldResource> field[2] = { + rg::FieldResource< std::array > field[2] = { rg::FieldResource>(new std::array()), rg::FieldResource>(new std::array()), }; @@ -100,14 +98,15 @@ int main() // initialize rg::emplace_task( - [](auto buf, auto mpi_config) + []( auto buf, auto mpi_config ) { int offset = 3 * mpi_config->world_rank; - for(size_t i = 0; i < buf->size(); ++i) + for( size_t i = 0; i < buf->size(); ++i ) buf[{i}] = offset + i; }, field[current].write(), - mpi_config.read()); + mpi_config.read() + ); for(size_t i = 0; i < 1; ++i) { @@ -118,67 +117,77 @@ int main() */ // Send - rg::emplace_task( - [i, current, mpi_request_pool](auto field, auto mpi_config) + rg::emplace_continuable_task( + [i, current, mpi_request_pool]( auto field, auto mpi_config ) { - int dst = (mpi_config->world_rank + 1) % mpi_config->world_size; + int dst = ( mpi_config->world_rank + 1 ) % mpi_config->world_size; MPI_Request request; - MPI_Isend(&field[{3}], sizeof(int), MPI_CHAR, dst, current, MPI_COMM_WORLD, &request); + MPI_Isend( &field[{3}], sizeof(int), MPI_CHAR, dst, current, MPI_COMM_WORLD, &request ); - mpi_request_pool->get_status(request); + mpi_request_pool->get_status( request ); }, field[current].at({3}).read(), mpi_config.read()) - .scheduling_tags({SCHED_MPI}) - .enable_stack_switching(); + .scheduling_tags({ SCHED_MPI }); // Receive - rg::emplace_task( - [i, current, mpi_request_pool](auto field, auto mpi_config) + rg::emplace_continuable_task( + [i, current, mpi_request_pool]( auto field, auto mpi_config ) { - int src = (mpi_config->world_rank - 1) % mpi_config->world_size; + int src = ( mpi_config->world_rank - 1 ) % mpi_config->world_size; MPI_Request request; - MPI_Irecv(&field[{0}], sizeof(int), MPI_CHAR, src, current, MPI_COMM_WORLD, &request); + MPI_Irecv( &field[{0}], sizeof(int), MPI_CHAR, src, current, MPI_COMM_WORLD, &request ); - MPI_Status status = mpi_request_pool->get_status(request); + MPI_Status status = mpi_request_pool->get_status( request ); int recv_data_count; - MPI_Get_count(&status, MPI_CHAR, &recv_data_count); + MPI_Get_count( &status, MPI_CHAR, &recv_data_count ); }, field[current].at({0}).write(), mpi_config.read()) - .scheduling_tags({SCHED_MPI}) - .enable_stack_switching(); + .scheduling_tags({ SCHED_MPI }); /* * Compute iteration */ - for(size_t i = 1; i < field[current]->size(); ++i) + for( size_t i = 1; i < field[current]->size(); ++i ) rg::emplace_task( - [i](auto dst, auto src) { dst[{i}] = src[{i - 1}]; }, + [i]( auto dst, auto src ) + { + dst[{i}] = src[{i - 1}]; + }, field[next].at({i}).write(), - field[current].at({i - 1}).read()); + field[current].at({i-1}).read() + ); /* * Write Output */ rg::emplace_task( - [i](auto buf, auto mpi_config) - { + [i]( auto buf, auto mpi_config ) + { std::cout << "Step[" << i << "], rank[" << mpi_config->world_rank << "] :: "; - for(size_t i = 0; i < buf->size(); ++i) + for( size_t i = 0; i < buf->size(); ++i ) std::cout << buf[{i}] << "; "; std::cout << std::endl; }, field[current].read(), - mpi_config.read()); + mpi_config.read() + ); current = next; } - rg::emplace_task([](auto m) { MPI_Finalize(); }, mpi_config.write()).scheduling_tags({SCHED_MPI}); + rg::emplace_task( + []( auto m ) + { + MPI_Finalize(); + }, + mpi_config.write() + ).scheduling_tags({ SCHED_MPI }); rg::finalize(); } + diff --git a/redGrapes/dispatch/thread/execute.cpp b/redGrapes/dispatch/thread/execute.cpp index 2e2dc254..c18a3080 100644 --- a/redGrapes/dispatch/thread/execute.cpp +++ b/redGrapes/dispatch/thread/execute.cpp @@ -11,11 +11,6 @@ #include #include -#include -#include - -#include - namespace redGrapes { /* @@ -38,8 +33,8 @@ namespace thread if(event) { - event->get_event().waker_id = current_worker->get_waker_id(); - task.sg_pause(*event); + event.get_event().waker_id = current_waker_id; + task.sg_pause(event); task.pre_event.up(); task.get_pre_event().notify(); diff --git a/redGrapes/dispatch/thread/worker_pool.cpp b/redGrapes/dispatch/thread/worker_pool.cpp index 194df68c..44ae51f8 100644 --- a/redGrapes/dispatch/thread/worker_pool.cpp +++ b/redGrapes/dispatch/thread/worker_pool.cpp @@ -45,7 +45,7 @@ namespace redGrapes hwloc_obj_t obj = hwloc_get_obj_by_type(hwloc_ctx.topology, HWLOC_OBJ_PU, pu_id); allocs.emplace_back(memory::HwlocAlloc(hwloc_ctx, obj), REDGRAPES_ALLOC_CHUNKSIZE); - SingletonContext::get().current_arena = pu_id; + SingletonContext::get().current_arena = worker_id; auto worker = memory::alloc_shared_bind(pu_id, get_alloc(pu_id), hwloc_ctx, obj, worker_id); // auto worker = std::make_shared< WorkerThread >( get_alloc(i), hwloc_ctx, obj, i ); diff --git a/redGrapes/memory/chunked_bump_alloc.hpp b/redGrapes/memory/chunked_bump_alloc.hpp index 20a6e876..60ea5ed1 100644 --- a/redGrapes/memory/chunked_bump_alloc.hpp +++ b/redGrapes/memory/chunked_bump_alloc.hpp @@ -84,7 +84,7 @@ namespace redGrapes TRACE_EVENT("Allocator", "ChunkedBumpAlloc::allocate()"); size_t alloc_size = roundup_to_poweroftwo(n); - size_t const chunk_capacity = bump_allocators.get_chunk_capacity(); + size_t const chunk_capacity = chunk_size; // bump_allocators.get_chunk_capacity(); if(alloc_size <= chunk_capacity) { diff --git a/redGrapes/memory/refcounted.hpp b/redGrapes/memory/refcounted.hpp new file mode 100644 index 00000000..ab185463 --- /dev/null +++ b/redGrapes/memory/refcounted.hpp @@ -0,0 +1,132 @@ + +#pragma once + +#include + +namespace redGrapes +{ + + namespace memory + { + + template + struct Refcounted + { + std::atomic refcount; + + Refcounted() : refcount(0) + { + } + + inline void acquire() + { + Refcount old_refcount = refcount.fetch_add(1); + } + + inline bool release() + { + Refcount old_refcount = refcount.fetch_sub(1); + return old_refcount == 0; + } + + struct Guard + { + private: + std::atomic ptr; + + public: + inline Guard() : ptr(nullptr) + { + } + + inline Guard(Derived* ptr) : ptr(ptr) + { + } + + inline Guard(Guard const& other) + { + acquire(other.ptr.load()); + } + + inline Guard(Guard&& other) : ptr(other.ptr.load()) + { + other.ptr = nullptr; + } + + inline Guard& operator=(Guard const& other) + { + release(); + acquire(other.ptr.load()); + return *this; + } + + inline Guard& operator=(Guard&& other) + { + release(); + ptr = other.ptr; + other.ptr = nullptr; + return *this; + } + + bool compare_exchange_strong(Derived* expected_ptr, Guard new_guard) + { + Derived* desired_ptr = new_guard.ptr.load(); + new_guard.ptr = expected_ptr; + return ptr.compare_exchange_strong(expected_ptr, desired_ptr); + } + + inline Derived* get() const + { + return ptr.load(); + } + + inline Derived& operator*() const + { + return *ptr.load(); + } + + inline Derived* operator->() const + { + return ptr.load(); + } + + inline bool operator==(Guard const& other) const + { + return ptr == other.ptr; + } + + inline bool operator!=(Guard const& other) const + { + return ptr != other.ptr; + } + + inline operator bool() const + { + return ptr.load(); + } + + inline void acquire(Derived* nw_ptr) + { + ptr = nw_ptr; + if(nw_ptr) + nw_ptr->acquire(); + } + + inline void release() + { + Derived* p = ptr.load(); + if(p) + if(p->release()) + Deleter{}(p); + } + + ~Guard() + { + release(); + } + }; + }; + + } // namespace memory + +} // namespace redGrapes diff --git a/redGrapes/redGrapes.cpp b/redGrapes/redGrapes.cpp index 9d1e5025..c6e34788 100644 --- a/redGrapes/redGrapes.cpp +++ b/redGrapes/redGrapes.cpp @@ -41,13 +41,13 @@ namespace redGrapes { } - std::shared_ptr Context::current_task_space() const + memory::Refcounted::Guard Context::current_task_space() const { if(current_task) { if(!current_task->children) { - auto task_space = std::make_shared(current_task); + memory::Refcounted::Guard task_space(new TaskSpace(current_task)); SPDLOG_TRACE("create child space = {}", (void*) task_space.get()); current_task->children = task_space; @@ -120,7 +120,7 @@ namespace redGrapes worker_pool = std::make_shared(hwloc_ctx, n_workers); worker_pool->emplace_workers(n_workers); - root_space = std::make_shared(); + root_space.acquire(new TaskSpace()); this->scheduler = scheduler; worker_pool->start(); @@ -148,7 +148,6 @@ namespace redGrapes worker_pool->stop(); scheduler.reset(); - root_space.reset(); finalize_tracing(); } diff --git a/redGrapes/redGrapes.hpp b/redGrapes/redGrapes.hpp index 0fc5a5cf..4eddc60c 100644 --- a/redGrapes/redGrapes.hpp +++ b/redGrapes/redGrapes.hpp @@ -19,6 +19,7 @@ // #include #include #include +#include #include #include @@ -58,7 +59,7 @@ namespace redGrapes std::optional create_event(); unsigned scope_depth() const; - std::shared_ptr current_task_space() const; + memory::Refcounted::Guard current_task_space() const; void execute_task(Task& task); @@ -77,6 +78,9 @@ namespace redGrapes template auto emplace_task(Callable&& f, Args&&... args); + template + auto emplace_continuable_task(Callable&& f, Args&&... args); + static thread_local Task* current_task; static thread_local std::function idle; static thread_local unsigned next_worker; @@ -89,7 +93,7 @@ namespace redGrapes HwlocContext hwloc_ctx; std::shared_ptr worker_pool; - std::shared_ptr root_space; + memory::Refcounted::Guard root_space; std::shared_ptr scheduler; #if REDGRAPES_ENABLE_TRACE @@ -156,7 +160,7 @@ namespace redGrapes return SingletonContext::get().scope_depth(); } - inline std::shared_ptr current_task_space() + inline memory::Refcounted::Guard current_task_space() { return SingletonContext::get().current_task_space(); } @@ -167,6 +171,12 @@ namespace redGrapes return std::move(SingletonContext::get().emplace_task(std::move(f), std::forward(args)...)); } + template + inline auto emplace_continuable_task(Callable&& f, Args&&... args) + { + return std::move(SingletonContext::get().emplace_continuable_task(std::move(f), std::forward(args)...)); + } + } // namespace redGrapes // `TaskBuilder` needs "Context`, so can only include here after definiton @@ -177,18 +187,20 @@ namespace redGrapes template auto Context::emplace_task(Callable&& f, Args&&... args) { - dispatch::thread::WorkerId worker_id = - // linear - next_worker % worker_pool->size(); + dispatch::thread::WorkerId worker_id = next_worker++ % worker_pool->size(); + current_arena = worker_id; + SPDLOG_TRACE("emplace task to worker {} next_worker={}", worker_id, next_worker); - // interleaved - // 2*next_worker % worker_pool->size() + ((2*next_worker) / worker_pool->size())%2; + return std::move(TaskBuilder(false, std::move(f), std::forward(args)...)); + } - next_worker++; + template + auto Context::emplace_continuable_task(Callable&& f, Args&&... args) + { + dispatch::thread::WorkerId worker_id = next_worker++ % worker_pool->size(); current_arena = worker_id; - SPDLOG_TRACE("emplace task to worker {} next_worker={}", worker_id, next_worker); - return std::move(TaskBuilder(std::move(f), std::forward(args)...)); + return std::move(TaskBuilder(true, std::move(f), std::forward(args)...)); } } // namespace redGrapes diff --git a/redGrapes/resource/resource.hpp b/redGrapes/resource/resource.hpp index 52b23265..289b879c 100644 --- a/redGrapes/resource/resource.hpp +++ b/redGrapes/resource/resource.hpp @@ -47,11 +47,11 @@ namespace redGrapes static unsigned int generateID(); public: - unsigned int id; - unsigned int scope_level; - - SpinLock users_mutex; ChunkedList users; + SpinLock users_mutex; + + uint16_t id; + uint8_t scope_level; /** * Create a new resource with an unused ID. diff --git a/redGrapes/resource/resource_user.hpp b/redGrapes/resource/resource_user.hpp index 3edf6082..67b1ce6f 100644 --- a/redGrapes/resource/resource_user.hpp +++ b/redGrapes/resource/resource_user.hpp @@ -51,10 +51,10 @@ namespace redGrapes static bool is_superset(ResourceUser const& a, ResourceUser const& b); static bool is_serial(ResourceUser const& a, ResourceUser const& b); - uint8_t scope_level; - ChunkedList access_list; ChunkedList unique_resources; + + uint8_t scope_level; }; // class ResourceUser } // namespace redGrapes diff --git a/redGrapes/scheduler/event.hpp b/redGrapes/scheduler/event.hpp index 8ff2c5c3..05125963 100644 --- a/redGrapes/scheduler/event.hpp +++ b/redGrapes/scheduler/event.hpp @@ -22,7 +22,6 @@ # define REDGRAPES_EVENT_FOLLOWER_LIST_CHUNKSIZE 16 #endif - namespace redGrapes { @@ -45,9 +44,14 @@ namespace redGrapes struct EventPtr { - enum EventPtrTag tag; - Task* task; std::shared_ptr external_event; + Task* task = nullptr; + enum EventPtrTag tag = T_UNINITIALIZED; + + inline operator bool() const + { + return tag != T_UNINITIALIZED && (task || tag == T_EVT_EXT); + } inline bool operator==(EventPtr const& other) const { @@ -88,17 +92,17 @@ namespace redGrapes */ struct Event { + //! the set of subsequent events + ChunkedList followers; + /*! number of incoming edges * state == 0: event is reached and can be removed */ - std::atomic_uint16_t state; + std::atomic state; //! waker that is waiting for this event WakerId waker_id; - //! the set of subsequent events - ChunkedList followers; - Event(); Event(Event&); Event(Event&&); diff --git a/redGrapes/task/property/graph.cpp b/redGrapes/task/property/graph.cpp index 3680c795..a59b0bd9 100644 --- a/redGrapes/task/property/graph.cpp +++ b/redGrapes/task/property/graph.cpp @@ -25,7 +25,7 @@ namespace redGrapes { auto event = memory::alloc_shared(); event->add_follower(get_post_event()); - return scheduler::EventPtr{scheduler::T_EVT_EXT, nullptr, event}; + return scheduler::EventPtr{event, nullptr, scheduler::T_EVT_EXT}; } /*! diff --git a/redGrapes/task/property/graph.hpp b/redGrapes/task/property/graph.hpp index d7d7cadc..c0fe4614 100644 --- a/redGrapes/task/property/graph.hpp +++ b/redGrapes/task/property/graph.hpp @@ -7,6 +7,10 @@ #pragma once +#include +#include +#include + #include #include @@ -16,16 +20,8 @@ #include #include -// #include -#include -#include - namespace redGrapes { - - struct Task; - struct TaskSpace; - /*! * Each task associates with two events: * A Pre-Event and a Post-Event. @@ -57,47 +53,48 @@ namespace redGrapes return task; } - Task* task; - - //! number of parents - uint8_t scope_depth; - //! task space that contains this task, must not be null - std::shared_ptr space; + memory::Refcounted::Guard space; //! task space for children, may be null - std::shared_ptr children; + memory::Refcounted::Guard children; + /* - // in edges dont need a mutex because they are initialized - // once by `init_dependencies()` and only read afterwards. - // expired pointers (null) must be ignored - std::vector in_edges; - */ + // in edges dont need a mutex because they are initialized + // once by `init_dependencies()` and only read afterwards. + // expired pointers (null) must be ignored + std::vector in_edges; + */ scheduler::Event pre_event; scheduler::Event post_event; scheduler::Event result_set_event; scheduler::Event result_get_event; + Task* task; + + //! number of parents + uint8_t scope_depth; + inline scheduler::EventPtr get_pre_event() { - return scheduler::EventPtr{scheduler::T_EVT_PRE, this->task}; + return scheduler::EventPtr{nullptr, this->task, scheduler::T_EVT_PRE}; } inline scheduler::EventPtr get_post_event() { - return scheduler::EventPtr{scheduler::T_EVT_POST, this->task}; + return scheduler::EventPtr{nullptr, this->task, scheduler::T_EVT_POST}; } inline scheduler::EventPtr get_result_set_event() { - return scheduler::EventPtr{scheduler::T_EVT_RES_SET, this->task}; + return scheduler::EventPtr{nullptr, this->task, scheduler::T_EVT_RES_SET}; } inline scheduler::EventPtr get_result_get_event() { - return scheduler::EventPtr{scheduler::T_EVT_RES_GET, this->task}; + return scheduler::EventPtr{nullptr, this->task, scheduler::T_EVT_RES_GET}; } inline bool is_ready() diff --git a/redGrapes/task/task.hpp b/redGrapes/task/task.hpp index a6038877..8b74f6d2 100644 --- a/redGrapes/task/task.hpp +++ b/redGrapes/task/task.hpp @@ -12,7 +12,6 @@ #include #include #include -#include #include @@ -23,29 +22,39 @@ namespace redGrapes { using TaskProperties = TaskProperties1< - IDProperty, - ResourceProperty, - QueueProperty, - GraphProperty + GraphProperty, + ResourceProperty +// , QueueProperty #ifdef REDGRAPES_TASK_PROPERTIES , REDGRAPES_TASK_PROPERTIES #endif - >; + , + IDProperty>; - struct Task - : TaskBase - , TaskProperties + struct Task : TaskProperties { + uint16_t arena_id; + std::atomic removal_countdown; + + Task() : removal_countdown(2) + { + } + virtual ~Task() { } - unsigned arena_id; - std::atomic_int removal_countdown; + inline scheduler::EventPtr operator()() + { + return this->run(); + } + + virtual scheduler::EventPtr run() = 0; - Task() : removal_countdown(2) + virtual void yield(scheduler::EventPtr event) { + spdlog::error("Task {} does not support yield()", this->task_id); } virtual void* get_result_data() @@ -54,9 +63,6 @@ namespace redGrapes } }; - // TODO: fuse ResultTask and FunTask into one template - // ---> removes one layer of virtual function calls - template struct ResultTask : Task { @@ -73,10 +79,11 @@ namespace redGrapes virtual Result run_result() = 0; - void run() final + virtual scheduler::EventPtr run() { result_data = run_result(); get_result_set_event().notify(); // result event now ready + return scheduler::EventPtr{}; } }; @@ -91,10 +98,11 @@ namespace redGrapes { } - void run() final + virtual scheduler::EventPtr run() { run_result(); get_result_set_event().notify(); + return scheduler::EventPtr{}; } }; @@ -114,3 +122,46 @@ namespace redGrapes }; } // namespace redGrapes + +#include + +#include + +namespace redGrapes +{ + + template + struct ContinuableTask : FunTask + { + boost::context::continuation yield_cont; + boost::context::continuation resume_cont; + scheduler::EventPtr event; + + scheduler::EventPtr run() + { + if(!resume_cont) + { + resume_cont = boost::context::callcc( + [this](boost::context::continuation&& c) + { + this->yield_cont = std::move(c); + this->FunTask::run(); + this->event = scheduler::EventPtr{}; + + return std::move(this->yield_cont); + }); + } + else + resume_cont = resume_cont.resume(); + + return event; + } + + void yield(scheduler::EventPtr e) + { + this->event = e; + yield_cont = yield_cont.resume(); + } + }; + +} // namespace redGrapes diff --git a/redGrapes/task/task_base.hpp b/redGrapes/task/task_base.hpp deleted file mode 100644 index e9f40e5f..00000000 --- a/redGrapes/task/task_base.hpp +++ /dev/null @@ -1,104 +0,0 @@ -/* Copyright 2019-2020 Michael Sippel - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - */ - -#pragma once - -#include - -#include - -#include -#include -#include -#include - -namespace redGrapes -{ - - struct TaskBase - { - bool finished; - bool enable_stack_switching; - - virtual ~TaskBase() - { - } - - TaskBase() : finished(false), enable_stack_switching(false) - { - } - - virtual void run() = 0; - - std::optional operator()() - { - if(enable_stack_switching) - { - if(!resume_cont) - resume_cont = boost::context::callcc( - [this](boost::context::continuation&& c) - { - { - std::lock_guard lock(yield_cont_mutex); - this->yield_cont = std::move(c); - } - - this->run(); - this->event = std::nullopt; - - std::optional yield_cont; - - { - std::lock_guard lock(yield_cont_mutex); - this->yield_cont.swap(yield_cont); - } - - return std::move(*yield_cont); - }); - else - resume_cont = resume_cont->resume(); - } - else - { - this->run(); - } - - return event; - } - - void yield(scheduler::EventPtr event) - { - this->event = event; - - if(enable_stack_switching) - { - std::optional old_yield; - this->yield_cont.swap(old_yield); - - boost::context::continuation new_yield = old_yield->resume(); - - std::lock_guard lock(yield_cont_mutex); - if(!yield_cont) - yield_cont = std::move(new_yield); - // else: yield_cont already been set by another thread running this task - } - else - { - spdlog::error("called yield in task without stack switching!"); - } - } - - std::optional event; - - private: - std::mutex yield_cont_mutex; - - std::optional yield_cont; - std::optional resume_cont; - }; - -} // namespace redGrapes diff --git a/redGrapes/task/task_builder.hpp b/redGrapes/task/task_builder.hpp index 6f6af0c4..6e043b86 100644 --- a/redGrapes/task/task_builder.hpp +++ b/redGrapes/task/task_builder.hpp @@ -60,23 +60,29 @@ namespace redGrapes using Impl = typename std::result_of::type; using Result = typename std::result_of::type; - std::shared_ptr space; + memory::Refcounted::Guard space; FunTask* task; - TaskBuilder(Callable&& f, Args&&... args) + TaskBuilder(bool continuable, Callable&& f, Args&&... args) : TaskProperties::Builder(*this) - , space(current_task_space()) + , space(SingletonContext::get().current_task_space()) { // allocate redGrapes::memory::Allocator alloc; - memory::Block blk = alloc.allocate(sizeof(FunTask)); - task = (FunTask*) blk.ptr; + memory::Block blk; - if(!task) - throw std::runtime_error("out of memory"); - - // construct task in-place - new(task) FunTask(); + if(continuable) + { + blk = alloc.allocate(sizeof(ContinuableTask)); + task = (ContinuableTask*) blk.ptr; + new(task) ContinuableTask(); + } + else + { + blk = alloc.allocate(sizeof(FunTask)); + task = (FunTask*) blk.ptr; + new(task) FunTask(); + } task->arena_id = SingletonContext::get().current_arena; @@ -114,12 +120,6 @@ namespace redGrapes submit(); } - TaskBuilder& enable_stack_switching() - { - task->enable_stack_switching = true; - return *this; - } - auto submit() { Task* t = task; diff --git a/redGrapes/task/task_space.cpp b/redGrapes/task/task_space.cpp index c889d539..2a8a3a06 100644 --- a/redGrapes/task/task_space.cpp +++ b/redGrapes/task/task_space.cpp @@ -16,6 +16,11 @@ namespace redGrapes { + void TaskSpaceDeleter::operator()(TaskSpace* s) + { + delete s; + } + TaskSpace::~TaskSpace() { } @@ -70,7 +75,7 @@ namespace redGrapes void TaskSpace::submit(Task* task) { TRACE_EVENT("TaskSpace", "submit()"); - task->space = shared_from_this(); + task->space.acquire(this); task->task = task; ++task_count; diff --git a/redGrapes/task/task_space.hpp b/redGrapes/task/task_space.hpp index 71e7c987..88f1ac2a 100644 --- a/redGrapes/task/task_space.hpp +++ b/redGrapes/task/task_space.hpp @@ -10,26 +10,34 @@ #include #include #include -#include +// #include +#include #include -#include +#include #include namespace redGrapes { + struct Task; + + struct TaskSpace; + + struct TaskSpaceDeleter + { + void operator()(TaskSpace* space); + }; /*! TaskSpace handles sub-taskspaces of child tasks */ - struct TaskSpace : std::enable_shared_from_this + struct TaskSpace : memory::Refcounted { - std::atomic task_count; + std::vector::Guard> active_child_spaces; + std::shared_mutex active_child_spaces_mutex; - unsigned depth; + std::atomic task_count; Task* parent; - - std::shared_mutex active_child_spaces_mutex; - std::vector> active_child_spaces; + unsigned depth; virtual ~TaskSpace(); diff --git a/redGrapes/util/atomic_list.hpp b/redGrapes/util/atomic_list.hpp index f67cc008..7249bbc4 100644 --- a/redGrapes/util/atomic_list.hpp +++ b/redGrapes/util/atomic_list.hpp @@ -7,7 +7,9 @@ #pragma once +#include #include +#include #include #include @@ -25,6 +27,7 @@ namespace redGrapes namespace memory { + /* maintains a lockfree singly-linked list * with the following allowed operations: * - append new chunks at head @@ -41,14 +44,26 @@ namespace redGrapes template struct AtomicList { - // private: - struct ItemControlBlock + struct ItemControlBlock; + + struct ItemControlBlockDeleter + { + void operator()(ItemControlBlock*); + }; + + struct ItemControlBlock : Refcounted { - bool volatile deleted; - std::shared_ptr prev; + using Guard = typename Refcounted::Guard; + + Guard prev; uintptr_t item_data_ptr; + Allocator alloc; - ItemControlBlock(memory::Block blk) : deleted(false), item_data_ptr(blk.ptr) + template + ItemControlBlock(Allocator alloc, memory::Block blk) + : prev(nullptr) + , item_data_ptr(blk.ptr) + , alloc(alloc) { /* put Item at front and initialize it * with the remaining memory region @@ -65,9 +80,20 @@ namespace redGrapes /* flag this chunk as deleted and call ChunkData destructor */ - void erase() + inline void erase() + { + // set MSB (most significant bit) of item_data ptr + item_data_ptr |= ~(~(uintptr_t) 0 >> 1); + } + + inline bool is_deleted() const { - deleted = true; + return item_data_ptr & ~(~(uintptr_t) 0 >> 1); + } + + inline Item* get() const + { + return (Item*) (item_data_ptr & (~(uintptr_t) 0 >> 1)); } /* adjusts `prev` so that it points to a non-deleted chunk again @@ -76,55 +102,17 @@ namespace redGrapes */ void skip_deleted_prev() { - std::shared_ptr p = std::atomic_load(&prev); - while(p && p->deleted) - p = std::atomic_load(&p->prev); + Guard p = prev; + while(p && p->is_deleted()) + p = p->prev; - std::atomic_store(&prev, p); - } - - Item* get() const - { - return (Item*) item_data_ptr; + prev = p; } }; - Allocator alloc; - std::shared_ptr head; + typename ItemControlBlock::Guard head; size_t const chunk_capacity; - - /* keeps a single, predefined pointer - * and frees it on deallocate. - * used to spoof the allocated size to be bigger than requested. - */ - template - struct StaticAlloc - { - typedef T value_type; - - Allocator alloc; - T* ptr; - - StaticAlloc(Allocator alloc, size_t n_bytes) : alloc(alloc), ptr((T*) alloc.allocate(n_bytes)) - { - } - - template - constexpr StaticAlloc(StaticAlloc const& other) noexcept : alloc(other.alloc) - , ptr((T*) other.ptr) - { - } - - T* allocate(size_t n) noexcept - { - return ptr; - } - - void deallocate(T* p, std::size_t n) noexcept - { - alloc.deallocate(Block{.ptr = (uintptr_t) p, .len = sizeof(T) * n}); - } - }; + Allocator alloc; public: AtomicList(Allocator&& alloc, size_t chunk_capacity) @@ -136,10 +124,7 @@ namespace redGrapes static constexpr size_t get_controlblock_size() { - /* TODO: use sizeof( ...shared_ptr_inplace_something... ) - */ - size_t const shared_ptr_size = 512; - return sizeof(ItemControlBlock) + shared_ptr_size; + return sizeof(ItemControlBlock) + sizeof(Item); } constexpr size_t get_chunk_capacity() @@ -160,23 +145,14 @@ namespace redGrapes { TRACE_EVENT("Allocator", "AtomicList::allocate_item()"); - /* NOTE: we are relying on std::allocate_shared - * to do one *single* allocation which contains: - * - shared_ptr control block - * - chunk control block - * - chunk data - * whereby chunk data is not included by sizeof(ItemControlBlock), - * but reserved by StaticAlloc. - * This works because shared_ptr control block lies at lower address. - */ - StaticAlloc chunk_alloc(this->alloc, get_chunk_allocsize()); + memory::Block blk = this->alloc.allocate(get_chunk_allocsize()); + ItemControlBlock* item_ctl = (ItemControlBlock*) blk.ptr; - // this block will contain the Item-data of ItemControlBlock - memory::Block blk{ - .ptr = (uintptr_t) chunk_alloc.ptr + get_controlblock_size(), - .len = chunk_capacity - get_controlblock_size()}; + blk.ptr += sizeof(ItemControlBlock); + blk.len -= sizeof(ItemControlBlock); - return append_item(std::allocate_shared(chunk_alloc, blk)); + new(item_ctl) ItemControlBlock(alloc, blk); + return append_item(std::move(typename ItemControlBlock::Guard(item_ctl))); } /** allocate the first item if the list is empty @@ -185,16 +161,16 @@ namespace redGrapes */ bool try_allocate_first_item() { - TRACE_EVENT("Allocator", "AtomicList::allocate_first_item()"); - StaticAlloc chunk_alloc(this->alloc, get_chunk_allocsize()); + TRACE_EVENT("Allocator", "AtomicList::try_allocate_first_item()"); + + memory::Block blk = this->alloc.allocate(get_chunk_allocsize()); + ItemControlBlock* item_ctl = (ItemControlBlock*) blk.ptr; - // this block will contain the Item-data of ItemControlBlock - memory::Block blk{ - .ptr = (uintptr_t) chunk_alloc.ptr + get_controlblock_size(), - .len = chunk_capacity - get_controlblock_size()}; + blk.ptr += sizeof(ItemControlBlock); + blk.len -= sizeof(ItemControlBlock); - auto sharedChunk = std::allocate_shared(chunk_alloc, blk); - return try_append_first_item(std::move(sharedChunk)); + new(item_ctl) ItemControlBlock(alloc, blk); + return try_append_first_item(std::move(typename ItemControlBlock::Guard(item_ctl))); } /** @} */ @@ -202,7 +178,7 @@ namespace redGrapes template struct BackwardIterator { - std::shared_ptr c; + typename ItemControlBlock::Guard c; void erase() { @@ -255,22 +231,22 @@ namespace redGrapes */ MutBackwardIterator rbegin() const { - return MutBackwardIterator{std::atomic_load(&head)}; + return MutBackwardIterator{typename ItemControlBlock::Guard(head)}; } MutBackwardIterator rend() const { - return MutBackwardIterator{std::shared_ptr()}; + return MutBackwardIterator{typename ItemControlBlock::Guard()}; } ConstBackwardIterator crbegin() const { - return ConstBackwardIterator{std::atomic_load(&head)}; + return ConstBackwardIterator{typename ItemControlBlock::Guard(head)}; } ConstBackwardIterator crend() const { - return ConstBackwardIterator{std::shared_ptr()}; + return ConstBackwardIterator{typename ItemControlBlock::Guard()}; } /* Flags chunk at `pos` as erased. Actual removal is delayed until @@ -288,34 +264,41 @@ namespace redGrapes * and returns the previous head to which the new_head * is now linked. */ - auto append_item(std::shared_ptr new_head) + auto append_item(typename ItemControlBlock::Guard new_head) { TRACE_EVENT("Allocator", "AtomicList::append_item()"); - std::shared_ptr old_head; + typename ItemControlBlock::Guard old_head; bool append_successful = false; while(!append_successful) { - old_head = std::atomic_load(&head); - std::atomic_store(&new_head->prev, old_head); - append_successful - = std::atomic_compare_exchange_strong(&head, &old_head, new_head); + typename ItemControlBlock::Guard old_head(head); + new_head->prev = old_head; + append_successful = head.compare_exchange_strong(old_head.get(), new_head); } return MutBackwardIterator{old_head}; } // append the first head item if not already exists - bool try_append_first_item(std::shared_ptr new_head) + bool try_append_first_item(typename ItemControlBlock::Guard new_head) { TRACE_EVENT("Allocator", "AtomicList::append_first_item()"); - std::shared_ptr expected(nullptr); - std::shared_ptr const& desired = new_head; - return std::atomic_compare_exchange_strong(&head, &expected, desired); + return head.compare_exchange_strong(nullptr, new_head); } }; + template + void AtomicList::ItemControlBlockDeleter::operator()( + AtomicList::ItemControlBlock* e) + { + Allocator alloc = e->alloc; + e->~ItemControlBlock(); + memory::Block blk{(uintptr_t) e, 0}; + alloc.deallocate(blk); + } + } // namespace memory } // namespace redGrapes diff --git a/redGrapes/util/chunked_list.hpp b/redGrapes/util/chunked_list.hpp index 84629df5..6af44f6d 100644 --- a/redGrapes/util/chunked_list.hpp +++ b/redGrapes/util/chunked_list.hpp @@ -19,7 +19,6 @@ #include #include -#include #include #include #include @@ -495,8 +494,9 @@ namespace redGrapes memory::AtomicList chunks; public: - ChunkedList(Allocator&& alloc) : chunks(std::move(alloc), T_chunk_size * sizeof(Item) + sizeof(Chunk)) + ChunkedList(Allocator&& alloc = memory::Allocator()) : chunks(std::move(alloc), T_chunk_size * sizeof(Item)) { + chunks.allocate_item(); } ChunkedList(ChunkedList&& other) = default;