From 47ab0de161191b2369075561b2f32cc286c5936d Mon Sep 17 00:00:00 2001 From: Curle Date: Wed, 3 Apr 2024 23:42:43 +0100 Subject: [PATCH 01/12] Fix setup --- projs/shadow/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projs/shadow/CMakeLists.txt b/projs/shadow/CMakeLists.txt index 32c43d0a..2b3a89dd 100644 --- a/projs/shadow/CMakeLists.txt +++ b/projs/shadow/CMakeLists.txt @@ -1,5 +1,7 @@ project(shadow) +include(FetchContent) + # External code add_subdirectory(extern/dxmath) add_subdirectory(extern/glm) From d0d77a5dabac9273074632b6f58077e0a39cda1b Mon Sep 17 00:00:00 2001 From: Curle Date: Fri, 5 Apr 2024 00:36:20 +0100 Subject: [PATCH 02/12] Add Platform-specific Threads, and the Profiler interface. --- .../core/inc/shadow/core/Thread.h | 38 ++++++ .../core/inc/shadow/profile/Profiler.h | 117 ++++++++++++++++++ .../platforms/linux/src/Thread.cpp | 88 +++++++++++++ .../platforms/windows/src/Thread.cpp | 109 ++++++++++++++++ 4 files changed, 352 insertions(+) create mode 100644 projs/shadow/shadow-engine/core/inc/shadow/core/Thread.h create mode 100644 projs/shadow/shadow-engine/core/inc/shadow/profile/Profiler.h create mode 100644 projs/shadow/shadow-engine/platforms/linux/src/Thread.cpp create mode 100644 projs/shadow/shadow-engine/platforms/windows/src/Thread.cpp diff --git a/projs/shadow/shadow-engine/core/inc/shadow/core/Thread.h b/projs/shadow/shadow-engine/core/inc/shadow/core/Thread.h new file mode 100644 index 00000000..537f1107 --- /dev/null +++ b/projs/shadow/shadow-engine/core/inc/shadow/core/Thread.h @@ -0,0 +1,38 @@ +#pragma once +#include +#include +#include "shadow/util/Syncronization.h" + +namespace SH { + + // A thread of execution. Start & manage threads on the fly. + struct API Thread { + + explicit Thread(); + virtual ~Thread(); + + // The actual work to do. + virtual int Run() = 0; + + // Start the thread and start it running. + bool Start(const char* name, bool extended); + // Wait for the thread to finish, blocking main thread until it's done + bool Join(); + + // Update the core affinity (makes the process favor a certain core that may be available more often) + void SetAffinity(size_t mask); + + // Wait. The given mutex will lock on the current thread, and will not be avaiable until this thread restarts. + void Wait(struct ShadowEngine::Mutex& mut); + // Unlock current thread and resumes. Partner to wait. + void Notify(); + + // Whether we're currently taking up CPU time. + bool IsRunning() const; + // Whether our task is finished. + bool IsFinished() const; + + private: + struct ThreadImpl* implementation; + }; +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/core/inc/shadow/profile/Profiler.h b/projs/shadow/shadow-engine/core/inc/shadow/profile/Profiler.h new file mode 100644 index 00000000..f460aade --- /dev/null +++ b/projs/shadow/shadow-engine/core/inc/shadow/profile/Profiler.h @@ -0,0 +1,117 @@ +#pragma once + +#include +#include "DirectXMath.h" + +namespace SH { + + /** + * @brief A profiler object. + * Contains a lot of static utilities if you want more control, otherwise just create an instance (or use the ProfileFunction() macro), and the profiler will track what happens until the object goes out of scope. + * Provides utilities for renaming, recoloring, filtering in the UI. + * Allows marking out and timing GPU operations. + * + * Standard usage: + * + * int Function() { + * SH::Profiler::ProfileFunction() + * doEngineThings(); + * loadData(); + * } + * + */ + struct Profiler { + static API void Pause(bool paused); + static API void SetThreadName(const char *name); + static API void ShowInProfiler(bool show); + + static API void Begin(const char *name); + static API void BlockColor(DirectX::XMFLOAT3 c); + static API void End(); + static API void Frame(); + static API void PushJob(int32_t signal); + static API void PushString(const char *value); + static API void PushInt(const char *key, int value); + + static API uint32_t MakeCounter(const char *key, float min); + static API void PushCounter(uint32_t ID, float value); + + // Profiler: a handle to link the GPU process to a profiler thread. Obtain by calling CreateLink() + static API void BeginGPU(const char *name, size_t time, size_t profiler); + // End the GPU block that started at the given timestamp. + static API void EndGPU(size_t time); + // Get GPU stats directly. + static API void GPUStats(size_t primitive); + // Link the current profiler thread with the given GPU task. + static API void Link(size_t profiler); + // Create a handle that can be used to profile GPU operations. + static API size_t CreateLink(); + + // Start the profiler directly. + explicit Profiler(const char* name) { + Begin(name); + } + + // End the profiler when the object goes out of scope. + ~Profiler() { + End(); + } + + struct Block { + const char* name; + size_t ID; + }; + + struct CounterData { + const char* name; + float min; + }; + + struct Counter { + size_t ID; + float val; + }; + + struct Int { + const char* key; + int val; + }; + + struct GPU { + const char* name; + size_t timestamp; + size_t profiler; + }; + + enum class EventType { + Begin, + Color, + End, + Frame, + String, + Int, + GPUBegin, + GPUEnd, + Link, + Pause, + GPuStats, + Continue, + Counter + }; + + #pragma pack(1) + struct Event { + char size; + EventType type; + size_t time; + }; + #pragma pack() + + #define concat2(a, b) a ## b + #define concat(a, b) concat2(a,b) + + #define ProfileFunction() SH::Profiler scope(__FUNCTION__); + #define ProfileBlock(name) SH::Profiler concat(scope, __LINE__)(name); + }; + +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/platforms/linux/src/Thread.cpp b/projs/shadow/shadow-engine/platforms/linux/src/Thread.cpp new file mode 100644 index 00000000..b88372be --- /dev/null +++ b/projs/shadow/shadow-engine/platforms/linux/src/Thread.cpp @@ -0,0 +1,88 @@ +#include + +#include +#include +#include "shadow/profile/Profiler.h" +#include + +namespace SH { + + // Platform-specific implementation of a Thread. Provides most of the interface of a Thread, except Run. + struct ThreadImpl { + bool forceExit; + bool finished; + bool running; + pthread_t handle; + const char* name; + Thread* owner; + ShadowEngine::ConditionVariable cv; + }; + + static void* threadFunction(void* ptr) { + static ThreadImpl* impl = reinterpret_cast(ptr); + pthread_setname_np(pthread_self(), impl->name); + SH::Profiler::SetThreadName(impl->name); + uint32_t ret = 0xFFFFFFFF; + if (!impl->forceExit) ret = impl->owner->Task(); + impl->finished = true; + impl->running = false; + + return nullptr; + } + + Thread::Thread() { + auto impl = new ThreadImpl(); + impl->running = false; + impl->forceExit = false; + impl->finished = false; + impl->name = ""; + impl->owner = this; + + implementation = impl; + } + + Thread::~Thread() { + delete implementation; + } + + void Thread::Wait(struct ShadowEngine::Mutex &mut) { + assert(pthread_self() == implementation->handle); + implementation->cv.sleep(mut); + } + + void Thread::Notify() { + implementation->cv.wakeup(); + } + + bool Thread::Start(const char *name, bool extended) { + pthread_attr_t attr; + int res = pthread_attr_init(&attr); + assert(res == 0); + if (res != 0) return false; + res = pthread_create(&implementation->handle, &attr, threadFunction, implementation); + assert(res == 0); + if (res != 0) return false; + return 0; + } + + bool Thread::Join() { + return pthread_join(implementation->handle, nullptr) == 0; + } + + void Thread::SetAffinity(size_t mask) { + cpu_set_t set; + CPU_ZERO(&set); + for (int i = 0; i < 64; i++) + if (mask & ((size_t)1 << i)) + CPU_SET(i, &set); + pthread_setaffinity_np(implementation->handle, sizeof(set), &set); + } + + bool Thread::IsRunning() const { + return implementation->running; + } + + bool Thread::IsFinished() const { + return implementation->finished; + } +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/platforms/windows/src/Thread.cpp b/projs/shadow/shadow-engine/platforms/windows/src/Thread.cpp new file mode 100644 index 00000000..9eeaa763 --- /dev/null +++ b/projs/shadow/shadow-engine/platforms/windows/src/Thread.cpp @@ -0,0 +1,109 @@ +#include +#ifdef _WIN32 +#define INITGUID +#define NOGDI +#define WIN32_LEAN_AND_MEAN +#include +#include + +#endif +#include +#include "shadow/profile/Profiler.h" + +namespace SH { + static constexpr uint32_t STACK_SIZE = 0x8000; + + // Platform-specific implementation of a Thread. Provides most of the interface of a Thread, except Run. + struct ThreadImpl { + explicit ThreadImpl() = default; + + HANDLE handle; + DWORD threadID; + size_t affinity; + uint32_t priority; + volatile bool running; + volatile bool finished; + const char* name; + ShadowEngine::ConditionVariable cv; + Thread* owner; + }; + + // A wrapper function that calls Run after some initial setup. + // This is what Windows really calls when the thread starts. + static DWORD WINAPI threadFunction(LPVOID ptr) { + struct ThreadImpl* impl = reinterpret_cast(ptr); + SH::Profiler::SetThreadName(impl->name); + const uint32_t ret = impl->owner->Run(); + impl->finished = true; + impl->running = false; + return ret; + } + + Thread::Thread() { + ThreadImpl* impl = new ThreadImpl(); + impl->handle = nullptr; + impl->priority = ::GetThreadPriority(GetCurrentThread()); + impl->running = false; + impl->finished = false; + impl->name = ""; + impl->owner = this; + + implementation = impl; + } + + Thread::~Thread() { + delete implementation; + } + + bool Thread::Start(const char *name, bool extended) { + HANDLE handle = CreateThread(nullptr, STACK_SIZE, threadFunction, implementation, CREATE_SUSPENDED, &implementation->threadID); + if (handle) { + implementation->finished = false; + implementation->name = name; + implementation->handle = handle; + implementation->running = true; + + bool success = ::ResumeThread(implementation->handle) != 1; + if (success) + return true; + CloseHandle(implementation->handle); + implementation->handle = nullptr; + return false; + } + return false; + } + + bool Thread::Join() { + // Join thread, never kill it while running + while (implementation->running) ::Sleep(1); + + ::CloseHandle(implementation->handle); + implementation->handle = nullptr; + return true; + } + + void Thread::SetAffinity(size_t mask) { + implementation->affinity = mask; + if (implementation->handle) + ::SetThreadAffinityMask(implementation->handle, mask); + } + + void Thread::Wait(struct ShadowEngine::Mutex &mut) { + implementation->cv.sleep(mut); + } + + void Thread::Notify() { + implementation->cv.wakeup(); + } + + bool Thread::IsRunning() const { + return implementation->running; + } + + bool Thread::IsFinished() const { + return implementation->finished; + } + + + +} \ No newline at end of file From 936e9b037a92bd0eb0f84021b3176c2ad994a437 Mon Sep 17 00:00:00 2001 From: Curle Date: Fri, 5 Apr 2024 02:27:10 +0100 Subject: [PATCH 03/12] Finish Profiler, update Time with platform-specific precision method --- .../shadow-engine/core/inc/shadow/core/Time.h | 60 +- .../core/inc/shadow/profile/Profiler.h | 72 ++- .../core/src/profile/Profiler.cpp | 556 ++++++++++++++++++ .../platforms/linux/src/Time.cpp | 7 + .../platforms/windows/src/Time.cpp | 12 + 5 files changed, 692 insertions(+), 15 deletions(-) create mode 100644 projs/shadow/shadow-engine/core/src/profile/Profiler.cpp create mode 100644 projs/shadow/shadow-engine/platforms/linux/src/Time.cpp create mode 100644 projs/shadow/shadow-engine/platforms/windows/src/Time.cpp diff --git a/projs/shadow/shadow-engine/core/inc/shadow/core/Time.h b/projs/shadow/shadow-engine/core/inc/shadow/core/Time.h index f5934840..1ca241af 100644 --- a/projs/shadow/shadow-engine/core/inc/shadow/core/Time.h +++ b/projs/shadow/shadow-engine/core/inc/shadow/core/Time.h @@ -1,18 +1,58 @@ #pragma once #include "shadow/exports.h" +#include -class Time { +namespace SH { -public: - static API int NOW; - static API int LAST; + // A RAII-operated timer. + // Instantiate to start the timer. + // Call timer.elapsed() to get the number of milliseconds since the timer was created. + // Call timer.swap() to reset the timer and get the time since instantiation. + class Timer { - static API double deltaTime; - static API double deltaTime_ms; + public: + std::chrono::high_resolution_clock::time_point timestamp = std::chrono::high_resolution_clock::now(); - static API double timeSinceStart; - static API double startTime; + inline void record() { + timestamp = std::chrono::high_resolution_clock::now(); + } - static void UpdateTime(); -}; + inline double secondsSince(std::chrono::high_resolution_clock::time_point ts2) { + std::chrono::duration span = std::chrono::duration_cast>(ts2 - timestamp); + return span.count(); + } + + inline double elapsedSeconds() { + return secondsSince(std::chrono::high_resolution_clock::now()); + } + + inline double elapsedMillis() { + return elapsedSeconds() * 1000.0f; + } + + inline double elapsed() { + return elapsedMillis(); + } + + inline double swap() { + auto ts2 = std::chrono::high_resolution_clock::now(); + auto elapsed = secondsSince(ts2); + timestamp = ts2; + return elapsed; + } + + static size_t getTimestamp(); + + static API int NOW; + static API int LAST; + + static API double deltaTime; + static API double deltaTime_ms; + + static API double timeSinceStart; + static API double startTime; + + static void UpdateTime(); + }; +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/core/inc/shadow/profile/Profiler.h b/projs/shadow/shadow-engine/core/inc/shadow/profile/Profiler.h index f460aade..57abbf2f 100644 --- a/projs/shadow/shadow-engine/core/inc/shadow/profile/Profiler.h +++ b/projs/shadow/shadow-engine/core/inc/shadow/profile/Profiler.h @@ -21,20 +21,32 @@ namespace SH { * */ struct Profiler { + // Stop the thread being profiled. + // Useful for scripting. static API void Pause(bool paused); + // Set a specific name for the current thread in the profiler. static API void SetThreadName(const char *name); + // Set whether the current thread will show in the profiler. static API void ShowInProfiler(bool show); + // Start a new block. Will separate the named results out from all others. static API void Begin(const char *name); + // Set a color for the current block. static API void BlockColor(DirectX::XMFLOAT3 c); + // End the current block. static API void End(); + // End the current frame. static API void Frame(); - static API void PushJob(int32_t signal); + // Tell the profiler that a new job is starting on the current thread. Will allow filtering by job type. + static API void PushJob(size_t signal); + // Send arbitrary data to the profiler log. static API void PushString(const char *value); static API void PushInt(const char *key, int value); - static API uint32_t MakeCounter(const char *key, float min); - static API void PushCounter(uint32_t ID, float value); + // Create a counter; increase at whim. + static API size_t MakeCounter(const char *key, float min); + // Send the counter data to the profiler log. + static API void PushCounter(size_t ID, float value); // Profiler: a handle to link the GPU process to a profiler thread. Obtain by calling CreateLink() static API void BeginGPU(const char *name, size_t time, size_t profiler); @@ -47,6 +59,41 @@ namespace SH { // Create a handle that can be used to profile GPU operations. static API size_t CreateLink(); + // Data for a fiber thread (soft-switching, used for the job task system + struct FiberData { + uint32_t ID; + uint32_t blocks[16]; + uint32_t count; + size_t signal; + }; + + // Preparing for a fiber switch + static API void PreFiberSwitch(); + // Trigger a signal on a fiber thread + static API void Trigger(size_t signal); + // A fiber thread is about to start waiting + static API FiberData BeginFiberWait(size_t signal, bool isMutex); + // A fiber thread is about to come out of waiting + static API void EndFiberWait(const FiberData& data); + + // How long did we spend in the last frame? + static API float GetLastFrameDuration(); + + // Can we context switch while profiling? + // Requires system agreement (ie. Windows must let us attach a profiler thread monitor.) + static API bool IsContextSwitchEnabled(); + + // Get the effective frequency of the process. + static API size_t GetFrequency(); + + // Used within the profiler to track when we get context switched out of profiling. + struct ContextSwitchRecord { + uint32_t oldThreadID; + uint32_t newThreadID; + size_t timestamp; + uint8_t cause; + }; + // Start the profiler directly. explicit Profiler(const char* name) { Begin(name); @@ -77,6 +124,16 @@ namespace SH { int val; }; + struct Job { + size_t signal; + }; + + struct FiberWait { + uint32_t ID; + size_t signal; + bool mutex; + }; + struct GPU { const char* name; size_t timestamp; @@ -90,18 +147,23 @@ namespace SH { Frame, String, Int, + FiberWait, + FiberWake, + ContextSwitch, + Job, GPUBegin, GPUEnd, Link, Pause, - GPuStats, + GPUStats, Continue, + Signal, Counter }; #pragma pack(1) struct Event { - char size; + uint8_t size; EventType type; size_t time; }; diff --git a/projs/shadow/shadow-engine/core/src/profile/Profiler.cpp b/projs/shadow/shadow-engine/core/src/profile/Profiler.cpp new file mode 100644 index 00000000..7dfafaa4 --- /dev/null +++ b/projs/shadow/shadow-engine/core/src/profile/Profiler.cpp @@ -0,0 +1,556 @@ +#include +#include +#include "shadow/assets/fs/iostream.h" +#include "shadow/util/Syncronization.h" +#include "shadow/core/Time.h" +#include +#include + +#ifdef _WIN32 +#define INITGUID +#define NOGDI +#define WIN32_LEAN_AND_MEAN +#include +#include +#include +#include + +#endif + +namespace SH { + + // Allows the Profiler to be used on multiple threads simultaneously, separating the data collected. + // Every thread has its own set of open blocks, its own mutex, and its own visibility toggles. + // Consider the ThreadContext as a horizontal bar on a profiler menu, showing what one thread is doing at any given time. + struct ThreadContext { + + ThreadContext(size_t bufferSize) : oms(), openBlocks() { + oms.resize(bufferSize); + openBlocks.reserve(64); + } + + std::vector openBlocks; + // Used as the data buffer. + ShadowEngine::OutputMemoryStream oms; + size_t begin = 0; + size_t end = 0; + ShadowEngine::Mutex mut; + std::string name; + bool show = false; + size_t threadID; + }; + + // This ifdef contains a bunch of the infrastructure necessary for performance monitoring under Windows. + #ifdef _WIN32 + #define SWITCH_OPCODE 36 + #pragma pack(1) + struct TraceProperties { + EVENT_TRACE_PROPERTIES base; + char name[sizeof(KERNEL_LOGGER_NAME) + 1]; + }; + #pragma pack() + + struct ContextSwitch { + uint32_t NewThreadID; + uint32_t OldThreadID; + int8_t NewThreadPriority; + int8_t OldThreadPriority; + uint8_t PreviousContextSwitchState; + int8_t Padding; + int8_t OldThreadWaitReason; + int8_t OldThreadWaitMode; + int8_t OldThreadState; + int8_t OldThreadIdealProcessor; + uint32_t NewThreadWaitTime; + uint32_t Reserved; + }; + + struct TraceTask : Thread { + TraceTask(); + + int Run() override; + static void callback(PEVENT_RECORD event); + + TRACEHANDLE handle; + }; + #else + + struct TraceTask { + TraceTask() {} + void Destroy() {} + int handle; + }; + void CloseTrace(int) {} + #endif + + // The profiler instance. + // Manages the profiler threads. + static struct Instance { + Instance() : contexts(), task(), counters(), global(10 * 1024 * 1024) { + startTrace(); + } + + ~Instance() { + CloseTrace(task.handle); + task.Join(); + } + + void startTrace() { + #ifdef _WIN32 + static TRACEHANDLE handle; + static TraceProperties properties = { + .base = { + .Wnode = { + .BufferSize = sizeof(properties), + .Guid = SystemTraceControlGuid, + .ClientContext = 1, + .Flags = WNODE_FLAG_TRACED_GUID, + }, + .LogFileMode = EVENT_TRACE_REAL_TIME_MODE, + .EnableFlags = EVENT_TRACE_FLAG_CSWITCH, + .LoggerNameOffset = sizeof(properties.base) + } + }; + strcpy_s(properties.name, KERNEL_LOGGER_NAME); + + TraceProperties tmp = properties; + // Start a trace that never starts, so we can see whether the real one will work. + ControlTrace(NULL, KERNEL_LOGGER_NAME, &tmp.base, EVENT_TRACE_CONTROL_STOP); + ULONG res = StartTrace(&handle, KERNEL_LOGGER_NAME, &properties.base); + switch (res) { + // If we get these errors, then we can't be pre-empted, so disable context switching on our thread. + case ERROR_ALREADY_EXISTS: + case ERROR_ACCESS_DENIED: + case ERROR_BAD_LENGTH: + default: + contextSwitch = false; + break; + case ERROR_SUCCESS: + contextSwitch = true; + break; + } + + static EVENT_TRACE_LOGFILE trace = { + .LoggerName = (decltype(trace.LoggerName)) KERNEL_LOGGER_NAME, + .ProcessTraceMode = PROCESS_TRACE_MODE_RAW_TIMESTAMP | PROCESS_TRACE_MODE_REAL_TIME | PROCESS_TRACE_MODE_EVENT_RECORD, + .EventRecordCallback = TraceTask::callback + }; + + task.handle = OpenTrace(&trace); + task.Start("Profiler Trace", true); + #endif + } + + // Get the thread context for our thread. + // This is thread_local, so the initializer will run once per thread. + // If the initializer has already run on this thread, ctx is cached, so it's just returned. + ThreadContext* getNewThreadContext() { + thread_local ThreadContext* ctx = [&]() { + auto *newCtx = new ThreadContext(5 * 1024 * 1024); + newCtx->threadID = ifsystem(pthread_self(), reinterpret_cast(::GetCurrentThread()), pthread_self()); + ShadowEngine::MutexGuard lock(mut); + contexts.push_back((newCtx)); + return newCtx; + }(); + + return ctx; + }; + + // A list of counters on the current thread. + std::vector counters; + // Child threads of the current. + std::vector contexts; + ShadowEngine::Mutex mut; + // Whether the current thread is paused. + bool paused = false; + // Whether the current thread can be pre-empted. + bool contextSwitch = false; + // The time the thread was paused, if it was. + size_t pausedTime = 0; + size_t lastFrameElapsed = 0; + size_t lastFrameTime = 0; + + // The ID of the fiber waiting on the current. + std::atomic fiberWaiting = 0; + // The actual Thread doing work for the current profiler. + TraceTask task; + ThreadContext global; + } gInstance; + + + /** + * Append data to the ThreadContext. + * The timestamp may be in the past, and the UI will update accordingly. + * The EventType tells what the event actually was. + * val is placed in the ThreadContext's buffer along with the event header. + * Resizing is done on the buffer if necessary. + */ + template + void Write(ThreadContext& ctx, size_t timestamp, Profiler::EventType type, const T& val) { + if (gInstance.paused && timestamp > gInstance.pausedTime) return; + + // A short-lived struct that lets us read the size of T indirectly. + #pragma pack(1) + struct { + Profiler::Event header; + T value; + } v; + #pragma pack() + + v = { + .header = { + .size = sizeof(v), // sizeof(v) - sizeof(Profiler::Event) == sizeof(T), but we don't need sizeof(T) directly + .type = type, + .time = timestamp + }, + .value = val + }; + + // Lock the context so that we can write data to it + ShadowEngine::MutexGuard lock(ctx.mut); + uint8_t* buffer = ctx.oms.dataMut(); + const size_t bufferSize = ctx.oms.size(); + + // Make sure there's enough space in the buffer. + while(sizeof(v) + ctx.end - ctx.begin > bufferSize) { + const uint8_t size = buffer[ctx.begin % bufferSize]; + ctx.begin += size; + } + + const uint32_t end = ctx.end % bufferSize; + // If there's space at the end of the buffer, copy it in. + if (bufferSize - end >= sizeof(v)) { + memcpy(buffer + end, &v, sizeof(v)); + } else { + // If there's no space, expand the buffer and copy it in at the new end. + memcpy(buffer + end, &v, bufferSize - end); + memcpy(buffer, ((uint8_t * ) & v) + bufferSize - end, sizeof(v) - (bufferSize - end)); + } + + ctx.end += sizeof(v); + }; + + /** + * Append data to the ThreadContext. + * Timestamp is set to current time, so this is instantaneous event. + * The EventType tells what the event actually was. + * val is placed in the ThreadContext's buffer along with the event header. + * Resizing is done on the buffer if necessary. + */ + template + void Write(ThreadContext& ctx, Profiler::EventType type, const T& val) { + if (gInstance.paused) return; + + #pragma pack(1) + struct { + Profiler::Event header; + T value; + } v; + #pragma pack() + + v = { + .header = { + .size = sizeof(v), + .type = type, + .time = static_cast(SH::Timer::getTimestamp()) + }, + .value = val, + }; + + ShadowEngine::MutexGuard lock(ctx.mut); + uint8_t *buffer = ctx.oms.dataMut(); + const size_t bufferSize = ctx.oms.size(); + + while (sizeof(v) + ctx.end - ctx.begin > bufferSize) { + const uint8_t size = buffer[ctx.begin % bufferSize]; + ctx.begin += size; + } + + const uint32_t end = ctx.end % bufferSize; + if (bufferSize - end >= sizeof(v)) { + memcpy(buffer + end, &v, sizeof(v)); + } else { + memcpy(buffer + end, &v, bufferSize - end); + memcpy(buffer, ((uint8_t * ) & v) + bufferSize - end, sizeof(v) - (bufferSize - end)); + } + + ctx.end += sizeof(v); + } + + /** + * Append arbitrary data to the ThreadContext. + * The EventType tells what the event actually was. + * size bytes of data is placed in the ThreadContext's buffer along with the event header. + * Resizing is done on the buffer if necessary. + */ + void Write(ThreadContext& ctx, Profiler::EventType type, const uint8_t* data, int size){ + if (gInstance.paused) return; + + Profiler::Event header = { + .size = uint8_t(sizeof(header) + size), + .type = type, + .time = static_cast(SH::Timer::getTimestamp()) + }; + assert(sizeof(header) + size <= 0xffff); + + ShadowEngine::MutexGuard lock(ctx.mut); + uint8_t* buf = ctx.oms.dataMut(); + const size_t buf_size = ctx.oms.size(); + + while (header.size + ctx.end - ctx.begin > buf_size) { + const uint8_t size = buf[ctx.begin % buf_size]; + ctx.begin += size; + } + + auto doCopy = [&](const uint8_t* ptr, size_t size) { + const uint32_t end = ctx.end % buf_size; + if (buf_size - end >= size) { + memcpy(buf + end, ptr, size); + } + else { + memcpy(buf + end, ptr, buf_size - end); + memcpy(buf, ((uint8_t*)ptr) + buf_size - end, size - (buf_size - end)); + } + + ctx.end += size; + }; + + doCopy((uint8_t*)&header, sizeof(header)); + doCopy(data, size); + }; + + #ifdef _WIN32 + + TraceTask::TraceTask() : Thread() {} + + // The profiler's Thread code. + int TraceTask::Run() { + ProcessTrace(&handle, 1, nullptr, nullptr); + return 0; + } + + // Called by Windows when the trace is pre-empted by a context switch. + void TraceTask::callback(PEVENT_RECORD event) { + if (event->EventHeader.EventDescriptor.Opcode != SWITCH_OPCODE) return; + if (sizeof(ContextSwitch) != event->UserDataLength) return; + + const ContextSwitch* cs = reinterpret_cast(event->UserData); + Profiler::ContextSwitchRecord rec = { + .oldThreadID = cs->OldThreadID, + .newThreadID = cs->NewThreadID, + .timestamp = static_cast(event->EventHeader.TimeStamp.QuadPart), + .cause = static_cast(cs->OldThreadWaitReason) + }; + + Write(gInstance.global, rec.timestamp, Profiler::EventType::ContextSwitch, rec); + }; + #endif + + size_t Profiler::MakeCounter(const char* key, float min) { + Profiler::CounterData c {}; + c.name = static_cast(malloc(strlen(key) + 1)); + gInstance.counters.emplace_back(c); + memcpy(&c.name, key, strlen(key)); + c.min = min; + return gInstance.counters.size() - 1; + } + + void Profiler::PushCounter(size_t counter, float val) { + Profiler::Counter r = { + .ID = counter, + .val = val + }; + + Write(gInstance.global, EventType::Counter, (uint8_t*) &r, sizeof(r)); + } + + void Profiler::PushInt(const char* key, int val) { + ThreadContext* ctx = gInstance.getNewThreadContext(); + Int r = { + .key = key, + .val = val + }; + + Write(*ctx, EventType::Int, (uint8_t*) &r, sizeof(r)); + }; + + void Profiler::PushString(const char* value) { + ThreadContext *ctx = gInstance.getNewThreadContext(); + Write(*ctx, EventType::String, (uint8_t *) value, strlen(value) + 1); + } + + static constexpr char ToChar(float x) { + return (char) ((x) * std::numeric_limits::max()); + } + + void Profiler::BlockColor(DirectX::XMFLOAT3 c) { + const uint32_t color = 0xFF000000 + ToChar(c.x) + (ToChar(c.y) << 8) + (ToChar(c.z) << 16); + ThreadContext* ctx = gInstance.getNewThreadContext(); + Write(*ctx, EventType::Color, color); + } + + + static void ContinueBlock(uint32_t ID) { + ThreadContext *ctx = gInstance.getNewThreadContext(); + ctx->openBlocks.push_back(ID); + Write(*ctx, Profiler::EventType::Continue, ID); + } + + static std::atomic lastBlockID = 0; + + void Profiler::Begin(const char *name) { + Block r = { + .name = name, + .ID = lastBlockID++ + }; + ThreadContext* ctx = gInstance.getNewThreadContext(); + ctx->openBlocks.push_back(r.ID); + Write(*ctx, EventType::Begin, r); + } + + void Profiler::BeginGPU(const char *name, size_t time, size_t profiler) { + GPU data = { + .timestamp = time, + .profiler = profiler + }; + memcpy(&data.name, name, strlen(name)); + Write(gInstance.global, EventType::GPUBegin, data); + } + + void Profiler::GPUStats(size_t primitive) { + Write(gInstance.global, EventType::GPUStats, primitive); + } + + void Profiler::EndGPU(size_t time) { + Write(gInstance.global, EventType::GPUEnd, time); + } + + size_t Profiler::CreateLink() { + std::atomic counter = 0; + return counter++; + } + + void Profiler::Link(size_t profiler) { + ThreadContext* ctx = gInstance.getNewThreadContext(); + Write(*ctx, EventType::Link, profiler); + } + + float Profiler::GetLastFrameDuration() { + return float (gInstance.lastFrameElapsed / double(GetFrequency())); + } + + void Profiler::PreFiberSwitch() { + ThreadContext* ctx = gInstance.getNewThreadContext(); + // End all active blocks before we change out + while (!ctx->openBlocks.empty()) { + Write(*ctx, EventType::End, 0); + ctx->openBlocks.pop_back(); + } + } + + void Profiler::PushJob(size_t signal) { + Job r = { + .signal = signal + }; + ThreadContext* ctx = gInstance.getNewThreadContext(); + Write(*ctx, EventType::Job, r); + } + + void Profiler::Trigger(size_t signal) { + ThreadContext* ctx = gInstance.getNewThreadContext(); + Write(*ctx, EventType::Signal, signal); + } + + Profiler::FiberData Profiler::BeginFiberWait(size_t signal, bool isMutex) { + FiberWait r = { + .ID = gInstance.fiberWaiting++, + .signal = signal, + .mutex = isMutex + }; + + ThreadContext* ctx = gInstance.getNewThreadContext(); + FiberData res = { + .ID = r.ID, + .count = static_cast(ctx->openBlocks.size()), + .signal = signal + }; + std::memcpy(res.blocks, ctx->openBlocks.data(), std::min(res.count, 16u) * sizeof(res.blocks[0])); + Write(*ctx, EventType::FiberWait, r); + return res; + } + + void Profiler::EndFiberWait(const SH::Profiler::FiberData &data) { + ThreadContext* ctx = gInstance.getNewThreadContext(); + FiberWait r = { + .ID = data.ID, + .signal = data.signal, + .mutex = false + }; + + Write(*ctx, EventType::FiberWake, r); + + const uint32_t count = data.count; + + for (size_t i = 0; i < count; i++) + if (i < lengthOf(data.blocks)) + ContinueBlock(data.blocks[i]); + else + ContinueBlock(-1); + } + + void Profiler::End() { + ThreadContext* ctx = gInstance.getNewThreadContext(); + if (!ctx->openBlocks.empty()) { + ctx->openBlocks.pop_back(); + Write(*ctx, EventType::End, 0); + } + } + + size_t Profiler::GetFrequency() { + static size_t frequency; + #ifdef _WIN32 + LARGE_INTEGER f; + QueryPerformanceFrequency(&f); + frequency = f.QuadPart; + #else + frequency = 1'000'000'000; + #endif + + return frequency; + } + + bool Profiler::IsContextSwitchEnabled() { + return gInstance.contextSwitch; + } + + void Profiler::Frame() { + const size_t n = SH::Timer::getTimestamp(); + if (gInstance.lastFrameTime != 0) + gInstance.lastFrameElapsed = n - gInstance.lastFrameTime; + gInstance.lastFrameTime = n; + Write(gInstance.global, EventType::Frame, 0); + } + + void Profiler::ShowInProfiler(bool show) { + ThreadContext* ctx = gInstance.getNewThreadContext(); + ShadowEngine::MutexGuard lock(ctx->mut); + + ctx->show = show; + } + + void Profiler::SetThreadName(const char *name) { + ThreadContext* ctx = gInstance.getNewThreadContext(); + ShadowEngine::MutexGuard lock(ctx->mut); + + ctx->name = name; + } + + void Profiler::Pause(bool paused) { + if (paused) Write(gInstance.global, EventType::Pause, 0); + + gInstance.paused = true; + if (paused) gInstance.pausedTime = SH::Timer::getTimestamp(); + } + +} + diff --git a/projs/shadow/shadow-engine/platforms/linux/src/Time.cpp b/projs/shadow/shadow-engine/platforms/linux/src/Time.cpp new file mode 100644 index 00000000..ed9a509e --- /dev/null +++ b/projs/shadow/shadow-engine/platforms/linux/src/Time.cpp @@ -0,0 +1,7 @@ +#include + +size_t SH::Timer::getTimestamp() { + timespec tick; + clock_gettime(CLOCK_REALTIME, &tick); + return size_t(tick.tv_sec * 1000000000 + size_t(tick.tv_nsec)); +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/platforms/windows/src/Time.cpp b/projs/shadow/shadow-engine/platforms/windows/src/Time.cpp new file mode 100644 index 00000000..bd9a1ed2 --- /dev/null +++ b/projs/shadow/shadow-engine/platforms/windows/src/Time.cpp @@ -0,0 +1,12 @@ +#include + +#define INITGUID +#define NOGDI +#define WIN32_LEAN_AND_MEAN +#include + +size_t SH::Timer::getTimestamp() { + LARGE_INTEGER tick; + QueryPerformanceCounter(&tick); + return tick.QuadPart; +} \ No newline at end of file From b16c560708352acb4b10419a217509e9beb7bc0c Mon Sep 17 00:00:00 2001 From: Curle Date: Fri, 5 Apr 2024 16:02:46 +0100 Subject: [PATCH 04/12] Add missing Common header --- .../platforms/inc/shadow/platform/Common.h | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 projs/shadow/shadow-engine/platforms/inc/shadow/platform/Common.h diff --git a/projs/shadow/shadow-engine/platforms/inc/shadow/platform/Common.h b/projs/shadow/shadow-engine/platforms/inc/shadow/platform/Common.h new file mode 100644 index 00000000..3414399d --- /dev/null +++ b/projs/shadow/shadow-engine/platforms/inc/shadow/platform/Common.h @@ -0,0 +1,22 @@ +#pragma once + +#ifdef __linux__ +#define ifsystem(linux,windows,apple) linux +#elif __WIN32 +#define ifsystem(linux,windows,apple) windows +#elif __APPLE__ +#define ifsystem(linux,windows,apple) apple +#endif + +// Clang on macOS has __GNUC__ defined to 4, for some reason. +#ifdef __clang__ +#define ifcompiler(gcc,clang,msvc) clang +#elif __GNUC__ +#define ifcompiler(gcc,clang,msvc) gcc +#elif _MSC_VER +#define ifcompiler(gcc,clang,msvc) msvc +#endif + +template constexpr uint32_t lengthOf(const T (&)[count]) { + return count; +}; \ No newline at end of file From beeed424346f8a89495dfe80b029c2f8d818cb11 Mon Sep 17 00:00:00 2001 From: Curle Date: Fri, 10 Mar 2023 15:54:29 +0000 Subject: [PATCH 05/12] File & FileSystem abstractions --- .../shadow-engine/assets/CMakeLists.txt | 49 +++- .../SFFElement.natvis | 0 .../cmake/Catch.cmake | 0 .../cmake/CatchAddTests.cmake | 0 .../src/SFFElement.cpp | 0 .../src/SFFElement.h | 0 .../src/SFFParser.cpp | 0 .../src/SFFParser.h | 0 .../src/SFFVersion.h | 0 .../src/SFFWriter.cpp | 0 .../src/SFFWriter.h | 0 .../src/Shadow.FileFormat.ixx | 0 .../shadow/shadow-engine/assets/src/fs/file.h | 112 +++++++++ .../shadow-engine/assets/src/fs/hash.cpp | 148 ++++++++++++ .../shadow/shadow-engine/assets/src/fs/hash.h | 158 +++++++++++++ .../shadow-engine/assets/src/fs/iostream.cpp | 212 ++++++++++++++++++ .../shadow-engine/assets/src/fs/iostream.h | 129 +++++++++++ .../shadow-engine/assets/src/fs/path.cpp | 117 ++++++++++ .../shadow/shadow-engine/assets/src/fs/path.h | 66 ++++++ .../shadow-engine/assets/src/fs/xxhash.h | 182 +++++++++++++++ .../assets/src/management/synchronization.h | 77 +++++++ .../shadow-engine/assets/src/str/string.cpp | 11 + .../shadow-engine/assets/src/str/string.h | 12 + .../test/Catch2Test.cpp | 0 .../test/catch2/catch.hpp | 2 +- .../test/sff_writer_tests.ocpp | 0 .../test/test.ocpp | 0 .../shadow-file-format/CMakeLists.txt | 20 -- 28 files changed, 1267 insertions(+), 28 deletions(-) rename projs/shadow/shadow-engine/{shadow-file-format => assets}/SFFElement.natvis (100%) rename projs/shadow/shadow-engine/{shadow-file-format => assets}/cmake/Catch.cmake (100%) rename projs/shadow/shadow-engine/{shadow-file-format => assets}/cmake/CatchAddTests.cmake (100%) rename projs/shadow/shadow-engine/{shadow-file-format => assets}/src/SFFElement.cpp (100%) rename projs/shadow/shadow-engine/{shadow-file-format => assets}/src/SFFElement.h (100%) rename projs/shadow/shadow-engine/{shadow-file-format => assets}/src/SFFParser.cpp (100%) rename projs/shadow/shadow-engine/{shadow-file-format => assets}/src/SFFParser.h (100%) rename projs/shadow/shadow-engine/{shadow-file-format => assets}/src/SFFVersion.h (100%) rename projs/shadow/shadow-engine/{shadow-file-format => assets}/src/SFFWriter.cpp (100%) rename projs/shadow/shadow-engine/{shadow-file-format => assets}/src/SFFWriter.h (100%) rename projs/shadow/shadow-engine/{shadow-file-format => assets}/src/Shadow.FileFormat.ixx (100%) create mode 100644 projs/shadow/shadow-engine/assets/src/fs/file.h create mode 100644 projs/shadow/shadow-engine/assets/src/fs/hash.cpp create mode 100644 projs/shadow/shadow-engine/assets/src/fs/hash.h create mode 100644 projs/shadow/shadow-engine/assets/src/fs/iostream.cpp create mode 100644 projs/shadow/shadow-engine/assets/src/fs/iostream.h create mode 100644 projs/shadow/shadow-engine/assets/src/fs/path.cpp create mode 100644 projs/shadow/shadow-engine/assets/src/fs/path.h create mode 100644 projs/shadow/shadow-engine/assets/src/fs/xxhash.h create mode 100644 projs/shadow/shadow-engine/assets/src/management/synchronization.h create mode 100644 projs/shadow/shadow-engine/assets/src/str/string.cpp create mode 100644 projs/shadow/shadow-engine/assets/src/str/string.h rename projs/shadow/shadow-engine/{shadow-file-format => assets}/test/Catch2Test.cpp (100%) rename projs/shadow/shadow-engine/{shadow-file-format => assets}/test/catch2/catch.hpp (97%) rename projs/shadow/shadow-engine/{shadow-file-format => assets}/test/sff_writer_tests.ocpp (100%) rename projs/shadow/shadow-engine/{shadow-file-format => assets}/test/test.ocpp (100%) delete mode 100644 projs/shadow/shadow-engine/shadow-file-format/CMakeLists.txt diff --git a/projs/shadow/shadow-engine/assets/CMakeLists.txt b/projs/shadow/shadow-engine/assets/CMakeLists.txt index 73460bfa..29725e3c 100644 --- a/projs/shadow/shadow-engine/assets/CMakeLists.txt +++ b/projs/shadow/shadow-engine/assets/CMakeLists.txt @@ -1,9 +1,44 @@ +<<<<<<<< HEAD:projs/shadow/shadow-engine/shadow-file-format/CMakeLists.txt +set(CMAKE_CXX_STANDARD 20) -FILE(GLOB_RECURSE SOURCES - ${CMAKE_CURRENT_LIST_DIR}/src/*.cpp -) +# Set up Catch2 testing +list(APPEND CMAKE_MODULE_PATH "cmake") +enable_testing() -target_shadow_module(shadow-engine - SOURCES ${SOURCES} - INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/inc/ -) \ No newline at end of file +# Set up asset sourceset +FILE(GLOB_RECURSE SOURCES src/*.cpp src/*.h) +FILE(GLOB_RECURSE TESTS test/*.cpp) + +add_library(shadow-asset ${SOURCES}) + +# Set up test executable +add_executable(shadow-asset-test ${TESTS}) +target_link_libraries(shadow-asset-test PRIVATE Catch2::Catch2 shadow-utils) + +# Enable testing on the executable +include(CTest) +include(Catch) +catch_discover_tests(shadow-asset-test) +======== +set(CMAKE_CXX_STANDARD 20) + +# Set up Catch2 testing +list(APPEND CMAKE_MODULE_PATH "cmake") +enable_testing() + +# Set up asset sourceset +FILE(GLOB_RECURSE SOURCES src/**.cpp src/**.h) +FILE(GLOB_RECURSE TESTS test/*.cpp) + +include_directories(src/) +add_library(shadow-asset ${SOURCES}) + +# Set up test executable +add_executable(shadow-asset-test ${TESTS}) +target_link_libraries(shadow-asset-test PRIVATE Catch2::Catch2 shadow-utils) + +# Enable testing on the executable +#include(CTest) +#include(Catch2) +#catch_discover_tests(shadow-asset-test) +>>>>>>>> 0009dc7 (File & FileSystem abstractions):projs/shadow/shadow-engine/assets/CMakeLists.txt diff --git a/projs/shadow/shadow-engine/shadow-file-format/SFFElement.natvis b/projs/shadow/shadow-engine/assets/SFFElement.natvis similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/SFFElement.natvis rename to projs/shadow/shadow-engine/assets/SFFElement.natvis diff --git a/projs/shadow/shadow-engine/shadow-file-format/cmake/Catch.cmake b/projs/shadow/shadow-engine/assets/cmake/Catch.cmake similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/cmake/Catch.cmake rename to projs/shadow/shadow-engine/assets/cmake/Catch.cmake diff --git a/projs/shadow/shadow-engine/shadow-file-format/cmake/CatchAddTests.cmake b/projs/shadow/shadow-engine/assets/cmake/CatchAddTests.cmake similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/cmake/CatchAddTests.cmake rename to projs/shadow/shadow-engine/assets/cmake/CatchAddTests.cmake diff --git a/projs/shadow/shadow-engine/shadow-file-format/src/SFFElement.cpp b/projs/shadow/shadow-engine/assets/src/SFFElement.cpp similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/src/SFFElement.cpp rename to projs/shadow/shadow-engine/assets/src/SFFElement.cpp diff --git a/projs/shadow/shadow-engine/shadow-file-format/src/SFFElement.h b/projs/shadow/shadow-engine/assets/src/SFFElement.h similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/src/SFFElement.h rename to projs/shadow/shadow-engine/assets/src/SFFElement.h diff --git a/projs/shadow/shadow-engine/shadow-file-format/src/SFFParser.cpp b/projs/shadow/shadow-engine/assets/src/SFFParser.cpp similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/src/SFFParser.cpp rename to projs/shadow/shadow-engine/assets/src/SFFParser.cpp diff --git a/projs/shadow/shadow-engine/shadow-file-format/src/SFFParser.h b/projs/shadow/shadow-engine/assets/src/SFFParser.h similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/src/SFFParser.h rename to projs/shadow/shadow-engine/assets/src/SFFParser.h diff --git a/projs/shadow/shadow-engine/shadow-file-format/src/SFFVersion.h b/projs/shadow/shadow-engine/assets/src/SFFVersion.h similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/src/SFFVersion.h rename to projs/shadow/shadow-engine/assets/src/SFFVersion.h diff --git a/projs/shadow/shadow-engine/shadow-file-format/src/SFFWriter.cpp b/projs/shadow/shadow-engine/assets/src/SFFWriter.cpp similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/src/SFFWriter.cpp rename to projs/shadow/shadow-engine/assets/src/SFFWriter.cpp diff --git a/projs/shadow/shadow-engine/shadow-file-format/src/SFFWriter.h b/projs/shadow/shadow-engine/assets/src/SFFWriter.h similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/src/SFFWriter.h rename to projs/shadow/shadow-engine/assets/src/SFFWriter.h diff --git a/projs/shadow/shadow-engine/shadow-file-format/src/Shadow.FileFormat.ixx b/projs/shadow/shadow-engine/assets/src/Shadow.FileFormat.ixx similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/src/Shadow.FileFormat.ixx rename to projs/shadow/shadow-engine/assets/src/Shadow.FileFormat.ixx diff --git a/projs/shadow/shadow-engine/assets/src/fs/file.h b/projs/shadow/shadow-engine/assets/src/fs/file.h new file mode 100644 index 00000000..d3265cb7 --- /dev/null +++ b/projs/shadow/shadow-engine/assets/src/fs/file.h @@ -0,0 +1,112 @@ +#pragma once +#include +#include + +namespace ShadowEngine { + + // An input stream that can read a file on disk. + struct FileInput final : InputStream { + FileInput(); + ~FileInput() = default; + + [[nodiscard]] bool open(std::string& path); + void close(); + + using InputStream::read; + [[nodiscard]] bool read(void* data, size_t size) override; + const void* getBuffer() const override { return nullptr; } + + size_t size() const override; + size_t pos(); + + [[nodiscard]] bool seek(size_t pos); + + private: + void* handle; + }; + + // An output stream that can write to a file on disk. + struct FileOutput final : OutputStream { + FileOutput(); + ~FileOutput() = default; + + [[nodiscard]] bool open(std::string& path); + void close(); + void flush(); + bool errored() const { return error; } + using OutputStream::write; + [[nodiscard]] bool write(const void* data, size_t size) override; + + private: + FileOutput(const FileOutput&) = delete; + void* handle; + bool error; + }; + + struct FileInfo { + bool directory; + std::string filename; + }; + + template struct Delegate; + + /** + * A generic Filesystem API. + * Allows interacting with files on disk the same as files in our Virtual Package Format. + */ + struct FileSystem { + // A function called when the data of a file is updated, such as when an asynchronous operation completes. + using ContentCallback = Delegate; + // A handle for asynchronous data movement; such as reading or writing a file. + struct AsyncHandle { + static AsyncHandle invalid() { return AsyncHandle(0xffffffff); } + explicit AsyncHandle(uint32_t val) : value(val) {} + + [[nodiscard]] bool valid() const { return value != 0xffffffff; } + + uint32_t value; + }; + + // Create a Filesystem that interacts with files on disk. + static std::unique_ptr& createDiskFS(std::string& basePath); + // Create a Virtual Filesystem based on the given path. + static std::unique_ptr& createVFS(std::string& basePath); + + virtual ~FileSystem() {} + + // Open a file for reading. + virtual bool open(std::string& path, FileInput& input) = 0; + // Open a file for writing. + virtual bool open(std::string& path, FileOutput& output) = 0; + // Check whether a file exists at the given path. + virtual bool fileExists(std::string& path) = 0; + // Get the time a file at the given path was last modified. + virtual size_t getLastModified(std::string& path) = 0; + // Copy a file from one path to another. + virtual bool copyFile(std::string& from, std::string& to) = 0; + // Move a file from one path to another. + virtual bool moveFile(std::string& from, std::string& to) = 0; + // Disassociate any files at the given path (not an immediate delete) + virtual bool deleteFile(std::string& path) = 0; + + // Get the path that this FileSystem originates at. The default is "/" for VFS, and whatever the Executable Path is for Disk FS. + virtual std::string& getBasePath() = 0; + // Set a new base path for the FileSystem. Any operations involving file paths will be relative to this new path. + virtual void setBasePath(std::string& path) = 0; + + // Process all the callbacks for async file operations. + virtual void processCallbacks() = 0; + // Check whether there are any outstanding async operations that need work. + virtual bool hasWork() = 0; + + // Write new content to a file synchronously. The thread will be blocked when doing this. + virtual bool saveSync(const struct Path& file, const uint8_t* content) = 0; + // Read content from a file synchronously. The thread will be blocked when doing this. + virtual bool readSync(const struct Path& file, struct OutputMemoryStream& content) = 0; + + // Read a file asynchronously. The given callback will be called with the file content once it is available. + virtual AsyncHandle readAsync(const Path& file, const ContentCallback& callback) = 0; + // Cancel an asynchronous operation, if it is not already complete. The associated callback will be called with a special flag. + virtual void cancelAsync(AsyncHandle& handle) = 0; + }; +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/assets/src/fs/hash.cpp b/projs/shadow/shadow-engine/assets/src/fs/hash.cpp new file mode 100644 index 00000000..67970e91 --- /dev/null +++ b/projs/shadow/shadow-engine/assets/src/fs/hash.cpp @@ -0,0 +1,148 @@ +#include +#include "xxhash.h" + +namespace ShadowEngine { + + StableHash::StableHash(const void *data, uint32_t length) { + hash = XXHash64::hash(data, length, 0); + } + + StableHash::StableHash(std::string &str) { + hash = XXHash64::hash(str.data(), str.size(), 0); + } + + HeapHash::HeapHash(const void *data, uint32_t length) { + hash = XXHash64::hash(data, length, 0); + } + + HeapHash::HeapHash(std::string &str) { + hash = XXHash64::hash(str.data(), str.size(), 0); + } + + HeapHash HeapHash::fromLong(size_t hash) { + HeapHash heap; + heap.hash = hash; + return heap; + } + + HeapHash32 HeapHash32::fromInt(uint32_t hash) { + HeapHash32 heap; + heap.hash = hash; + return heap; + } + + StableHash StableHash::fromLong(size_t hash) { + StableHash stable; + stable.hash = hash; + return stable; + } + + StableHash32 StableHash32::fromInt(uint32_t hash) { + StableHash32 stable; + stable.hash = hash; + return stable; + } + + static uint32_t CRC[256] = { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, + 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, + 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, + 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, + 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, + 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, + 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, + 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, + 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, + 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, + 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, + 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, + 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, + 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, + 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, + 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, + 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, + 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d}; + + static uint32_t CRC32(const void* data, uint32_t length) { + const auto* c = static_cast(data); + uint32_t crcTemp = 0xFFFFFFFF; + uint32_t len = length; + while (len) { + crcTemp = (crcTemp >> 8) ^ CRC[(crcTemp & 0xFF) ^ *c]; + --len; ++c; + } + + return ~crcTemp; + } + + StableHash32::StableHash32(const void *data, uint32_t length) { + hash = CRC32(data, length); + } + + StableHash32::StableHash32(std::string &str) { + hash = CRC32(str.data(), str.size()); + } + + static XXHash64 DeferredHashState(0); + + DeferredHash::DeferredHash() { + DeferredHashState = XXHash64(0); + } + + void DeferredHash::insert(const void *data, uint32_t length) { + DeferredHashState.add(data, length); + } + + StableHash32 DeferredHash::submit32() { + const auto result = DeferredHashState.hash(); + return StableHash32::fromInt(uint32_t(result ^ (result >> 32))); + } + + StableHash DeferredHash::submit() { + const auto result = DeferredHashState.hash(); + return StableHash::fromLong(result); + } + + DeferredHeapHash::DeferredHeapHash() { + DeferredHashState = XXHash64(0); + } + + void DeferredHeapHash::insert(const void *data, uint32_t length) { + DeferredHashState.add(data, length); + } + + HeapHash32 DeferredHeapHash::submit32() { + const auto result = DeferredHashState.hash(); + return HeapHash32::fromInt(uint32_t(result ^ (result >> 32))); + } + + HeapHash DeferredHeapHash::submit() { + const auto result = DeferredHashState.hash(); + return HeapHash::fromLong(result); + } +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/assets/src/fs/hash.h b/projs/shadow/shadow-engine/assets/src/fs/hash.h new file mode 100644 index 00000000..39e477e5 --- /dev/null +++ b/projs/shadow/shadow-engine/assets/src/fs/hash.h @@ -0,0 +1,158 @@ +#pragma once +#include + +namespace ShadowEngine { + + /** + * A 64-bit hashing algorithm that uses the state of the allocation heap as a "salt". + * Outputs are NOT stable, so do not serialize this. + * However, because it uses the heap, it has a very low collision rate. + */ + struct HeapHash { + // For if you MUST recreate a hash exactly. + // Please only use this for testing. + static HeapHash fromLong(size_t hash); + + HeapHash() = default; + // Hash a string; for paths and such. + explicit HeapHash(std::string& str); + // Hash arbitrary data. + HeapHash(const void* data, uint32_t length); + + bool operator!= (HeapHash& other) const { return hash != other.hash; } + bool operator== (HeapHash& other) const { return hash == other.hash; } + + size_t getHash() const { return hash; } + private: + size_t hash = 0; + }; + + /** + * A 32-bit hashing algorithm that uses the state of the allocation heap as a "salt". + * Outputs are NOT stable, so do not serialize this. + * However, because it uses the heap, it has a very low collision rate. + */ + struct HeapHash32 { + // For if you MUST recreate a hash exactly. + // Please only use this for testing. + static HeapHash32 fromInt(uint32_t hash); + + HeapHash32() = default; + // Hash a string; for paths and such. + explicit HeapHash32(std::string& str); + // Hash arbitrary data. + HeapHash32(const void* data, uint32_t length); + + bool operator!= (HeapHash32& other) const { return hash != other.hash; } + bool operator== (HeapHash32& other) const { return hash == other.hash; } + + uint32_t getHash() const { return hash; } + private: + uint32_t hash = 0; + }; + + /** + * A 64-bit hashing algorithm that generates the same hash value per input every time. + * A little more likely to generate conflicts than the hash that uses the state of the heap as a salt. + * Suitable for serialization. + */ + struct StableHash { + static StableHash fromLong(size_t data); + StableHash() = default; + StableHash(std::string& str); + StableHash(const void* data, uint32_t length); + + bool operator!= (StableHash& other) const { return hash != other.hash; } + bool operator== (StableHash& other) const { return hash == other.hash; } + bool operator< (StableHash& other) const { return hash < other.hash; } + + size_t getHash() const { return hash; } + + private: + size_t hash = 0; + }; + + /** + * A 32-bit hashing algorithm that generates the same hash value per input every time. + * A little more likely to generate conflicts than the hash that uses the state of the heap as a salt. + * Suitable for serialization. + */ + struct StableHash32 { + static StableHash32 fromInt(uint32_t data); + StableHash32() = default; + StableHash32(std::string& str); + StableHash32(const void* data, uint32_t length); + + bool operator!= (StableHash32& other) const { return hash != other.hash; } + bool operator== (StableHash32& other) const { return hash == other.hash; } + bool operator< (StableHash32& other) const { return hash < other.hash; } + + uint32_t getHash() const { return hash; } + + private: + uint32_t hash = 0; + }; + + // File Paths are hashed using the 64-bit StableHash system. + using PathHash = StableHash; + + /** + * A hashing utility that lets you insert data piecemeal before committing to the hash. + * Useful for when you're parsing a file and need to wait for more data to be available before hashing. + * Generates a Stable Hash. + */ + struct DeferredHash { + DeferredHash(); + // Insert new data to be considered for hashing + void insert(const void* data, uint32_t length); + // Submit the data to the hashing algorithm, and return a value in 64-bit StableHash + StableHash submit(); + // Submit the data to the hashing algorithm, and return a value in 32-bit StableHash + StableHash32 submit32(); + }; + + /** + * A hashing utility that lets you insert data piecemeal before committing to the hash. + * Useful for when you're parsing a file and need to wait for more data to be available before hashing. + * Generates a Heap Hash. + */ + struct DeferredHeapHash { + DeferredHeapHash(); + // Insert new data to be considered for hashing + void insert(const void* data, uint32_t length); + // Submit the data to the hashing algorithm, and return a value in 64-bit HeapHash + HeapHash submit(); + // Submit the data to the hashing algorithm, and return a value in 32-bit HeapHash + HeapHash32 submit32(); + }; + + /** The implementations of these hashing algorithms */ + + template struct HashFunc; + + template<> struct HashFunc { + static uint32_t get(const HeapHash& h) { + const size_t hash = h.getHash(); + return uint32_t(hash & (hash >> 16)); + } + }; + + template<> struct HashFunc { + static uint32_t get(const StableHash& h) { + const size_t hash = h.getHash(); + return uint32_t(hash & (hash >> 16)); + } + }; + + template<> struct HashFunc { + static uint32_t get(const HeapHash32& h) { + return h.getHash(); + } + }; + + template<> struct HashFunc { + static uint32_t get(const StableHash& h) { + return h.getHash(); + } + }; +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/assets/src/fs/iostream.cpp b/projs/shadow/shadow-engine/assets/src/fs/iostream.cpp new file mode 100644 index 00000000..2c268a42 --- /dev/null +++ b/projs/shadow/shadow-engine/assets/src/fs/iostream.cpp @@ -0,0 +1,212 @@ +#include + +namespace ShadowEngine { + + OutputMemoryStream::OutputMemoryStream(void *data, size_t size) + : buffer(static_cast(data)), capacity(size), usage(0) {} + + OutputMemoryStream::OutputMemoryStream(ShadowEngine::OutputMemoryStream &&str) noexcept { + capacity = str.capacity; + buffer = str.buffer; + usage = str.usage; + + str.free(); + } + + void OutputMemoryStream::operator=(ShadowEngine::OutputMemoryStream &&str) noexcept { + capacity = str.capacity; + buffer = str.buffer; + usage = str.usage; + + str.free(); + } + + void OutputMemoryStream::operator=(const ShadowEngine::OutputMemoryStream &rhs) noexcept { + usage = rhs.usage; + + if (rhs.capacity > 0) { + buffer = (uint8_t*)malloc(rhs.capacity); + memcpy(buffer, rhs.buffer, rhs.capacity); + capacity = rhs.capacity; + } else { + buffer = nullptr; + capacity = 0; + } + } + + OutputMemoryStream::OutputMemoryStream(const ShadowEngine::OutputMemoryStream &rhs) noexcept { + usage = rhs.usage; + + if (rhs.capacity > 0) { + buffer = (uint8_t*)malloc(rhs.capacity); + memcpy(buffer, rhs.buffer, rhs.capacity); + capacity = rhs.capacity; + } else { + buffer = nullptr; + capacity = 0; + } + } + + + OutputMemoryStream::~OutputMemoryStream() = default; + + OutputStream &OutputStream::operator<<(std::string &str) { + write(str.data(), str.length()); + return *this; + } + + OutputStream &OutputStream::operator<<(const char* str) { + write(str, strlen(str)); + return *this; + } + + OutputStream &OutputStream::operator<<(uint32_t val) { + std::string str = std::to_string(val); + write(str.c_str(), str.length()); + return *this; + } + + OutputStream &OutputStream::operator<<(int32_t val) { + std::string str = std::to_string(val); + write(str.c_str(), str.length()); + return *this; + } + + OutputStream &OutputStream::operator<<(uint64_t val) { + std::string str = std::to_string(val); + write(str.c_str(), str.length()); + return *this; + } + + OutputStream &OutputStream::operator<<(int64_t val) { + std::string str = std::to_string(val); + write(str.c_str(), str.length()); + return *this; + } + + OutputStream &OutputStream::operator<<(float val) { + std::string str = std::to_string(val); + write(str.c_str(), str.length()); + return *this; + } + + OutputStream &OutputStream::operator<<(double val) { + std::string str = std::to_string(val); + write(str.c_str(), str.length()); + return *this; + } + + void OutputMemoryStream::write(std::string &str) { + write(str.c_str(), str.length()); + } + + void *OutputMemoryStream::skip(size_t size) { + if (size + usage > capacity) { + reserve((size + usage) << 1); + } + + void* ret = (uint8_t*)buffer + usage; + usage += size; + return ret; + } + + OutputMemoryStream& OutputMemoryStream::operator+=(size_t size) { + skip(size); + return *this; + } + + OutputMemoryStream& OutputMemoryStream::operator++() { + skip(1); + return *this; + } + + uint8_t OutputMemoryStream::operator[](size_t index) const { + return buffer[index]; + } + + uint8_t &OutputMemoryStream::operator[](size_t index) { + return buffer[index]; + } + + bool OutputMemoryStream::write(const void *data, size_t size) { + if (!size) return true; + + if (usage + size > capacity) { + reserve((usage + size) << 1); + } + + memcpy((uint8_t*)data + usage, data, size); + usage += size; + return true; + } + + void OutputMemoryStream::clear() { usage = 0; } + + void OutputMemoryStream::free() { + usage = 0; + capacity = 0; + delete[] buffer; + buffer = nullptr; + } + + void OutputMemoryStream::reserve(size_t size) { + if (size < capacity) return; + + auto* temp = static_cast(malloc(size)); + memcpy(temp, buffer, capacity); + delete[] buffer; + buffer = temp; + capacity = size; + } + + uint8_t *OutputMemoryStream::release() { + auto* temp = static_cast(malloc(usage)); + memcpy(temp, buffer, usage); + free(); + return temp; + } + + InputMemoryStream::InputMemoryStream(const void *data, size_t size) + : data(static_cast(data)), capacity(size), position(0) {} + + InputMemoryStream::InputMemoryStream(const ShadowEngine::OutputMemoryStream &blob) + : data(blob.data()), capacity(blob.size()), position(0) {} + + void InputMemoryStream::set(const void *newData, size_t size) { + data = (uint8_t*) newData; capacity = size; position = 0; + } + + const void *InputMemoryStream::skip(size_t size) { + auto* pos = data + position; + position += size; + if (position > capacity) { + position = capacity; + } + + return (const void*) pos; + } + + bool InputMemoryStream::read(void *out, size_t size) { + if (position + (uint32_t) size > capacity) { + for (int32_t i = 0; i < size; i++) + ((unsigned char*)out)[i] = 0; + return false; + } + + if (size) { + memcpy(out, ((char*)data) + position, capacity); + } + + position += size; + + return true; + } + + std::string InputMemoryStream::readString() { + const char* ret = (const char*) data + position; + while (position < capacity && data[position]) ++position; + ++position; + + return { ret }; + } +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/assets/src/fs/iostream.h b/projs/shadow/shadow-engine/assets/src/fs/iostream.h new file mode 100644 index 00000000..6ac41a69 --- /dev/null +++ b/projs/shadow/shadow-engine/assets/src/fs/iostream.h @@ -0,0 +1,129 @@ +#pragma once +#include + +namespace ShadowEngine { + // A custom OutputStream that can be implemented to output to any arbitrary data structure. + // The idea is that it can write to a file, or into memory, or into a temporary buffer that is copied to both. + // As opposed to the hardcoded streams that exist in C++, which have a single purpose for their entire lifetime. + struct OutputStream { + virtual bool write(const void* data, size_t size) = 0; + + OutputStream& operator<< (std::string& str); + OutputStream& operator<< (const char* str); + OutputStream& operator<< (size_t val); + OutputStream& operator<< (int64_t val); + OutputStream& operator<< (uint32_t val); + OutputStream& operator<< (int32_t val); + OutputStream& operator<< (float val); + OutputStream& operator<< (double val); + template bool write(const T& val); + }; + + // A custom InputStream that can be implemented to read from any arbitrary data structure. + // The idea is that it can read from a file, or from memory, or from a temporary buffer that is merged from both. + // As opposed to the hardcoded streams that exist in C++, which have a single purpose for their entire lifetime. + struct InputStream { + virtual bool read(void* buffer, size_t size) = 0; + virtual const void* getBuffer() const = 0; + virtual size_t size() const = 0; + + template void read(T& val) { read(&val, sizeof(T)); } + template T read(); + }; + + // A custom OutputStream that writes to memory. + struct OutputMemoryStream final : OutputStream { + + OutputMemoryStream(void* data, size_t size); + OutputMemoryStream(OutputMemoryStream&& str) noexcept; + OutputMemoryStream(const OutputMemoryStream& rhs) noexcept; + ~OutputMemoryStream(); + + void operator= (const OutputMemoryStream& rhs) noexcept; + void operator= (OutputMemoryStream&& rhs) noexcept; + + uint8_t operator[] (size_t index) const; + uint8_t& operator[] (size_t index); + OutputMemoryStream& operator+= (size_t index); + OutputMemoryStream& operator++ (); + + bool write(const void* data, size_t size) override; + + uint8_t* release(); + void resize(size_t size); + void reserve(size_t size); + const uint8_t* data() const { return buffer; }; + uint8_t* dataMut() { return buffer; }; + size_t size() const { return usage; }; + void clear(); + void* skip(size_t size); + bool empty() const { return usage == 0; }; + void free(); + + void write(std::string& str); + template void write(const T& val); + + private: + uint8_t* buffer; + size_t capacity; + size_t usage; + }; + + template void OutputMemoryStream::write(const T& val){ + write(&val, sizeof(T)); + } + + + template <> inline void OutputMemoryStream::write(const bool& val) { + uint8_t v = val; + write(&v, sizeof(v)); + } + + struct InputMemoryStream final : InputStream { + InputMemoryStream(const void* data, size_t size); + explicit InputMemoryStream(const OutputMemoryStream& blob); + + void set(const void* data, size_t size); + bool read(void* data, size_t size) override; + std::string readString(); + const void* skip(size_t size); + const void* getData() const { return data; }; + const void* getBuffer() const override { return data; }; + size_t size() const override { return capacity; }; + size_t pos() const { return position; }; + void setPos(size_t pos) { position = pos; }; + void restart() { position = 0; }; + uint8_t readChar() { position++; return data[position-1]; }; + + template + T getAs() const { + static_assert(position + sizeof(T) < capacity); + return *(T*)(data + position); + } + + using InputStream::read; + + private: + const uint8_t* data; + size_t capacity; + size_t position; + }; + + template + T InputStream::read() { + T v; + read(&v, sizeof(T)); + return v; + } + + template<> inline bool InputStream::read() { + uint8_t v; + read(&v, sizeof(bool)); + return v; + } + + template + bool OutputStream::write(const T &val) { + return write(&val, sizeof(T)); + } +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/assets/src/fs/path.cpp b/projs/shadow/shadow-engine/assets/src/fs/path.cpp new file mode 100644 index 00000000..01bad0d6 --- /dev/null +++ b/projs/shadow/shadow-engine/assets/src/fs/path.cpp @@ -0,0 +1,117 @@ +#include +#include +#include + +namespace ShadowEngine { + + Path::Path() : path {} { } + + Path::Path(const std::string &str) { + set(normalise((std::string&) str)); + } + + void Path::set(const std::string &str) { +#ifdef _WIN32 + std::string temp = Str::toLower((std::string&) str); + hash = PathHash(temp); +#else + hash = PathHash(str); +#endif + path = str; + } + + void Path::operator=(const std::string &rhs) { + set(rhs); + } + + bool Path::operator==(const std::string &rhs) { + return path == rhs; + } + + bool Path::operator==(const ShadowEngine::Path &rhs) { + return path == rhs.path; + } + + bool Path::operator!=(const ShadowEngine::Path &rhs) { + return path != rhs.path; + } + + std::string Path::normalise(std::string &str) { + bool prevSlash = false; + + std::string temp; + const char* path = str.c_str(); + size_t len = str.length(); + size_t i = 0; + + // Skip initial stuff. + size_t ind = str.find_first_of(":"); + path += ind; + if (path[0] == '.' && (path[1] == '\\' || path[1] == '/')) + path += 2; +#ifdef _WIN32 + if (path[0] == '\\' || path[0] == '/') + ++path; +#endif + + while (*path != '\0' && i < len) { + bool slash = *path == '\\' || *path == '/'; + + // Skip double slashes. + if (slash && prevSlash) { + path++; continue; + } + + // Convert backslashes to forward slashes. + temp.append(std::to_string(*path == '\\' ? '/' : *path)); + + path++; i++; prevSlash = slash; + } + + return temp; + } + + std::string Path::getPrelude(std::string &path) { + return path.substr(0, path.find_first_of(":")); + } + + std::string Path::getDomain(std::string &path) { + return path.substr(path.find_first_of(":"), path.find_first_of("/")); + } + + std::string Path::getDirectory(std::string &path) { + return path.substr(path.find_first_of(":"), path.find_last_of("/")); + } + + std::string Path::getFilename(std::string &path) { + return path.substr(path.find_last_of("/"), path.find_last_of(".")); + } + + std::string Path::getExtension(std::string &path) { + return path.substr(path.find_last_of("."), path.length()); + } + + std::string Path::replaceExtension(std::string &path, std::string &newExt) { + return path.substr(0, path.length() - newExt.length()).append(newExt); + } + + bool Path::hasExtension(std::string &path, std::string &ext) { + return path.find_last_of(ext) == path.length() - ext.length(); + } + + PathInfo::PathInfo(std::string &str) { + std::string normalised = Path::normalise(str); + + std::string preludeS = Path::getPrelude(normalised); + memcpy_s(prelude, 10, preludeS.c_str(), preludeS.length()); + std::string domainS = Path::getDomain(normalised); + memcpy_s(domain, 256, domainS.c_str(), domainS.length()); + std::string directoryS = Path::getDirectory(normalised); + memcpy_s(directory, 256, directoryS.c_str(), directoryS.length()); + std::string filenameS = Path::getFilename(normalised); + memcpy_s(baseName, 256, filenameS.c_str(), filenameS.length()); + std::string extensionS = Path::getExtension(normalised); + memcpy_s(extension, 10, extensionS.c_str(), extensionS.length()); + } + +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/assets/src/fs/path.h b/projs/shadow/shadow-engine/assets/src/fs/path.h new file mode 100644 index 00000000..8ce66288 --- /dev/null +++ b/projs/shadow/shadow-engine/assets/src/fs/path.h @@ -0,0 +1,66 @@ +#pragma once + +#include +#include + +namespace ShadowEngine { + /** + * Stores split data about a path, for easy referencing and decomposition. + * Not to be used as a replacement for the Path class. + */ + struct PathInfo { + explicit PathInfo(std::string& str); + + char extension[10]; + char baseName[256]; + char directory[256]; + char domain[256]; + char prelude[10]; + }; + + /** + * Stores and handles paths in the VFS. + * All operations are copy-instantiated, nothing works in-place. + * A typical path is of the form: + * prelude:/domain/directory/filename.extension + */ + struct Path { + // Make sure the path is valid. + // Always from the root. + // One slash separating. + static std::string normalise(std::string& path); + // Get the prelude of the given path. + static std::string getPrelude(std::string& path); + // Get the domain of the given path. + static std::string getDomain(std::string& path); + // Get the directory of the given path. + static std::string getDirectory(std::string& path); + // Get the name of the file of the given path. + static std::string getFilename(std::string& path); + // Get the file extension of the given path. + static std::string getExtension(std::string& path); + // Check if the path has the given extension. + static bool hasExtension(std::string& path, std::string& ext); + // Replace the extension of the given path. + static std::string replaceExtension(std::string& path, std::string& newExt); + + Path(); + explicit Path(const std::string& str); + + void operator=(const std::string& rhs); + bool operator==(const std::string& rhs); + bool operator==(const Path& rhs); + bool operator!=(const Path& rhs); + + // Use this to set a new value into the path; it handles the hash too. + void set(const std::string& path); + + uint32_t length() const { return path.length(); }; + PathHash getHash() { return hash; } + const char* c_str() const { return path.data(); } + bool isEmpty() const { return path.length() == 0; } + private: + std::string path; + PathHash hash; + }; +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/assets/src/fs/xxhash.h b/projs/shadow/shadow-engine/assets/src/fs/xxhash.h new file mode 100644 index 00000000..e75b9c9c --- /dev/null +++ b/projs/shadow/shadow-engine/assets/src/fs/xxhash.h @@ -0,0 +1,182 @@ +#pragma once +#include // for uint32_t and uint64_t + +class XXHash64 +{ +public: + /// create new XXHash (64 bit) + /** @param seed your seed value, even zero is a valid seed **/ + explicit XXHash64(uint64_t seed) + { + state[0] = seed + Prime1 + Prime2; + state[1] = seed + Prime2; + state[2] = seed; + state[3] = seed - Prime1; + bufferSize = 0; + totalLength = 0; + } + + /// add a chunk of bytes + /** @param input pointer to a continuous block of data + @param length number of bytes + @return false if parameters are invalid / zero **/ + bool add(const void* input, uint64_t length) + { + // no data ? + if (!input || length == 0) + return false; + + totalLength += length; + // byte-wise access + const unsigned char* data = (const unsigned char*)input; + + // unprocessed old data plus new data still fit in temporary buffer ? + if (bufferSize + length < MaxBufferSize) + { + // just add new data + while (length-- > 0) + buffer[bufferSize++] = *data++; + return true; + } + + // point beyond last byte + const unsigned char* stop = data + length; + const unsigned char* stopBlock = stop - MaxBufferSize; + + // some data left from previous update ? + if (bufferSize > 0) + { + // make sure temporary buffer is full (16 bytes) + while (bufferSize < MaxBufferSize) + buffer[bufferSize++] = *data++; + + // process these 32 bytes (4x8) + process(buffer, state[0], state[1], state[2], state[3]); + } + + // copying state to local variables helps optimizer A LOT + uint64_t s0 = state[0], s1 = state[1], s2 = state[2], s3 = state[3]; + // 32 bytes at once + while (data <= stopBlock) + { + // local variables s0..s3 instead of state[0]..state[3] are much faster + process(data, s0, s1, s2, s3); + data += 32; + } + // copy back + state[0] = s0; state[1] = s1; state[2] = s2; state[3] = s3; + + // copy remainder to temporary buffer + bufferSize = stop - data; + for (uint64_t i = 0; i < bufferSize; i++) + buffer[i] = data[i]; + + // done + return true; + } + + /// get current hash + /** @return 64 bit XXHash **/ + uint64_t hash() const + { + // fold 256 bit state into one single 64 bit value + uint64_t result; + if (totalLength >= MaxBufferSize) + { + result = rotateLeft(state[0], 1) + + rotateLeft(state[1], 7) + + rotateLeft(state[2], 12) + + rotateLeft(state[3], 18); + result = (result ^ processSingle(0, state[0])) * Prime1 + Prime4; + result = (result ^ processSingle(0, state[1])) * Prime1 + Prime4; + result = (result ^ processSingle(0, state[2])) * Prime1 + Prime4; + result = (result ^ processSingle(0, state[3])) * Prime1 + Prime4; + } + else + { + // internal state wasn't set in add(), therefore original seed is still stored in state2 + result = state[2] + Prime5; + } + + result += totalLength; + + // process remaining bytes in temporary buffer + const unsigned char* data = buffer; + // point beyond last byte + const unsigned char* stop = data + bufferSize; + + // at least 8 bytes left ? => eat 8 bytes per step + for (; data + 8 <= stop; data += 8) + result = rotateLeft(result ^ processSingle(0, *(uint64_t*)data), 27) * Prime1 + Prime4; + + // 4 bytes left ? => eat those + if (data + 4 <= stop) + { + result = rotateLeft(result ^ (*(uint32_t*)data) * Prime1, 23) * Prime2 + Prime3; + data += 4; + } + + // take care of remaining 0..3 bytes, eat 1 byte per step + while (data != stop) + result = rotateLeft(result ^ (*data++) * Prime5, 11) * Prime1; + + // mix bits + result ^= result >> 33; + result *= Prime2; + result ^= result >> 29; + result *= Prime3; + result ^= result >> 32; + return result; + } + + + /// combine constructor, add() and hash() in one static function (C style) + /** @param input pointer to a continuous block of data + @param length number of bytes + @param seed your seed value, e.g. zero is a valid seed + @return 64 bit XXHash **/ + static uint64_t hash(const void* input, uint64_t length, uint64_t seed) + { + XXHash64 hasher(seed); + hasher.add(input, length); + return hasher.hash(); + } + +private: + /// magic constants :-) + static const uint64_t Prime1 = 11400714785074694791ULL; + static const uint64_t Prime2 = 14029467366897019727ULL; + static const uint64_t Prime3 = 1609587929392839161ULL; + static const uint64_t Prime4 = 9650029242287828579ULL; + static const uint64_t Prime5 = 2870177450012600261ULL; + + /// temporarily store up to 31 bytes between multiple add() calls + static const uint64_t MaxBufferSize = 31+1; + + uint64_t state[4]; + unsigned char buffer[MaxBufferSize]; + uint64_t bufferSize; + uint64_t totalLength; + + /// rotate bits, should compile to a single CPU instruction (ROL) + static inline uint64_t rotateLeft(uint64_t x, unsigned char bits) + { + return (x << bits) | (x >> (64 - bits)); + } + + /// process a single 64 bit value + static inline uint64_t processSingle(uint64_t previous, uint64_t input) + { + return rotateLeft(previous + input * Prime2, 31) * Prime1; + } + + /// process a block of 4x4 bytes, this is the main part of the XXHash32 algorithm + static inline void process(const void* data, uint64_t& state0, uint64_t& state1, uint64_t& state2, uint64_t& state3) + { + const uint64_t* block = (const uint64_t*) data; + state0 = processSingle(state0, block[0]); + state1 = processSingle(state1, block[1]); + state2 = processSingle(state2, block[2]); + state3 = processSingle(state3, block[3]); + } +}; \ No newline at end of file diff --git a/projs/shadow/shadow-engine/assets/src/management/synchronization.h b/projs/shadow/shadow-engine/assets/src/management/synchronization.h new file mode 100644 index 00000000..3d3278e7 --- /dev/null +++ b/projs/shadow/shadow-engine/assets/src/management/synchronization.h @@ -0,0 +1,77 @@ +#pragma once + +#include + +#ifdef __linux__ +#include +#endif + +namespace ShadowEngine { + // A simple synchronization system that allows one "accessing thread" at a time. + struct alignas(8) Mutex { + friend struct ConditionVariable; + Mutex(); + Mutex(const Mutex&) = delete; + ~Mutex(); + + void enter(); + void exit(); + private: +#ifdef _WIN32 + uint8_t data[8]; +#else + pthread_mutex_t mutex; +#endif + }; + + // A simple synchronization system that allows many threads to wait for one thread to complete an operation. + struct Semaphore { + Semaphore(int initcount, int maxcount); + Semaphore(const Semaphore&) = delete; + ~Semaphore(); + + void raise(); + void wait(); + private: +#ifdef _WIN32 + void* id; +#else + struct { + pthread_mutex_t mutex; + pthread_mutex_cond cond; + volatile int32_t count; + } id; +#endif + }; + + struct ConditionVariable { + ConditionVariable(); + ConditionVariable(const ConditionVariable&) = delete; + ~ConditionVariable(); + + void sleep(Mutex& mut); + void wake(); + + private: +#ifdef _WIN32 + uint8_t data[64]; +#else + pthread_cond_t cond; +#endif + }; + + // A simple RAII wrapper for Mutexes, that locks and unlocks as the Guard goes in and out of scope. + struct MutexGuard { + explicit MutexGuard(Mutex& mut) : mut(mut) { + mut.enter(); + } + + ~MutexGuard() { mut.exit(); } + + MutexGuard(const MutexGuard&) = delete; + void operator=(const MutexGuard&) = delete; + + private: + Mutex& mut; + }; +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/assets/src/str/string.cpp b/projs/shadow/shadow-engine/assets/src/str/string.cpp new file mode 100644 index 00000000..6cb12eea --- /dev/null +++ b/projs/shadow/shadow-engine/assets/src/str/string.cpp @@ -0,0 +1,11 @@ +#include + +namespace ShadowEngine::Str { + std::string toLower(const std::string& str) { + std::string temp; + for (auto c : str) { + temp.append(std::to_string(tolower(c))); + } + return temp; + } +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/assets/src/str/string.h b/projs/shadow/shadow-engine/assets/src/str/string.h new file mode 100644 index 00000000..02749482 --- /dev/null +++ b/projs/shadow/shadow-engine/assets/src/str/string.h @@ -0,0 +1,12 @@ +#pragma once +#include + +namespace ShadowEngine { + // String manipluation utilities. + // Because std::string is heavily lacking. + namespace Str { + // Convert the string to lower case, return a new string. + // This only works in ASCII encoding. + std::string toLower(std::string& str); + } +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-file-format/test/Catch2Test.cpp b/projs/shadow/shadow-engine/assets/test/Catch2Test.cpp similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/test/Catch2Test.cpp rename to projs/shadow/shadow-engine/assets/test/Catch2Test.cpp diff --git a/projs/shadow/shadow-engine/shadow-file-format/test/catch2/catch.hpp b/projs/shadow/shadow-engine/assets/test/catch2/catch.hpp similarity index 97% rename from projs/shadow/shadow-engine/shadow-file-format/test/catch2/catch.hpp rename to projs/shadow/shadow-engine/assets/test/catch2/catch.hpp index 64a7b458..ac53e71b 100644 --- a/projs/shadow/shadow-engine/shadow-file-format/test/catch2/catch.hpp +++ b/projs/shadow/shadow-engine/assets/test/catch2/catch.hpp @@ -356,7 +356,7 @@ namespace Catch { #if defined(CATCH_INTERNAL_CONFIG_WINDOWS_SEH) && !defined(CATCH_CONFIG_NO_WINDOWS_SEH) && !defined(CATCH_CONFIG_WINDOWS_SEH) && !defined(CATCH_INTERNAL_CONFIG_NO_WINDOWS_SEH) # define CATCH_CONFIG_WINDOWS_SEH #endif -// This is set by default, because we assume that linux compilers are posix-signal-compatible by default. +// This is set by default, because we assume that unix compilers are posix-signal-compatible by default. #if defined(CATCH_INTERNAL_CONFIG_POSIX_SIGNALS) && !defined(CATCH_INTERNAL_CONFIG_NO_POSIX_SIGNALS) && !defined(CATCH_CONFIG_NO_POSIX_SIGNALS) && !defined(CATCH_CONFIG_POSIX_SIGNALS) # define CATCH_CONFIG_POSIX_SIGNALS #endif diff --git a/projs/shadow/shadow-engine/shadow-file-format/test/sff_writer_tests.ocpp b/projs/shadow/shadow-engine/assets/test/sff_writer_tests.ocpp similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/test/sff_writer_tests.ocpp rename to projs/shadow/shadow-engine/assets/test/sff_writer_tests.ocpp diff --git a/projs/shadow/shadow-engine/shadow-file-format/test/test.ocpp b/projs/shadow/shadow-engine/assets/test/test.ocpp similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/test/test.ocpp rename to projs/shadow/shadow-engine/assets/test/test.ocpp diff --git a/projs/shadow/shadow-engine/shadow-file-format/CMakeLists.txt b/projs/shadow/shadow-engine/shadow-file-format/CMakeLists.txt deleted file mode 100644 index 00308d24..00000000 --- a/projs/shadow/shadow-engine/shadow-file-format/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -set(CMAKE_CXX_STANDARD 20) - -# Set up Catch2 testing -list(APPEND CMAKE_MODULE_PATH "cmake") -enable_testing() - -# Set up asset sourceset -FILE(GLOB_RECURSE SOURCES src/*.cpp src/*.h) -FILE(GLOB_RECURSE TESTS test/*.cpp) - -add_library(shadow-asset ${SOURCES}) - -# Set up test executable -add_executable(shadow-asset-test ${TESTS}) -target_link_libraries(shadow-asset-test PRIVATE Catch2::Catch2 shadow-utils) - -# Enable testing on the executable -include(CTest) -include(Catch) -catch_discover_tests(shadow-asset-test) \ No newline at end of file From 19b6c52dc03caa0b2174fd37e4a0b0c120c00af7 Mon Sep 17 00:00:00 2001 From: Curle Date: Sun, 12 Mar 2023 16:44:10 +0000 Subject: [PATCH 06/12] FileSystem specializations for Disk & VFS operations. --- .../shadow-engine/assets/src/fs/file.cpp | 244 ++++++++++++++++++ .../shadow/shadow-engine/assets/src/fs/file.h | 15 +- .../shadow/shadow-engine/assets/src/fs/hash.h | 10 +- .../shadow-engine/assets/src/fs/iostream.h | 6 +- .../shadow-engine/assets/src/fs/path.cpp | 3 +- .../shadow/shadow-engine/assets/src/fs/path.h | 11 +- .../assets/src/management/delegate.h | 92 +++++++ 7 files changed, 362 insertions(+), 19 deletions(-) create mode 100644 projs/shadow/shadow-engine/assets/src/fs/file.cpp create mode 100644 projs/shadow/shadow-engine/assets/src/management/delegate.h diff --git a/projs/shadow/shadow-engine/assets/src/fs/file.cpp b/projs/shadow/shadow-engine/assets/src/fs/file.cpp new file mode 100644 index 00000000..6af00837 --- /dev/null +++ b/projs/shadow/shadow-engine/assets/src/fs/file.cpp @@ -0,0 +1,244 @@ +#include +#include +#include "management/synchronization.h" +#include "../../../../../../cmake-build-debug/_deps/spdlog-src/include/spdlog/spdlog.h" +#include +#include +#include + +namespace ShadowEngine { + + /** + * An async operation to be performed. + * For reading files from disk into memory. + */ + + struct AsyncRead { + enum class Flags : uint32_t { + FAILED = 0, // The read failed due to some error. + CANCELLED // The read was cancelled due to the resource not being needed any more. + }; + + AsyncRead() : data() {} + + bool isFailed() const { return flags == Flags::FAILED; } + bool isCancelled() const { return flags == Flags::CANCELLED; } + + FileSystem::ContentCallback callback; + OutputMemoryStream data; + std::string path; + uint32_t id = 0; + Flags flags; + }; + + // The FileSystem that operates on raw on-disk files. + struct DiskFS; + + struct DiskFS : FileSystem { + + explicit DiskFS(std::string& path) : sem(0, 0xffff) { + setBasePath(path); + } + + bool hasWork() override { + return workCounter != 0; + } + + std::string const& getBasePath() const override { return basePath; } + void setBasePath(std::string& path) final { + basePath = Path::normalise(path); + if (!basePath.ends_with('/') && !basePath.ends_with('\\')) + basePath.append("/"); + } + + bool saveSync(const Path& path, const uint8_t* data, const size_t size) override { + FileOutput file; + std::string fullPath(basePath.append(path.c_str())); + + if (!file.open(fullPath)) return false; + bool res = file.write(data, size); + file.close(); + + return res; + } + + bool readSync(const Path& path, struct OutputMemoryStream& content) override { + FileInput file; + std::string fullPath(basePath.append(path.c_str())); + + if (!file.open(fullPath)) return false; + + content.resize(file.size()); + if (!file.read(content.dataMut(), content.size())) { + file.close(); + return false; + } + + file.close(); + return true; + } + + AsyncHandle readAsync(const Path& file, const ContentCallback& callback) override { + if (!file.isEmpty()) return AsyncHandle::invalid(); + + MutexGuard lock(mutex); + workCounter++; + + AsyncRead& read = queue.emplace_back(); + if (++lastID == 0) lastID++; + + read.id = lastID; + read.path = file.c_str(); + read.callback = callback; + sem.raise(); + + return AsyncHandle(read.id); + } + + void cancelAsync(AsyncHandle& handle) override { + MutexGuard lock(mutex); + + for (AsyncRead& read : queue) { + if (read.id == handle.value) { + read.flags = AsyncRead::Flags::CANCELLED; + workCounter--; + return; + } + } + + for (AsyncRead& read : finished) { + if (read.id == handle.value) { + read.flags = AsyncRead::Flags::CANCELLED; + return; + } + } + } + + bool open(std::string& path, FileInput& file) override { + return file.open(basePath.append(path)); + } + + bool open(std::string& path, FileOutput& file) override { + return file.open(basePath.append(path)); + } + + bool deleteFile(std::string& path) override { + return std::remove((basePath.append(path).c_str())); + } + + bool moveFile(std::string& from, std::string& to) override { + try { + std::rename(basePath.append(from).c_str(), basePath.append(to).c_str()); + } catch (std::filesystem::filesystem_error& e) { + return false; + } + return true; + } + + bool copyFile(std::string& from, std::string& to) override { + try { + std::filesystem::copy(basePath.append(from).c_str(), basePath.append(to).c_str()); + } catch (std::filesystem::filesystem_error& e) { + return false; + } + + return true; + } + + bool fileExists(std::string& path) override { + return std::filesystem::exists(path); + } + + size_t getLastModified(std::string& path) override { + return std::filesystem::last_write_time(path).time_since_epoch().count(); + } + + // TODO: File iterators + + void processCallbacks() override { + // TODO: Timeout this function! + for (;;) { + mutex.enter(); + if (finished.empty() || workCounter == 0) { + mutex.exit(); + break; + } + + AsyncRead item = finished[0]; + finished.erase(finished.begin()); + --workCounter; + + mutex.exit(); + + if (!item.isCancelled()) + item.callback.invoke(item.data.size(), (const uint8_t*) item.data.data(), !item.isFailed()); + + } + } + + // TODO: Task Management + std::string basePath; + std::vector queue; + uint64_t workCounter; + std::vector finished; + Mutex mutex; + Semaphore sem; + + uint32_t lastID; + + }; + + struct VFS : DiskFS { + VFS(std::string& root_pack_path) : DiskFS((std::string &) "vfs:/") { + if (!pack.open(root_pack_path)) { + spdlog::error("Unable to open " + root_pack_path + ", please check paths"); + return; + } + + const auto count = pack.read(); + for (size_t i = 0; i < count; i++) { + const auto hash = pack.read(); + PackFile& file = packFiles[hash]; + file.offset = pack.read(); + file.size = pack.read(); + } + } + + ~VFS() { pack.close(); } + + bool readSync(const Path& path, OutputMemoryStream& content) override { + std::string basename = Path::getFilename(const_cast(path.get())); + PathHash hash = path.getHash(); + + auto i = packFiles.find(hash); + if (i == packFiles.end()) return false; + + content.resize(i->second.size); + MutexGuard lock(mutex); + + const size_t headerSize = sizeof(uint32_t) + packFiles.size() * (3 * sizeof(size_t)); + if (pack.seek(i->second.offset + headerSize) || !pack.read(content.dataMut(), content.size())) { + spdlog::error("Could not read file " + path.get() + " from the pack file."); + return false; + } + + return true; + } + + struct PackFile { + size_t offset; + size_t size; + }; + + std::map packFiles; + FileInput pack; + }; + + std::unique_ptr FileSystem::createDiskFS(std::string &basePath) { + return std::make_unique(basePath); + } + + std::unique_ptr FileSystem::createVFS(std::string& basePath) { + return std::make_unique(basePath); + } +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/assets/src/fs/file.h b/projs/shadow/shadow-engine/assets/src/fs/file.h index d3265cb7..9c6750a9 100644 --- a/projs/shadow/shadow-engine/assets/src/fs/file.h +++ b/projs/shadow/shadow-engine/assets/src/fs/file.h @@ -1,7 +1,11 @@ #pragma once #include +#include +#include #include +template struct Delegate; + namespace ShadowEngine { // An input stream that can read a file on disk. @@ -48,7 +52,6 @@ namespace ShadowEngine { std::string filename; }; - template struct Delegate; /** * A generic Filesystem API. @@ -68,9 +71,9 @@ namespace ShadowEngine { }; // Create a Filesystem that interacts with files on disk. - static std::unique_ptr& createDiskFS(std::string& basePath); + static std::unique_ptr createDiskFS(std::string& basePath); // Create a Virtual Filesystem based on the given path. - static std::unique_ptr& createVFS(std::string& basePath); + static std::unique_ptr createVFS(std::string& basePath); virtual ~FileSystem() {} @@ -90,7 +93,7 @@ namespace ShadowEngine { virtual bool deleteFile(std::string& path) = 0; // Get the path that this FileSystem originates at. The default is "/" for VFS, and whatever the Executable Path is for Disk FS. - virtual std::string& getBasePath() = 0; + virtual std::string const& getBasePath() const = 0; // Set a new base path for the FileSystem. Any operations involving file paths will be relative to this new path. virtual void setBasePath(std::string& path) = 0; @@ -100,9 +103,9 @@ namespace ShadowEngine { virtual bool hasWork() = 0; // Write new content to a file synchronously. The thread will be blocked when doing this. - virtual bool saveSync(const struct Path& file, const uint8_t* content) = 0; + virtual bool saveSync(const Path& file, const uint8_t* content, const size_t size) = 0; // Read content from a file synchronously. The thread will be blocked when doing this. - virtual bool readSync(const struct Path& file, struct OutputMemoryStream& content) = 0; + virtual bool readSync(const Path& file, struct OutputMemoryStream& content) = 0; // Read a file asynchronously. The given callback will be called with the file content once it is available. virtual AsyncHandle readAsync(const Path& file, const ContentCallback& callback) = 0; diff --git a/projs/shadow/shadow-engine/assets/src/fs/hash.h b/projs/shadow/shadow-engine/assets/src/fs/hash.h index 39e477e5..0b581878 100644 --- a/projs/shadow/shadow-engine/assets/src/fs/hash.h +++ b/projs/shadow/shadow-engine/assets/src/fs/hash.h @@ -59,14 +59,14 @@ namespace ShadowEngine { struct StableHash { static StableHash fromLong(size_t data); StableHash() = default; - StableHash(std::string& str); + explicit StableHash(std::string& str); StableHash(const void* data, uint32_t length); - bool operator!= (StableHash& other) const { return hash != other.hash; } - bool operator== (StableHash& other) const { return hash == other.hash; } - bool operator< (StableHash& other) const { return hash < other.hash; } + bool operator!= (const StableHash& other) const { return hash != other.hash; } + bool operator== (const StableHash& other) const { return hash == other.hash; } + bool operator< (const StableHash& other) const { return hash < other.hash; } - size_t getHash() const { return hash; } + [[nodiscard]] size_t getHash() const { return hash; } private: size_t hash = 0; diff --git a/projs/shadow/shadow-engine/assets/src/fs/iostream.h b/projs/shadow/shadow-engine/assets/src/fs/iostream.h index 6ac41a69..1c5c7c81 100644 --- a/projs/shadow/shadow-engine/assets/src/fs/iostream.h +++ b/projs/shadow/shadow-engine/assets/src/fs/iostream.h @@ -34,13 +34,14 @@ namespace ShadowEngine { // A custom OutputStream that writes to memory. struct OutputMemoryStream final : OutputStream { + OutputMemoryStream(); OutputMemoryStream(void* data, size_t size); OutputMemoryStream(OutputMemoryStream&& str) noexcept; OutputMemoryStream(const OutputMemoryStream& rhs) noexcept; ~OutputMemoryStream(); - void operator= (const OutputMemoryStream& rhs) noexcept; - void operator= (OutputMemoryStream&& rhs) noexcept; + OutputMemoryStream& operator= (const OutputMemoryStream& rhs) noexcept; + OutputMemoryStream& operator= (OutputMemoryStream&& rhs) noexcept; uint8_t operator[] (size_t index) const; uint8_t& operator[] (size_t index); @@ -79,6 +80,7 @@ namespace ShadowEngine { write(&v, sizeof(v)); } + // A custom InputStream that writes from memory. struct InputMemoryStream final : InputStream { InputMemoryStream(const void* data, size_t size); explicit InputMemoryStream(const OutputMemoryStream& blob); diff --git a/projs/shadow/shadow-engine/assets/src/fs/path.cpp b/projs/shadow/shadow-engine/assets/src/fs/path.cpp index 01bad0d6..8766231d 100644 --- a/projs/shadow/shadow-engine/assets/src/fs/path.cpp +++ b/projs/shadow/shadow-engine/assets/src/fs/path.cpp @@ -20,8 +20,9 @@ namespace ShadowEngine { path = str; } - void Path::operator=(const std::string &rhs) { + Path& Path::operator=(const std::string &rhs) { set(rhs); + return *this; } bool Path::operator==(const std::string &rhs) { diff --git a/projs/shadow/shadow-engine/assets/src/fs/path.h b/projs/shadow/shadow-engine/assets/src/fs/path.h index 8ce66288..5c943a21 100644 --- a/projs/shadow/shadow-engine/assets/src/fs/path.h +++ b/projs/shadow/shadow-engine/assets/src/fs/path.h @@ -47,7 +47,7 @@ namespace ShadowEngine { Path(); explicit Path(const std::string& str); - void operator=(const std::string& rhs); + Path& operator=(const std::string& rhs); bool operator==(const std::string& rhs); bool operator==(const Path& rhs); bool operator!=(const Path& rhs); @@ -55,10 +55,11 @@ namespace ShadowEngine { // Use this to set a new value into the path; it handles the hash too. void set(const std::string& path); - uint32_t length() const { return path.length(); }; - PathHash getHash() { return hash; } - const char* c_str() const { return path.data(); } - bool isEmpty() const { return path.length() == 0; } + [[nodiscard]] uint32_t length() const { return path.length(); }; + [[nodiscard]] PathHash getHash() const { return hash; } + [[nodiscard]] const char* c_str() const { return path.data(); } + [[nodiscard]] std::string const& get() const { return path; } + [[nodiscard]] bool isEmpty() const { return path.length() == 0; } private: std::string path; PathHash hash; diff --git a/projs/shadow/shadow-engine/assets/src/management/delegate.h b/projs/shadow/shadow-engine/assets/src/management/delegate.h new file mode 100644 index 00000000..5ef30c5e --- /dev/null +++ b/projs/shadow/shadow-engine/assets/src/management/delegate.h @@ -0,0 +1,92 @@ +#pragma once + +namespace ShadowEngine { + + template struct Delegate; + + /** + * A simple, generic callback template. + * A Delegate takes the form of a std::function that can be called, redirected and bound freely. + * Advantages over std::function includes the ability to reference types that aren't yet fully qualified. + * + * Idea taken from https://blog.molecular-matters.com/2011/09/19/generic-type-safe-delegates-and-events-in-c/ + * + * @tparam R the return type + * @tparam Args the arguments to the function + */ + template struct Delegate { + private: + using InstancePtr = void*; + using InternalFunction = R (*)(InstancePtr, Args...); + struct Stub { + InstancePtr first; + InternalFunction second; + }; + + template static R FunctionStub(InstancePtr, Args... args) { + return (Function)(args...); + } + + template + static R ClassMethodStub(InstancePtr instance, Args... args) { + return (static_cast(instance)->*Function)(args...); + } + + template + static R ClassMethodStub(InstancePtr instance, Args... args) { + return (static_cast(instance)->*Function)(args...); + } + + public: + Delegate() { + m_stub.first = nullptr; + m_stub.second = nullptr; + } + + template + Delegate(const T& obj) { + m_stub.first = (InstancePtr)&obj; + m_stub.second = [](InstancePtr inst, Args... args) -> R { + const T& obj = *(const T*)inst; + return obj(args...); + }; + } + + bool isValid() { return m_stub.second != nullptr; } + + template void bind() { + m_stub.first = nullptr; + m_stub.second = &FunctionStub; + } + + template void bind(C* instance) { + m_stub.first = instance; + m_stub.second = &ClassMethodStub; + } + + R invoke(Args... args) const { + return m_stub.second(m_stub.first, args...); + } + + bool operator==(const Delegate& rhs) { + return m_stub.first == rhs.m_stub.first && m_stub.second == rhs.m_stub.second; + } + + private: + Stub m_stub; + }; + + template struct ToDelegate_T; + template struct ToDelegate_T { + using Type = Delegate; + }; + + template using ToDelegate = typename ToDelegate_T::Type; + + template + auto makeDelegate(C* inst) { + ToDelegate res; + res.template bind(inst); + return res; + }; +} \ No newline at end of file From 413a37d13d7819e654d399c73253f83fbb60689b Mon Sep 17 00:00:00 2001 From: Curle Date: Mon, 13 Mar 2023 15:25:31 +0000 Subject: [PATCH 07/12] Resource and Resource Managers. --- .../shadow/shadow-engine/assets/src/fs/hash.h | 4 +- .../assets/src/management/delegate_list.h | 58 ++++++ .../shadow-assets/src/resource/Resource.cpp | 177 ++++++++++++++++ .../shadow-assets/src/resource/Resource.h | 128 ++++++++++++ .../src/resource/ResourceManager.cpp | 195 ++++++++++++++++++ .../src/resource/ResourceManager.h | 95 +++++++++ 6 files changed, 655 insertions(+), 2 deletions(-) create mode 100644 projs/shadow/shadow-engine/assets/src/management/delegate_list.h create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.cpp create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.h create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.cpp create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.h diff --git a/projs/shadow/shadow-engine/assets/src/fs/hash.h b/projs/shadow/shadow-engine/assets/src/fs/hash.h index 0b581878..f5b9f04f 100644 --- a/projs/shadow/shadow-engine/assets/src/fs/hash.h +++ b/projs/shadow/shadow-engine/assets/src/fs/hash.h @@ -19,8 +19,8 @@ namespace ShadowEngine { // Hash arbitrary data. HeapHash(const void* data, uint32_t length); - bool operator!= (HeapHash& other) const { return hash != other.hash; } - bool operator== (HeapHash& other) const { return hash == other.hash; } + bool operator!= (const HeapHash& other) const { return hash != other.hash; } + bool operator== (const HeapHash& other) const { return hash == other.hash; } size_t getHash() const { return hash; } private: diff --git a/projs/shadow/shadow-engine/assets/src/management/delegate_list.h b/projs/shadow/shadow-engine/assets/src/management/delegate_list.h new file mode 100644 index 00000000..42d85b37 --- /dev/null +++ b/projs/shadow/shadow-engine/assets/src/management/delegate_list.h @@ -0,0 +1,58 @@ +#pragma once +#include +#include +#include + +namespace ShadowEngine { + + template struct DelegateList; + + template struct DelegateList { + DelegateList() = default; + + template void bind(C* instance) { + Delegate cb; + cb.template bind(instance); + m_delegates.push_back(cb); + } + + template void bind() { + Delegate cb; + cb.template bind(); + m_delegates.push_back(cb); + } + + template void unbind() { + Delegate cb; + cb.template bind(); + for (int i = 0; i < m_delegates.size(); ++i) + { + if (m_delegates[i] == cb) + { + m_delegates.swapAndPop(i); + break; + } + } + } + + template void unbind(C* instance) { + Delegate cb; + cb.template bind(instance); + for (int i = 0; i < m_delegates.size(); ++i) + { + if (m_delegates[i] == cb) + { + m_delegates.swapAndPop(i); + break; + } + } + } + + void invoke(Args... args) { + for (uint32_t i = 0, c = m_delegates.size(); i < c; ++i) m_delegates[i].invoke(args...); + } + + private: + std::vector> m_delegates; + }; +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.cpp b/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.cpp new file mode 100644 index 00000000..409e68dc --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.cpp @@ -0,0 +1,177 @@ +#include +#include +#include + +namespace ShadowEngine { + + ResourceType::ResourceType(std::string& name) { + hash = HeapHash(name); + } + + Resource::Resource(const ShadowEngine::Path& path, ShadowEngine::ResourceTypeManager &manager) + : references(0), + emptyDependencies(0), + failedDependencies(0), + state(State::EMPTY), + desiredState(State::EMPTY), + path(path), + size(), + callback(), + manager(manager), + handle(FileSystem::AsyncHandle::invalid()) { + } + + Resource::~Resource() = default; + + void Resource::refresh() { + if (state == State::EMPTY) return; + + const State old = state; + state = State::EMPTY; + callback.invoke(old, state, *this); + checkState(); + } + + void Resource::checkState() { + State old = state; + if (failedDependencies > 0 && state != State::FAILED) { + state = State::FAILED; + } else if (failedDependencies == 0) { + if (emptyDependencies > 0 && state != State::EMPTY) + state = State::EMPTY; + + if (emptyDependencies == 0 && state != State::READY && desiredState != State::EMPTY) { + onReadying(); + + if (emptyDependencies != 0 || state == State::READY || desiredState == State::EMPTY) + return; + + if (failedDependencies != 0) { + checkState(); + return; + } + + state = State::READY; + } + } + callback.invoke(old, state, *this); + } + + void Resource::fileLoaded(size_t fileSize, const uint8_t *mem, bool success) { + handle = FileSystem::AsyncHandle::invalid(); + if (desiredState != State::READY) return; + + if (!success) { + ResourceManager& owner = getManager().getOwner(); + if (!hooked && owner.isHooked()) { + if (owner.onLoad(*this) == ResourceManager::LoadHook::Action::DEFERRED) { + hooked = true; + desiredState = State::READY; + increaseReferences(); + return; + } + } + + --emptyDependencies; + ++failedDependencies; + checkState(); + handle = FileSystem::AsyncHandle::invalid(); + return; + } + + const ResourceHeader* header = (const ResourceHeader*) mem; + + if (size < sizeof(*header)) { + spdlog::error("Invalid resource: ", path.get(), ": size mismatch. Expected ", fileSize, ", got " , sizeof(*header)); + failedDependencies++; + } else if (header->magic != ResourceHeader::MAGIC) { + spdlog::error("Invalid resource: " , path.get(), ": magic number mismatch. Expected " , ResourceHeader::MAGIC, ", got ", header->magic); + failedDependencies++; + } else if (header->version > 0) { + spdlog::error("Invalid resource: ", path.get(), ": verison mismatch. Expected 0, got ", header->version); + failedDependencies++; + } else { + // TODO: Compression? + if (!load(size - sizeof(*header), mem + sizeof(*header))) + failedDependencies++; + size = header->decompressedSize; + } + + emptyDependencies--; + checkState(); + handle = FileSystem::AsyncHandle::invalid(); + } + + void Resource::performUnload() { + if (handle.valid()) { + FileSystem& fs = manager.getOwner().getFileSystem(); + fs.cancelAsync(handle); + handle = FileSystem::AsyncHandle::invalid(); + } + + hooked = false; + desiredState = State::EMPTY; + unload(); + + size = 0; + emptyDependencies = 1; + failedDependencies = 0; + checkState(); + } + + void Resource::onCreated(ShadowEngine::Resource::State newState) { + state = newState; + desiredState = State::READY; + failedDependencies = state == State::FAILED ? 1 : 0; + emptyDependencies = 0; + } + + void Resource::doLoad() { + if (desiredState == State::READY) return; + desiredState = State::READY; + + if (handle.valid()) return; + + FileSystem& fs = manager.getOwner().getFileSystem(); + FileSystem::ContentCallback cb = makeDelegate<&Resource::fileLoaded>(this); + + const PathHash hash = path.getHash(); + Path resourcePath("./resources/" + std::to_string(hash.getHash()) + ".res"); + handle = fs.readAsync(resourcePath, cb); + } + + void Resource::addDependency(ShadowEngine::Resource &dependent) { + dependent.callback.bind<&Resource::stateChanged>(this); + if (dependent.isEmpty()) emptyDependencies++; + if (dependent.isFailure()) failedDependencies++; + + checkState(); + } + + void Resource::removeDependency(ShadowEngine::Resource &dependent) { + dependent.callback.unbind<&Resource::stateChanged>(this); + if (dependent.isEmpty()) --emptyDependencies; + if (dependent.isFailure()) --failedDependencies; + + checkState(); + } + + uint32_t Resource::decreaseReferences() { + --references; + if (references == 0 && manager.unloadEnabled) + performUnload(); + + return references; + } + + void Resource::stateChanged(ShadowEngine::Resource::State old, ShadowEngine::Resource::State newState, + ShadowEngine::Resource &) { + if (old == State::EMPTY) --emptyDependencies; + if (old == State::FAILED) --failedDependencies; + + if (newState == State::EMPTY) ++emptyDependencies; + if (newState == State::FAILED) ++failedDependencies; + + checkState(); + } +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.h b/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.h new file mode 100644 index 00000000..c91b2a46 --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.h @@ -0,0 +1,128 @@ +#pragma once + +#include "fs/hash.h" +#include "fs/path.h" +#include "fs/file.h" +#include + +namespace ShadowEngine { + + /** + * A runtime-only struct that determines the type of a resource - whether it be a texture, mesh, animation, or other data. + * Provides some specializations for living in a map. + */ + struct ResourceType { + ResourceType() = default; + explicit ResourceType(std::string& name); + bool operator!=(const ResourceType& o) const { return o.hash != hash; } + bool operator==(const ResourceType& o) const { return o.hash == hash; } + bool operator< (const ResourceType& o) const { return o.hash.getHash() < hash.getHash(); } + bool isValid() const { return hash.getHash() != 0; } + + HeapHash hash; + }; + + // A Resource Type that is guaranteed to be invalid. + const ResourceType INVALID_RESOURCE((std::string &) ""); + + // A specialization of HashFunc for ResourceTypes, since they already have a HeapHash within. + template<> struct HashFunc { + static uint32_t get(const ResourceType& key) { return HashFunc::get(key.hash); } + }; + +#pragma pack(1) + struct ResourceHeader { + static const uint32_t MAGIC = 'VXIP'; + uint32_t magic = MAGIC; // VXI Package header + uint32_t version = 0; + uint32_t flags = 0; + uint32_t padding = 0; + uint32_t decompressedSize = 0; + }; +#pragma pack() + + /** + * A basic Resource type. + * Represents a single file loaded from disk. + * May have dependencies on other Resources, and other Resources may depend on this. + * Resources are reference-counted, and are removed when they go out of usage. + */ + + struct Resource { + + friend struct ResourceTypeManager; + friend struct ResourceManager; + + enum class State : uint32_t { + EMPTY = 0, + READY, + FAILED + }; + + using Observer = DelegateList; + + virtual ~Resource(); + virtual ResourceType getType() const = 0; + State getState() const { return state; } + + bool isEmpty() const { return state == State::EMPTY; } + bool isReady() const { return state == State::READY; } + bool isFailure() const { return state == State::FAILED; } + + uint32_t getReferenceCount() const { return references; } + + Observer const& getCallback() const { return callback; } + size_t getSize() const { return size; } + + const Path& getPath() const { return path; } + + struct ResourceTypeManager& getManager() { return manager; } + + uint32_t decreaseReferences(); + uint32_t increaseReferences() { return references++; } + + bool toInitialize() const { return desiredState == State::READY; } + bool isHooked() const { return hooked; } + + template void onLoaded(C* instance) { + callback.bind(instance); + if (isReady()) (instance->*Function)(State::READY, State::READY, *this); + } + + protected: + Resource(const Path& path, ResourceTypeManager& manager); + + virtual void onReadying() {} + virtual void unload() = 0; + virtual bool load(size_t size, const uint8_t* mem) = 0; + + void onCreated(State newState); + void performUnload(); + void addDependency(Resource& dependent); + void removeDependency(Resource& dependent); + void checkState(); + void refresh(); + + State desiredState; + uint16_t emptyDependencies; + ResourceTypeManager& manager; + + private: + + void doLoad(); + void fileLoaded(size_t fileSize, const uint8_t* mem, bool success); + void stateChanged(State old, State newState, Resource&); + + Resource(const Resource&) = delete; + void operator=(const Resource&) = delete; + + Observer callback; + size_t size; + Path path; + uint32_t references; + uint16_t failedDependencies; + FileSystem::AsyncHandle handle; + State state; + bool hooked = false; + }; +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.cpp b/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.cpp new file mode 100644 index 00000000..8aa98bf4 --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.cpp @@ -0,0 +1,195 @@ +#include "ResourceManager.h" +#include "Resource.h" +#include "spdlog/spdlog.h" + +namespace ShadowEngine { + + void ResourceTypeManager::create(struct ResourceType type, struct ResourceManager &manager) { + manager.add(type, this); + owner = &manager; + } + + void ResourceTypeManager::destroy() { + for (auto iter = resources.begin(), end = resources.end(); iter != end; ++iter) { + Resource* res = iter->second; + if (!res->isEmpty()) + spdlog::error("Resource Type Manager destruction leaks ", res->path.get()); + + destroyResource(*res); + } + resources.clear(); + } + + Resource* ResourceTypeManager::get(const Path& path) { + auto it = resources.find(path.getHash()); + if (it != resources.end()) return it->second; + return nullptr; + } + + Resource* ResourceTypeManager::load(const Path &path) { + if (path.isEmpty()) return nullptr; + Resource* res = get(path); + if (res == nullptr) { + res = createResource(path); + resources[path.getHash()] = res; + } + + if (res->isEmpty() && res->desiredState == Resource::State::EMPTY) { + if (owner->onLoad(*res) == ResourceManager::LoadHook::Action::DEFERRED) { + res->hooked = true; + res->desiredState = Resource::State::READY; + res->increaseReferences(); + res->increaseReferences(); + return res; + } + + res->doLoad(); + } + + res->increaseReferences(); + return res; + } + + void ResourceTypeManager::removeUnreferencedResources() { + if (!unloadEnabled) return; + + std::vector toRemove; + for (auto i : resources) + if (i.second->getReferenceCount() == 0) toRemove.push_back(i.second); + + for (auto i : toRemove) { + auto iter = resources.find(i->getPath().getHash()); + if (iter->second->isReady()) iter->second->performUnload(); + } + } + + void ResourceTypeManager::reload(const Path &path) { + Resource* res = get(path); + if (res) reload(*res); + } + + void ResourceTypeManager::reload(Resource& res) { + if (res.state != Resource::State::EMPTY) + res.performUnload(); + else if (res.desiredState == Resource::State::READY) + return; + + if (owner->onLoad(res) == ResourceManager::LoadHook::Action::DEFERRED) { + res.hooked = true; + res.desiredState = Resource::State::READY; + res.increaseReferences(); + res.increaseReferences(); + } else { + res.performUnload(); + } + } + + void ResourceTypeManager::setUnloadable(bool status) { + unloadEnabled = status; + if (!unloadEnabled) return; + + for (auto res : resources) + if (res.second->getReferenceCount() == 0) + res.second->performUnload(); + } + + ResourceTypeManager::ResourceTypeManager() : + resources(), + owner(nullptr), + unloadEnabled(true) { + + } + + ResourceTypeManager::~ResourceTypeManager() { + + } + + ResourceManager::ResourceManager() : + managers(), + hook(nullptr), + filesystem(nullptr) { + + } + + ResourceManager::~ResourceManager() = default; + + void ResourceManager::init(FileSystem &fs) { + filesystem = &fs; + } + + Resource* ResourceManager::load(ResourceType type, const Path& path) { + ResourceTypeManager* manager = get(type); + if (!manager) return nullptr; + return load(*manager, path); + } + + Resource* ResourceManager::load(ResourceTypeManager& manager, const Path& path) { + return manager.load(path); + } + + ResourceTypeManager* ResourceManager::get(ResourceType type) { + auto iter = managers.find(type); + if (iter == managers.end()) return nullptr; + return iter->second; + } + + void ResourceManager::LoadHook::continueLoad(Resource &res) { + res.decreaseReferences(); + res.hooked = false; + res.desiredState = Resource::State::EMPTY; + res.doLoad(); + } + + void ResourceManager::setLoadHook(LoadHook *loadHook) { + hook = loadHook; + + if (hook) + for (auto manager : managers) + for (auto res : manager.second->getResources()) + if (res.second->isFailure()) + manager.second->reload(*res.second); + } + + ResourceManager::LoadHook::Action ResourceManager::onLoad(Resource &res) const { + return hook ? hook->load(res) : LoadHook::Action::IMMEDIATE; + } + + void ResourceManager::add(ResourceType type, ResourceTypeManager* manager) { + managers[type] = manager; + } + + void ResourceManager::remove(ResourceType type) { + managers.erase(type); + } + + void ResourceManager::removeUnreferenced() { + for (auto manager : managers) + manager.second->removeUnreferencedResources(); + } + + void ResourceManager::setUnloadable(bool enable) { + for (auto manager : managers) + manager.second->setUnloadable(enable); + } + + void ResourceManager::reloadAll() { + while (filesystem->hasWork()) filesystem->processCallbacks(); + + std::vector toReload; + for (auto manager : managers) { + ResourceTypeManager::ResourceTable& resources = manager.second->getResources(); + for (auto res : resources) { + if (res.second->isReady()) { + res.second->performUnload(); + toReload.push_back(res.second); + } + } + } + } + + void ResourceManager::reload(const Path& path) { + for (auto manager : managers) + manager.second->reload(path); + } + +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.h b/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.h new file mode 100644 index 00000000..98fc6b1e --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.h @@ -0,0 +1,95 @@ +#pragma once +#include +#include +#include + +namespace ShadowEngine { + + /** + * Handles all of the Resources of a single Type. + * Handles reference counting, hot reloading, and etc. + */ + + struct ResourceTypeManager { + friend struct Resource; + friend struct ResourceManager; + + using ResourceTable = std::map; + + void create(struct ResourceType type, struct ResourceManager& manager); + void destroy(); + + void setUnloadable(bool status); + + void removeUnreferencedResources(); + + void reload(const Path& path); + void reload(Resource& resource); + + ResourceTable& getResources() { return resources; } + + ResourceTypeManager(); + virtual ~ResourceTypeManager(); + ResourceManager& getOwner() const { return *owner; } + + protected: + Resource* load(const Path& path); + virtual Resource* createResource(const Path& path) = 0; + virtual void destroyResource(Resource& res) = 0; + Resource* get(const Path& path); + + ResourceTable resources; + ResourceManager* owner; + bool unloadEnabled; + }; + + /** + * Handles all of the ResourceTypeManagers, for every ResourceType with at least one applicable Resource + */ + + struct ResourceManager { + using ResourceTypeManagers = std::map; + + struct LoadHook { + enum class Action { IMMEDIATE, DEFERRED }; + virtual ~LoadHook(); + virtual Action load(Resource& res) = 0; + void continueLoad(Resource& res); + }; + + ResourceManager(); + ~ResourceManager(); + ResourceManager(const ResourceManager& o) = delete; + + void init(struct FileSystem& fs); + + ResourceTypeManager* get(ResourceType); + const ResourceTypeManagers& getAll() const { return managers; } + + template + R* load(const Path& path) { + return static_cast(load(R::TYPE, path)); + } + + Resource* load(ResourceTypeManager& manager, const Path& path); + Resource* load(ResourceType type, const Path& path); + + void setLoadHook(LoadHook* hook); + bool isHooked() const { return hook; } + LoadHook::Action onLoad(Resource& res) const; + void add(ResourceType, ResourceTypeManager* manager); + void remove(ResourceType type); + void reload(const Path& path); + void reloadAll(); + void removeUnreferenced(); + void setUnloadable(bool enable); + + FileSystem& getFileSystem() { return *filesystem; } + + private: + ResourceTypeManagers managers; + FileSystem* filesystem; + LoadHook* hook; + + }; +} \ No newline at end of file From 957b62f36737d8fd196a911dbd93c3913e1b01da Mon Sep 17 00:00:00 2001 From: Curle Date: Wed, 16 Aug 2023 19:43:53 +0100 Subject: [PATCH 08/12] Resource Fixes --- .../shadow-engine/assets/src/fs/file.cpp | 23 +++++- .../shadow/shadow-engine/assets/src/fs/file.h | 4 +- .../shadow-engine/assets/src/fs/iostream.cpp | 8 +- .../assets/src/management/synchronization.cpp | 73 +++++++++++++++++++ .../assets/src/management/synchronization.h | 2 +- .../shadow-assets/src/resource/Resource.cpp | 17 +++++ .../shadow-assets/src/resource/Resource.h | 14 +++- .../src/resource/ResourceManager.cpp | 1 - 8 files changed, 134 insertions(+), 8 deletions(-) create mode 100644 projs/shadow/shadow-engine/assets/src/management/synchronization.cpp diff --git a/projs/shadow/shadow-engine/assets/src/fs/file.cpp b/projs/shadow/shadow-engine/assets/src/fs/file.cpp index 6af00837..6b944c0b 100644 --- a/projs/shadow/shadow-engine/assets/src/fs/file.cpp +++ b/projs/shadow/shadow-engine/assets/src/fs/file.cpp @@ -1,13 +1,34 @@ #include #include #include "management/synchronization.h" -#include "../../../../../../cmake-build-debug/_deps/spdlog-src/include/spdlog/spdlog.h" +#include #include #include #include namespace ShadowEngine { + // Because fuck Linux? Need platform-specific source files! +#ifdef _WIN32 + +#define WIN32_LEAN_AND_MEAN +#include + + FileInput::FileInput() { + handle = (void*) INVALID_HANDLE_VALUE; + } + + FileOutput::FileOutput() { + error = false; + handle = (void*) INVALID_HANDLE_VALUE; + } + + bool FileOutput::open(std::string& path) { + + } + +#endif + /** * An async operation to be performed. * For reading files from disk into memory. diff --git a/projs/shadow/shadow-engine/assets/src/fs/file.h b/projs/shadow/shadow-engine/assets/src/fs/file.h index 9c6750a9..0bc97aeb 100644 --- a/projs/shadow/shadow-engine/assets/src/fs/file.h +++ b/projs/shadow/shadow-engine/assets/src/fs/file.h @@ -75,7 +75,7 @@ namespace ShadowEngine { // Create a Virtual Filesystem based on the given path. static std::unique_ptr createVFS(std::string& basePath); - virtual ~FileSystem() {} + virtual ~FileSystem() = default; // Open a file for reading. virtual bool open(std::string& path, FileInput& input) = 0; @@ -103,7 +103,7 @@ namespace ShadowEngine { virtual bool hasWork() = 0; // Write new content to a file synchronously. The thread will be blocked when doing this. - virtual bool saveSync(const Path& file, const uint8_t* content, const size_t size) = 0; + virtual bool saveSync(const Path& file, const uint8_t* content, size_t size) = 0; // Read content from a file synchronously. The thread will be blocked when doing this. virtual bool readSync(const Path& file, struct OutputMemoryStream& content) = 0; diff --git a/projs/shadow/shadow-engine/assets/src/fs/iostream.cpp b/projs/shadow/shadow-engine/assets/src/fs/iostream.cpp index 2c268a42..a8c1e93a 100644 --- a/projs/shadow/shadow-engine/assets/src/fs/iostream.cpp +++ b/projs/shadow/shadow-engine/assets/src/fs/iostream.cpp @@ -1,4 +1,5 @@ #include +#include namespace ShadowEngine { @@ -13,15 +14,16 @@ namespace ShadowEngine { str.free(); } - void OutputMemoryStream::operator=(ShadowEngine::OutputMemoryStream &&str) noexcept { + OutputMemoryStream& OutputMemoryStream::operator=(ShadowEngine::OutputMemoryStream &&str) noexcept { capacity = str.capacity; buffer = str.buffer; usage = str.usage; str.free(); + return *this; } - void OutputMemoryStream::operator=(const ShadowEngine::OutputMemoryStream &rhs) noexcept { + OutputMemoryStream& OutputMemoryStream::operator=(const ShadowEngine::OutputMemoryStream &rhs) noexcept { usage = rhs.usage; if (rhs.capacity > 0) { @@ -32,6 +34,8 @@ namespace ShadowEngine { buffer = nullptr; capacity = 0; } + + return *this; } OutputMemoryStream::OutputMemoryStream(const ShadowEngine::OutputMemoryStream &rhs) noexcept { diff --git a/projs/shadow/shadow-engine/assets/src/management/synchronization.cpp b/projs/shadow/shadow-engine/assets/src/management/synchronization.cpp new file mode 100644 index 00000000..985109e3 --- /dev/null +++ b/projs/shadow/shadow-engine/assets/src/management/synchronization.cpp @@ -0,0 +1,73 @@ +#include + +// This doesn't work on Linux. Sucks to be you? dpeter won't let me do system-specific source files. +#ifdef _WIN32 + +#include +#define WIN32_LEAN_AND_MEAN +#include + + +namespace ShadowEngine { struct NewPlaceholder {}; } +inline void* operator new(size_t, ShadowEngine::NewPlaceholder, void* where) { return where; } +inline void operator delete(void*, ShadowEngine::NewPlaceholder, void*) { } + +namespace ShadowEngine { + + Semaphore::Semaphore(int initCount, int maxCount) { + id = ::CreateSemaphore(nullptr, initCount, maxCount, nullptr); + } + + Semaphore::~Semaphore() { + ::CloseHandle(id); + } + + void Semaphore::raise() { + ::ReleaseSemaphore(id, 1, nullptr); + } + + void Semaphore::wait() { + ::WaitForSingleObject(id, INFINITE); + } + + ConditionVariable::ConditionVariable() { + memset(data, 0, sizeof(data)); + auto* var = new (NewPlaceholder(), data) CONDITION_VARIABLE; + InitializeConditionVariable(var); + } + + ConditionVariable::~ConditionVariable() { + ((CONDITION_VARIABLE*)data)->~CONDITION_VARIABLE(); + } + + void ConditionVariable::sleep(ShadowEngine::Mutex &mut) { + ::SleepConditionVariableSRW((CONDITION_VARIABLE*) data, (SRWLOCK*) mut.data, INFINITE, 0); + } + + void ConditionVariable::wake() { + ::WakeConditionVariable((CONDITION_VARIABLE*) data); + } + + Mutex::Mutex() { + memset(data, 0, sizeof(data)); + auto* lock = new (NewPlaceholder(), data) SRWLOCK; + ::InitializeSRWLock(lock); + } + + Mutex::~Mutex() { + auto* lock = (SRWLOCK*) data; + lock->~SRWLOCK(); + } + + void Mutex::enter() { + auto* lock = (SRWLOCK*) data; + ::AcquireSRWLockExclusive(lock); + } + + void Mutex::exit() { + auto* lock = (SRWLOCK*) data; + ::ReleaseSRWLockExclusive(lock); + } +} + +#endif \ No newline at end of file diff --git a/projs/shadow/shadow-engine/assets/src/management/synchronization.h b/projs/shadow/shadow-engine/assets/src/management/synchronization.h index 3d3278e7..364bf4fd 100644 --- a/projs/shadow/shadow-engine/assets/src/management/synchronization.h +++ b/projs/shadow/shadow-engine/assets/src/management/synchronization.h @@ -18,7 +18,7 @@ namespace ShadowEngine { void exit(); private: #ifdef _WIN32 - uint8_t data[8]; + uint8_t data[8] {}; #else pthread_mutex_t mutex; #endif diff --git a/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.cpp b/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.cpp index 409e68dc..46a2323c 100644 --- a/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.cpp +++ b/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.cpp @@ -174,4 +174,21 @@ namespace ShadowEngine { checkState(); } + + static std::string type("prefab"); + const ResourceType PrefabResource::TYPE(type); + + PrefabResource::PrefabResource(const ShadowEngine::Path &path, + ShadowEngine::ResourceTypeManager &resource_manager) : Resource(path, resource_manager) {} + + ResourceType PrefabResource::getType() const { return TYPE; } + + void PrefabResource::unload() { data.clear(); } + + bool PrefabResource::load(size_t size, const uint8_t *mem) { + data.resize(size); + memcpy(data.dataMut(), mem, size); + hash = StableHash(mem, size); + return true; + } } \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.h b/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.h index c91b2a46..a26af607 100644 --- a/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.h +++ b/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.h @@ -23,7 +23,8 @@ namespace ShadowEngine { }; // A Resource Type that is guaranteed to be invalid. - const ResourceType INVALID_RESOURCE((std::string &) ""); + static std::string empty; + const ResourceType INVALID_RESOURCE(empty); // A specialization of HashFunc for ResourceTypes, since they already have a HeapHash within. template<> struct HashFunc { @@ -125,4 +126,15 @@ namespace ShadowEngine { State state; bool hooked = false; }; + + struct PrefabResource : Resource { + PrefabResource(const Path& path, ResourceTypeManager& resource_manager); + ResourceType getType() const override; + void unload() override; + bool load(size_t size, const uint8_t* data) override; + + OutputMemoryStream data; + StableHash hash; + static const ResourceType TYPE; + }; } \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.cpp b/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.cpp index 8aa98bf4..163183d5 100644 --- a/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.cpp +++ b/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.cpp @@ -191,5 +191,4 @@ namespace ShadowEngine { for (auto manager : managers) manager.second->reload(path); } - } \ No newline at end of file From 231f9900c7845509455cf0853a3904bfeeb90af9 Mon Sep 17 00:00:00 2001 From: Curle Date: Sat, 6 Apr 2024 16:31:13 +0100 Subject: [PATCH 09/12] Move asset management, cleanup other files --- .../{src => inc/shadow/assets}/fs/file.h | 29 +- .../{src => inc/shadow/assets}/fs/hash.h | 8 +- .../{src => inc/shadow/assets}/fs/iostream.h | 0 .../{src => inc/shadow/assets}/fs/path.h | 20 +- .../{src => inc/shadow/assets}/fs/xxhash.h | 0 .../shadow/assets}/management/delegate.h | 0 .../shadow/assets}/management/delegate_list.h | 0 .../assets}/management/synchronization.h | 0 .../inc/shadow/assets/resource/Resource.h | 139 ++++++ .../shadow/assets}/resource/ResourceManager.h | 2 - .../shadow/assets/sff}/SFFElement.h | 70 +-- .../shadow/assets/sff}/SFFParser.h | 44 +- .../shadow/assets/sff}/SFFVersion.h | 54 +-- .../shadow/assets/sff}/SFFWriter.h | 52 +-- .../{src => inc/shadow/assets}/str/string.h | 0 .../shadow-engine/assets/src/fs/file.cpp | 411 ++++++++++-------- .../shadow-engine/assets/src/fs/hash.cpp | 288 ++++++------ .../shadow-engine/assets/src/fs/iostream.cpp | 3 +- .../shadow-engine/assets/src/fs/path.cpp | 27 +- .../assets/src/management/synchronization.cpp | 3 +- .../assets/src/resource/Resource.cpp | 198 +++++++++ .../src/resource/ResourceManager.cpp | 6 +- .../shadow-engine/assets/src/str/string.cpp | 4 +- .../core/inc/shadow/core/Module.h | 16 - .../core/inc/shadow/core/ShadowApplication.h | 8 +- .../core/inc/shadow/core/ShadowWindow.h | 4 - .../core/inc/shadow/event-bus/events.h | 8 - .../core/src/core/ShadowApplication.cpp | 20 +- .../core/src/event_bus/event_bus.cpp | 8 - .../shadow-assets/src/resource/Resource.cpp | 194 --------- .../shadow-assets/src/resource/Resource.h | 140 ------ .../utility/inc/shadow/util/string-helpers.h | 14 + .../utility/src/string-helpers.cpp | 74 ++++ 33 files changed, 976 insertions(+), 868 deletions(-) rename projs/shadow/shadow-engine/assets/{src => inc/shadow/assets}/fs/file.h (80%) rename projs/shadow/shadow-engine/assets/{src => inc/shadow/assets}/fs/hash.h (96%) rename projs/shadow/shadow-engine/assets/{src => inc/shadow/assets}/fs/iostream.h (100%) rename projs/shadow/shadow-engine/assets/{src => inc/shadow/assets}/fs/path.h (74%) rename projs/shadow/shadow-engine/assets/{src => inc/shadow/assets}/fs/xxhash.h (100%) rename projs/shadow/shadow-engine/assets/{src => inc/shadow/assets}/management/delegate.h (100%) rename projs/shadow/shadow-engine/assets/{src => inc/shadow/assets}/management/delegate_list.h (100%) rename projs/shadow/shadow-engine/assets/{src => inc/shadow/assets}/management/synchronization.h (100%) create mode 100644 projs/shadow/shadow-engine/assets/inc/shadow/assets/resource/Resource.h rename projs/shadow/shadow-engine/{shadow-assets/src => assets/inc/shadow/assets}/resource/ResourceManager.h (98%) rename projs/shadow/shadow-engine/assets/{src => inc/shadow/assets/sff}/SFFElement.h (93%) rename projs/shadow/shadow-engine/assets/{src => inc/shadow/assets/sff}/SFFParser.h (93%) rename projs/shadow/shadow-engine/assets/{src => inc/shadow/assets/sff}/SFFVersion.h (92%) rename projs/shadow/shadow-engine/assets/{src => inc/shadow/assets/sff}/SFFWriter.h (92%) rename projs/shadow/shadow-engine/assets/{src => inc/shadow/assets}/str/string.h (100%) create mode 100644 projs/shadow/shadow-engine/assets/src/resource/Resource.cpp rename projs/shadow/shadow-engine/{shadow-assets => assets}/src/resource/ResourceManager.cpp (97%) delete mode 100644 projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.cpp delete mode 100644 projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.h diff --git a/projs/shadow/shadow-engine/assets/src/fs/file.h b/projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/file.h similarity index 80% rename from projs/shadow/shadow-engine/assets/src/fs/file.h rename to projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/file.h index 0bc97aeb..5a8947a6 100644 --- a/projs/shadow/shadow-engine/assets/src/fs/file.h +++ b/projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/file.h @@ -1,8 +1,7 @@ #pragma once -#include -#include -#include #include +#include "iostream.h" +#include "path.h" template struct Delegate; @@ -13,7 +12,7 @@ namespace ShadowEngine { FileInput(); ~FileInput() = default; - [[nodiscard]] bool open(std::string& path); + [[nodiscard]] bool open(const std::string& path); void close(); using InputStream::read; @@ -34,7 +33,7 @@ namespace ShadowEngine { FileOutput(); ~FileOutput() = default; - [[nodiscard]] bool open(std::string& path); + [[nodiscard]] bool open(const std::string& path); void close(); void flush(); bool errored() const { return error; } @@ -71,31 +70,31 @@ namespace ShadowEngine { }; // Create a Filesystem that interacts with files on disk. - static std::unique_ptr createDiskFS(std::string& basePath); + static std::unique_ptr createDiskFS(const std::string& basePath); // Create a Virtual Filesystem based on the given path. - static std::unique_ptr createVFS(std::string& basePath); + static std::unique_ptr createVFS(const std::string& basePath); virtual ~FileSystem() = default; // Open a file for reading. - virtual bool open(std::string& path, FileInput& input) = 0; + virtual bool open(const std::string& path, FileInput& input) = 0; // Open a file for writing. - virtual bool open(std::string& path, FileOutput& output) = 0; + virtual bool open(const std::string& path, FileOutput& output) = 0; // Check whether a file exists at the given path. - virtual bool fileExists(std::string& path) = 0; + virtual bool fileExists(const std::string& path) = 0; // Get the time a file at the given path was last modified. - virtual size_t getLastModified(std::string& path) = 0; + virtual size_t getLastModified(const std::string& path) = 0; // Copy a file from one path to another. - virtual bool copyFile(std::string& from, std::string& to) = 0; + virtual bool copyFile(const std::string& from, const std::string& to) = 0; // Move a file from one path to another. - virtual bool moveFile(std::string& from, std::string& to) = 0; + virtual bool moveFile(const std::string& from, const std::string& to) = 0; // Disassociate any files at the given path (not an immediate delete) - virtual bool deleteFile(std::string& path) = 0; + virtual bool deleteFile(const std::string& path) = 0; // Get the path that this FileSystem originates at. The default is "/" for VFS, and whatever the Executable Path is for Disk FS. virtual std::string const& getBasePath() const = 0; // Set a new base path for the FileSystem. Any operations involving file paths will be relative to this new path. - virtual void setBasePath(std::string& path) = 0; + virtual void setBasePath(const std::string& path) = 0; // Process all the callbacks for async file operations. virtual void processCallbacks() = 0; diff --git a/projs/shadow/shadow-engine/assets/src/fs/hash.h b/projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/hash.h similarity index 96% rename from projs/shadow/shadow-engine/assets/src/fs/hash.h rename to projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/hash.h index f5b9f04f..e1abd33c 100644 --- a/projs/shadow/shadow-engine/assets/src/fs/hash.h +++ b/projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/hash.h @@ -15,7 +15,7 @@ namespace ShadowEngine { HeapHash() = default; // Hash a string; for paths and such. - explicit HeapHash(std::string& str); + explicit HeapHash(const std::string& str); // Hash arbitrary data. HeapHash(const void* data, uint32_t length); @@ -39,7 +39,7 @@ namespace ShadowEngine { HeapHash32() = default; // Hash a string; for paths and such. - explicit HeapHash32(std::string& str); + explicit HeapHash32(const std::string& str); // Hash arbitrary data. HeapHash32(const void* data, uint32_t length); @@ -59,7 +59,7 @@ namespace ShadowEngine { struct StableHash { static StableHash fromLong(size_t data); StableHash() = default; - explicit StableHash(std::string& str); + explicit StableHash(const std::string& str); StableHash(const void* data, uint32_t length); bool operator!= (const StableHash& other) const { return hash != other.hash; } @@ -80,7 +80,7 @@ namespace ShadowEngine { struct StableHash32 { static StableHash32 fromInt(uint32_t data); StableHash32() = default; - StableHash32(std::string& str); + StableHash32(const std::string& str); StableHash32(const void* data, uint32_t length); bool operator!= (StableHash32& other) const { return hash != other.hash; } diff --git a/projs/shadow/shadow-engine/assets/src/fs/iostream.h b/projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/iostream.h similarity index 100% rename from projs/shadow/shadow-engine/assets/src/fs/iostream.h rename to projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/iostream.h diff --git a/projs/shadow/shadow-engine/assets/src/fs/path.h b/projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/path.h similarity index 74% rename from projs/shadow/shadow-engine/assets/src/fs/path.h rename to projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/path.h index 5c943a21..47f9cd6b 100644 --- a/projs/shadow/shadow-engine/assets/src/fs/path.h +++ b/projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/path.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include "hash.h" namespace ShadowEngine { /** @@ -9,7 +9,7 @@ namespace ShadowEngine { * Not to be used as a replacement for the Path class. */ struct PathInfo { - explicit PathInfo(std::string& str); + explicit PathInfo(const std::string& str); char extension[10]; char baseName[256]; @@ -28,21 +28,21 @@ namespace ShadowEngine { // Make sure the path is valid. // Always from the root. // One slash separating. - static std::string normalise(std::string& path); + static std::string normalise(const std::string& path); // Get the prelude of the given path. - static std::string getPrelude(std::string& path); + static std::string getPrelude(const std::string& path); // Get the domain of the given path. - static std::string getDomain(std::string& path); + static std::string getDomain(const std::string& path); // Get the directory of the given path. - static std::string getDirectory(std::string& path); + static std::string getDirectory(const std::string& path); // Get the name of the file of the given path. - static std::string getFilename(std::string& path); + static std::string getFilename(const std::string& path); // Get the file extension of the given path. - static std::string getExtension(std::string& path); + static std::string getExtension(const std::string& path); // Check if the path has the given extension. - static bool hasExtension(std::string& path, std::string& ext); + static bool hasExtension(const std::string& path, const std::string& ext); // Replace the extension of the given path. - static std::string replaceExtension(std::string& path, std::string& newExt); + static std::string replaceExtension(const std::string& path, const std::string& newExt); Path(); explicit Path(const std::string& str); diff --git a/projs/shadow/shadow-engine/assets/src/fs/xxhash.h b/projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/xxhash.h similarity index 100% rename from projs/shadow/shadow-engine/assets/src/fs/xxhash.h rename to projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/xxhash.h diff --git a/projs/shadow/shadow-engine/assets/src/management/delegate.h b/projs/shadow/shadow-engine/assets/inc/shadow/assets/management/delegate.h similarity index 100% rename from projs/shadow/shadow-engine/assets/src/management/delegate.h rename to projs/shadow/shadow-engine/assets/inc/shadow/assets/management/delegate.h diff --git a/projs/shadow/shadow-engine/assets/src/management/delegate_list.h b/projs/shadow/shadow-engine/assets/inc/shadow/assets/management/delegate_list.h similarity index 100% rename from projs/shadow/shadow-engine/assets/src/management/delegate_list.h rename to projs/shadow/shadow-engine/assets/inc/shadow/assets/management/delegate_list.h diff --git a/projs/shadow/shadow-engine/assets/src/management/synchronization.h b/projs/shadow/shadow-engine/assets/inc/shadow/assets/management/synchronization.h similarity index 100% rename from projs/shadow/shadow-engine/assets/src/management/synchronization.h rename to projs/shadow/shadow-engine/assets/inc/shadow/assets/management/synchronization.h diff --git a/projs/shadow/shadow-engine/assets/inc/shadow/assets/resource/Resource.h b/projs/shadow/shadow-engine/assets/inc/shadow/assets/resource/Resource.h new file mode 100644 index 00000000..5ac64114 --- /dev/null +++ b/projs/shadow/shadow-engine/assets/inc/shadow/assets/resource/Resource.h @@ -0,0 +1,139 @@ +#pragma once + +#include "shadow/assets/fs/hash.h" +#include "shadow/assets/fs/path.h" +#include "shadow/assets/fs/file.h" +#include + +namespace ShadowEngine { + + /** + * A runtime-only struct that determines the type of a resource - whether it be a texture, mesh, animation, or other data. + * Provides some specializations for living in a map. + */ + struct ResourceType { + ResourceType() = default; + explicit ResourceType(const std::string& name); + bool operator!=(const ResourceType& o) const { return o.hash != hash; } + bool operator==(const ResourceType& o) const { return o.hash == hash; } + bool operator< (const ResourceType& o) const { return o.hash.getHash() < hash.getHash(); } + bool isValid() const { return hash.getHash() != 0; } + + HeapHash hash; + }; + + // A Resource Type that is guaranteed to be invalid. + static std::string empty; + const ResourceType INVALID_RESOURCE(empty); + + // A specialization of HashFunc for ResourceTypes, since they already have a HeapHash within. + template<> struct HashFunc { + static uint32_t get(const ResourceType& key) { return HashFunc::get(key.hash); } + }; + +#pragma pack(1) + struct ResourceHeader { + static const uint32_t MAGIC; + uint32_t magic = MAGIC; // VXI Package header + uint32_t version = 0; + uint32_t flags = 0; + uint32_t padding = 0; + uint32_t decompressedSize = 0; + }; +#pragma pack() + + /** + * A basic Resource type. + * Represents a single file loaded from disk. + * May have dependencies on other Resources, and other Resources may depend on this. + * Resources are reference-counted, and are removed when they go out of usage. + */ + + struct Resource { + friend struct ResourceTypeManager; + friend struct ResourceManager; + + enum class State : uint32_t { + EMPTY = 0, + READY, + FAILED + }; + + using Observer = DelegateList; + + virtual ~Resource(); + virtual ResourceType getType() const = 0; + State getState() const { return state; } + + bool isEmpty() const { return state == State::EMPTY; } + bool isReady() const { return state == State::READY; } + bool isFailure() const { return state == State::FAILED; } + + uint32_t getReferenceCount() const { return references; } + + Observer const& getCallback() const { return callback; } + size_t getSize() const { return size; } + + const Path& getPath() const { return path; } + + struct ResourceTypeManager& getManager() { return manager; } + + uint32_t decreaseReferences(); + uint32_t increaseReferences() { return references++; } + + bool toInitialize() const { return desiredState == State::READY; } + bool isHooked() const { return hooked; } + + template void onLoaded(C* instance) { + callback.bind(instance); + if (isReady()) (instance->*Function)(State::READY, State::READY, *this); + } + + protected: + Resource(Path path, ResourceTypeManager& manager); + + virtual void onReadying() {} + virtual void unload() = 0; + virtual bool load(size_t size, const uint8_t* mem) = 0; + + void onCreated(State newState); + void performUnload(); + void addDependency(Resource& dependent); + void removeDependency(Resource& dependent); + void checkState(); + void refresh(); + + State desiredState; + uint16_t emptyDependencies; + ResourceTypeManager& manager; + + private: + + void doLoad(); + void fileLoaded(size_t fileSize, const uint8_t* mem, bool success); + void stateChanged(State old, State newState, Resource&); + + Resource(const Resource&) = delete; + void operator=(const Resource&) = delete; + + Observer callback; + size_t size; + Path path; + uint32_t references; + uint16_t failedDependencies; + FileSystem::AsyncHandle handle; + State state; + bool hooked = false; + }; + + struct PrefabResource : Resource { + PrefabResource(const Path& path, ResourceTypeManager& resource_manager); + ResourceType getType() const override; + void unload() override; + bool load(size_t size, const uint8_t* data) override; + + OutputMemoryStream data; + StableHash hash; + static const ResourceType TYPE; + }; +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.h b/projs/shadow/shadow-engine/assets/inc/shadow/assets/resource/ResourceManager.h similarity index 98% rename from projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.h rename to projs/shadow/shadow-engine/assets/inc/shadow/assets/resource/ResourceManager.h index 98fc6b1e..6acb5c1c 100644 --- a/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.h +++ b/projs/shadow/shadow-engine/assets/inc/shadow/assets/resource/ResourceManager.h @@ -1,7 +1,5 @@ #pragma once #include -#include -#include namespace ShadowEngine { diff --git a/projs/shadow/shadow-engine/assets/src/SFFElement.h b/projs/shadow/shadow-engine/assets/inc/shadow/assets/sff/SFFElement.h similarity index 93% rename from projs/shadow/shadow-engine/assets/src/SFFElement.h rename to projs/shadow/shadow-engine/assets/inc/shadow/assets/sff/SFFElement.h index b3bbe88c..bf8946dc 100644 --- a/projs/shadow/shadow-engine/assets/src/SFFElement.h +++ b/projs/shadow/shadow-engine/assets/inc/shadow/assets/sff/SFFElement.h @@ -1,36 +1,36 @@ -#pragma once - -#include -#include -#include - - - namespace Shadow::SFF { - - class SFFElement - { - public: - SFFElement* parent; - - std::string name; - - bool isBlock; - - std::string value; - typedef std::map ChildrenMap; - - ChildrenMap children; - - std::string GetStringProperty(std::string name); - - SFFElement* GetFirstChild(); - - SFFElement* GetChildByIndex(int index); - - SFFElement* GetChildByName(std::string name); - - ~SFFElement(); - - }; - +#pragma once + +#include +#include +#include + + + namespace Shadow::SFF { + + class SFFElement + { + public: + SFFElement* parent; + + std::string name; + + bool isBlock; + + std::string value; + typedef std::map ChildrenMap; + + ChildrenMap children; + + std::string GetStringProperty(std::string name); + + SFFElement* GetFirstChild(); + + SFFElement* GetChildByIndex(int index); + + SFFElement* GetChildByName(std::string name); + + ~SFFElement(); + + }; + } \ No newline at end of file diff --git a/projs/shadow/shadow-engine/assets/src/SFFParser.h b/projs/shadow/shadow-engine/assets/inc/shadow/assets/sff/SFFParser.h similarity index 93% rename from projs/shadow/shadow-engine/assets/src/SFFParser.h rename to projs/shadow/shadow-engine/assets/inc/shadow/assets/sff/SFFParser.h index 542e3dec..d6aa8a3a 100644 --- a/projs/shadow/shadow-engine/assets/src/SFFParser.h +++ b/projs/shadow/shadow-engine/assets/inc/shadow/assets/sff/SFFParser.h @@ -1,22 +1,22 @@ -#pragma once - -#include -#include - -#include "SFFElement.h" -#include "SFFVersion.h" - -namespace Shadow::SFF { - - class SFFParser - { - public: - - static SFFElement* ReadFromStream(std::istream& stream); - - static SFFVersion ReadVersionFromHeader(std::istream& stream); - - static SFFElement* ReadFromFile(std::string path); - }; - -} +#pragma once + +#include +#include + +#include "SFFElement.h" +#include "SFFVersion.h" + +namespace Shadow::SFF { + + class SFFParser + { + public: + + static SFFElement* ReadFromStream(std::istream& stream); + + static SFFVersion ReadVersionFromHeader(std::istream& stream); + + static SFFElement* ReadFromFile(std::string path); + }; + +} diff --git a/projs/shadow/shadow-engine/assets/src/SFFVersion.h b/projs/shadow/shadow-engine/assets/inc/shadow/assets/sff/SFFVersion.h similarity index 92% rename from projs/shadow/shadow-engine/assets/src/SFFVersion.h rename to projs/shadow/shadow-engine/assets/inc/shadow/assets/sff/SFFVersion.h index 2afff832..136b0bf6 100644 --- a/projs/shadow/shadow-engine/assets/src/SFFVersion.h +++ b/projs/shadow/shadow-engine/assets/inc/shadow/assets/sff/SFFVersion.h @@ -1,28 +1,28 @@ -#pragma once - -namespace Shadow::SFF { - - struct SFFVersion { - public: - int mayor; - int minor; - int patch; - - bool invalid; - - SFFVersion(int ma, int mi, int pa) - { - this->mayor = ma; - this->minor = mi; - this->patch = pa; - - if (ma >= 0 && mi >= 0 && pa >= 0) { - this->invalid = false; - } - else - { - invalid = true; - } - } - }; +#pragma once + +namespace Shadow::SFF { + + struct SFFVersion { + public: + int mayor; + int minor; + int patch; + + bool invalid; + + SFFVersion(int ma, int mi, int pa) + { + this->mayor = ma; + this->minor = mi; + this->patch = pa; + + if (ma >= 0 && mi >= 0 && pa >= 0) { + this->invalid = false; + } + else + { + invalid = true; + } + } + }; } \ No newline at end of file diff --git a/projs/shadow/shadow-engine/assets/src/SFFWriter.h b/projs/shadow/shadow-engine/assets/inc/shadow/assets/sff/SFFWriter.h similarity index 92% rename from projs/shadow/shadow-engine/assets/src/SFFWriter.h rename to projs/shadow/shadow-engine/assets/inc/shadow/assets/sff/SFFWriter.h index 31b254f4..2ee034ac 100644 --- a/projs/shadow/shadow-engine/assets/src/SFFWriter.h +++ b/projs/shadow/shadow-engine/assets/inc/shadow/assets/sff/SFFWriter.h @@ -1,26 +1,26 @@ -#pragma once - -#include -#include -#include - -#include "SFFElement.h" -#include "SFFVersion.h" - -namespace Shadow::SFF { - - class SFFWriter - { - public: - - static void WriteFile(SFFElement& root, std::string path); - - static void WriteElement(std::ostream& w, SFFElement& e, int &depth); - - - - }; - -} - - +#pragma once + +#include +#include +#include + +#include "SFFElement.h" +#include "SFFVersion.h" + +namespace Shadow::SFF { + + class SFFWriter + { + public: + + static void WriteFile(SFFElement& root, std::string path); + + static void WriteElement(std::ostream& w, SFFElement& e, int &depth); + + + + }; + +} + + diff --git a/projs/shadow/shadow-engine/assets/src/str/string.h b/projs/shadow/shadow-engine/assets/inc/shadow/assets/str/string.h similarity index 100% rename from projs/shadow/shadow-engine/assets/src/str/string.h rename to projs/shadow/shadow-engine/assets/inc/shadow/assets/str/string.h diff --git a/projs/shadow/shadow-engine/assets/src/fs/file.cpp b/projs/shadow/shadow-engine/assets/src/fs/file.cpp index 6b944c0b..ccda9a3b 100644 --- a/projs/shadow/shadow-engine/assets/src/fs/file.cpp +++ b/projs/shadow/shadow-engine/assets/src/fs/file.cpp @@ -1,17 +1,20 @@ -#include +#include #include -#include "management/synchronization.h" #include #include -#include #include +#include "shadow/assets/fs/iostream.h" +#include "shadow/assets/fs/path.h" +#include "shadow/assets/fs/hash.h" +#include +#include namespace ShadowEngine { - // Because fuck Linux? Need platform-specific source files! + // Because fuck Linux? Need platform-specific source files! #ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN + #define WIN32_LEAN_AND_MEAN #include FileInput::FileInput() { @@ -23,243 +26,299 @@ namespace ShadowEngine { handle = (void*) INVALID_HANDLE_VALUE; } - bool FileOutput::open(std::string& path) { + bool FileOutput::open(const std::string& path) { + handle = (HANDLE) CreateFile(path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + return INVALID_HANDLE_VALUE != handle; + } + bool FileInput::open(const std::string& path) { + handle = (HANDLE) CreateFile(path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + return INVALID_HANDLE_VALUE != handle; } -#endif + void FileInput::close() { + if (INVALID_HANDLE_VALUE != (HANDLE) handle) { + CloseHandle((HANDLE) handle); + handle = (void*) INVALID_HANDLE_VALUE; + } + } - /** - * An async operation to be performed. - * For reading files from disk into memory. - */ + void FileOutput::close() { + if (INVALID_HANDLE_VALUE != (HANDLE) handle) { + CloseHandle((HANDLE) handle); + handle = (void*) INVALID_HANDLE_VALUE; + } + } + + size_t FileInput::size() const { + assert(INVALID_HANDLE_VALUE != handle); + return GetFileSize((HANDLE) handle, 0); + } - struct AsyncRead { - enum class Flags : uint32_t { - FAILED = 0, // The read failed due to some error. - CANCELLED // The read was cancelled due to the resource not being needed any more. - }; + size_t FileInput::pos() { + assert(INVALID_HANDLE_VALUE != handle); + return SetFilePointer((HANDLE) handle, 0, nullptr, FILE_CURRENT); + } + + bool FileInput::seek(size_t pos) { + assert(INVALID_HANDLE_VALUE != handle); + LARGE_INTEGER distance; + distance.QuadPart = pos; + return SetFilePointer((HANDLE) handle, distance.u.LowPart, &distance.u.HighPart, FILE_BEGIN) != INVALID_SET_FILE_POINTER; + } + + bool FileInput::read(void* data, size_t size) { + assert(INVALID_HANDLE_VALUE != handle); + DWORD read = 0; + BOOL success = ReadFile((HANDLE) handle, data, (DWORD) size, (LPDWORD) &read, nullptr); + return success && size == read; + } + + void FileOutput::flush() { + assert(handle != nullptr); + FlushFileBuffers((HANDLE) handle); + } + + bool FileOutput::write(const void* data, size_t size) { + assert(handle != INVALID_HANDLE_VALUE); + size_t written = 0; + WriteFile((HANDLE) handle, data, (DWORD) size, (LPDWORD) &written, nullptr); + error = error | size != written; + return !error; + } - AsyncRead() : data() {} +#endif - bool isFailed() const { return flags == Flags::FAILED; } - bool isCancelled() const { return flags == Flags::CANCELLED; } + /** + * An async operation to be performed. + * For reading files from disk into memory. + */ - FileSystem::ContentCallback callback; - OutputMemoryStream data; - std::string path; - uint32_t id = 0; - Flags flags; + struct AsyncRead { + enum class Flags : uint32_t { + FAILED = 0, // The read failed due to some error. + CANCELLED // The read was cancelled due to the resource not being needed any more. }; - // The FileSystem that operates on raw on-disk files. - struct DiskFS; + AsyncRead() : data() {} - struct DiskFS : FileSystem { + bool isFailed() const { return flags == Flags::FAILED; } + bool isCancelled() const { return flags == Flags::CANCELLED; } - explicit DiskFS(std::string& path) : sem(0, 0xffff) { - setBasePath(path); - } + FileSystem::ContentCallback callback; + OutputMemoryStream data; + std::string path; + uint32_t id = 0; + Flags flags; + }; - bool hasWork() override { - return workCounter != 0; - } + // The FileSystem that operates on raw on-disk files. + struct DiskFS; - std::string const& getBasePath() const override { return basePath; } - void setBasePath(std::string& path) final { - basePath = Path::normalise(path); - if (!basePath.ends_with('/') && !basePath.ends_with('\\')) - basePath.append("/"); - } + struct DiskFS : FileSystem { - bool saveSync(const Path& path, const uint8_t* data, const size_t size) override { - FileOutput file; - std::string fullPath(basePath.append(path.c_str())); + explicit DiskFS(const std::string& path) : sem(0, 0xffff) { + setBasePath(path); + } - if (!file.open(fullPath)) return false; - bool res = file.write(data, size); - file.close(); + bool hasWork() override { + return workCounter != 0; + } - return res; - } + std::string const& getBasePath() const override { return basePath; } + void setBasePath(const std::string& path) final { + basePath = Path::normalise(path); + if (!basePath.ends_with('/') && !basePath.ends_with('\\')) + basePath.append("/"); + } - bool readSync(const Path& path, struct OutputMemoryStream& content) override { - FileInput file; - std::string fullPath(basePath.append(path.c_str())); + bool saveSync(const Path& path, const uint8_t* data, const size_t size) override { + FileOutput file; + std::string fullPath(basePath.append(path.c_str())); - if (!file.open(fullPath)) return false; + if (!file.open(fullPath)) return false; + bool res = file.write(data, size); + file.close(); - content.resize(file.size()); - if (!file.read(content.dataMut(), content.size())) { - file.close(); - return false; - } + return res; + } + + bool readSync(const Path& path, struct OutputMemoryStream& content) override { + FileInput file; + std::string fullPath(basePath.append(path.c_str())); + + if (!file.open(fullPath)) return false; + content.resize(file.size()); + if (!file.read(content.dataMut(), content.size())) { file.close(); - return true; + return false; } - AsyncHandle readAsync(const Path& file, const ContentCallback& callback) override { - if (!file.isEmpty()) return AsyncHandle::invalid(); + file.close(); + return true; + } - MutexGuard lock(mutex); - workCounter++; + AsyncHandle readAsync(const Path& file, const ContentCallback& callback) override { + if (!file.isEmpty()) return AsyncHandle::invalid(); - AsyncRead& read = queue.emplace_back(); - if (++lastID == 0) lastID++; + ShadowEngine::MutexGuard lock(mutex); + workCounter++; - read.id = lastID; - read.path = file.c_str(); - read.callback = callback; - sem.raise(); + AsyncRead& read = queue.emplace_back(); + if (++lastID == 0) lastID++; - return AsyncHandle(read.id); - } + read.id = lastID; + read.path = file.c_str(); + read.callback = callback; + sem.raise(); - void cancelAsync(AsyncHandle& handle) override { - MutexGuard lock(mutex); + return AsyncHandle(read.id); + } - for (AsyncRead& read : queue) { - if (read.id == handle.value) { - read.flags = AsyncRead::Flags::CANCELLED; - workCounter--; - return; - } - } + void cancelAsync(AsyncHandle& handle) override { + MutexGuard lock(mutex); - for (AsyncRead& read : finished) { - if (read.id == handle.value) { - read.flags = AsyncRead::Flags::CANCELLED; - return; - } + for (AsyncRead& read : queue) { + if (read.id == handle.value) { + read.flags = AsyncRead::Flags::CANCELLED; + workCounter--; + return; } } - bool open(std::string& path, FileInput& file) override { - return file.open(basePath.append(path)); - } - - bool open(std::string& path, FileOutput& file) override { - return file.open(basePath.append(path)); + for (AsyncRead& read : finished) { + if (read.id == handle.value) { + read.flags = AsyncRead::Flags::CANCELLED; + return; + } } + } - bool deleteFile(std::string& path) override { - return std::remove((basePath.append(path).c_str())); - } + bool open(const std::string& path, FileInput& file) override { + return file.open(basePath.append(path)); + } - bool moveFile(std::string& from, std::string& to) override { - try { - std::rename(basePath.append(from).c_str(), basePath.append(to).c_str()); - } catch (std::filesystem::filesystem_error& e) { - return false; - } - return true; - } + bool open(const std::string& path, FileOutput& file) override { + return file.open(basePath.append(path)); + } - bool copyFile(std::string& from, std::string& to) override { - try { - std::filesystem::copy(basePath.append(from).c_str(), basePath.append(to).c_str()); - } catch (std::filesystem::filesystem_error& e) { - return false; - } + bool deleteFile(const std::string& path) override { + return std::remove((basePath.append(path).c_str())); + } - return true; + bool moveFile(const std::string& from, const std::string& to) override { + try { + std::rename(basePath.append(from).c_str(), basePath.append(to).c_str()); + } catch (std::filesystem::filesystem_error& e) { + return false; } + return true; + } - bool fileExists(std::string& path) override { - return std::filesystem::exists(path); + bool copyFile(const std::string& from, const std::string& to) override { + try { + std::filesystem::copy(basePath.append(from).c_str(), basePath.append(to).c_str()); + } catch (std::filesystem::filesystem_error& e) { + return false; } - size_t getLastModified(std::string& path) override { - return std::filesystem::last_write_time(path).time_since_epoch().count(); - } + return true; + } - // TODO: File iterators + bool fileExists(const std::string& path) override { + return std::filesystem::exists(path); + } - void processCallbacks() override { - // TODO: Timeout this function! - for (;;) { - mutex.enter(); - if (finished.empty() || workCounter == 0) { - mutex.exit(); - break; - } + size_t getLastModified(const std::string& path) override { + return std::filesystem::last_write_time(path).time_since_epoch().count(); + } - AsyncRead item = finished[0]; - finished.erase(finished.begin()); - --workCounter; + // TODO: File iterators + void processCallbacks() override { + // TODO: Timeout this function! + for (;;) { + mutex.enter(); + if (finished.empty() || workCounter == 0) { mutex.exit(); + break; + } - if (!item.isCancelled()) - item.callback.invoke(item.data.size(), (const uint8_t*) item.data.data(), !item.isFailed()); + AsyncRead item = finished[0]; + finished.erase(finished.begin()); + --workCounter; - } - } + mutex.exit(); - // TODO: Task Management - std::string basePath; - std::vector queue; - uint64_t workCounter; - std::vector finished; - Mutex mutex; - Semaphore sem; + if (!item.isCancelled()) + item.callback.invoke(item.data.size(), (const uint8_t*) item.data.data(), !item.isFailed()); + } + } - uint32_t lastID; + // TODO: Run Management + std::string basePath; + std::vector queue; + uint64_t workCounter; + std::vector finished; + Mutex mutex; + Semaphore sem; - }; + uint32_t lastID; - struct VFS : DiskFS { - VFS(std::string& root_pack_path) : DiskFS((std::string &) "vfs:/") { - if (!pack.open(root_pack_path)) { - spdlog::error("Unable to open " + root_pack_path + ", please check paths"); - return; - } + }; - const auto count = pack.read(); - for (size_t i = 0; i < count; i++) { - const auto hash = pack.read(); - PackFile& file = packFiles[hash]; - file.offset = pack.read(); - file.size = pack.read(); - } + struct VFS : DiskFS { + VFS(const std::string& root_pack_path) : DiskFS((std::string &) "vfs:/") { + if (!pack.open(root_pack_path)) { + spdlog::error("Unable to open " + root_pack_path + ", please check paths"); + return; } - ~VFS() { pack.close(); } + const auto count = pack.read(); + for (size_t i = 0; i < count; i++) { + const auto hash = pack.read(); + PackFile& file = packFiles[hash]; + file.offset = pack.read(); + file.size = pack.read(); + } + } - bool readSync(const Path& path, OutputMemoryStream& content) override { - std::string basename = Path::getFilename(const_cast(path.get())); - PathHash hash = path.getHash(); + ~VFS() { pack.close(); } - auto i = packFiles.find(hash); - if (i == packFiles.end()) return false; + bool readSync(const Path& path, OutputMemoryStream& content) override { + std::string basename = Path::getFilename(const_cast(path.get())); + PathHash hash = path.getHash(); - content.resize(i->second.size); - MutexGuard lock(mutex); + auto i = packFiles.find(hash); + if (i == packFiles.end()) return false; - const size_t headerSize = sizeof(uint32_t) + packFiles.size() * (3 * sizeof(size_t)); - if (pack.seek(i->second.offset + headerSize) || !pack.read(content.dataMut(), content.size())) { - spdlog::error("Could not read file " + path.get() + " from the pack file."); - return false; - } + content.resize(i->second.size); + MutexGuard lock(mutex); - return true; + const size_t headerSize = sizeof(uint32_t) + packFiles.size() * (3 * sizeof(size_t)); + if (pack.seek(i->second.offset + headerSize) || !pack.read(content.dataMut(), content.size())) { + spdlog::error("Could not read file " + path.get() + " from the pack file."); + return false; } - struct PackFile { - size_t offset; - size_t size; - }; + return true; + } - std::map packFiles; - FileInput pack; + struct PackFile { + size_t offset; + size_t size; }; - std::unique_ptr FileSystem::createDiskFS(std::string &basePath) { - return std::make_unique(basePath); - } + std::map packFiles; + FileInput pack; + }; - std::unique_ptr FileSystem::createVFS(std::string& basePath) { - return std::make_unique(basePath); - } + std::unique_ptr FileSystem::createDiskFS(const std::string& basePath) { + return std::make_unique(basePath); + } + + std::unique_ptr FileSystem::createVFS(const std::string& basePath) { + return std::make_unique(basePath); + } } \ No newline at end of file diff --git a/projs/shadow/shadow-engine/assets/src/fs/hash.cpp b/projs/shadow/shadow-engine/assets/src/fs/hash.cpp index 67970e91..2f01069c 100644 --- a/projs/shadow/shadow-engine/assets/src/fs/hash.cpp +++ b/projs/shadow/shadow-engine/assets/src/fs/hash.cpp @@ -1,148 +1,148 @@ -#include -#include "xxhash.h" +#include +#include namespace ShadowEngine { - StableHash::StableHash(const void *data, uint32_t length) { - hash = XXHash64::hash(data, length, 0); - } - - StableHash::StableHash(std::string &str) { - hash = XXHash64::hash(str.data(), str.size(), 0); - } - - HeapHash::HeapHash(const void *data, uint32_t length) { - hash = XXHash64::hash(data, length, 0); - } - - HeapHash::HeapHash(std::string &str) { - hash = XXHash64::hash(str.data(), str.size(), 0); - } - - HeapHash HeapHash::fromLong(size_t hash) { - HeapHash heap; - heap.hash = hash; - return heap; - } - - HeapHash32 HeapHash32::fromInt(uint32_t hash) { - HeapHash32 heap; - heap.hash = hash; - return heap; - } - - StableHash StableHash::fromLong(size_t hash) { - StableHash stable; - stable.hash = hash; - return stable; - } - - StableHash32 StableHash32::fromInt(uint32_t hash) { - StableHash32 stable; - stable.hash = hash; - return stable; - } - - static uint32_t CRC[256] = { - 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, - 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, - 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, - 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, - 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, - 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, - 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, - 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, - 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, - 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, - 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, - 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, - 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, - 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, - 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, - 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, - 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, - 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, - 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, - 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, - 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, - 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, - 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, - 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, - 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, - 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, - 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, - 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, - 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, - 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, - 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, - 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, - 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, - 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, - 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, - 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, - 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, - 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, - 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, - 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, - 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, - 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, - 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d}; - - static uint32_t CRC32(const void* data, uint32_t length) { - const auto* c = static_cast(data); - uint32_t crcTemp = 0xFFFFFFFF; - uint32_t len = length; - while (len) { - crcTemp = (crcTemp >> 8) ^ CRC[(crcTemp & 0xFF) ^ *c]; - --len; ++c; - } - - return ~crcTemp; - } - - StableHash32::StableHash32(const void *data, uint32_t length) { - hash = CRC32(data, length); - } - - StableHash32::StableHash32(std::string &str) { - hash = CRC32(str.data(), str.size()); - } - - static XXHash64 DeferredHashState(0); - - DeferredHash::DeferredHash() { - DeferredHashState = XXHash64(0); - } - - void DeferredHash::insert(const void *data, uint32_t length) { - DeferredHashState.add(data, length); - } - - StableHash32 DeferredHash::submit32() { - const auto result = DeferredHashState.hash(); - return StableHash32::fromInt(uint32_t(result ^ (result >> 32))); - } - - StableHash DeferredHash::submit() { - const auto result = DeferredHashState.hash(); - return StableHash::fromLong(result); - } - - DeferredHeapHash::DeferredHeapHash() { - DeferredHashState = XXHash64(0); - } - - void DeferredHeapHash::insert(const void *data, uint32_t length) { - DeferredHashState.add(data, length); - } - - HeapHash32 DeferredHeapHash::submit32() { - const auto result = DeferredHashState.hash(); - return HeapHash32::fromInt(uint32_t(result ^ (result >> 32))); - } - - HeapHash DeferredHeapHash::submit() { - const auto result = DeferredHashState.hash(); - return HeapHash::fromLong(result); - } + StableHash::StableHash(const void *data, uint32_t length) { + hash = XXHash64::hash(data, length, 0); + } + + StableHash::StableHash(const std::string &str) { + hash = XXHash64::hash(str.data(), str.size(), 0); + } + + HeapHash::HeapHash(const void *data, uint32_t length) { + hash = XXHash64::hash(data, length, 0); + } + + HeapHash::HeapHash(const std::string& str) { + hash = XXHash64::hash(str.data(), str.size(), 0); + } + + HeapHash HeapHash::fromLong(size_t hash) { + HeapHash heap; + heap.hash = hash; + return heap; + } + + HeapHash32 HeapHash32::fromInt(uint32_t hash) { + HeapHash32 heap; + heap.hash = hash; + return heap; + } + + StableHash StableHash::fromLong(size_t hash) { + StableHash stable; + stable.hash = hash; + return stable; + } + + StableHash32 StableHash32::fromInt(uint32_t hash) { + StableHash32 stable; + stable.hash = hash; + return stable; + } + + static uint32_t CRC[256] = { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, + 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, + 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, + 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, + 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, + 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, + 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, + 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, + 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, + 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, + 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, + 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, + 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, + 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, + 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, + 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, + 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, + 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d}; + + static uint32_t CRC32(const void* data, uint32_t length) { + const auto* c = static_cast(data); + uint32_t crcTemp = 0xFFFFFFFF; + uint32_t len = length; + while (len) { + crcTemp = (crcTemp >> 8) ^ CRC[(crcTemp & 0xFF) ^ *c]; + --len; ++c; + } + + return ~crcTemp; + } + + StableHash32::StableHash32(const void *data, uint32_t length) { + hash = CRC32(data, length); + } + + StableHash32::StableHash32(const std::string &str) { + hash = CRC32(str.data(), str.size()); + } + + static XXHash64 DeferredHashState(0); + + DeferredHash::DeferredHash() { + DeferredHashState = XXHash64(0); + } + + void DeferredHash::insert(const void *data, uint32_t length) { + DeferredHashState.add(data, length); + } + + StableHash32 DeferredHash::submit32() { + const auto result = DeferredHashState.hash(); + return StableHash32::fromInt(uint32_t(result ^ (result >> 32))); + } + + StableHash DeferredHash::submit() { + const auto result = DeferredHashState.hash(); + return StableHash::fromLong(result); + } + + DeferredHeapHash::DeferredHeapHash() { + DeferredHashState = XXHash64(0); + } + + void DeferredHeapHash::insert(const void *data, uint32_t length) { + DeferredHashState.add(data, length); + } + + HeapHash32 DeferredHeapHash::submit32() { + const auto result = DeferredHashState.hash(); + return HeapHash32::fromInt(uint32_t(result ^ (result >> 32))); + } + + HeapHash DeferredHeapHash::submit() { + const auto result = DeferredHashState.hash(); + return HeapHash::fromLong(result); + } } \ No newline at end of file diff --git a/projs/shadow/shadow-engine/assets/src/fs/iostream.cpp b/projs/shadow/shadow-engine/assets/src/fs/iostream.cpp index a8c1e93a..f02f7bb6 100644 --- a/projs/shadow/shadow-engine/assets/src/fs/iostream.cpp +++ b/projs/shadow/shadow-engine/assets/src/fs/iostream.cpp @@ -1,5 +1,6 @@ -#include + #include +#include "shadow/assets/fs/iostream.h" namespace ShadowEngine { diff --git a/projs/shadow/shadow-engine/assets/src/fs/path.cpp b/projs/shadow/shadow-engine/assets/src/fs/path.cpp index 8766231d..d63ca3d9 100644 --- a/projs/shadow/shadow-engine/assets/src/fs/path.cpp +++ b/projs/shadow/shadow-engine/assets/src/fs/path.cpp @@ -1,6 +1,9 @@ -#include -#include -#include + +#include +#include +#include +#include +#include namespace ShadowEngine { @@ -37,7 +40,7 @@ namespace ShadowEngine { return path != rhs.path; } - std::string Path::normalise(std::string &str) { + std::string Path::normalise(const std::string &str) { bool prevSlash = false; std::string temp; @@ -72,35 +75,35 @@ namespace ShadowEngine { return temp; } - std::string Path::getPrelude(std::string &path) { + std::string Path::getPrelude(const std::string &path) { return path.substr(0, path.find_first_of(":")); } - std::string Path::getDomain(std::string &path) { + std::string Path::getDomain(const std::string &path) { return path.substr(path.find_first_of(":"), path.find_first_of("/")); } - std::string Path::getDirectory(std::string &path) { + std::string Path::getDirectory(const std::string &path) { return path.substr(path.find_first_of(":"), path.find_last_of("/")); } - std::string Path::getFilename(std::string &path) { + std::string Path::getFilename(const std::string &path) { return path.substr(path.find_last_of("/"), path.find_last_of(".")); } - std::string Path::getExtension(std::string &path) { + std::string Path::getExtension(const std::string &path) { return path.substr(path.find_last_of("."), path.length()); } - std::string Path::replaceExtension(std::string &path, std::string &newExt) { + std::string Path::replaceExtension(const std::string &path, const std::string &newExt) { return path.substr(0, path.length() - newExt.length()).append(newExt); } - bool Path::hasExtension(std::string &path, std::string &ext) { + bool Path::hasExtension(const std::string &path, const std::string &ext) { return path.find_last_of(ext) == path.length() - ext.length(); } - PathInfo::PathInfo(std::string &str) { + PathInfo::PathInfo(const std::string &str) { std::string normalised = Path::normalise(str); std::string preludeS = Path::getPrelude(normalised); diff --git a/projs/shadow/shadow-engine/assets/src/management/synchronization.cpp b/projs/shadow/shadow-engine/assets/src/management/synchronization.cpp index 985109e3..910db104 100644 --- a/projs/shadow/shadow-engine/assets/src/management/synchronization.cpp +++ b/projs/shadow/shadow-engine/assets/src/management/synchronization.cpp @@ -1,4 +1,3 @@ -#include // This doesn't work on Linux. Sucks to be you? dpeter won't let me do system-specific source files. #ifdef _WIN32 @@ -6,7 +5,7 @@ #include #define WIN32_LEAN_AND_MEAN #include - +#include "shadow/assets/management/synchronization.h" namespace ShadowEngine { struct NewPlaceholder {}; } inline void* operator new(size_t, ShadowEngine::NewPlaceholder, void* where) { return where; } diff --git a/projs/shadow/shadow-engine/assets/src/resource/Resource.cpp b/projs/shadow/shadow-engine/assets/src/resource/Resource.cpp new file mode 100644 index 00000000..4319b22a --- /dev/null +++ b/projs/shadow/shadow-engine/assets/src/resource/Resource.cpp @@ -0,0 +1,198 @@ +#include "shadow/assets/resource/Resource.h" +#include "shadow/assets/fs/file.h" +#include "shadow/assets/management/delegate.h" +#include +#include + +#include + +namespace ShadowEngine { + const uint32_t ResourceHeader::MAGIC = 'VXIP'; + + ResourceType::ResourceType(const std::string& name) { + hash = HeapHash(name); + } + + Resource::Resource(ShadowEngine::Path path, ShadowEngine::ResourceTypeManager &manager) + : references(0), + emptyDependencies(0), + failedDependencies(0), + state(State::EMPTY), + desiredState(State::EMPTY), + path(std::move(path)), + size(), + callback(), + manager(manager), + handle(FileSystem::AsyncHandle::invalid()) { + } + + Resource::~Resource() = default; + + void Resource::refresh() { + if (state == State::EMPTY) return; + + const State old = state; + state = State::EMPTY; + callback.invoke(old, state, *this); + checkState(); + } + + void Resource::checkState() { + State old = state; + if (failedDependencies > 0 && state != State::FAILED) { + state = State::FAILED; + } else if (failedDependencies == 0) { + if (emptyDependencies > 0 && state != State::EMPTY) + state = State::EMPTY; + + if (emptyDependencies == 0 && state != State::READY && desiredState != State::EMPTY) { + onReadying(); + + if (emptyDependencies != 0 || state == State::READY || desiredState == State::EMPTY) + return; + + if (failedDependencies != 0) { + checkState(); + return; + } + + state = State::READY; + } + } + callback.invoke(old, state, *this); + } + + void Resource::fileLoaded(size_t fileSize, const uint8_t *mem, bool success) { + handle = FileSystem::AsyncHandle::invalid(); + if (desiredState != State::READY) return; + + if (!success) { + ResourceManager& owner = getManager().getOwner(); + if (!hooked && owner.isHooked()) { + if (owner.onLoad(*this) == ResourceManager::LoadHook::Action::DEFERRED) { + hooked = true; + desiredState = State::READY; + increaseReferences(); + return; + } + } + + --emptyDependencies; + ++failedDependencies; + checkState(); + handle = FileSystem::AsyncHandle::invalid(); + return; + } + + const auto* header = (const ResourceHeader*) mem; + + if (size < sizeof(*header)) { + spdlog::error("Invalid resource: ", path.get(), ": size mismatch. Expected ", fileSize, ", got " , sizeof(*header)); + failedDependencies++; + } else if (header->magic != ResourceHeader::MAGIC) { + spdlog::error("Invalid resource: " , path.get(), ": magic number mismatch. Expected " , ResourceHeader::MAGIC, ", got ", header->magic); + failedDependencies++; + } else if (header->version > 0) { + spdlog::error("Invalid resource: ", path.get(), ": verison mismatch. Expected 0, got ", header->version); + failedDependencies++; + } else { + // TODO: Compression? + if (!load(size - sizeof(*header), mem + sizeof(*header))) + failedDependencies++; + size = header->decompressedSize; + } + + emptyDependencies--; + checkState(); + handle = FileSystem::AsyncHandle::invalid(); + } + + void Resource::performUnload() { + if (handle.valid()) { + FileSystem& fs = manager.getOwner().getFileSystem(); + fs.cancelAsync(handle); + handle = FileSystem::AsyncHandle::invalid(); + } + + hooked = false; + desiredState = State::EMPTY; + unload(); + + size = 0; + emptyDependencies = 1; + failedDependencies = 0; + checkState(); + } + + void Resource::onCreated(ShadowEngine::Resource::State newState) { + state = newState; + desiredState = State::READY; + failedDependencies = state == State::FAILED ? 1 : 0; + emptyDependencies = 0; + } + + void Resource::doLoad() { + if (desiredState == State::READY) return; + desiredState = State::READY; + + if (handle.valid()) return; + + FileSystem& fs = manager.getOwner().getFileSystem(); + FileSystem::ContentCallback cb = makeDelegate<&Resource::fileLoaded>(this); + + const PathHash hash = path.getHash(); + Path resourcePath("./resources/" + std::to_string(hash.getHash()) + ".res"); + handle = fs.readAsync(resourcePath, cb); + } + + void Resource::addDependency(ShadowEngine::Resource &dependent) { + dependent.callback.bind<&Resource::stateChanged>(this); + if (dependent.isEmpty()) emptyDependencies++; + if (dependent.isFailure()) failedDependencies++; + + checkState(); + } + + void Resource::removeDependency(ShadowEngine::Resource &dependent) { + dependent.callback.unbind<&Resource::stateChanged>(this); + if (dependent.isEmpty()) --emptyDependencies; + if (dependent.isFailure()) --failedDependencies; + + checkState(); + } + + uint32_t Resource::decreaseReferences() { + --references; + if (references == 0 && manager.unloadEnabled) + performUnload(); + + return references; + } + + void Resource::stateChanged(ShadowEngine::Resource::State old, ShadowEngine::Resource::State newState, + ShadowEngine::Resource &) { + if (old == State::EMPTY) --emptyDependencies; + if (old == State::FAILED) --failedDependencies; + + if (newState == State::EMPTY) ++emptyDependencies; + if (newState == State::FAILED) ++failedDependencies; + + checkState(); + } + + const ResourceType PrefabResource::TYPE("prefab"); + + PrefabResource::PrefabResource(const ShadowEngine::Path &path, + ShadowEngine::ResourceTypeManager &resource_manager) : Resource(path, resource_manager) {} + + ResourceType PrefabResource::getType() const { return TYPE; } + + void PrefabResource::unload() { data.clear(); } + + bool PrefabResource::load(size_t size, const uint8_t *mem) { + data.resize(size); + memcpy(data.dataMut(), mem, size); + hash = StableHash(mem, size); + return true; + } +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.cpp b/projs/shadow/shadow-engine/assets/src/resource/ResourceManager.cpp similarity index 97% rename from projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.cpp rename to projs/shadow/shadow-engine/assets/src/resource/ResourceManager.cpp index 163183d5..d52e16b9 100644 --- a/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.cpp +++ b/projs/shadow/shadow-engine/assets/src/resource/ResourceManager.cpp @@ -1,6 +1,8 @@ -#include "ResourceManager.h" -#include "Resource.h" + #include "spdlog/spdlog.h" +#include "shadow/assets/resource/Resource.h" +#include "shadow/assets/resource/ResourceManager.h" +#include "shadow/assets/fs/path.h" namespace ShadowEngine { diff --git a/projs/shadow/shadow-engine/assets/src/str/string.cpp b/projs/shadow/shadow-engine/assets/src/str/string.cpp index 6cb12eea..96fcc596 100644 --- a/projs/shadow/shadow-engine/assets/src/str/string.cpp +++ b/projs/shadow/shadow-engine/assets/src/str/string.cpp @@ -1,4 +1,6 @@ -#include +#include + + namespace ShadowEngine::Str { std::string toLower(const std::string& str) { diff --git a/projs/shadow/shadow-engine/core/inc/shadow/core/Module.h b/projs/shadow/shadow-engine/core/inc/shadow/core/Module.h index 3f081a44..9a1bcdff 100644 --- a/projs/shadow/shadow-engine/core/inc/shadow/core/Module.h +++ b/projs/shadow/shadow-engine/core/inc/shadow/core/Module.h @@ -4,7 +4,6 @@ #include #include "shadow/SHObject.h" -#include "shadow/renderer/vulkan/vlkx/vulkan/abstraction/Commands.h" namespace SH { @@ -43,20 +42,5 @@ namespace SH { }; }; -/** - * A class especially for modules that are renderers. - * Allows the engine to access state from the renderer independent of implementation. - */ - class RendererModule : public Module { - public: - // Begin the render pass using the given commands. - // Will call out through the regular modules to gather geometry to render. - virtual void BeginRenderPass(const std::unique_ptr &commands) = 0; - - virtual void EnableEditor() = 0; - - virtual VkExtent2D GetRenderExtent() = 0; - }; - } diff --git a/projs/shadow/shadow-engine/core/inc/shadow/core/ShadowApplication.h b/projs/shadow/shadow-engine/core/inc/shadow/core/ShadowApplication.h index 2a18382f..f04bcbf9 100644 --- a/projs/shadow/shadow-engine/core/inc/shadow/core/ShadowApplication.h +++ b/projs/shadow/shadow-engine/core/inc/shadow/core/ShadowApplication.h @@ -6,6 +6,7 @@ #include "imgui_impl_sdl2.h" #include "imgui_impl_vulkan.h" #include "shadow/event-bus/event_bus.h" +#include "shadow/assets/fs/file.h" #include @@ -16,12 +17,15 @@ namespace SH { /// class API ShadowApplication : SH::SHObject { SHObject_Base(ShadowApplication) - /// + public: + static std::unique_ptr diskFS; + /// /// This is the singleton instance /// static ShadowApplication *instance; + private: - /// + /// /// The module manager instance /// ModuleManager moduleManager; diff --git a/projs/shadow/shadow-engine/core/inc/shadow/core/ShadowWindow.h b/projs/shadow/shadow-engine/core/inc/shadow/core/ShadowWindow.h index 67e3e36c..ea26db2a 100644 --- a/projs/shadow/shadow-engine/core/inc/shadow/core/ShadowWindow.h +++ b/projs/shadow/shadow-engine/core/inc/shadow/core/ShadowWindow.h @@ -6,7 +6,6 @@ namespace SH { class ShadowWindow { public: - int Height; int Width; @@ -14,9 +13,6 @@ namespace SH { SDL_Surface *sdlSurface = NULL; - - //ShadowEngine::Ref context; - ShadowWindow(int W, int H); ~ShadowWindow(); diff --git a/projs/shadow/shadow-engine/core/inc/shadow/event-bus/events.h b/projs/shadow/shadow-engine/core/inc/shadow/event-bus/events.h index d7eb08e4..01e75b3a 100644 --- a/projs/shadow/shadow-engine/core/inc/shadow/event-bus/events.h +++ b/projs/shadow/shadow-engine/core/inc/shadow/event-bus/events.h @@ -18,14 +18,6 @@ namespace SH::Events { SDL_Event event; }; - class OverlayRender : public Event { - SHObject_Base(OverlayRender) - }; - - class PreRender : public Event { - SHObject_Base(PreRender) - }; - class Recreate : public Event { SHObject_Base(Recreate) }; diff --git a/projs/shadow/shadow-engine/core/src/core/ShadowApplication.cpp b/projs/shadow/shadow-engine/core/src/core/ShadowApplication.cpp index 6f2b50f6..84e1a272 100644 --- a/projs/shadow/shadow-engine/core/src/core/ShadowApplication.cpp +++ b/projs/shadow/shadow-engine/core/src/core/ShadowApplication.cpp @@ -5,10 +5,9 @@ #include "runtime/Runtime.h" #include "shadow/core/ShadowApplication.h" #include "shadow/core/Time.h" -#include "shadow/renderer/vulkan/vlkx/vulkan/abstraction/Commands.h" -#include "shadow/renderer/vulkan/vlkx/vulkan/VulkanModule.h" #include "shadow/platform/console-setup.h" +#include "shadow/assets/fs/file.h" #define CATCH(x) \ try { x } catch (std::exception& e) { spdlog::error(e.what()); exit(0); } @@ -21,9 +20,7 @@ namespace SH { ShadowApplication *ShadowApplication::instance = nullptr; - std::unique_ptr renderCommands; - - std::weak_ptr renderer; + std::unique_ptr ShadowApplication::diskFS = ShadowEngine::FileSystem::createDiskFS("./"); ShadowApplication::ShadowApplication(int argc, char *argv[]) { instance = this; @@ -64,9 +61,7 @@ namespace SH { moduleManager.Init(); - renderer = moduleManager.GetById("module:/renderer/vulkan"); - renderCommands = std::make_unique(2); } void ShadowApplication::Start() { @@ -79,16 +74,7 @@ namespace SH { if (event.type == SDL_QUIT) running = false; } - - eventBus.fire(SH::Events::PreRender()); - - if (!renderer.expired()) { - auto r = renderer.lock(); - r->BeginRenderPass(renderCommands); - } - - renderCommands->nextFrame(); - Time::UpdateTime(); + Timer::UpdateTime(); } //moduleManager.Destroy(); diff --git a/projs/shadow/shadow-engine/core/src/event_bus/event_bus.cpp b/projs/shadow/shadow-engine/core/src/event_bus/event_bus.cpp index 44783998..943201ae 100644 --- a/projs/shadow/shadow-engine/core/src/event_bus/event_bus.cpp +++ b/projs/shadow/shadow-engine/core/src/event_bus/event_bus.cpp @@ -9,14 +9,6 @@ namespace SH::Events { Event_Impl(SDLEvent) - SHObject_Base_Impl(OverlayRender) - - Event_Impl(OverlayRender) - - SHObject_Base_Impl(PreRender) - - Event_Impl(PreRender) - SHObject_Base_Impl(Recreate) Event_Impl(Recreate) diff --git a/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.cpp b/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.cpp deleted file mode 100644 index 46a2323c..00000000 --- a/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.cpp +++ /dev/null @@ -1,194 +0,0 @@ -#include -#include -#include - -namespace ShadowEngine { - - ResourceType::ResourceType(std::string& name) { - hash = HeapHash(name); - } - - Resource::Resource(const ShadowEngine::Path& path, ShadowEngine::ResourceTypeManager &manager) - : references(0), - emptyDependencies(0), - failedDependencies(0), - state(State::EMPTY), - desiredState(State::EMPTY), - path(path), - size(), - callback(), - manager(manager), - handle(FileSystem::AsyncHandle::invalid()) { - } - - Resource::~Resource() = default; - - void Resource::refresh() { - if (state == State::EMPTY) return; - - const State old = state; - state = State::EMPTY; - callback.invoke(old, state, *this); - checkState(); - } - - void Resource::checkState() { - State old = state; - if (failedDependencies > 0 && state != State::FAILED) { - state = State::FAILED; - } else if (failedDependencies == 0) { - if (emptyDependencies > 0 && state != State::EMPTY) - state = State::EMPTY; - - if (emptyDependencies == 0 && state != State::READY && desiredState != State::EMPTY) { - onReadying(); - - if (emptyDependencies != 0 || state == State::READY || desiredState == State::EMPTY) - return; - - if (failedDependencies != 0) { - checkState(); - return; - } - - state = State::READY; - } - } - callback.invoke(old, state, *this); - } - - void Resource::fileLoaded(size_t fileSize, const uint8_t *mem, bool success) { - handle = FileSystem::AsyncHandle::invalid(); - if (desiredState != State::READY) return; - - if (!success) { - ResourceManager& owner = getManager().getOwner(); - if (!hooked && owner.isHooked()) { - if (owner.onLoad(*this) == ResourceManager::LoadHook::Action::DEFERRED) { - hooked = true; - desiredState = State::READY; - increaseReferences(); - return; - } - } - - --emptyDependencies; - ++failedDependencies; - checkState(); - handle = FileSystem::AsyncHandle::invalid(); - return; - } - - const ResourceHeader* header = (const ResourceHeader*) mem; - - if (size < sizeof(*header)) { - spdlog::error("Invalid resource: ", path.get(), ": size mismatch. Expected ", fileSize, ", got " , sizeof(*header)); - failedDependencies++; - } else if (header->magic != ResourceHeader::MAGIC) { - spdlog::error("Invalid resource: " , path.get(), ": magic number mismatch. Expected " , ResourceHeader::MAGIC, ", got ", header->magic); - failedDependencies++; - } else if (header->version > 0) { - spdlog::error("Invalid resource: ", path.get(), ": verison mismatch. Expected 0, got ", header->version); - failedDependencies++; - } else { - // TODO: Compression? - if (!load(size - sizeof(*header), mem + sizeof(*header))) - failedDependencies++; - size = header->decompressedSize; - } - - emptyDependencies--; - checkState(); - handle = FileSystem::AsyncHandle::invalid(); - } - - void Resource::performUnload() { - if (handle.valid()) { - FileSystem& fs = manager.getOwner().getFileSystem(); - fs.cancelAsync(handle); - handle = FileSystem::AsyncHandle::invalid(); - } - - hooked = false; - desiredState = State::EMPTY; - unload(); - - size = 0; - emptyDependencies = 1; - failedDependencies = 0; - checkState(); - } - - void Resource::onCreated(ShadowEngine::Resource::State newState) { - state = newState; - desiredState = State::READY; - failedDependencies = state == State::FAILED ? 1 : 0; - emptyDependencies = 0; - } - - void Resource::doLoad() { - if (desiredState == State::READY) return; - desiredState = State::READY; - - if (handle.valid()) return; - - FileSystem& fs = manager.getOwner().getFileSystem(); - FileSystem::ContentCallback cb = makeDelegate<&Resource::fileLoaded>(this); - - const PathHash hash = path.getHash(); - Path resourcePath("./resources/" + std::to_string(hash.getHash()) + ".res"); - handle = fs.readAsync(resourcePath, cb); - } - - void Resource::addDependency(ShadowEngine::Resource &dependent) { - dependent.callback.bind<&Resource::stateChanged>(this); - if (dependent.isEmpty()) emptyDependencies++; - if (dependent.isFailure()) failedDependencies++; - - checkState(); - } - - void Resource::removeDependency(ShadowEngine::Resource &dependent) { - dependent.callback.unbind<&Resource::stateChanged>(this); - if (dependent.isEmpty()) --emptyDependencies; - if (dependent.isFailure()) --failedDependencies; - - checkState(); - } - - uint32_t Resource::decreaseReferences() { - --references; - if (references == 0 && manager.unloadEnabled) - performUnload(); - - return references; - } - - void Resource::stateChanged(ShadowEngine::Resource::State old, ShadowEngine::Resource::State newState, - ShadowEngine::Resource &) { - if (old == State::EMPTY) --emptyDependencies; - if (old == State::FAILED) --failedDependencies; - - if (newState == State::EMPTY) ++emptyDependencies; - if (newState == State::FAILED) ++failedDependencies; - - checkState(); - } - - static std::string type("prefab"); - const ResourceType PrefabResource::TYPE(type); - - PrefabResource::PrefabResource(const ShadowEngine::Path &path, - ShadowEngine::ResourceTypeManager &resource_manager) : Resource(path, resource_manager) {} - - ResourceType PrefabResource::getType() const { return TYPE; } - - void PrefabResource::unload() { data.clear(); } - - bool PrefabResource::load(size_t size, const uint8_t *mem) { - data.resize(size); - memcpy(data.dataMut(), mem, size); - hash = StableHash(mem, size); - return true; - } -} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.h b/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.h deleted file mode 100644 index a26af607..00000000 --- a/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.h +++ /dev/null @@ -1,140 +0,0 @@ -#pragma once - -#include "fs/hash.h" -#include "fs/path.h" -#include "fs/file.h" -#include - -namespace ShadowEngine { - - /** - * A runtime-only struct that determines the type of a resource - whether it be a texture, mesh, animation, or other data. - * Provides some specializations for living in a map. - */ - struct ResourceType { - ResourceType() = default; - explicit ResourceType(std::string& name); - bool operator!=(const ResourceType& o) const { return o.hash != hash; } - bool operator==(const ResourceType& o) const { return o.hash == hash; } - bool operator< (const ResourceType& o) const { return o.hash.getHash() < hash.getHash(); } - bool isValid() const { return hash.getHash() != 0; } - - HeapHash hash; - }; - - // A Resource Type that is guaranteed to be invalid. - static std::string empty; - const ResourceType INVALID_RESOURCE(empty); - - // A specialization of HashFunc for ResourceTypes, since they already have a HeapHash within. - template<> struct HashFunc { - static uint32_t get(const ResourceType& key) { return HashFunc::get(key.hash); } - }; - -#pragma pack(1) - struct ResourceHeader { - static const uint32_t MAGIC = 'VXIP'; - uint32_t magic = MAGIC; // VXI Package header - uint32_t version = 0; - uint32_t flags = 0; - uint32_t padding = 0; - uint32_t decompressedSize = 0; - }; -#pragma pack() - - /** - * A basic Resource type. - * Represents a single file loaded from disk. - * May have dependencies on other Resources, and other Resources may depend on this. - * Resources are reference-counted, and are removed when they go out of usage. - */ - - struct Resource { - - friend struct ResourceTypeManager; - friend struct ResourceManager; - - enum class State : uint32_t { - EMPTY = 0, - READY, - FAILED - }; - - using Observer = DelegateList; - - virtual ~Resource(); - virtual ResourceType getType() const = 0; - State getState() const { return state; } - - bool isEmpty() const { return state == State::EMPTY; } - bool isReady() const { return state == State::READY; } - bool isFailure() const { return state == State::FAILED; } - - uint32_t getReferenceCount() const { return references; } - - Observer const& getCallback() const { return callback; } - size_t getSize() const { return size; } - - const Path& getPath() const { return path; } - - struct ResourceTypeManager& getManager() { return manager; } - - uint32_t decreaseReferences(); - uint32_t increaseReferences() { return references++; } - - bool toInitialize() const { return desiredState == State::READY; } - bool isHooked() const { return hooked; } - - template void onLoaded(C* instance) { - callback.bind(instance); - if (isReady()) (instance->*Function)(State::READY, State::READY, *this); - } - - protected: - Resource(const Path& path, ResourceTypeManager& manager); - - virtual void onReadying() {} - virtual void unload() = 0; - virtual bool load(size_t size, const uint8_t* mem) = 0; - - void onCreated(State newState); - void performUnload(); - void addDependency(Resource& dependent); - void removeDependency(Resource& dependent); - void checkState(); - void refresh(); - - State desiredState; - uint16_t emptyDependencies; - ResourceTypeManager& manager; - - private: - - void doLoad(); - void fileLoaded(size_t fileSize, const uint8_t* mem, bool success); - void stateChanged(State old, State newState, Resource&); - - Resource(const Resource&) = delete; - void operator=(const Resource&) = delete; - - Observer callback; - size_t size; - Path path; - uint32_t references; - uint16_t failedDependencies; - FileSystem::AsyncHandle handle; - State state; - bool hooked = false; - }; - - struct PrefabResource : Resource { - PrefabResource(const Path& path, ResourceTypeManager& resource_manager); - ResourceType getType() const override; - void unload() override; - bool load(size_t size, const uint8_t* data) override; - - OutputMemoryStream data; - StableHash hash; - static const ResourceType TYPE; - }; -} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/utility/inc/shadow/util/string-helpers.h b/projs/shadow/shadow-engine/utility/inc/shadow/util/string-helpers.h index 68f05e19..671793c0 100644 --- a/projs/shadow/shadow-engine/utility/inc/shadow/util/string-helpers.h +++ b/projs/shadow/shadow-engine/utility/inc/shadow/util/string-helpers.h @@ -16,5 +16,19 @@ namespace SH { std::vector API explode(const std::string &s, const char &c); std::string API substr_range(std::string const &str, size_t start, size_t end); + + void StringConvert(const std::string& from, std::wstring& to); + + void StringConvert(const std::wstring& from, std::string& to); + +// Parameter - to - must be pre-allocated! +// dest_size_in_characters : number of characters in the pre-allocated string memory +// returns result string length + int StringConvert(const char* from, wchar_t* to, int dest_size_in_characters = -1); + +// Parameter - to - must be pre-allocated! +// dest_size_in_characters : number of characters in the pre-allocated string memory +// returns result string length + int StringConvert(const wchar_t* from, char* to, int dest_size_in_characters = -1); } } \ No newline at end of file diff --git a/projs/shadow/shadow-engine/utility/src/string-helpers.cpp b/projs/shadow/shadow-engine/utility/src/string-helpers.cpp index f768df66..324f7412 100644 --- a/projs/shadow/shadow-engine/utility/src/string-helpers.cpp +++ b/projs/shadow/shadow-engine/utility/src/string-helpers.cpp @@ -1,8 +1,11 @@ +#include #include "shadow/util/string-helpers.h" namespace SH::Util::Str { + #define CP_UTF8 65001 + std::vector explode(const std::string &s, const char &c) { std::string buff; std::vector v; @@ -30,4 +33,75 @@ namespace SH::Util::Str { std::string substr_range(std::string const &str, size_t start, size_t end) { return str.substr(start, end - start); } + + + void StringConvert(const std::string& from, std::wstring& to) { +#ifdef _WIN32 + int num = MultiByteToWideChar(CP_UTF8, 0, from.c_str(), -1, NULL, 0); + if (num > 0) { + to.resize(size_t(num) - 1); + MultiByteToWideChar(CP_UTF8, 0, from.c_str(), -1, &to[0], num); + } +#else + std::wstring_convert> cv; + to = cv.from_bytes(from); +#endif // _WIN32 + } + + void StringConvert(const std::wstring& from, std::string& to) { +#ifdef _WIN32 + int num = WideCharToMultiByte(CP_UTF8, 0, from.c_str(), -1, NULL, 0, NULL, NULL); + if (num > 0) { + to.resize(size_t(num) - 1); + WideCharToMultiByte(CP_UTF8, 0, from.c_str(), -1, &to[0], num, NULL, NULL); + } +#else + std::wstring_convert> cv; + to = cv.to_bytes(from); +#endif // _WIN32 + } + + int StringConvert(const char* from, wchar_t* to, int dest_size_in_characters) { +#ifdef _WIN32 + int num = MultiByteToWideChar(CP_UTF8, 0, from, -1, NULL, 0); + if (num > 0) { + if (dest_size_in_characters >= 0) { + num = std::min(num, dest_size_in_characters); + } + MultiByteToWideChar(CP_UTF8, 0, from, -1, &to[0], num); + } + return num; +#else + std::wstring_convert> cv; + auto result = cv.from_bytes(from).c_str(); + int num = (int)cv.converted(); + if (dest_size_in_characters >= 0) { + num = std::min(num, dest_size_in_characters); + } + std::memcpy(to, result, num * sizeof(wchar_t)); + return num; +#endif // _WIN32 + } + + int StringConvert(const wchar_t* from, char* to, int dest_size_in_characters) { +#ifdef _WIN32 + int num = WideCharToMultiByte(CP_UTF8, 0, from, -1, NULL, 0, NULL, NULL); + if (num > 0) { + if (dest_size_in_characters >= 0) { + num = std::min(num, dest_size_in_characters); + } + WideCharToMultiByte(CP_UTF8, 0, from, -1, &to[0], num, NULL, NULL); + } + return num; +#else + std::wstring_convert> cv; + auto result = cv.to_bytes(from).c_str(); + int num = (size_t)cv.converted(); + if (dest_size_in_characters >= 0) { + num = std::min(num, dest_size_in_characters); + } + std::memcpy(to, result, num * sizeof(char)); + return num; +#endif // _WIN32 + } } \ No newline at end of file From c4897c09e071b3abc4c4a4d59bcd360eb2dbea85 Mon Sep 17 00:00:00 2001 From: Curle Date: Sat, 6 Apr 2024 18:03:58 +0100 Subject: [PATCH 10/12] Fix mistakenly overwritten builscripts --- .../shadow-engine/assets/CMakeLists.txt | 49 +++---------------- 1 file changed, 7 insertions(+), 42 deletions(-) diff --git a/projs/shadow/shadow-engine/assets/CMakeLists.txt b/projs/shadow/shadow-engine/assets/CMakeLists.txt index 29725e3c..031da61c 100644 --- a/projs/shadow/shadow-engine/assets/CMakeLists.txt +++ b/projs/shadow/shadow-engine/assets/CMakeLists.txt @@ -1,44 +1,9 @@ -<<<<<<<< HEAD:projs/shadow/shadow-engine/shadow-file-format/CMakeLists.txt -set(CMAKE_CXX_STANDARD 20) -# Set up Catch2 testing -list(APPEND CMAKE_MODULE_PATH "cmake") -enable_testing() +FILE(GLOB_RECURSE SOURCES + ${CMAKE_CURRENT_LIST_DIR}/src/*.cpp + ) -# Set up asset sourceset -FILE(GLOB_RECURSE SOURCES src/*.cpp src/*.h) -FILE(GLOB_RECURSE TESTS test/*.cpp) - -add_library(shadow-asset ${SOURCES}) - -# Set up test executable -add_executable(shadow-asset-test ${TESTS}) -target_link_libraries(shadow-asset-test PRIVATE Catch2::Catch2 shadow-utils) - -# Enable testing on the executable -include(CTest) -include(Catch) -catch_discover_tests(shadow-asset-test) -======== -set(CMAKE_CXX_STANDARD 20) - -# Set up Catch2 testing -list(APPEND CMAKE_MODULE_PATH "cmake") -enable_testing() - -# Set up asset sourceset -FILE(GLOB_RECURSE SOURCES src/**.cpp src/**.h) -FILE(GLOB_RECURSE TESTS test/*.cpp) - -include_directories(src/) -add_library(shadow-asset ${SOURCES}) - -# Set up test executable -add_executable(shadow-asset-test ${TESTS}) -target_link_libraries(shadow-asset-test PRIVATE Catch2::Catch2 shadow-utils) - -# Enable testing on the executable -#include(CTest) -#include(Catch2) -#catch_discover_tests(shadow-asset-test) ->>>>>>>> 0009dc7 (File & FileSystem abstractions):projs/shadow/shadow-engine/assets/CMakeLists.txt +target_shadow_module(shadow-engine + SOURCES ${SOURCES} + INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/inc/ + ) \ No newline at end of file From 88b3c7752d1b617156bd0ff4e21c40c6233ddb2c Mon Sep 17 00:00:00 2001 From: Curle Date: Sun, 7 Apr 2024 20:12:21 +0100 Subject: [PATCH 11/12] Make compile --- .../shadow/shadow-editor/src/EditorModule.cpp | 14 +++++++------- .../assets/inc/shadow/assets/fs/file.h | 3 +-- .../shadow-engine/assets/src/fs/iostream.cpp | 14 ++++++++++++++ .../shadow-engine/assets/src/fs/path.cpp | 3 ++- .../core/inc/shadow/core/Thread.h | 2 +- .../core/src/core/ShadowApplication.cpp | 19 ++++++++++++++++++- .../shadow-engine/core/src/core/Time.cpp | 15 +++++++-------- .../core/src/debug/DebugModule.cpp | 4 ++-- .../core/src/profile/Profiler.cpp | 2 +- .../shadow/entitiy/entities/MeshComponent.h | 19 ------------------- .../entity/src/entities/entities.cpp | 3 --- .../platforms/windows/src/Thread.cpp | 2 +- .../shadow-engine/reflection/src/Runtime.cpp | 1 + .../Vulkan/src/vulkan/VulkanModule.cpp | 18 +++++++++--------- projs/test-game/src/TestScene.cpp | 3 --- 15 files changed, 64 insertions(+), 58 deletions(-) delete mode 100644 projs/shadow/shadow-engine/entity/inc/shadow/entitiy/entities/MeshComponent.h diff --git a/projs/shadow/shadow-editor/src/EditorModule.cpp b/projs/shadow/shadow-editor/src/EditorModule.cpp index fb2aa73f..ac690755 100644 --- a/projs/shadow/shadow-editor/src/EditorModule.cpp +++ b/projs/shadow/shadow-editor/src/EditorModule.cpp @@ -16,13 +16,13 @@ namespace SH::Editor { void EditorModule::OverlayRender(SH::Events::OverlayRender &) { static bool dockspaceOpen = true; - ImGuiWindowFlags window_flags = ImGuiWindowFlags_MenuBar | ImGuiWindowFlags_NoDocking; - static ImGuiDockNodeFlags dockspace_flags = ImGuiDockNodeFlags_None; + ImGuiWindowFlags window_flags = ImGuiWindowFlags_MenuBar; //| ImGuiWindowFlags_NoDocking; + //static ImGuiDockNodeFlags dockspace_flags = ImGuiDockNodeFlags_None; ImGuiViewport *viewport = ImGui::GetMainViewport(); ImGui::SetNextWindowPos(viewport->Pos); ImGui::SetNextWindowSize(viewport->Size); - ImGui::SetNextWindowViewport(viewport->ID); + //ImGui::SetNextWindowViewport(viewport->ID); ImGui::PushStyleVar(ImGuiStyleVar_WindowRounding, 0.0f); ImGui::PushStyleVar(ImGuiStyleVar_WindowBorderSize, 0.0f); window_flags |= ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_NoCollapse | ImGuiWindowFlags_NoResize @@ -38,10 +38,10 @@ namespace SH::Editor { ImGuiStyle &style = ImGui::GetStyle(); float minWinSizeX = style.WindowMinSize.x; style.WindowMinSize.x = 370.0f; - if (io.ConfigFlags & ImGuiConfigFlags_DockingEnable) { - ImGuiID dockspace_id = ImGui::GetID("MyDockSpace"); - ImGui::DockSpace(dockspace_id, ImVec2(0.0f, 0.0f), dockspace_flags); - } + //if (io.ConfigFlags & ImGuiConfigFlags_DockingEnable) { + // ImGuiID dockspace_id = ImGui::GetID("MyDockSpace"); + // ImGui::DockSpace(dockspace_id, ImVec2(0.0f, 0.0f), dockspace_flags); + //} if (ImGui::BeginMenuBar()) { DrawMenu(); diff --git a/projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/file.h b/projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/file.h index 5a8947a6..08ee48df 100644 --- a/projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/file.h +++ b/projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/file.h @@ -2,8 +2,7 @@ #include #include "iostream.h" #include "path.h" - -template struct Delegate; +#include "shadow/assets/management/delegate.h" namespace ShadowEngine { diff --git a/projs/shadow/shadow-engine/assets/src/fs/iostream.cpp b/projs/shadow/shadow-engine/assets/src/fs/iostream.cpp index f02f7bb6..ca6cb4ab 100644 --- a/projs/shadow/shadow-engine/assets/src/fs/iostream.cpp +++ b/projs/shadow/shadow-engine/assets/src/fs/iostream.cpp @@ -7,6 +7,8 @@ namespace ShadowEngine { OutputMemoryStream::OutputMemoryStream(void *data, size_t size) : buffer(static_cast(data)), capacity(size), usage(0) {} + OutputMemoryStream::OutputMemoryStream() : buffer(), capacity(0), usage(0) {} + OutputMemoryStream::OutputMemoryStream(ShadowEngine::OutputMemoryStream &&str) noexcept { capacity = str.capacity; buffer = str.buffer; @@ -101,6 +103,18 @@ namespace ShadowEngine { return *this; } + void OutputMemoryStream::resize(size_t size) { + if (size > 0) { + auto* newbuffer = (uint8_t*)malloc(size); + memcpy(newbuffer, buffer, usage); + capacity = size; + delete[] buffer; + } else { + delete[] buffer; + capacity = 0; + } + } + void OutputMemoryStream::write(std::string &str) { write(str.c_str(), str.length()); } diff --git a/projs/shadow/shadow-engine/assets/src/fs/path.cpp b/projs/shadow/shadow-engine/assets/src/fs/path.cpp index d63ca3d9..84926080 100644 --- a/projs/shadow/shadow-engine/assets/src/fs/path.cpp +++ b/projs/shadow/shadow-engine/assets/src/fs/path.cpp @@ -4,6 +4,7 @@ #include #include #include +#include "shadow/util/string-helpers.h" namespace ShadowEngine { @@ -15,7 +16,7 @@ namespace ShadowEngine { void Path::set(const std::string &str) { #ifdef _WIN32 - std::string temp = Str::toLower((std::string&) str); + std::string temp = SH::Util::Str::toLower((std::string&) str); hash = PathHash(temp); #else hash = PathHash(str); diff --git a/projs/shadow/shadow-engine/core/inc/shadow/core/Thread.h b/projs/shadow/shadow-engine/core/inc/shadow/core/Thread.h index 537f1107..9b60a606 100644 --- a/projs/shadow/shadow-engine/core/inc/shadow/core/Thread.h +++ b/projs/shadow/shadow-engine/core/inc/shadow/core/Thread.h @@ -1,7 +1,7 @@ #pragma once #include #include -#include "shadow/util/Syncronization.h" +#include "shadow/assets/management/synchronization.h" namespace SH { diff --git a/projs/shadow/shadow-engine/core/src/core/ShadowApplication.cpp b/projs/shadow/shadow-engine/core/src/core/ShadowApplication.cpp index 84e1a272..776079ae 100644 --- a/projs/shadow/shadow-engine/core/src/core/ShadowApplication.cpp +++ b/projs/shadow/shadow-engine/core/src/core/ShadowApplication.cpp @@ -5,6 +5,8 @@ #include "runtime/Runtime.h" #include "shadow/core/ShadowApplication.h" #include "shadow/core/Time.h" +#include "shadow/renderer/vulkan/vlkx/vulkan/abstraction/Commands.h" +#include "shadow/renderer/vulkan/vlkx/vulkan/VulkanModule.h" #include "shadow/platform/console-setup.h" #include "shadow/assets/fs/file.h" @@ -22,6 +24,10 @@ namespace SH { std::unique_ptr ShadowApplication::diskFS = ShadowEngine::FileSystem::createDiskFS("./"); + std::unique_ptr renderCommands; + + std::weak_ptr renderer; + ShadowApplication::ShadowApplication(int argc, char *argv[]) { instance = this; @@ -61,7 +67,9 @@ namespace SH { moduleManager.Init(); + renderer = moduleManager.GetById("module:/renderer/vulkan"); + renderCommands = std::make_unique(2); } void ShadowApplication::Start() { @@ -74,7 +82,16 @@ namespace SH { if (event.type == SDL_QUIT) running = false; } - Timer::UpdateTime(); + + eventBus.fire(SH::Events::PreRender()); + + if (!renderer.expired()) { + auto r = renderer.lock(); + r->BeginRenderPass(renderCommands); + } + + renderCommands->nextFrame(); + SH::Timer::UpdateTime(); } //moduleManager.Destroy(); diff --git a/projs/shadow/shadow-engine/core/src/core/Time.cpp b/projs/shadow/shadow-engine/core/src/core/Time.cpp index 748c7ee0..43999116 100644 --- a/projs/shadow/shadow-engine/core/src/core/Time.cpp +++ b/projs/shadow/shadow-engine/core/src/core/Time.cpp @@ -1,16 +1,15 @@ #include "shadow/core/Time.h" #include -API int Time::NOW = 0;//SDL_GetPerformanceCounter(); -API int Time::LAST = 0; +API int SH::Timer::NOW = 0;//SDL_GetPerformanceCounter(); +API int SH::Timer::LAST = 0; API double lastFrame = 0; -API double Time::deltaTime_ms = 0; -API double Time::deltaTime = 0; -API double Time::startTime = 0; -API double Time::timeSinceStart = 0; +API double SH::Timer::deltaTime_ms = 0; +API double SH::Timer::deltaTime = 0; +API double SH::Timer::startTime = 0; +API double SH::Timer::timeSinceStart = 0; -void Time::UpdateTime() -{ +void SH::Timer::UpdateTime() { using namespace std::chrono; auto now = system_clock::now(); auto now_ms = time_point_cast(now); diff --git a/projs/shadow/shadow-engine/core/src/debug/DebugModule.cpp b/projs/shadow/shadow-engine/core/src/debug/DebugModule.cpp index d5b67a89..59764f90 100644 --- a/projs/shadow/shadow-engine/core/src/debug/DebugModule.cpp +++ b/projs/shadow/shadow-engine/core/src/debug/DebugModule.cpp @@ -15,8 +15,8 @@ void SH::Debug::DebugModule::DrawTimeWindow() { return; if (ImGui::Begin("Time", &w_time, ImGuiWindowFlags_MenuBar)) { - ImGui::Text("Time since start: %lf", Time::deltaTime_ms); - ImGui::Text("Delta time in ms: %lf", Time::deltaTime); + ImGui::Text("Time since start: %lf", SH::Timer::deltaTime_ms); + ImGui::Text("Delta time in ms: %lf", SH::Timer::deltaTime); } ImGui::End(); diff --git a/projs/shadow/shadow-engine/core/src/profile/Profiler.cpp b/projs/shadow/shadow-engine/core/src/profile/Profiler.cpp index 7dfafaa4..cc5a2210 100644 --- a/projs/shadow/shadow-engine/core/src/profile/Profiler.cpp +++ b/projs/shadow/shadow-engine/core/src/profile/Profiler.cpp @@ -1,8 +1,8 @@ #include #include #include "shadow/assets/fs/iostream.h" -#include "shadow/util/Syncronization.h" #include "shadow/core/Time.h" +#include "shadow/assets/management/synchronization.h" #include #include diff --git a/projs/shadow/shadow-engine/entity/inc/shadow/entitiy/entities/MeshComponent.h b/projs/shadow/shadow-engine/entity/inc/shadow/entitiy/entities/MeshComponent.h deleted file mode 100644 index 75b4785b..00000000 --- a/projs/shadow/shadow-engine/entity/inc/shadow/entitiy/entities/MeshComponent.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#include "shadow/entitiy/graph/graph.h" - -#include "shadow/assets/Mesh.h" - -namespace SH::Entities::Builtin { - - //A component that holds a mesh reference - class API MeshComponent : public SH::Entities::Component { - SHObject_Base(MeshComponent) - public: - MeshComponent() : Component() {} - - private: - std::shared_ptr mesh; - }; - -} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/entity/src/entities/entities.cpp b/projs/shadow/shadow-engine/entity/src/entities/entities.cpp index 25cd832b..7d648596 100644 --- a/projs/shadow/shadow-engine/entity/src/entities/entities.cpp +++ b/projs/shadow/shadow-engine/entity/src/entities/entities.cpp @@ -1,7 +1,6 @@ #include "shadow/entitiy/entities/NullActor.h" #include "shadow/entitiy/entities/Position.h" -#include "shadow/entitiy/entities/MeshComponent.h" #include "shadow/entitiy/entities/Light.h" namespace SH::Entities::Builtin { @@ -10,8 +9,6 @@ namespace SH::Entities::Builtin { SHObject_Base_Impl(Position) - SHObject_Base_Impl(MeshComponent) - SHObject_Base_Impl(Light) } // Builtin diff --git a/projs/shadow/shadow-engine/platforms/windows/src/Thread.cpp b/projs/shadow/shadow-engine/platforms/windows/src/Thread.cpp index 9eeaa763..228adfc8 100644 --- a/projs/shadow/shadow-engine/platforms/windows/src/Thread.cpp +++ b/projs/shadow/shadow-engine/platforms/windows/src/Thread.cpp @@ -93,7 +93,7 @@ namespace SH { } void Thread::Notify() { - implementation->cv.wakeup(); + implementation->cv.wake(); } bool Thread::IsRunning() const { diff --git a/projs/shadow/shadow-engine/reflection/src/Runtime.cpp b/projs/shadow/shadow-engine/reflection/src/Runtime.cpp index 00d92c97..a15964c4 100644 --- a/projs/shadow/shadow-engine/reflection/src/Runtime.cpp +++ b/projs/shadow/shadow-engine/reflection/src/Runtime.cpp @@ -1,3 +1,4 @@ +#include #include "runtime/Runtime.h" #include "spdlog/spdlog.h" #include "runtime/native_loading.h" diff --git a/projs/shadow/shadow-engine/renderer/Vulkan/src/vulkan/VulkanModule.cpp b/projs/shadow/shadow-engine/renderer/Vulkan/src/vulkan/VulkanModule.cpp index 7a4857ea..d0346a70 100644 --- a/projs/shadow/shadow-engine/renderer/Vulkan/src/vulkan/VulkanModule.cpp +++ b/projs/shadow/shadow-engine/renderer/Vulkan/src/vulkan/VulkanModule.cpp @@ -101,8 +101,8 @@ void VulkanModule::PreInit() { ImGui::CreateContext(); ImGuiIO &io = ImGui::GetIO(); (void) io; - io.ConfigFlags |= ImGuiConfigFlags_DockingEnable; // Enable Docking - io.ConfigFlags |= ImGuiConfigFlags_ViewportsEnable; // Enable Multi-Viewport / Platform Windows + //io.ConfigFlags |= ImGuiConfigFlags_DockingEnable; // Enable Docking + //io.ConfigFlags |= ImGuiConfigFlags_ViewportsEnable; // Enable Multi-Viewport / Platform Windows // Setup Dear ImGui style ImGui::StyleColorsDark(); @@ -168,9 +168,9 @@ void VulkanModule::PreInit() { editorPass = std::make_unique(vlkx::RendererConfig{2, swapchain->images, true}); editorPass->initializeRenderPass(); - ImGui_ImplVulkan_Init(&init_info, **(editorEnabled ? editorPass : renderPass)->getPass()); + ImGui_ImplVulkan_Init(&init_info); - VkTools::immediateExecute([](const VkCommandBuffer &commands) { ImGui_ImplVulkan_CreateFontsTexture(commands); }, + VkTools::immediateExecute([](const VkCommandBuffer &commands) { ImGui_ImplVulkan_CreateFontsTexture(); }, getDevice()); if (editorEnabled) { @@ -232,11 +232,11 @@ void VulkanModule::BeginRenderPass(const std::unique_ptr &c commands); // Update and Render additional Platform Windows - if (io.ConfigFlags - & ImGuiConfigFlags_ViewportsEnable) { - ImGui::UpdatePlatformWindows(); - ImGui::RenderPlatformWindowsDefault(); - } + //if (io.ConfigFlags + // & ImGuiConfigFlags_ViewportsEnable) { + // ImGui::UpdatePlatformWindows(); + // ImGui::RenderPlatformWindowsDefault(); + //} } }); } diff --git a/projs/test-game/src/TestScene.cpp b/projs/test-game/src/TestScene.cpp index dffb3907..b3a93fde 100644 --- a/projs/test-game/src/TestScene.cpp +++ b/projs/test-game/src/TestScene.cpp @@ -2,7 +2,6 @@ #include "entities/Player.h" #include "shadow/entitiy/entities/NullActor.h" #include "shadow/entitiy/entities/Position.h" -#include "shadow/entitiy/entities/MeshComponent.h" #include "shadow/entitiy/entities/Light.h" SHObject_Base_Impl(TestScene) @@ -12,12 +11,10 @@ void TestScene::Build() { auto cube1 = this->Add({}); cube1->SetName("Cube 1"); cube1->Add({-5, 0, 0}); - cube1->Add({}); auto cube2 = this->Add({}); cube2->SetName("Cube 2"); cube2->Add({5, 0, 0}); - cube2->Add({}); //Add a light to the center of the scene auto light = this->Add({}); From dbc3372cc5ad0fa2e326633797982604814c0819 Mon Sep 17 00:00:00 2001 From: Curle Date: Sun, 7 Apr 2024 20:25:44 +0100 Subject: [PATCH 12/12] Make compile v2 --- projs/shadow/extern/vulkan_memory_allocator | 2 +- projs/shadow/shadow-editor/inc/EditorModule.h | 2 -- projs/shadow/shadow-editor/src/EditorModule.cpp | 7 ++++--- .../inc/shadow/assets/management/delegate_list.h | 4 ++-- .../assets/inc/shadow/assets/resource/Resource.h | 2 +- .../shadow/shadow-engine/assets/src/SFFElement.cpp | 3 +-- .../shadow/shadow-engine/assets/src/SFFParser.cpp | 7 ++++--- .../shadow/shadow-engine/assets/src/SFFWriter.cpp | 3 +-- .../core/inc/shadow/debug/DebugModule.h | 2 -- .../core/src/core/ShadowApplication.cpp | 2 +- .../shadow-engine/core/src/debug/DebugModule.cpp | 10 ---------- .../entity/inc/shadow/entitiy/EntitySystem.h | 2 -- .../shadow-engine/entity/src/EntitySystem.cpp | 14 +++++++------- .../renderer/vulkan/vlkx/vulkan/VulkanModule.h | 12 ++++++------ .../renderer/Vulkan/src/vulkan/VulkanModule.cpp | 10 +++++----- 15 files changed, 33 insertions(+), 49 deletions(-) diff --git a/projs/shadow/extern/vulkan_memory_allocator b/projs/shadow/extern/vulkan_memory_allocator index e88fff95..60fe740c 160000 --- a/projs/shadow/extern/vulkan_memory_allocator +++ b/projs/shadow/extern/vulkan_memory_allocator @@ -1 +1 @@ -Subproject commit e88fff957b94f4b541ccac67a4290f07e52aa610 +Subproject commit 60fe740c77437a933456b00894336028ecf10885 diff --git a/projs/shadow/shadow-editor/inc/EditorModule.h b/projs/shadow/shadow-editor/inc/EditorModule.h index 0ec2e3f1..ed7ac67f 100644 --- a/projs/shadow/shadow-editor/inc/EditorModule.h +++ b/projs/shadow/shadow-editor/inc/EditorModule.h @@ -25,8 +25,6 @@ namespace SH::Editor { void RegisterMenu(std::string path, Menu m); - void OverlayRender(SH::Events::OverlayRender &); - void DrawMenu(); private: diff --git a/projs/shadow/shadow-editor/src/EditorModule.cpp b/projs/shadow/shadow-editor/src/EditorModule.cpp index ac690755..dfff1efb 100644 --- a/projs/shadow/shadow-editor/src/EditorModule.cpp +++ b/projs/shadow/shadow-editor/src/EditorModule.cpp @@ -13,6 +13,7 @@ namespace SH::Editor { MODULE_ENTRY(SH::Editor::EditorModule, EditorModule) + /* void EditorModule::OverlayRender(SH::Events::OverlayRender &) { static bool dockspaceOpen = true; @@ -54,7 +55,7 @@ namespace SH::Editor { } ImGui::End(); - } + } */ void EditorModule::DrawMenu() { @@ -79,8 +80,8 @@ namespace SH::Editor { } void EditorModule::Init() { - SH::ShadowApplication::Get().GetEventBus() - .subscribe(this, &EditorModule::OverlayRender); + //SH::ShadowApplication::Get().GetEventBus() + // .subscribe(this, &EditorModule::OverlayRender); windows.push_back(std::make_shared()); windows.push_back(std::make_shared()); diff --git a/projs/shadow/shadow-engine/assets/inc/shadow/assets/management/delegate_list.h b/projs/shadow/shadow-engine/assets/inc/shadow/assets/management/delegate_list.h index 42d85b37..947c5bd2 100644 --- a/projs/shadow/shadow-engine/assets/inc/shadow/assets/management/delegate_list.h +++ b/projs/shadow/shadow-engine/assets/inc/shadow/assets/management/delegate_list.h @@ -1,5 +1,5 @@ #pragma once -#include +#include #include #include @@ -42,7 +42,7 @@ namespace ShadowEngine { { if (m_delegates[i] == cb) { - m_delegates.swapAndPop(i); + //m_delegates.swapAndPop(i); break; } } diff --git a/projs/shadow/shadow-engine/assets/inc/shadow/assets/resource/Resource.h b/projs/shadow/shadow-engine/assets/inc/shadow/assets/resource/Resource.h index 5ac64114..64ec7028 100644 --- a/projs/shadow/shadow-engine/assets/inc/shadow/assets/resource/Resource.h +++ b/projs/shadow/shadow-engine/assets/inc/shadow/assets/resource/Resource.h @@ -3,7 +3,7 @@ #include "shadow/assets/fs/hash.h" #include "shadow/assets/fs/path.h" #include "shadow/assets/fs/file.h" -#include +#include namespace ShadowEngine { diff --git a/projs/shadow/shadow-engine/assets/src/SFFElement.cpp b/projs/shadow/shadow-engine/assets/src/SFFElement.cpp index 2cbc30d1..c5bbeb3a 100644 --- a/projs/shadow/shadow-engine/assets/src/SFFElement.cpp +++ b/projs/shadow/shadow-engine/assets/src/SFFElement.cpp @@ -1,5 +1,4 @@ -#include "SFFElement.h" - +#include namespace Shadow::SFF { diff --git a/projs/shadow/shadow-engine/assets/src/SFFParser.cpp b/projs/shadow/shadow-engine/assets/src/SFFParser.cpp index efde0707..c652948b 100644 --- a/projs/shadow/shadow-engine/assets/src/SFFParser.cpp +++ b/projs/shadow/shadow-engine/assets/src/SFFParser.cpp @@ -1,5 +1,6 @@ -#include "SFFParser.h" -#include "string-helpers.h" +#include "shadow/assets/sff/SFFElement.h" +#include "shadow/util/string-helpers.h" +#include #include @@ -95,7 +96,7 @@ namespace Shadow::SFF { SFFVersion SFFParser::ReadVersionFromHeader(std::istream& stream) { std::string line; std::getline(stream, line); - auto parts = explode(line, '_'); + auto parts = SH::Util::Str::explode(line, '_'); if (parts[0] != "ShadowFileFormat") { return SFFVersion(-1, -1, -1); } diff --git a/projs/shadow/shadow-engine/assets/src/SFFWriter.cpp b/projs/shadow/shadow-engine/assets/src/SFFWriter.cpp index 9daace19..e8ce1b4c 100644 --- a/projs/shadow/shadow-engine/assets/src/SFFWriter.cpp +++ b/projs/shadow/shadow-engine/assets/src/SFFWriter.cpp @@ -1,5 +1,4 @@ -#include "SFFWriter.h" - +#include namespace Shadow::SFF { diff --git a/projs/shadow/shadow-engine/core/inc/shadow/debug/DebugModule.h b/projs/shadow/shadow-engine/core/inc/shadow/debug/DebugModule.h index 37dc4de9..77cb9f8b 100644 --- a/projs/shadow/shadow-engine/core/inc/shadow/debug/DebugModule.h +++ b/projs/shadow/shadow-engine/core/inc/shadow/debug/DebugModule.h @@ -18,8 +18,6 @@ namespace SH::Debug { bool w_modules = true; bool w_imguiDemo = true; - void DrawDirect(SH::Events::OverlayRender &); - void DrawModuleWindow(); void DrawTimeWindow(); diff --git a/projs/shadow/shadow-engine/core/src/core/ShadowApplication.cpp b/projs/shadow/shadow-engine/core/src/core/ShadowApplication.cpp index 776079ae..4c3e7bff 100644 --- a/projs/shadow/shadow-engine/core/src/core/ShadowApplication.cpp +++ b/projs/shadow/shadow-engine/core/src/core/ShadowApplication.cpp @@ -83,7 +83,7 @@ namespace SH { running = false; } - eventBus.fire(SH::Events::PreRender()); + //eventBus.fire(SH::Events::PreRender()); if (!renderer.expired()) { auto r = renderer.lock(); diff --git a/projs/shadow/shadow-engine/core/src/debug/DebugModule.cpp b/projs/shadow/shadow-engine/core/src/debug/DebugModule.cpp index 59764f90..028d024b 100644 --- a/projs/shadow/shadow-engine/core/src/debug/DebugModule.cpp +++ b/projs/shadow/shadow-engine/core/src/debug/DebugModule.cpp @@ -52,14 +52,4 @@ void SH::Debug::DebugModule::DrawImguiDemo() { } void SH::Debug::DebugModule::Init() { - SH::ShadowApplication::Get().GetEventBus().subscribe( - this, - &DebugModule::DrawDirect - ); -} - -void SH::Debug::DebugModule::DrawDirect(SH::Events::OverlayRender &) { - //this->DrawModuleWindow(); - //this->DrawImguiDemo(); - //this->DrawTimeWindow(); } diff --git a/projs/shadow/shadow-engine/entity/inc/shadow/entitiy/EntitySystem.h b/projs/shadow/shadow-engine/entity/inc/shadow/entitiy/EntitySystem.h index ba23e4ca..6d868a56 100644 --- a/projs/shadow/shadow-engine/entity/inc/shadow/entitiy/EntitySystem.h +++ b/projs/shadow/shadow-engine/entity/inc/shadow/entitiy/EntitySystem.h @@ -30,8 +30,6 @@ namespace SH::Entities { void Init() override; void Update(int frame) override; - - void OverlayRender(SH::Events::OverlayRender &); }; } \ No newline at end of file diff --git a/projs/shadow/shadow-engine/entity/src/EntitySystem.cpp b/projs/shadow/shadow-engine/entity/src/EntitySystem.cpp index d71577c0..5dc4a977 100644 --- a/projs/shadow/shadow-engine/entity/src/EntitySystem.cpp +++ b/projs/shadow/shadow-engine/entity/src/EntitySystem.cpp @@ -13,11 +13,6 @@ namespace SH::Entities { MODULE_ENTRY(SH::Entities::EntitySystem, EntitySystem) - void EntitySystem::OverlayRender(SH::Events::OverlayRender &) { - SH::Entities::Debugger::AllocationDebugger::Draw(); - SH::Entities::Editor::HierarchyWindow::Draw(); - } - EntitySystem::EntitySystem() { //AddChild a new scene to the world auto scene = world.AddScene({"Generated Scene"}); @@ -39,9 +34,14 @@ namespace SH::Entities { } + //void Render(SH::Events::Render& r) { + // SH::Entities::Debugger::AllocationDebugger::Draw(); + // SH::Entities::Editor::HierarchyWindow::Draw(); + //} + void EntitySystem::Init() { - SH::ShadowApplication::Get().GetEventBus() - .subscribe(this, &EntitySystem::OverlayRender); + //SH::ShadowApplication::Get().GetEventBus() + // .subscribe(this, &Render); } void EntitySystem::Update(int frame) { diff --git a/projs/shadow/shadow-engine/renderer/Vulkan/inc/shadow/renderer/vulkan/vlkx/vulkan/VulkanModule.h b/projs/shadow/shadow-engine/renderer/Vulkan/inc/shadow/renderer/vulkan/vlkx/vulkan/VulkanModule.h index ddef5491..992cb5ac 100644 --- a/projs/shadow/shadow-engine/renderer/Vulkan/inc/shadow/renderer/vulkan/vlkx/vulkan/VulkanModule.h +++ b/projs/shadow/shadow-engine/renderer/Vulkan/inc/shadow/renderer/vulkan/vlkx/vulkan/VulkanModule.h @@ -10,14 +10,15 @@ #include "shadow/core/Module.h" #include "SwapChain.h" #include "shadow/event-bus/events.h" +#include "shadow/renderer/vulkan/vlkx/vulkan/abstraction/Commands.h" namespace vlkx { class ScreenRenderPassManager; } -class VulkanModule : public SH::RendererModule { +class VulkanModule : public SH::Module { SHObject_Base(VulkanModule); public: - VulkanModule() : RendererModule() { instance = this; } + VulkanModule() : Module() { instance = this; } ~VulkanModule() override; @@ -33,15 +34,14 @@ SHObject_Base(VulkanModule); void Recreate(); - void PreRender(SH::Events::PreRender); void Destroy() override; - void BeginRenderPass(const std::unique_ptr &commands) override; + void BeginRenderPass(const std::unique_ptr &commands); - void EnableEditor() override; + void EnableEditor(); - VkExtent2D GetRenderExtent() override; + VkExtent2D GetRenderExtent(); // VulkanModule is a singleton class. static VulkanModule *instance; diff --git a/projs/shadow/shadow-engine/renderer/Vulkan/src/vulkan/VulkanModule.cpp b/projs/shadow/shadow-engine/renderer/Vulkan/src/vulkan/VulkanModule.cpp index d0346a70..5f1f399c 100644 --- a/projs/shadow/shadow-engine/renderer/Vulkan/src/vulkan/VulkanModule.cpp +++ b/projs/shadow/shadow-engine/renderer/Vulkan/src/vulkan/VulkanModule.cpp @@ -221,7 +221,7 @@ void VulkanModule::BeginRenderPass(const std::unique_ptr &c ImGui_ImplSDL2_NewFrame(); ImGui::NewFrame(); - SH::ShadowApplication::Get().GetEventBus().fire(SH::Events::OverlayRender()); + //SH::ShadowApplication::Get().GetEventBus().fire(SH::Events::OverlayRender()); ImGui::Render(); ImGuiIO &io = ImGui::GetIO(); @@ -247,7 +247,7 @@ void VulkanModule::BeginRenderPass(const std::unique_ptr &c } } -void VulkanModule::PreRender(SH::Events::PreRender) { +/*void VulkanModule::PreRender(SH::Events::PreRender) { if (editorEnabled) { editorRenderCommands->executeSimple(editorRenderCommands->getFrame(), [](const int frame) { @@ -267,7 +267,7 @@ void VulkanModule::PreRender(SH::Events::PreRender) { ); } -} +}*/ void VulkanModule::Destroy() { ImGui_ImplVulkan_Shutdown(); @@ -374,6 +374,6 @@ VkDescriptorSet VulkanModule::getEditorRenderPlanes() { } void VulkanModule::Init() { - SH::ShadowApplication::Get().GetEventBus() - .subscribe(std::bind(&VulkanModule::PreRender, this, std::placeholders::_1)); + //SH::ShadowApplication::Get().GetEventBus() + // .subscribe(std::bind(&VulkanModule::PreRender, this, std::placeholders::_1)); }