From 49ee9fc10dd9857ccac9cd12e462d901648db884 Mon Sep 17 00:00:00 2001 From: Curle Date: Mon, 13 Mar 2023 15:25:31 +0000 Subject: [PATCH] Resource and Resource Managers. --- projs/shadow/shadow-engine/CMakeLists.txt | 5 +- .../shadow-assets/CMakeLists.txt | 21 -- .../shadow-engine/shadow-assets/src/fs/hash.h | 4 +- .../src/management/delegate_list.h | 58 ++++++ .../shadow-assets/src/resource/Resource.cpp | 177 ++++++++++++++++ .../shadow-assets/src/resource/Resource.h | 128 ++++++++++++ .../src/resource/ResourceManager.cpp | 195 ++++++++++++++++++ .../src/resource/ResourceManager.h | 95 +++++++++ 8 files changed, 658 insertions(+), 25 deletions(-) delete mode 100644 projs/shadow/shadow-engine/shadow-assets/CMakeLists.txt create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/management/delegate_list.h create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.cpp create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.h create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.cpp create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.h diff --git a/projs/shadow/shadow-engine/CMakeLists.txt b/projs/shadow/shadow-engine/CMakeLists.txt index 0738aa1..5fe36e5 100644 --- a/projs/shadow/shadow-engine/CMakeLists.txt +++ b/projs/shadow/shadow-engine/CMakeLists.txt @@ -5,19 +5,19 @@ find_package(imgui REQUIRED) set(CMAKE_CXX_STANDARD 20) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) -add_subdirectory(shadow-assets) - FILE(GLOB_RECURSE SOURCES core/src/*.cpp shadow-renderer/src/*.cpp shadow-reflection/src/*.cpp shadow-utility/src/*.cpp + shadow-assets/src/*.cpp ) FILE(GLOB_RECURSE HEADERS core/inc/*.h shadow-renderer/inc/*.h shadow-reflection/inc/*.h shadow-utility/inc/*.h + shadow-assets/src/*.h ) add_library(shadow-engine SHARED ${SOURCES} $) @@ -29,6 +29,7 @@ target_include_directories(shadow-engine shadow-renderer/inc shadow-reflection/inc shadow-utility/inc + shadow-assets/src ${glm_SOURCE_DIR} INTERFACE ${imgui_SOURCE_DIR} diff --git a/projs/shadow/shadow-engine/shadow-assets/CMakeLists.txt b/projs/shadow/shadow-engine/shadow-assets/CMakeLists.txt deleted file mode 100644 index 1c81c2c..0000000 --- a/projs/shadow/shadow-engine/shadow-assets/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -set(CMAKE_CXX_STANDARD 20) - -# Set up Catch2 testing -list(APPEND CMAKE_MODULE_PATH "cmake") -enable_testing() - -# Set up asset sourceset -FILE(GLOB_RECURSE SOURCES src/**.cpp src/**.h) -FILE(GLOB_RECURSE TESTS test/*.cpp) - -include_directories(src/) -add_library(shadow-asset ${SOURCES}) - -# Set up test executable -add_executable(shadow-asset-test ${TESTS}) -target_link_libraries(shadow-asset-test PRIVATE Catch2::Catch2 shadow-utils) - -# Enable testing on the executable -#include(CTest) -#include(Catch2) -#catch_discover_tests(shadow-asset-test) \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/fs/hash.h b/projs/shadow/shadow-engine/shadow-assets/src/fs/hash.h index 0b58187..f5b9f04 100644 --- a/projs/shadow/shadow-engine/shadow-assets/src/fs/hash.h +++ b/projs/shadow/shadow-engine/shadow-assets/src/fs/hash.h @@ -19,8 +19,8 @@ namespace ShadowEngine { // Hash arbitrary data. HeapHash(const void* data, uint32_t length); - bool operator!= (HeapHash& other) const { return hash != other.hash; } - bool operator== (HeapHash& other) const { return hash == other.hash; } + bool operator!= (const HeapHash& other) const { return hash != other.hash; } + bool operator== (const HeapHash& other) const { return hash == other.hash; } size_t getHash() const { return hash; } private: diff --git a/projs/shadow/shadow-engine/shadow-assets/src/management/delegate_list.h b/projs/shadow/shadow-engine/shadow-assets/src/management/delegate_list.h new file mode 100644 index 0000000..42d85b3 --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/management/delegate_list.h @@ -0,0 +1,58 @@ +#pragma once +#include +#include +#include + +namespace ShadowEngine { + + template struct DelegateList; + + template struct DelegateList { + DelegateList() = default; + + template void bind(C* instance) { + Delegate cb; + cb.template bind(instance); + m_delegates.push_back(cb); + } + + template void bind() { + Delegate cb; + cb.template bind(); + m_delegates.push_back(cb); + } + + template void unbind() { + Delegate cb; + cb.template bind(); + for (int i = 0; i < m_delegates.size(); ++i) + { + if (m_delegates[i] == cb) + { + m_delegates.swapAndPop(i); + break; + } + } + } + + template void unbind(C* instance) { + Delegate cb; + cb.template bind(instance); + for (int i = 0; i < m_delegates.size(); ++i) + { + if (m_delegates[i] == cb) + { + m_delegates.swapAndPop(i); + break; + } + } + } + + void invoke(Args... args) { + for (uint32_t i = 0, c = m_delegates.size(); i < c; ++i) m_delegates[i].invoke(args...); + } + + private: + std::vector> m_delegates; + }; +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.cpp b/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.cpp new file mode 100644 index 0000000..409e68d --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.cpp @@ -0,0 +1,177 @@ +#include +#include +#include + +namespace ShadowEngine { + + ResourceType::ResourceType(std::string& name) { + hash = HeapHash(name); + } + + Resource::Resource(const ShadowEngine::Path& path, ShadowEngine::ResourceTypeManager &manager) + : references(0), + emptyDependencies(0), + failedDependencies(0), + state(State::EMPTY), + desiredState(State::EMPTY), + path(path), + size(), + callback(), + manager(manager), + handle(FileSystem::AsyncHandle::invalid()) { + } + + Resource::~Resource() = default; + + void Resource::refresh() { + if (state == State::EMPTY) return; + + const State old = state; + state = State::EMPTY; + callback.invoke(old, state, *this); + checkState(); + } + + void Resource::checkState() { + State old = state; + if (failedDependencies > 0 && state != State::FAILED) { + state = State::FAILED; + } else if (failedDependencies == 0) { + if (emptyDependencies > 0 && state != State::EMPTY) + state = State::EMPTY; + + if (emptyDependencies == 0 && state != State::READY && desiredState != State::EMPTY) { + onReadying(); + + if (emptyDependencies != 0 || state == State::READY || desiredState == State::EMPTY) + return; + + if (failedDependencies != 0) { + checkState(); + return; + } + + state = State::READY; + } + } + callback.invoke(old, state, *this); + } + + void Resource::fileLoaded(size_t fileSize, const uint8_t *mem, bool success) { + handle = FileSystem::AsyncHandle::invalid(); + if (desiredState != State::READY) return; + + if (!success) { + ResourceManager& owner = getManager().getOwner(); + if (!hooked && owner.isHooked()) { + if (owner.onLoad(*this) == ResourceManager::LoadHook::Action::DEFERRED) { + hooked = true; + desiredState = State::READY; + increaseReferences(); + return; + } + } + + --emptyDependencies; + ++failedDependencies; + checkState(); + handle = FileSystem::AsyncHandle::invalid(); + return; + } + + const ResourceHeader* header = (const ResourceHeader*) mem; + + if (size < sizeof(*header)) { + spdlog::error("Invalid resource: ", path.get(), ": size mismatch. Expected ", fileSize, ", got " , sizeof(*header)); + failedDependencies++; + } else if (header->magic != ResourceHeader::MAGIC) { + spdlog::error("Invalid resource: " , path.get(), ": magic number mismatch. Expected " , ResourceHeader::MAGIC, ", got ", header->magic); + failedDependencies++; + } else if (header->version > 0) { + spdlog::error("Invalid resource: ", path.get(), ": verison mismatch. Expected 0, got ", header->version); + failedDependencies++; + } else { + // TODO: Compression? + if (!load(size - sizeof(*header), mem + sizeof(*header))) + failedDependencies++; + size = header->decompressedSize; + } + + emptyDependencies--; + checkState(); + handle = FileSystem::AsyncHandle::invalid(); + } + + void Resource::performUnload() { + if (handle.valid()) { + FileSystem& fs = manager.getOwner().getFileSystem(); + fs.cancelAsync(handle); + handle = FileSystem::AsyncHandle::invalid(); + } + + hooked = false; + desiredState = State::EMPTY; + unload(); + + size = 0; + emptyDependencies = 1; + failedDependencies = 0; + checkState(); + } + + void Resource::onCreated(ShadowEngine::Resource::State newState) { + state = newState; + desiredState = State::READY; + failedDependencies = state == State::FAILED ? 1 : 0; + emptyDependencies = 0; + } + + void Resource::doLoad() { + if (desiredState == State::READY) return; + desiredState = State::READY; + + if (handle.valid()) return; + + FileSystem& fs = manager.getOwner().getFileSystem(); + FileSystem::ContentCallback cb = makeDelegate<&Resource::fileLoaded>(this); + + const PathHash hash = path.getHash(); + Path resourcePath("./resources/" + std::to_string(hash.getHash()) + ".res"); + handle = fs.readAsync(resourcePath, cb); + } + + void Resource::addDependency(ShadowEngine::Resource &dependent) { + dependent.callback.bind<&Resource::stateChanged>(this); + if (dependent.isEmpty()) emptyDependencies++; + if (dependent.isFailure()) failedDependencies++; + + checkState(); + } + + void Resource::removeDependency(ShadowEngine::Resource &dependent) { + dependent.callback.unbind<&Resource::stateChanged>(this); + if (dependent.isEmpty()) --emptyDependencies; + if (dependent.isFailure()) --failedDependencies; + + checkState(); + } + + uint32_t Resource::decreaseReferences() { + --references; + if (references == 0 && manager.unloadEnabled) + performUnload(); + + return references; + } + + void Resource::stateChanged(ShadowEngine::Resource::State old, ShadowEngine::Resource::State newState, + ShadowEngine::Resource &) { + if (old == State::EMPTY) --emptyDependencies; + if (old == State::FAILED) --failedDependencies; + + if (newState == State::EMPTY) ++emptyDependencies; + if (newState == State::FAILED) ++failedDependencies; + + checkState(); + } +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.h b/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.h new file mode 100644 index 0000000..c91b2a4 --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.h @@ -0,0 +1,128 @@ +#pragma once + +#include "fs/hash.h" +#include "fs/path.h" +#include "fs/file.h" +#include + +namespace ShadowEngine { + + /** + * A runtime-only struct that determines the type of a resource - whether it be a texture, mesh, animation, or other data. + * Provides some specializations for living in a map. + */ + struct ResourceType { + ResourceType() = default; + explicit ResourceType(std::string& name); + bool operator!=(const ResourceType& o) const { return o.hash != hash; } + bool operator==(const ResourceType& o) const { return o.hash == hash; } + bool operator< (const ResourceType& o) const { return o.hash.getHash() < hash.getHash(); } + bool isValid() const { return hash.getHash() != 0; } + + HeapHash hash; + }; + + // A Resource Type that is guaranteed to be invalid. + const ResourceType INVALID_RESOURCE((std::string &) ""); + + // A specialization of HashFunc for ResourceTypes, since they already have a HeapHash within. + template<> struct HashFunc { + static uint32_t get(const ResourceType& key) { return HashFunc::get(key.hash); } + }; + +#pragma pack(1) + struct ResourceHeader { + static const uint32_t MAGIC = 'VXIP'; + uint32_t magic = MAGIC; // VXI Package header + uint32_t version = 0; + uint32_t flags = 0; + uint32_t padding = 0; + uint32_t decompressedSize = 0; + }; +#pragma pack() + + /** + * A basic Resource type. + * Represents a single file loaded from disk. + * May have dependencies on other Resources, and other Resources may depend on this. + * Resources are reference-counted, and are removed when they go out of usage. + */ + + struct Resource { + + friend struct ResourceTypeManager; + friend struct ResourceManager; + + enum class State : uint32_t { + EMPTY = 0, + READY, + FAILED + }; + + using Observer = DelegateList; + + virtual ~Resource(); + virtual ResourceType getType() const = 0; + State getState() const { return state; } + + bool isEmpty() const { return state == State::EMPTY; } + bool isReady() const { return state == State::READY; } + bool isFailure() const { return state == State::FAILED; } + + uint32_t getReferenceCount() const { return references; } + + Observer const& getCallback() const { return callback; } + size_t getSize() const { return size; } + + const Path& getPath() const { return path; } + + struct ResourceTypeManager& getManager() { return manager; } + + uint32_t decreaseReferences(); + uint32_t increaseReferences() { return references++; } + + bool toInitialize() const { return desiredState == State::READY; } + bool isHooked() const { return hooked; } + + template void onLoaded(C* instance) { + callback.bind(instance); + if (isReady()) (instance->*Function)(State::READY, State::READY, *this); + } + + protected: + Resource(const Path& path, ResourceTypeManager& manager); + + virtual void onReadying() {} + virtual void unload() = 0; + virtual bool load(size_t size, const uint8_t* mem) = 0; + + void onCreated(State newState); + void performUnload(); + void addDependency(Resource& dependent); + void removeDependency(Resource& dependent); + void checkState(); + void refresh(); + + State desiredState; + uint16_t emptyDependencies; + ResourceTypeManager& manager; + + private: + + void doLoad(); + void fileLoaded(size_t fileSize, const uint8_t* mem, bool success); + void stateChanged(State old, State newState, Resource&); + + Resource(const Resource&) = delete; + void operator=(const Resource&) = delete; + + Observer callback; + size_t size; + Path path; + uint32_t references; + uint16_t failedDependencies; + FileSystem::AsyncHandle handle; + State state; + bool hooked = false; + }; +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.cpp b/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.cpp new file mode 100644 index 0000000..8aa98bf --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.cpp @@ -0,0 +1,195 @@ +#include "ResourceManager.h" +#include "Resource.h" +#include "spdlog/spdlog.h" + +namespace ShadowEngine { + + void ResourceTypeManager::create(struct ResourceType type, struct ResourceManager &manager) { + manager.add(type, this); + owner = &manager; + } + + void ResourceTypeManager::destroy() { + for (auto iter = resources.begin(), end = resources.end(); iter != end; ++iter) { + Resource* res = iter->second; + if (!res->isEmpty()) + spdlog::error("Resource Type Manager destruction leaks ", res->path.get()); + + destroyResource(*res); + } + resources.clear(); + } + + Resource* ResourceTypeManager::get(const Path& path) { + auto it = resources.find(path.getHash()); + if (it != resources.end()) return it->second; + return nullptr; + } + + Resource* ResourceTypeManager::load(const Path &path) { + if (path.isEmpty()) return nullptr; + Resource* res = get(path); + if (res == nullptr) { + res = createResource(path); + resources[path.getHash()] = res; + } + + if (res->isEmpty() && res->desiredState == Resource::State::EMPTY) { + if (owner->onLoad(*res) == ResourceManager::LoadHook::Action::DEFERRED) { + res->hooked = true; + res->desiredState = Resource::State::READY; + res->increaseReferences(); + res->increaseReferences(); + return res; + } + + res->doLoad(); + } + + res->increaseReferences(); + return res; + } + + void ResourceTypeManager::removeUnreferencedResources() { + if (!unloadEnabled) return; + + std::vector toRemove; + for (auto i : resources) + if (i.second->getReferenceCount() == 0) toRemove.push_back(i.second); + + for (auto i : toRemove) { + auto iter = resources.find(i->getPath().getHash()); + if (iter->second->isReady()) iter->second->performUnload(); + } + } + + void ResourceTypeManager::reload(const Path &path) { + Resource* res = get(path); + if (res) reload(*res); + } + + void ResourceTypeManager::reload(Resource& res) { + if (res.state != Resource::State::EMPTY) + res.performUnload(); + else if (res.desiredState == Resource::State::READY) + return; + + if (owner->onLoad(res) == ResourceManager::LoadHook::Action::DEFERRED) { + res.hooked = true; + res.desiredState = Resource::State::READY; + res.increaseReferences(); + res.increaseReferences(); + } else { + res.performUnload(); + } + } + + void ResourceTypeManager::setUnloadable(bool status) { + unloadEnabled = status; + if (!unloadEnabled) return; + + for (auto res : resources) + if (res.second->getReferenceCount() == 0) + res.second->performUnload(); + } + + ResourceTypeManager::ResourceTypeManager() : + resources(), + owner(nullptr), + unloadEnabled(true) { + + } + + ResourceTypeManager::~ResourceTypeManager() { + + } + + ResourceManager::ResourceManager() : + managers(), + hook(nullptr), + filesystem(nullptr) { + + } + + ResourceManager::~ResourceManager() = default; + + void ResourceManager::init(FileSystem &fs) { + filesystem = &fs; + } + + Resource* ResourceManager::load(ResourceType type, const Path& path) { + ResourceTypeManager* manager = get(type); + if (!manager) return nullptr; + return load(*manager, path); + } + + Resource* ResourceManager::load(ResourceTypeManager& manager, const Path& path) { + return manager.load(path); + } + + ResourceTypeManager* ResourceManager::get(ResourceType type) { + auto iter = managers.find(type); + if (iter == managers.end()) return nullptr; + return iter->second; + } + + void ResourceManager::LoadHook::continueLoad(Resource &res) { + res.decreaseReferences(); + res.hooked = false; + res.desiredState = Resource::State::EMPTY; + res.doLoad(); + } + + void ResourceManager::setLoadHook(LoadHook *loadHook) { + hook = loadHook; + + if (hook) + for (auto manager : managers) + for (auto res : manager.second->getResources()) + if (res.second->isFailure()) + manager.second->reload(*res.second); + } + + ResourceManager::LoadHook::Action ResourceManager::onLoad(Resource &res) const { + return hook ? hook->load(res) : LoadHook::Action::IMMEDIATE; + } + + void ResourceManager::add(ResourceType type, ResourceTypeManager* manager) { + managers[type] = manager; + } + + void ResourceManager::remove(ResourceType type) { + managers.erase(type); + } + + void ResourceManager::removeUnreferenced() { + for (auto manager : managers) + manager.second->removeUnreferencedResources(); + } + + void ResourceManager::setUnloadable(bool enable) { + for (auto manager : managers) + manager.second->setUnloadable(enable); + } + + void ResourceManager::reloadAll() { + while (filesystem->hasWork()) filesystem->processCallbacks(); + + std::vector toReload; + for (auto manager : managers) { + ResourceTypeManager::ResourceTable& resources = manager.second->getResources(); + for (auto res : resources) { + if (res.second->isReady()) { + res.second->performUnload(); + toReload.push_back(res.second); + } + } + } + } + + void ResourceManager::reload(const Path& path) { + for (auto manager : managers) + manager.second->reload(path); + } + +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.h b/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.h new file mode 100644 index 0000000..98fc6b1 --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.h @@ -0,0 +1,95 @@ +#pragma once +#include +#include +#include + +namespace ShadowEngine { + + /** + * Handles all of the Resources of a single Type. + * Handles reference counting, hot reloading, and etc. + */ + + struct ResourceTypeManager { + friend struct Resource; + friend struct ResourceManager; + + using ResourceTable = std::map; + + void create(struct ResourceType type, struct ResourceManager& manager); + void destroy(); + + void setUnloadable(bool status); + + void removeUnreferencedResources(); + + void reload(const Path& path); + void reload(Resource& resource); + + ResourceTable& getResources() { return resources; } + + ResourceTypeManager(); + virtual ~ResourceTypeManager(); + ResourceManager& getOwner() const { return *owner; } + + protected: + Resource* load(const Path& path); + virtual Resource* createResource(const Path& path) = 0; + virtual void destroyResource(Resource& res) = 0; + Resource* get(const Path& path); + + ResourceTable resources; + ResourceManager* owner; + bool unloadEnabled; + }; + + /** + * Handles all of the ResourceTypeManagers, for every ResourceType with at least one applicable Resource + */ + + struct ResourceManager { + using ResourceTypeManagers = std::map; + + struct LoadHook { + enum class Action { IMMEDIATE, DEFERRED }; + virtual ~LoadHook(); + virtual Action load(Resource& res) = 0; + void continueLoad(Resource& res); + }; + + ResourceManager(); + ~ResourceManager(); + ResourceManager(const ResourceManager& o) = delete; + + void init(struct FileSystem& fs); + + ResourceTypeManager* get(ResourceType); + const ResourceTypeManagers& getAll() const { return managers; } + + template + R* load(const Path& path) { + return static_cast(load(R::TYPE, path)); + } + + Resource* load(ResourceTypeManager& manager, const Path& path); + Resource* load(ResourceType type, const Path& path); + + void setLoadHook(LoadHook* hook); + bool isHooked() const { return hook; } + LoadHook::Action onLoad(Resource& res) const; + void add(ResourceType, ResourceTypeManager* manager); + void remove(ResourceType type); + void reload(const Path& path); + void reloadAll(); + void removeUnreferenced(); + void setUnloadable(bool enable); + + FileSystem& getFileSystem() { return *filesystem; } + + private: + ResourceTypeManagers managers; + FileSystem* filesystem; + LoadHook* hook; + + }; +} \ No newline at end of file