Resource and Resource Managers.

This commit is contained in:
Curle 2023-03-13 15:25:31 +00:00
parent b723a0778f
commit 49ee9fc10d
8 changed files with 658 additions and 25 deletions

View File

@ -5,19 +5,19 @@ find_package(imgui REQUIRED)
set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD 20)
set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
add_subdirectory(shadow-assets)
FILE(GLOB_RECURSE SOURCES FILE(GLOB_RECURSE SOURCES
core/src/*.cpp core/src/*.cpp
shadow-renderer/src/*.cpp shadow-renderer/src/*.cpp
shadow-reflection/src/*.cpp shadow-reflection/src/*.cpp
shadow-utility/src/*.cpp shadow-utility/src/*.cpp
shadow-assets/src/*.cpp
) )
FILE(GLOB_RECURSE HEADERS FILE(GLOB_RECURSE HEADERS
core/inc/*.h core/inc/*.h
shadow-renderer/inc/*.h shadow-renderer/inc/*.h
shadow-reflection/inc/*.h shadow-reflection/inc/*.h
shadow-utility/inc/*.h shadow-utility/inc/*.h
shadow-assets/src/*.h
) )
add_library(shadow-engine SHARED ${SOURCES} $<TARGET_OBJECTS:imgui>) add_library(shadow-engine SHARED ${SOURCES} $<TARGET_OBJECTS:imgui>)
@ -29,6 +29,7 @@ target_include_directories(shadow-engine
shadow-renderer/inc shadow-renderer/inc
shadow-reflection/inc shadow-reflection/inc
shadow-utility/inc shadow-utility/inc
shadow-assets/src
${glm_SOURCE_DIR} ${glm_SOURCE_DIR}
INTERFACE INTERFACE
${imgui_SOURCE_DIR} ${imgui_SOURCE_DIR}

View File

@ -1,21 +0,0 @@
set(CMAKE_CXX_STANDARD 20)
# Set up Catch2 testing
list(APPEND CMAKE_MODULE_PATH "cmake")
enable_testing()
# Set up asset sourceset
FILE(GLOB_RECURSE SOURCES src/**.cpp src/**.h)
FILE(GLOB_RECURSE TESTS test/*.cpp)
include_directories(src/)
add_library(shadow-asset ${SOURCES})
# Set up test executable
add_executable(shadow-asset-test ${TESTS})
target_link_libraries(shadow-asset-test PRIVATE Catch2::Catch2 shadow-utils)
# Enable testing on the executable
#include(CTest)
#include(Catch2)
#catch_discover_tests(shadow-asset-test)

View File

@ -19,8 +19,8 @@ namespace ShadowEngine {
// Hash arbitrary data. // Hash arbitrary data.
HeapHash(const void* data, uint32_t length); HeapHash(const void* data, uint32_t length);
bool operator!= (HeapHash& other) const { return hash != other.hash; } bool operator!= (const HeapHash& other) const { return hash != other.hash; }
bool operator== (HeapHash& other) const { return hash == other.hash; } bool operator== (const HeapHash& other) const { return hash == other.hash; }
size_t getHash() const { return hash; } size_t getHash() const { return hash; }
private: private:

View File

@ -0,0 +1,58 @@
#pragma once
#include <management/delegate.h>
#include <vector>
#include <cstdint>
namespace ShadowEngine {
template <typename T> struct DelegateList;
template <typename R, typename... Args> struct DelegateList<R(Args...)> {
DelegateList() = default;
template <auto Function, typename C> void bind(C* instance) {
Delegate<R(Args...)> cb;
cb.template bind<Function>(instance);
m_delegates.push_back(cb);
}
template <R (*Function)(Args...)> void bind() {
Delegate<R(Args...)> cb;
cb.template bind<Function>();
m_delegates.push_back(cb);
}
template <R (*Function)(Args...)> void unbind() {
Delegate<R(Args...)> cb;
cb.template bind<Function>();
for (int i = 0; i < m_delegates.size(); ++i)
{
if (m_delegates[i] == cb)
{
m_delegates.swapAndPop(i);
break;
}
}
}
template <auto Function, typename C> void unbind(C* instance) {
Delegate<R(Args...)> cb;
cb.template bind<Function>(instance);
for (int i = 0; i < m_delegates.size(); ++i)
{
if (m_delegates[i] == cb)
{
m_delegates.swapAndPop(i);
break;
}
}
}
void invoke(Args... args) {
for (uint32_t i = 0, c = m_delegates.size(); i < c; ++i) m_delegates[i].invoke(args...);
}
private:
std::vector<Delegate<R(Args...)>> m_delegates;
};
}

View File

@ -0,0 +1,177 @@
#include <resource/Resource.h>
#include <resource/ResourceManager.h>
#include <spdlog/spdlog.h>
namespace ShadowEngine {
ResourceType::ResourceType(std::string& name) {
hash = HeapHash(name);
}
Resource::Resource(const ShadowEngine::Path& path, ShadowEngine::ResourceTypeManager &manager)
: references(0),
emptyDependencies(0),
failedDependencies(0),
state(State::EMPTY),
desiredState(State::EMPTY),
path(path),
size(),
callback(),
manager(manager),
handle(FileSystem::AsyncHandle::invalid()) {
}
Resource::~Resource() = default;
void Resource::refresh() {
if (state == State::EMPTY) return;
const State old = state;
state = State::EMPTY;
callback.invoke(old, state, *this);
checkState();
}
void Resource::checkState() {
State old = state;
if (failedDependencies > 0 && state != State::FAILED) {
state = State::FAILED;
} else if (failedDependencies == 0) {
if (emptyDependencies > 0 && state != State::EMPTY)
state = State::EMPTY;
if (emptyDependencies == 0 && state != State::READY && desiredState != State::EMPTY) {
onReadying();
if (emptyDependencies != 0 || state == State::READY || desiredState == State::EMPTY)
return;
if (failedDependencies != 0) {
checkState();
return;
}
state = State::READY;
}
}
callback.invoke(old, state, *this);
}
void Resource::fileLoaded(size_t fileSize, const uint8_t *mem, bool success) {
handle = FileSystem::AsyncHandle::invalid();
if (desiredState != State::READY) return;
if (!success) {
ResourceManager& owner = getManager().getOwner();
if (!hooked && owner.isHooked()) {
if (owner.onLoad(*this) == ResourceManager::LoadHook::Action::DEFERRED) {
hooked = true;
desiredState = State::READY;
increaseReferences();
return;
}
}
--emptyDependencies;
++failedDependencies;
checkState();
handle = FileSystem::AsyncHandle::invalid();
return;
}
const ResourceHeader* header = (const ResourceHeader*) mem;
if (size < sizeof(*header)) {
spdlog::error("Invalid resource: ", path.get(), ": size mismatch. Expected ", fileSize, ", got " , sizeof(*header));
failedDependencies++;
} else if (header->magic != ResourceHeader::MAGIC) {
spdlog::error("Invalid resource: " , path.get(), ": magic number mismatch. Expected " , ResourceHeader::MAGIC, ", got ", header->magic);
failedDependencies++;
} else if (header->version > 0) {
spdlog::error("Invalid resource: ", path.get(), ": verison mismatch. Expected 0, got ", header->version);
failedDependencies++;
} else {
// TODO: Compression?
if (!load(size - sizeof(*header), mem + sizeof(*header)))
failedDependencies++;
size = header->decompressedSize;
}
emptyDependencies--;
checkState();
handle = FileSystem::AsyncHandle::invalid();
}
void Resource::performUnload() {
if (handle.valid()) {
FileSystem& fs = manager.getOwner().getFileSystem();
fs.cancelAsync(handle);
handle = FileSystem::AsyncHandle::invalid();
}
hooked = false;
desiredState = State::EMPTY;
unload();
size = 0;
emptyDependencies = 1;
failedDependencies = 0;
checkState();
}
void Resource::onCreated(ShadowEngine::Resource::State newState) {
state = newState;
desiredState = State::READY;
failedDependencies = state == State::FAILED ? 1 : 0;
emptyDependencies = 0;
}
void Resource::doLoad() {
if (desiredState == State::READY) return;
desiredState = State::READY;
if (handle.valid()) return;
FileSystem& fs = manager.getOwner().getFileSystem();
FileSystem::ContentCallback cb = makeDelegate<&Resource::fileLoaded>(this);
const PathHash hash = path.getHash();
Path resourcePath("./resources/" + std::to_string(hash.getHash()) + ".res");
handle = fs.readAsync(resourcePath, cb);
}
void Resource::addDependency(ShadowEngine::Resource &dependent) {
dependent.callback.bind<&Resource::stateChanged>(this);
if (dependent.isEmpty()) emptyDependencies++;
if (dependent.isFailure()) failedDependencies++;
checkState();
}
void Resource::removeDependency(ShadowEngine::Resource &dependent) {
dependent.callback.unbind<&Resource::stateChanged>(this);
if (dependent.isEmpty()) --emptyDependencies;
if (dependent.isFailure()) --failedDependencies;
checkState();
}
uint32_t Resource::decreaseReferences() {
--references;
if (references == 0 && manager.unloadEnabled)
performUnload();
return references;
}
void Resource::stateChanged(ShadowEngine::Resource::State old, ShadowEngine::Resource::State newState,
ShadowEngine::Resource &) {
if (old == State::EMPTY) --emptyDependencies;
if (old == State::FAILED) --failedDependencies;
if (newState == State::EMPTY) ++emptyDependencies;
if (newState == State::FAILED) ++failedDependencies;
checkState();
}
}

View File

@ -0,0 +1,128 @@
#pragma once
#include "fs/hash.h"
#include "fs/path.h"
#include "fs/file.h"
#include <management/delegate_list.h>
namespace ShadowEngine {
/**
* A runtime-only struct that determines the type of a resource - whether it be a texture, mesh, animation, or other data.
* Provides some specializations for living in a map.
*/
struct ResourceType {
ResourceType() = default;
explicit ResourceType(std::string& name);
bool operator!=(const ResourceType& o) const { return o.hash != hash; }
bool operator==(const ResourceType& o) const { return o.hash == hash; }
bool operator< (const ResourceType& o) const { return o.hash.getHash() < hash.getHash(); }
bool isValid() const { return hash.getHash() != 0; }
HeapHash hash;
};
// A Resource Type that is guaranteed to be invalid.
const ResourceType INVALID_RESOURCE((std::string &) "");
// A specialization of HashFunc for ResourceTypes, since they already have a HeapHash within.
template<> struct HashFunc<ResourceType> {
static uint32_t get(const ResourceType& key) { return HashFunc<HeapHash>::get(key.hash); }
};
#pragma pack(1)
struct ResourceHeader {
static const uint32_t MAGIC = 'VXIP';
uint32_t magic = MAGIC; // VXI Package header
uint32_t version = 0;
uint32_t flags = 0;
uint32_t padding = 0;
uint32_t decompressedSize = 0;
};
#pragma pack()
/**
* A basic Resource type.
* Represents a single file loaded from disk.
* May have dependencies on other Resources, and other Resources may depend on this.
* Resources are reference-counted, and are removed when they go out of usage.
*/
struct Resource {
friend struct ResourceTypeManager;
friend struct ResourceManager;
enum class State : uint32_t {
EMPTY = 0,
READY,
FAILED
};
using Observer = DelegateList<void(State, State, Resource&)>;
virtual ~Resource();
virtual ResourceType getType() const = 0;
State getState() const { return state; }
bool isEmpty() const { return state == State::EMPTY; }
bool isReady() const { return state == State::READY; }
bool isFailure() const { return state == State::FAILED; }
uint32_t getReferenceCount() const { return references; }
Observer const& getCallback() const { return callback; }
size_t getSize() const { return size; }
const Path& getPath() const { return path; }
struct ResourceTypeManager& getManager() { return manager; }
uint32_t decreaseReferences();
uint32_t increaseReferences() { return references++; }
bool toInitialize() const { return desiredState == State::READY; }
bool isHooked() const { return hooked; }
template <auto Function, typename C> void onLoaded(C* instance) {
callback.bind<Function>(instance);
if (isReady()) (instance->*Function)(State::READY, State::READY, *this);
}
protected:
Resource(const Path& path, ResourceTypeManager& manager);
virtual void onReadying() {}
virtual void unload() = 0;
virtual bool load(size_t size, const uint8_t* mem) = 0;
void onCreated(State newState);
void performUnload();
void addDependency(Resource& dependent);
void removeDependency(Resource& dependent);
void checkState();
void refresh();
State desiredState;
uint16_t emptyDependencies;
ResourceTypeManager& manager;
private:
void doLoad();
void fileLoaded(size_t fileSize, const uint8_t* mem, bool success);
void stateChanged(State old, State newState, Resource&);
Resource(const Resource&) = delete;
void operator=(const Resource&) = delete;
Observer callback;
size_t size;
Path path;
uint32_t references;
uint16_t failedDependencies;
FileSystem::AsyncHandle handle;
State state;
bool hooked = false;
};
}

View File

@ -0,0 +1,195 @@
#include "ResourceManager.h"
#include "Resource.h"
#include "spdlog/spdlog.h"
namespace ShadowEngine {
void ResourceTypeManager::create(struct ResourceType type, struct ResourceManager &manager) {
manager.add(type, this);
owner = &manager;
}
void ResourceTypeManager::destroy() {
for (auto iter = resources.begin(), end = resources.end(); iter != end; ++iter) {
Resource* res = iter->second;
if (!res->isEmpty())
spdlog::error("Resource Type Manager destruction leaks ", res->path.get());
destroyResource(*res);
}
resources.clear();
}
Resource* ResourceTypeManager::get(const Path& path) {
auto it = resources.find(path.getHash());
if (it != resources.end()) return it->second;
return nullptr;
}
Resource* ResourceTypeManager::load(const Path &path) {
if (path.isEmpty()) return nullptr;
Resource* res = get(path);
if (res == nullptr) {
res = createResource(path);
resources[path.getHash()] = res;
}
if (res->isEmpty() && res->desiredState == Resource::State::EMPTY) {
if (owner->onLoad(*res) == ResourceManager::LoadHook::Action::DEFERRED) {
res->hooked = true;
res->desiredState = Resource::State::READY;
res->increaseReferences();
res->increaseReferences();
return res;
}
res->doLoad();
}
res->increaseReferences();
return res;
}
void ResourceTypeManager::removeUnreferencedResources() {
if (!unloadEnabled) return;
std::vector<Resource*> toRemove;
for (auto i : resources)
if (i.second->getReferenceCount() == 0) toRemove.push_back(i.second);
for (auto i : toRemove) {
auto iter = resources.find(i->getPath().getHash());
if (iter->second->isReady()) iter->second->performUnload();
}
}
void ResourceTypeManager::reload(const Path &path) {
Resource* res = get(path);
if (res) reload(*res);
}
void ResourceTypeManager::reload(Resource& res) {
if (res.state != Resource::State::EMPTY)
res.performUnload();
else if (res.desiredState == Resource::State::READY)
return;
if (owner->onLoad(res) == ResourceManager::LoadHook::Action::DEFERRED) {
res.hooked = true;
res.desiredState = Resource::State::READY;
res.increaseReferences();
res.increaseReferences();
} else {
res.performUnload();
}
}
void ResourceTypeManager::setUnloadable(bool status) {
unloadEnabled = status;
if (!unloadEnabled) return;
for (auto res : resources)
if (res.second->getReferenceCount() == 0)
res.second->performUnload();
}
ResourceTypeManager::ResourceTypeManager() :
resources(),
owner(nullptr),
unloadEnabled(true) {
}
ResourceTypeManager::~ResourceTypeManager() {
}
ResourceManager::ResourceManager() :
managers(),
hook(nullptr),
filesystem(nullptr) {
}
ResourceManager::~ResourceManager() = default;
void ResourceManager::init(FileSystem &fs) {
filesystem = &fs;
}
Resource* ResourceManager::load(ResourceType type, const Path& path) {
ResourceTypeManager* manager = get(type);
if (!manager) return nullptr;
return load(*manager, path);
}
Resource* ResourceManager::load(ResourceTypeManager& manager, const Path& path) {
return manager.load(path);
}
ResourceTypeManager* ResourceManager::get(ResourceType type) {
auto iter = managers.find(type);
if (iter == managers.end()) return nullptr;
return iter->second;
}
void ResourceManager::LoadHook::continueLoad(Resource &res) {
res.decreaseReferences();
res.hooked = false;
res.desiredState = Resource::State::EMPTY;
res.doLoad();
}
void ResourceManager::setLoadHook(LoadHook *loadHook) {
hook = loadHook;
if (hook)
for (auto manager : managers)
for (auto res : manager.second->getResources())
if (res.second->isFailure())
manager.second->reload(*res.second);
}
ResourceManager::LoadHook::Action ResourceManager::onLoad(Resource &res) const {
return hook ? hook->load(res) : LoadHook::Action::IMMEDIATE;
}
void ResourceManager::add(ResourceType type, ResourceTypeManager* manager) {
managers[type] = manager;
}
void ResourceManager::remove(ResourceType type) {
managers.erase(type);
}
void ResourceManager::removeUnreferenced() {
for (auto manager : managers)
manager.second->removeUnreferencedResources();
}
void ResourceManager::setUnloadable(bool enable) {
for (auto manager : managers)
manager.second->setUnloadable(enable);
}
void ResourceManager::reloadAll() {
while (filesystem->hasWork()) filesystem->processCallbacks();
std::vector<Resource*> toReload;
for (auto manager : managers) {
ResourceTypeManager::ResourceTable& resources = manager.second->getResources();
for (auto res : resources) {
if (res.second->isReady()) {
res.second->performUnload();
toReload.push_back(res.second);
}
}
}
}
void ResourceManager::reload(const Path& path) {
for (auto manager : managers)
manager.second->reload(path);
}
}

View File

@ -0,0 +1,95 @@
#pragma once
#include <map>
#include <fs/hash.h>
#include <fs/path.h>
namespace ShadowEngine {
/**
* Handles all of the Resources of a single Type.
* Handles reference counting, hot reloading, and etc.
*/
struct ResourceTypeManager {
friend struct Resource;
friend struct ResourceManager;
using ResourceTable = std::map<PathHash, struct Resource*>;
void create(struct ResourceType type, struct ResourceManager& manager);
void destroy();
void setUnloadable(bool status);
void removeUnreferencedResources();
void reload(const Path& path);
void reload(Resource& resource);
ResourceTable& getResources() { return resources; }
ResourceTypeManager();
virtual ~ResourceTypeManager();
ResourceManager& getOwner() const { return *owner; }
protected:
Resource* load(const Path& path);
virtual Resource* createResource(const Path& path) = 0;
virtual void destroyResource(Resource& res) = 0;
Resource* get(const Path& path);
ResourceTable resources;
ResourceManager* owner;
bool unloadEnabled;
};
/**
* Handles all of the ResourceTypeManagers, for every ResourceType with at least one applicable Resource
*/
struct ResourceManager {
using ResourceTypeManagers = std::map<ResourceType, ResourceTypeManager*>;
struct LoadHook {
enum class Action { IMMEDIATE, DEFERRED };
virtual ~LoadHook();
virtual Action load(Resource& res) = 0;
void continueLoad(Resource& res);
};
ResourceManager();
~ResourceManager();
ResourceManager(const ResourceManager& o) = delete;
void init(struct FileSystem& fs);
ResourceTypeManager* get(ResourceType);
const ResourceTypeManagers& getAll() const { return managers; }
template <typename R>
R* load(const Path& path) {
return static_cast<R*>(load(R::TYPE, path));
}
Resource* load(ResourceTypeManager& manager, const Path& path);
Resource* load(ResourceType type, const Path& path);
void setLoadHook(LoadHook* hook);
bool isHooked() const { return hook; }
LoadHook::Action onLoad(Resource& res) const;
void add(ResourceType, ResourceTypeManager* manager);
void remove(ResourceType type);
void reload(const Path& path);
void reloadAll();
void removeUnreferenced();
void setUnloadable(bool enable);
FileSystem& getFileSystem() { return *filesystem; }
private:
ResourceTypeManagers managers;
FileSystem* filesystem;
LoadHook* hook;
};
}