Resource and Resource Managers.

projs/shadow/shadow-engine/CMakeLists.txt

@@ -5,19 +5,19 @@ find_package(imgui REQUIRED)
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 
-add_subdirectory(shadow-assets)
-
 FILE(GLOB_RECURSE SOURCES
         core/src/*.cpp
         shadow-renderer/src/*.cpp
         shadow-reflection/src/*.cpp
         shadow-utility/src/*.cpp
+        shadow-assets/src/*.cpp
         )
 FILE(GLOB_RECURSE HEADERS
         core/inc/*.h
         shadow-renderer/inc/*.h
         shadow-reflection/inc/*.h
         shadow-utility/inc/*.h
+        shadow-assets/src/*.h
         )
 
 add_library(shadow-engine SHARED ${SOURCES} $<TARGET_OBJECTS:imgui>)
@@ -29,6 +29,7 @@ target_include_directories(shadow-engine
             shadow-renderer/inc
             shadow-reflection/inc
             shadow-utility/inc
+            shadow-assets/src
         ${glm_SOURCE_DIR}
         INTERFACE
             ${imgui_SOURCE_DIR}

projs/shadow/shadow-engine/shadow-assets/CMakeLists.txt

@@ -1,21 +0,0 @@
-set(CMAKE_CXX_STANDARD 20)
-
-# Set up Catch2 testing
-list(APPEND CMAKE_MODULE_PATH "cmake")
-enable_testing()
-
-# Set up asset sourceset
-FILE(GLOB_RECURSE SOURCES src/**.cpp src/**.h)
-FILE(GLOB_RECURSE TESTS test/*.cpp)
-
-include_directories(src/)
-add_library(shadow-asset ${SOURCES})
-
-# Set up test executable
-add_executable(shadow-asset-test ${TESTS})
-target_link_libraries(shadow-asset-test PRIVATE Catch2::Catch2 shadow-utils)
-
-# Enable testing on the executable
-#include(CTest)
-#include(Catch2)
-#catch_discover_tests(shadow-asset-test)

projs/shadow/shadow-engine/shadow-assets/src/fs/hash.h

@@ -19,8 +19,8 @@ namespace ShadowEngine {
         // Hash arbitrary data.
         HeapHash(const void* data, uint32_t length);
 
-        bool operator!= (HeapHash& other) const { return hash != other.hash; }
+        bool operator!= (const HeapHash& other) const { return hash != other.hash; }
-        bool operator== (HeapHash& other) const { return hash == other.hash; }
+        bool operator== (const HeapHash& other) const { return hash == other.hash; }
 
         size_t getHash() const { return hash; }
     private:

projs/shadow/shadow-engine/shadow-assets/src/management/delegate_list.h

@@ -0,0 +1,58 @@
+#pragma once
+#include <management/delegate.h>
+#include <vector>
+#include <cstdint>
+
+namespace ShadowEngine {
+
+    template <typename T> struct DelegateList;
+
+    template <typename R, typename... Args> struct DelegateList<R(Args...)> {
+        DelegateList() = default;
+
+        template <auto Function, typename C> void bind(C* instance) {
+            Delegate<R(Args...)> cb;
+            cb.template bind<Function>(instance);
+            m_delegates.push_back(cb);
+        }
+
+        template <R (*Function)(Args...)> void bind() {
+            Delegate<R(Args...)> cb;
+            cb.template bind<Function>();
+            m_delegates.push_back(cb);
+        }
+
+        template <R (*Function)(Args...)> void unbind() {
+            Delegate<R(Args...)> cb;
+            cb.template bind<Function>();
+            for (int i = 0; i < m_delegates.size(); ++i)
+            {
+                if (m_delegates[i] == cb)
+                {
+                    m_delegates.swapAndPop(i);
+                    break;
+                }
+            }
+        }
+
+        template <auto Function, typename C> void unbind(C* instance) {
+            Delegate<R(Args...)> cb;
+            cb.template bind<Function>(instance);
+            for (int i = 0; i < m_delegates.size(); ++i)
+            {
+                if (m_delegates[i] == cb)
+                {
+                    m_delegates.swapAndPop(i);
+                    break;
+                }
+            }
+        }
+
+        void invoke(Args... args) {
+            for (uint32_t i = 0, c = m_delegates.size(); i < c; ++i) m_delegates[i].invoke(args...);
+        }
+
+    private:
+        std::vector<Delegate<R(Args...)>> m_delegates;
+    };
+}

projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.cpp

@@ -0,0 +1,177 @@
+#include <resource/Resource.h>
+#include <resource/ResourceManager.h>
+#include <spdlog/spdlog.h>
+
+namespace ShadowEngine {
+
+    ResourceType::ResourceType(std::string& name) {
+        hash = HeapHash(name);
+    }
+
+    Resource::Resource(const ShadowEngine::Path& path, ShadowEngine::ResourceTypeManager &manager)
+        : references(0),
+          emptyDependencies(0),
+          failedDependencies(0),
+          state(State::EMPTY),
+          desiredState(State::EMPTY),
+          path(path),
+          size(),
+          callback(),
+          manager(manager),
+          handle(FileSystem::AsyncHandle::invalid()) {
+    }
+
+    Resource::~Resource() = default;
+
+    void Resource::refresh() {
+        if (state == State::EMPTY) return;
+
+        const State old = state;
+        state = State::EMPTY;
+        callback.invoke(old, state, *this);
+        checkState();
+    }
+
+    void Resource::checkState() {
+        State old = state;
+        if (failedDependencies > 0 && state != State::FAILED) {
+            state = State::FAILED;
+        } else if (failedDependencies == 0) {
+            if (emptyDependencies > 0 && state != State::EMPTY)
+                state = State::EMPTY;
+
+            if (emptyDependencies == 0 && state != State::READY && desiredState != State::EMPTY) {
+                onReadying();
+
+                if (emptyDependencies != 0 || state == State::READY || desiredState == State::EMPTY)
+                    return;
+
+                if (failedDependencies != 0) {
+                    checkState();
+                    return;
+                }
+
+                state = State::READY;
+            }
+        }
+        callback.invoke(old, state, *this);
+    }
+
+    void Resource::fileLoaded(size_t fileSize, const uint8_t *mem, bool success) {
+        handle = FileSystem::AsyncHandle::invalid();
+        if (desiredState != State::READY) return;
+
+        if (!success) {
+            ResourceManager& owner = getManager().getOwner();
+            if (!hooked && owner.isHooked()) {
+                if (owner.onLoad(*this) == ResourceManager::LoadHook::Action::DEFERRED) {
+                    hooked = true;
+                    desiredState = State::READY;
+                    increaseReferences();
+                    return;
+                }
+            }
+
+            --emptyDependencies;
+            ++failedDependencies;
+            checkState();
+            handle = FileSystem::AsyncHandle::invalid();
+            return;
+        }
+
+        const ResourceHeader* header = (const ResourceHeader*) mem;
+
+        if (size < sizeof(*header)) {
+            spdlog::error("Invalid resource: ", path.get(), ": size mismatch. Expected ", fileSize, ", got " , sizeof(*header));
+            failedDependencies++;
+        } else if (header->magic != ResourceHeader::MAGIC) {
+            spdlog::error("Invalid resource: " , path.get(), ": magic number mismatch. Expected " , ResourceHeader::MAGIC, ", got ", header->magic);
+            failedDependencies++;
+        } else if (header->version > 0) {
+            spdlog::error("Invalid resource: ", path.get(), ": verison mismatch. Expected 0, got ", header->version);
+            failedDependencies++;
+        } else {
+          // TODO: Compression?
+          if (!load(size - sizeof(*header), mem + sizeof(*header)))
+              failedDependencies++;
+          size = header->decompressedSize;
+        }
+
+        emptyDependencies--;
+        checkState();
+        handle = FileSystem::AsyncHandle::invalid();
+    }
+
+    void Resource::performUnload() {
+        if (handle.valid()) {
+            FileSystem& fs = manager.getOwner().getFileSystem();
+            fs.cancelAsync(handle);
+            handle = FileSystem::AsyncHandle::invalid();
+        }
+
+        hooked = false;
+        desiredState = State::EMPTY;
+        unload();
+
+        size = 0;
+        emptyDependencies = 1;
+        failedDependencies = 0;
+        checkState();
+    }
+
+    void Resource::onCreated(ShadowEngine::Resource::State newState) {
+        state = newState;
+        desiredState = State::READY;
+        failedDependencies = state == State::FAILED ? 1 : 0;
+        emptyDependencies = 0;
+    }
+
+    void Resource::doLoad() {
+        if (desiredState == State::READY) return;
+        desiredState = State::READY;
+
+        if (handle.valid()) return;
+
+        FileSystem& fs = manager.getOwner().getFileSystem();
+        FileSystem::ContentCallback cb = makeDelegate<&Resource::fileLoaded>(this);
+
+        const PathHash hash = path.getHash();
+        Path resourcePath("./resources/" + std::to_string(hash.getHash()) + ".res");
+        handle = fs.readAsync(resourcePath, cb);
+    }
+
+    void Resource::addDependency(ShadowEngine::Resource &dependent) {
+        dependent.callback.bind<&Resource::stateChanged>(this);
+        if (dependent.isEmpty()) emptyDependencies++;
+        if (dependent.isFailure()) failedDependencies++;
+
+        checkState();
+    }
+
+    void Resource::removeDependency(ShadowEngine::Resource &dependent) {
+        dependent.callback.unbind<&Resource::stateChanged>(this);
+        if (dependent.isEmpty()) --emptyDependencies;
+        if (dependent.isFailure()) --failedDependencies;
+
+        checkState();
+    }
+
+    uint32_t Resource::decreaseReferences() {
+        --references;
+        if (references == 0 && manager.unloadEnabled)
+            performUnload();
+
+        return references;
+    }
+
+    void Resource::stateChanged(ShadowEngine::Resource::State old, ShadowEngine::Resource::State newState,
+                                ShadowEngine::Resource &) {
+        if (old == State::EMPTY) --emptyDependencies;
+        if (old == State::FAILED) --failedDependencies;
+
+        if (newState == State::EMPTY) ++emptyDependencies;
+        if (newState == State::FAILED) ++failedDependencies;
+
+        checkState();
+    }
+}

projs/shadow/shadow-engine/shadow-assets/src/resource/Resource.h

@@ -0,0 +1,128 @@
+#pragma once
+
+#include "fs/hash.h"
+#include "fs/path.h"
+#include "fs/file.h"
+#include <management/delegate_list.h>
+
+namespace ShadowEngine {
+
+    /**
+     * A runtime-only struct that determines the type of a resource - whether it be a texture, mesh, animation, or other data.
+     * Provides some specializations for living in a map.
+     */
+    struct ResourceType {
+        ResourceType() = default;
+        explicit ResourceType(std::string& name);
+        bool operator!=(const ResourceType& o) const { return o.hash != hash; }
+        bool operator==(const ResourceType& o) const { return o.hash == hash; }
+        bool operator< (const ResourceType& o) const { return o.hash.getHash() < hash.getHash(); }
+        bool isValid() const { return hash.getHash() != 0; }
+
+        HeapHash hash;
+    };
+
+    // A Resource Type that is guaranteed to be invalid.
+    const ResourceType INVALID_RESOURCE((std::string &) "");
+
+    // A specialization of HashFunc for ResourceTypes, since they already have a HeapHash within.
+    template<> struct HashFunc<ResourceType> {
+        static uint32_t get(const ResourceType& key) { return HashFunc<HeapHash>::get(key.hash); }
+    };
+
+#pragma pack(1)
+    struct ResourceHeader {
+        static const uint32_t MAGIC = 'VXIP';
+        uint32_t magic = MAGIC; // VXI Package header
+        uint32_t version = 0;
+        uint32_t flags = 0;
+        uint32_t padding = 0;
+        uint32_t decompressedSize = 0;
+    };
+#pragma pack()
+
+    /**
+     * A basic Resource type.
+     * Represents a single file loaded from disk.
+     * May have dependencies on other Resources, and other Resources may depend on this.
+     * Resources are reference-counted, and are removed when they go out of usage.
+     */
+
+    struct Resource {
+
+        friend struct ResourceTypeManager;
+        friend struct ResourceManager;
+
+        enum class State : uint32_t {
+            EMPTY = 0,
+            READY,
+            FAILED
+        };
+
+        using Observer = DelegateList<void(State, State, Resource&)>;
+
+        virtual ~Resource();
+        virtual ResourceType getType() const = 0;
+        State getState() const { return state; }
+
+        bool isEmpty() const { return state == State::EMPTY; }
+        bool isReady() const { return state == State::READY; }
+        bool isFailure() const { return state == State::FAILED; }
+
+        uint32_t getReferenceCount() const { return references; }
+
+        Observer const& getCallback() const { return callback; }
+        size_t getSize() const { return size; }
+
+        const Path& getPath() const { return path; }
+
+        struct ResourceTypeManager& getManager() { return manager; }
+
+        uint32_t decreaseReferences();
+        uint32_t increaseReferences() { return references++; }
+
+        bool toInitialize() const { return desiredState == State::READY; }
+        bool isHooked() const { return hooked; }
+
+        template <auto Function, typename C> void onLoaded(C* instance) {
+            callback.bind<Function>(instance);
+            if (isReady()) (instance->*Function)(State::READY, State::READY, *this);
+        }
+
+    protected:
+        Resource(const Path& path, ResourceTypeManager& manager);
+
+        virtual void onReadying() {}
+        virtual void unload() = 0;
+        virtual bool load(size_t size, const uint8_t* mem) = 0;
+
+        void onCreated(State newState);
+        void performUnload();
+        void addDependency(Resource& dependent);
+        void removeDependency(Resource& dependent);
+        void checkState();
+        void refresh();
+
+        State desiredState;
+        uint16_t emptyDependencies;
+        ResourceTypeManager& manager;
+
+    private:
+
+        void doLoad();
+        void fileLoaded(size_t fileSize, const uint8_t* mem, bool success);
+        void stateChanged(State old, State newState, Resource&);
+
+        Resource(const Resource&) = delete;
+        void operator=(const Resource&) = delete;
+
+        Observer callback;
+        size_t size;
+        Path path;
+        uint32_t references;
+        uint16_t failedDependencies;
+        FileSystem::AsyncHandle handle;
+        State state;
+        bool hooked = false;
+    };
+}

projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.cpp

@@ -0,0 +1,195 @@
+#include "ResourceManager.h"
+#include "Resource.h"
+#include "spdlog/spdlog.h"
+
+namespace ShadowEngine {
+
+    void ResourceTypeManager::create(struct ResourceType type, struct ResourceManager &manager) {
+        manager.add(type, this);
+        owner = &manager;
+    }
+
+    void ResourceTypeManager::destroy() {
+        for (auto iter = resources.begin(), end = resources.end(); iter != end; ++iter) {
+            Resource* res = iter->second;
+            if (!res->isEmpty())
+                spdlog::error("Resource Type Manager destruction leaks ", res->path.get());
+
+            destroyResource(*res);
+        }
+        resources.clear();
+    }
+
+    Resource* ResourceTypeManager::get(const Path& path) {
+        auto it = resources.find(path.getHash());
+        if (it != resources.end()) return it->second;
+        return nullptr;
+    }
+
+    Resource* ResourceTypeManager::load(const Path &path) {
+        if (path.isEmpty()) return nullptr;
+        Resource* res = get(path);
+        if (res == nullptr) {
+            res = createResource(path);
+            resources[path.getHash()] = res;
+        }
+
+        if (res->isEmpty() && res->desiredState == Resource::State::EMPTY) {
+            if (owner->onLoad(*res) == ResourceManager::LoadHook::Action::DEFERRED) {
+                res->hooked = true;
+                res->desiredState = Resource::State::READY;
+                res->increaseReferences();
+                res->increaseReferences();
+                return res;
+            }
+
+            res->doLoad();
+        }
+
+        res->increaseReferences();
+        return res;
+    }
+
+    void ResourceTypeManager::removeUnreferencedResources() {
+        if (!unloadEnabled) return;
+
+        std::vector<Resource*> toRemove;
+        for (auto i : resources)
+            if (i.second->getReferenceCount() == 0) toRemove.push_back(i.second);
+
+        for (auto i : toRemove) {
+            auto iter = resources.find(i->getPath().getHash());
+            if (iter->second->isReady()) iter->second->performUnload();
+        }
+    }
+
+    void ResourceTypeManager::reload(const Path &path) {
+        Resource* res = get(path);
+        if (res) reload(*res);
+    }
+
+    void ResourceTypeManager::reload(Resource& res) {
+        if (res.state != Resource::State::EMPTY)
+            res.performUnload();
+        else if (res.desiredState == Resource::State::READY)
+            return;
+
+        if (owner->onLoad(res) == ResourceManager::LoadHook::Action::DEFERRED) {
+            res.hooked = true;
+            res.desiredState = Resource::State::READY;
+            res.increaseReferences();
+            res.increaseReferences();
+        } else {
+            res.performUnload();
+        }
+    }
+
+    void ResourceTypeManager::setUnloadable(bool status) {
+        unloadEnabled = status;
+        if (!unloadEnabled) return;
+
+        for (auto res : resources)
+            if (res.second->getReferenceCount() == 0)
+                res.second->performUnload();
+    }
+
+    ResourceTypeManager::ResourceTypeManager() :
+        resources(),
+        owner(nullptr),
+        unloadEnabled(true) {
+
+    }
+
+    ResourceTypeManager::~ResourceTypeManager() {
+
+    }
+
+    ResourceManager::ResourceManager() :
+        managers(),
+        hook(nullptr),
+        filesystem(nullptr) {
+
+    }
+
+    ResourceManager::~ResourceManager() = default;
+
+    void ResourceManager::init(FileSystem &fs) {
+        filesystem = &fs;
+    }
+
+    Resource* ResourceManager::load(ResourceType type, const Path& path) {
+        ResourceTypeManager* manager = get(type);
+        if (!manager) return nullptr;
+        return load(*manager, path);
+    }
+
+    Resource* ResourceManager::load(ResourceTypeManager& manager, const Path& path) {
+        return manager.load(path);
+    }
+
+    ResourceTypeManager* ResourceManager::get(ResourceType type) {
+        auto iter = managers.find(type);
+        if (iter == managers.end()) return nullptr;
+        return iter->second;
+    }
+
+    void ResourceManager::LoadHook::continueLoad(Resource &res) {
+        res.decreaseReferences();
+        res.hooked = false;
+        res.desiredState = Resource::State::EMPTY;
+        res.doLoad();
+    }
+
+    void ResourceManager::setLoadHook(LoadHook *loadHook) {
+        hook = loadHook;
+
+        if (hook)
+            for (auto manager : managers)
+                for (auto res : manager.second->getResources())
+                    if (res.second->isFailure())
+                        manager.second->reload(*res.second);
+    }
+
+    ResourceManager::LoadHook::Action ResourceManager::onLoad(Resource &res) const {
+        return hook ? hook->load(res) : LoadHook::Action::IMMEDIATE;
+    }
+
+    void ResourceManager::add(ResourceType type, ResourceTypeManager* manager) {
+        managers[type] = manager;
+    }
+
+    void ResourceManager::remove(ResourceType type) {
+        managers.erase(type);
+    }
+
+    void ResourceManager::removeUnreferenced() {
+        for (auto manager : managers)
+            manager.second->removeUnreferencedResources();
+    }
+
+    void ResourceManager::setUnloadable(bool enable) {
+        for (auto manager : managers)
+            manager.second->setUnloadable(enable);
+    }
+
+    void ResourceManager::reloadAll() {
+        while (filesystem->hasWork()) filesystem->processCallbacks();
+
+        std::vector<Resource*> toReload;
+        for (auto manager : managers) {
+            ResourceTypeManager::ResourceTable& resources = manager.second->getResources();
+            for (auto res : resources) {
+                if (res.second->isReady()) {
+                    res.second->performUnload();
+                    toReload.push_back(res.second);
+                }
+            }
+        }
+    }
+
+    void ResourceManager::reload(const Path& path) {
+        for (auto manager : managers)
+            manager.second->reload(path);
+    }
+
+}

projs/shadow/shadow-engine/shadow-assets/src/resource/ResourceManager.h

@@ -0,0 +1,95 @@
+#pragma once
+#include <map>
+#include <fs/hash.h>
+#include <fs/path.h>
+
+namespace ShadowEngine {
+
+    /**
+     * Handles all of the Resources of a single Type.
+     * Handles reference counting, hot reloading, and etc.
+     */
+
+    struct ResourceTypeManager {
+        friend struct Resource;
+        friend struct ResourceManager;
+
+        using ResourceTable = std::map<PathHash, struct Resource*>;
+
+        void create(struct ResourceType type, struct ResourceManager& manager);
+        void destroy();
+
+        void setUnloadable(bool status);
+
+        void removeUnreferencedResources();
+
+        void reload(const Path& path);
+        void reload(Resource& resource);
+
+        ResourceTable& getResources() { return resources; }
+
+        ResourceTypeManager();
+        virtual ~ResourceTypeManager();
+        ResourceManager& getOwner() const { return *owner; }
+
+    protected:
+        Resource* load(const Path& path);
+        virtual Resource* createResource(const Path& path) = 0;
+        virtual void destroyResource(Resource& res) = 0;
+        Resource* get(const Path& path);
+
+        ResourceTable resources;
+        ResourceManager* owner;
+        bool unloadEnabled;
+    };
+
+    /**
+     * Handles all of the ResourceTypeManagers, for every ResourceType with at least one applicable Resource
+     */
+
+    struct ResourceManager {
+        using ResourceTypeManagers = std::map<ResourceType, ResourceTypeManager*>;
+
+        struct LoadHook {
+            enum class Action { IMMEDIATE, DEFERRED };
+            virtual ~LoadHook();
+            virtual Action load(Resource& res) = 0;
+            void continueLoad(Resource& res);
+        };
+
+        ResourceManager();
+        ~ResourceManager();
+        ResourceManager(const ResourceManager& o) = delete;
+
+        void init(struct FileSystem& fs);
+
+        ResourceTypeManager* get(ResourceType);
+        const ResourceTypeManagers& getAll() const { return managers; }
+
+        template <typename R>
+        R* load(const Path& path) {
+            return static_cast<R*>(load(R::TYPE, path));
+        }
+
+        Resource* load(ResourceTypeManager& manager, const Path& path);
+        Resource* load(ResourceType type, const Path& path);
+
+        void setLoadHook(LoadHook* hook);
+        bool isHooked() const { return hook; }
+        LoadHook::Action onLoad(Resource& res) const;
+        void add(ResourceType, ResourceTypeManager* manager);
+        void remove(ResourceType type);
+        void reload(const Path& path);
+        void reloadAll();
+        void removeUnreferenced();
+        void setUnloadable(bool enable);
+
+        FileSystem& getFileSystem() { return *filesystem; }
+
+    private:
+        ResourceTypeManagers managers;
+        FileSystem* filesystem;
+        LoadHook* hook;
+
+    };
+}