File & FileSystem abstractions

.github/workflows/build.yml

@@ -58,8 +58,8 @@ jobs:
     - name: Make output folder
       run: mkdir ./test-results
 
-    - name: Test shadow-file-format
-      run: ./bazel-bin/projs/shadow-file-format/test.exe -r junit -o ./test-results/shadow-file-format-test.xml
+    - name: Test shadow-assets
+      run: ./bazel-bin/projs/shadow-assets/test.exe -r junit -o ./test-results/shadow-assets-test.xml
 
     - name: Publish Test Results
       uses: EnricoMi/publish-unit-test-result-action/composite@v1

imgui.ini

@@ -5,37 +5,36 @@ Collapsed=0
 DockId=0x00000008,0
 
 [Window][Dear ImGui Demo]
-Pos=1193,178
-Size=62,25
+Pos=1193,111
+Size=62,92
 Collapsed=0
-DockId=0x00000007,0
+DockId=0x00000003,0
 
 [Window][Game module window]
-Pos=1193,8
-Size=62,50
+Pos=956,275
+Size=302,121
 Collapsed=0
-DockId=0x00000004,0
 
 [Window][Time]
-Pos=1193,60
-Size=62,49
+Pos=1193,8
+Size=62,101
 Collapsed=0
-DockId=0x00000005,0
+DockId=0x00000002,0
 
 [Window][Active Modules]
-Pos=1193,111
-Size=62,65
+Pos=829,449
+Size=362,187
+Collapsed=0
+
+[Window][Game View]
+Pos=60,60
+Size=656,515
 Collapsed=0
-DockId=0x00000006,0
 
 [Docking][Data]
 DockNode      ID=0x00000001 Pos=1196,188 Size=379,195 Split=X
   DockNode    ID=0x00000008 Parent=0x00000001 SizeRef=652,419 HiddenTabBar=1 Selected=0x55954704
   DockNode    ID=0x00000009 Parent=0x00000001 SizeRef=129,419 Split=Y
-    DockNode    ID=0x00000002 Parent=0x00000009 SizeRef=219,34 Split=Y Selected=0xFC1D20C0
-      DockNode  ID=0x00000004 Parent=0x00000002 SizeRef=219,64 Selected=0xFC1D20C0
-      DockNode  ID=0x00000005 Parent=0x00000002 SizeRef=219,62 Selected=0xE75A179E
-    DockNode    ID=0x00000003 Parent=0x00000009 SizeRef=219,31 Split=Y Selected=0xEE305C78
-      DockNode  ID=0x00000006 Parent=0x00000003 SizeRef=219,142 Selected=0xEE305C78
-      DockNode  ID=0x00000007 Parent=0x00000003 SizeRef=219,55 Selected=0xE87781F4
+    DockNode  ID=0x00000002 Parent=0x00000009 SizeRef=219,34 Selected=0xE75A179E
+    DockNode  ID=0x00000003 Parent=0x00000009 SizeRef=219,31 Selected=0xE87781F4
 

projs/shadow/shadow-engine/CMakeLists.txt

@@ -5,6 +5,8 @@ find_package(imgui REQUIRED)
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 
+add_subdirectory(shadow-assets)
+
 FILE(GLOB_RECURSE SOURCES
         core/src/*.cpp
         shadow-renderer/src/*.cpp

projs/shadow/shadow-engine/shadow-assets/CMakeLists.txt

@@ -5,9 +5,10 @@ list(APPEND CMAKE_MODULE_PATH "cmake")
 enable_testing()
 
 # Set up asset sourceset
-FILE(GLOB_RECURSE SOURCES src/*.cpp src/*.h)
+FILE(GLOB_RECURSE SOURCES src/**.cpp src/**.h)
 FILE(GLOB_RECURSE TESTS test/*.cpp)
 
+include_directories(src/)
 add_library(shadow-asset ${SOURCES})
 
 # Set up test executable
@@ -15,6 +16,6 @@ add_executable(shadow-asset-test ${TESTS})
 target_link_libraries(shadow-asset-test PRIVATE Catch2::Catch2 shadow-utils)
 
 # Enable testing on the executable
-include(CTest)
-include(Catch)
-catch_discover_tests(shadow-asset-test)
+#include(CTest)
+#include(Catch2)
+#catch_discover_tests(shadow-asset-test)

projs/shadow/shadow-engine/shadow-assets/src/fs/file.h

@@ -0,0 +1,112 @@
+#pragma once
+#include <fs/iostream.h>
+#include <memory>
+
+namespace ShadowEngine {
+
+    // An input stream that can read a file on disk.
+    struct FileInput final : InputStream {
+        FileInput();
+        ~FileInput() = default;
+
+        [[nodiscard]] bool open(std::string& path);
+        void close();
+
+        using InputStream::read;
+        [[nodiscard]] bool read(void* data, size_t size) override;
+        const void* getBuffer() const override { return nullptr; }
+
+        size_t size() const override;
+        size_t pos();
+
+        [[nodiscard]] bool seek(size_t pos);
+
+    private:
+        void* handle;
+    };
+
+    // An output stream that can write to a file on disk.
+    struct FileOutput final : OutputStream {
+        FileOutput();
+        ~FileOutput() = default;
+
+        [[nodiscard]] bool open(std::string& path);
+        void close();
+        void flush();
+        bool errored() const { return error; }
+        using OutputStream::write;
+        [[nodiscard]] bool write(const void* data, size_t size) override;
+
+    private:
+        FileOutput(const FileOutput&) = delete;
+        void* handle;
+        bool error;
+    };
+
+    struct FileInfo {
+        bool directory;
+        std::string filename;
+    };
+
+    template <class T> struct Delegate;
+
+    /**
+     * A generic Filesystem API.
+     * Allows interacting with files on disk the same as files in our Virtual Package Format.
+     */
+    struct FileSystem {
+        // A function called when the data of a file is updated, such as when an asynchronous operation completes.
+        using ContentCallback = Delegate<void(size_t, const uint8_t*, bool)>;
+        // A handle for asynchronous data movement; such as reading or writing a file.
+        struct AsyncHandle {
+            static AsyncHandle invalid() { return AsyncHandle(0xffffffff); }
+            explicit AsyncHandle(uint32_t val) : value(val) {}
+
+            [[nodiscard]] bool valid() const { return value != 0xffffffff; }
+
+            uint32_t value;
+        };
+
+        // Create a Filesystem that interacts with files on disk.
+        static std::unique_ptr<FileSystem>& createDiskFS(std::string& basePath);
+        // Create a Virtual Filesystem based on the given path.
+        static std::unique_ptr<FileSystem>& createVFS(std::string& basePath);
+
+        virtual ~FileSystem() {}
+
+        // Open a file for reading.
+        virtual bool open(std::string& path, FileInput& input) = 0;
+        // Open a file for writing.
+        virtual bool open(std::string& path, FileOutput& output) = 0;
+        // Check whether a file exists at the given path.
+        virtual bool fileExists(std::string& path) = 0;
+        // Get the time a file at the given path was last modified.
+        virtual size_t getLastModified(std::string& path) = 0;
+        // Copy a file from one path to another.
+        virtual bool copyFile(std::string& from, std::string& to) = 0;
+        // Move a file from one path to another.
+        virtual bool moveFile(std::string& from, std::string& to) = 0;
+        // Disassociate any files at the given path (not an immediate delete)
+        virtual bool deleteFile(std::string& path) = 0;
+
+        // Get the path that this FileSystem originates at. The default is "/" for VFS, and whatever the Executable Path is for Disk FS.
+        virtual std::string& getBasePath() = 0;
+        // Set a new base path for the FileSystem. Any operations involving file paths will be relative to this new path.
+        virtual void setBasePath(std::string& path) = 0;
+
+        // Process all the callbacks for async file operations.
+        virtual void processCallbacks() = 0;
+        // Check whether there are any outstanding async operations that need work.
+        virtual bool hasWork() = 0;
+
+        // Write new content to a file synchronously. The thread will be blocked when doing this.
+        virtual bool saveSync(const struct Path& file, const uint8_t* content) = 0;
+        // Read content from a file synchronously. The thread will be blocked when doing this.
+        virtual bool readSync(const struct Path& file, struct OutputMemoryStream& content) = 0;
+
+        // Read a file asynchronously. The given callback will be called with the file content once it is available.
+        virtual AsyncHandle readAsync(const Path& file, const ContentCallback& callback) = 0;
+        // Cancel an asynchronous operation, if it is not already complete. The associated callback will be called with a special flag.
+        virtual void cancelAsync(AsyncHandle& handle) = 0;
+    };
+}

projs/shadow/shadow-engine/shadow-assets/src/fs/hash.cpp

@@ -0,0 +1,148 @@
+#include <fs/hash.h>
+#include "xxhash.h"
+
+namespace ShadowEngine {
+
+    StableHash::StableHash(const void *data, uint32_t length) {
+        hash = XXHash64::hash(data, length, 0);
+    }
+
+    StableHash::StableHash(std::string &str) {
+        hash = XXHash64::hash(str.data(), str.size(), 0);
+    }
+
+    HeapHash::HeapHash(const void *data, uint32_t length) {
+        hash = XXHash64::hash(data, length, 0);
+    }
+
+    HeapHash::HeapHash(std::string &str) {
+        hash = XXHash64::hash(str.data(), str.size(), 0);
+    }
+
+    HeapHash HeapHash::fromLong(size_t hash) {
+        HeapHash heap;
+        heap.hash = hash;
+        return heap;
+    }
+
+    HeapHash32 HeapHash32::fromInt(uint32_t hash) {
+        HeapHash32 heap;
+        heap.hash = hash;
+        return heap;
+    }
+
+    StableHash StableHash::fromLong(size_t hash) {
+        StableHash stable;
+        stable.hash = hash;
+        return stable;
+    }
+
+    StableHash32 StableHash32::fromInt(uint32_t hash) {
+        StableHash32 stable;
+        stable.hash = hash;
+        return stable;
+    }
+
+    static uint32_t CRC[256] = {
+            0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f,
+            0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
+            0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2,
+            0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
+            0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9,
+            0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
+            0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c,
+            0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
+            0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423,
+            0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
+            0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106,
+            0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
+            0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d,
+            0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
+            0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950,
+            0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
+            0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7,
+            0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
+            0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa,
+            0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
+            0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81,
+            0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
+            0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84,
+            0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
+            0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb,
+            0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
+            0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e,
+            0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
+            0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55,
+            0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
+            0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28,
+            0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
+            0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f,
+            0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
+            0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
+            0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
+            0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69,
+            0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
+            0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc,
+            0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
+            0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693,
+            0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
+            0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d};
+
+    static uint32_t CRC32(const void* data, uint32_t length) {
+        const auto* c = static_cast<const uint8_t*>(data);
+        uint32_t crcTemp = 0xFFFFFFFF;
+        uint32_t len = length;
+        while (len) {
+            crcTemp = (crcTemp >> 8) ^ CRC[(crcTemp & 0xFF) ^ *c];
+            --len; ++c;
+        }
+
+        return ~crcTemp;
+    }
+
+    StableHash32::StableHash32(const void *data, uint32_t length) {
+        hash = CRC32(data, length);
+    }
+
+    StableHash32::StableHash32(std::string &str) {
+        hash = CRC32(str.data(), str.size());
+    }
+
+    static XXHash64 DeferredHashState(0);
+
+    DeferredHash::DeferredHash() {
+        DeferredHashState = XXHash64(0);
+    }
+
+    void DeferredHash::insert(const void *data, uint32_t length) {
+        DeferredHashState.add(data, length);
+    }
+
+    StableHash32 DeferredHash::submit32() {
+        const auto result = DeferredHashState.hash();
+        return StableHash32::fromInt(uint32_t(result ^ (result >> 32)));
+    }
+
+    StableHash DeferredHash::submit() {
+        const auto result = DeferredHashState.hash();
+        return StableHash::fromLong(result);
+    }
+
+    DeferredHeapHash::DeferredHeapHash() {
+        DeferredHashState = XXHash64(0);
+    }
+
+    void DeferredHeapHash::insert(const void *data, uint32_t length) {
+        DeferredHashState.add(data, length);
+    }
+
+    HeapHash32 DeferredHeapHash::submit32() {
+        const auto result = DeferredHashState.hash();
+        return HeapHash32::fromInt(uint32_t(result ^ (result >> 32)));
+    }
+
+    HeapHash DeferredHeapHash::submit() {
+        const auto result = DeferredHashState.hash();
+        return HeapHash::fromLong(result);
+    }
+}

projs/shadow/shadow-engine/shadow-assets/src/fs/hash.h

@@ -0,0 +1,158 @@
+#pragma once
+#include <string>
+
+namespace ShadowEngine {
+
+    /**
+     * A 64-bit hashing algorithm that uses the state of the allocation heap as a "salt".
+     * Outputs are NOT stable, so do not serialize this.
+     * However, because it uses the heap, it has a very low collision rate.
+     */
+    struct HeapHash {
+        // For if you MUST recreate a hash exactly.
+        // Please only use this for testing.
+        static HeapHash fromLong(size_t hash);
+
+        HeapHash() = default;
+        // Hash a string; for paths and such.
+        explicit HeapHash(std::string& str);
+        // Hash arbitrary data.
+        HeapHash(const void* data, uint32_t length);
+
+        bool operator!= (HeapHash& other) const { return hash != other.hash; }
+        bool operator== (HeapHash& other) const { return hash == other.hash; }
+
+        size_t getHash() const { return hash; }
+    private:
+        size_t hash = 0;
+    };
+
+    /**
+    * A 32-bit hashing algorithm that uses the state of the allocation heap as a "salt".
+    * Outputs are NOT stable, so do not serialize this.
+    * However, because it uses the heap, it has a very low collision rate.
+    */
+    struct HeapHash32 {
+        // For if you MUST recreate a hash exactly.
+        // Please only use this for testing.
+        static HeapHash32 fromInt(uint32_t hash);
+
+        HeapHash32() = default;
+        // Hash a string; for paths and such.
+        explicit HeapHash32(std::string& str);
+        // Hash arbitrary data.
+        HeapHash32(const void* data, uint32_t length);
+
+        bool operator!= (HeapHash32& other) const { return hash != other.hash; }
+        bool operator== (HeapHash32& other) const { return hash == other.hash; }
+
+        uint32_t getHash() const { return hash; }
+    private:
+        uint32_t hash = 0;
+    };
+
+    /**
+     * A 64-bit hashing algorithm that generates the same hash value per input every time.
+     * A little more likely to generate conflicts than the hash that uses the state of the heap as a salt.
+     * Suitable for serialization.
+     */
+    struct StableHash {
+        static StableHash fromLong(size_t data);
+        StableHash() = default;
+        StableHash(std::string& str);
+        StableHash(const void* data, uint32_t length);
+
+        bool operator!= (StableHash& other) const { return hash != other.hash; }
+        bool operator== (StableHash& other) const { return hash == other.hash; }
+        bool operator< (StableHash& other) const { return hash < other.hash; }
+
+        size_t getHash() const { return hash; }
+
+    private:
+        size_t hash = 0;
+    };
+
+    /**
+     * A 32-bit hashing algorithm that generates the same hash value per input every time.
+     * A little more likely to generate conflicts than the hash that uses the state of the heap as a salt.
+     * Suitable for serialization.
+     */
+    struct StableHash32 {
+        static StableHash32 fromInt(uint32_t data);
+        StableHash32() = default;
+        StableHash32(std::string& str);
+        StableHash32(const void* data, uint32_t length);
+
+        bool operator!= (StableHash32& other) const { return hash != other.hash; }
+        bool operator== (StableHash32& other) const { return hash == other.hash; }
+        bool operator< (StableHash32& other) const { return hash < other.hash; }
+
+        uint32_t getHash() const { return hash; }
+
+    private:
+        uint32_t hash = 0;
+    };
+
+    // File Paths are hashed using the 64-bit StableHash system.
+    using PathHash = StableHash;
+
+    /**
+     * A hashing utility that lets you insert data piecemeal before committing to the hash.
+     * Useful for when you're parsing a file and need to wait for more data to be available before hashing.
+     * Generates a Stable Hash.
+     */
+    struct DeferredHash {
+        DeferredHash();
+        // Insert new data to be considered for hashing
+        void insert(const void* data, uint32_t length);
+        // Submit the data to the hashing algorithm, and return a value in 64-bit StableHash
+        StableHash submit();
+        // Submit the data to the hashing algorithm, and return a value in 32-bit StableHash
+        StableHash32 submit32();
+    };
+
+    /**
+     * A hashing utility that lets you insert data piecemeal before committing to the hash.
+     * Useful for when you're parsing a file and need to wait for more data to be available before hashing.
+     * Generates a Heap Hash.
+     */
+    struct DeferredHeapHash {
+        DeferredHeapHash();
+        // Insert new data to be considered for hashing
+        void insert(const void* data, uint32_t length);
+        // Submit the data to the hashing algorithm, and return a value in 64-bit HeapHash
+        HeapHash submit();
+        // Submit the data to the hashing algorithm, and return a value in 32-bit HeapHash
+        HeapHash32 submit32();
+    };
+
+    /** The implementations of these hashing algorithms */
+
+    template <class Hash> struct HashFunc;
+
+    template<> struct HashFunc<HeapHash> {
+        static uint32_t get(const HeapHash& h) {
+            const size_t hash = h.getHash();
+            return uint32_t(hash & (hash >> 16));
+        }
+    };
+
+    template<> struct HashFunc<StableHash> {
+        static uint32_t get(const StableHash& h) {
+            const size_t hash = h.getHash();
+            return uint32_t(hash & (hash >> 16));
+        }
+    };
+
+    template<> struct HashFunc<HeapHash32> {
+        static uint32_t get(const HeapHash32& h) {
+            return h.getHash();
+        }
+    };
+
+    template<> struct HashFunc<StableHash32> {
+        static uint32_t get(const StableHash& h) {
+            return h.getHash();
+        }
+    };
+}

projs/shadow/shadow-engine/shadow-assets/src/fs/iostream.cpp

@@ -0,0 +1,212 @@
+#include <fs/iostream.h>
+
+namespace ShadowEngine {
+
+    OutputMemoryStream::OutputMemoryStream(void *data, size_t size)
+        : buffer(static_cast<uint8_t *>(data)), capacity(size), usage(0) {}
+
+    OutputMemoryStream::OutputMemoryStream(ShadowEngine::OutputMemoryStream &&str) noexcept {
+        capacity = str.capacity;
+        buffer = str.buffer;
+        usage = str.usage;
+
+        str.free();
+    }
+
+    void OutputMemoryStream::operator=(ShadowEngine::OutputMemoryStream &&str) noexcept {
+        capacity = str.capacity;
+        buffer = str.buffer;
+        usage = str.usage;
+
+        str.free();
+    }
+
+    void OutputMemoryStream::operator=(const ShadowEngine::OutputMemoryStream &rhs) noexcept {
+        usage = rhs.usage;
+
+        if (rhs.capacity > 0) {
+            buffer = (uint8_t*)malloc(rhs.capacity);
+            memcpy(buffer, rhs.buffer, rhs.capacity);
+            capacity = rhs.capacity;
+        } else {
+            buffer = nullptr;
+            capacity = 0;
+        }
+    }
+
+    OutputMemoryStream::OutputMemoryStream(const ShadowEngine::OutputMemoryStream &rhs) noexcept {
+        usage = rhs.usage;
+
+        if (rhs.capacity > 0) {
+            buffer = (uint8_t*)malloc(rhs.capacity);
+            memcpy(buffer, rhs.buffer, rhs.capacity);
+            capacity = rhs.capacity;
+        } else {
+            buffer = nullptr;
+            capacity = 0;
+        }
+    }
+
+
+    OutputMemoryStream::~OutputMemoryStream() = default;
+
+    OutputStream &OutputStream::operator<<(std::string &str) {
+        write(str.data(), str.length());
+        return *this;
+    }
+
+    OutputStream &OutputStream::operator<<(const char* str) {
+        write(str, strlen(str));
+        return *this;
+    }
+
+    OutputStream &OutputStream::operator<<(uint32_t val) {
+        std::string str = std::to_string(val);
+        write(str.c_str(), str.length());
+        return *this;
+    }
+
+    OutputStream &OutputStream::operator<<(int32_t val) {
+        std::string str = std::to_string(val);
+        write(str.c_str(), str.length());
+        return *this;
+    }
+
+    OutputStream &OutputStream::operator<<(uint64_t val) {
+        std::string str = std::to_string(val);
+        write(str.c_str(), str.length());
+        return *this;
+    }
+
+    OutputStream &OutputStream::operator<<(int64_t val) {
+        std::string str = std::to_string(val);
+        write(str.c_str(), str.length());
+        return *this;
+    }
+
+    OutputStream &OutputStream::operator<<(float val) {
+        std::string str = std::to_string(val);
+        write(str.c_str(), str.length());
+        return *this;
+    }
+
+    OutputStream &OutputStream::operator<<(double val) {
+        std::string str = std::to_string(val);
+        write(str.c_str(), str.length());
+        return *this;
+    }
+
+    void OutputMemoryStream::write(std::string &str) {
+        write(str.c_str(), str.length());
+    }
+
+    void *OutputMemoryStream::skip(size_t size) {
+        if (size + usage > capacity) {
+            reserve((size + usage) << 1);
+        }
+
+        void* ret = (uint8_t*)buffer + usage;
+        usage += size;
+        return ret;
+    }
+
+    OutputMemoryStream& OutputMemoryStream::operator+=(size_t size) {
+        skip(size);
+        return *this;
+    }
+
+    OutputMemoryStream& OutputMemoryStream::operator++() {
+        skip(1);
+        return *this;
+    }
+
+    uint8_t OutputMemoryStream::operator[](size_t index) const {
+        return buffer[index];
+    }
+
+    uint8_t &OutputMemoryStream::operator[](size_t index) {
+        return buffer[index];
+    }
+
+    bool OutputMemoryStream::write(const void *data, size_t size) {
+        if (!size) return true;
+
+        if (usage + size > capacity) {
+            reserve((usage + size) << 1);
+        }
+
+        memcpy((uint8_t*)data + usage, data, size);
+        usage += size;
+        return true;
+    }
+
+    void OutputMemoryStream::clear() { usage = 0; }
+
+    void OutputMemoryStream::free() {
+        usage = 0;
+        capacity = 0;
+        delete[] buffer;
+        buffer = nullptr;
+    }
+
+    void OutputMemoryStream::reserve(size_t size) {
+        if (size < capacity) return;
+
+        auto* temp = static_cast<uint8_t *>(malloc(size));
+        memcpy(temp, buffer, capacity);
+        delete[] buffer;
+        buffer = temp;
+        capacity = size;
+    }
+
+    uint8_t *OutputMemoryStream::release() {
+        auto* temp = static_cast<uint8_t *>(malloc(usage));
+        memcpy(temp, buffer, usage);
+        free();
+        return temp;
+    }
+
+    InputMemoryStream::InputMemoryStream(const void *data, size_t size)
+        : data(static_cast<const uint8_t *>(data)), capacity(size), position(0) {}
+
+    InputMemoryStream::InputMemoryStream(const ShadowEngine::OutputMemoryStream &blob)
+        : data(blob.data()), capacity(blob.size()), position(0) {}
+
+    void InputMemoryStream::set(const void *newData, size_t size) {
+        data = (uint8_t*) newData; capacity = size; position = 0;
+    }
+
+    const void *InputMemoryStream::skip(size_t size) {
+        auto* pos = data + position;
+        position += size;
+        if (position > capacity) {
+            position = capacity;
+        }
+
+        return (const void*) pos;
+    }
+
+    bool InputMemoryStream::read(void *out, size_t size) {
+        if (position + (uint32_t) size > capacity) {
+            for (int32_t i = 0; i < size; i++)
+                ((unsigned char*)out)[i] = 0;
+            return false;
+        }
+
+        if (size) {
+            memcpy(out, ((char*)data) + position, capacity);
+        }
+
+        position += size;
+
+        return true;
+    }
+
+    std::string InputMemoryStream::readString() {
+        const char* ret = (const char*) data + position;
+        while (position < capacity && data[position]) ++position;
+        ++position;
+
+        return { ret };
+    }
+}

projs/shadow/shadow-engine/shadow-assets/src/fs/iostream.h

@@ -0,0 +1,129 @@
+#pragma once
+#include <string>
+
+namespace ShadowEngine {
+    // A custom OutputStream that can be implemented to output to any arbitrary data structure.
+    // The idea is that it can write to a file, or into memory, or into a temporary buffer that is copied to both.
+    // As opposed to the hardcoded streams that exist in C++, which have a single purpose for their entire lifetime.
+    struct OutputStream {
+        virtual bool write(const void* data, size_t size) = 0;
+
+        OutputStream& operator<< (std::string& str);
+        OutputStream& operator<< (const char* str);
+        OutputStream& operator<< (size_t val);
+        OutputStream& operator<< (int64_t val);
+        OutputStream& operator<< (uint32_t val);
+        OutputStream& operator<< (int32_t val);
+        OutputStream& operator<< (float val);
+        OutputStream& operator<< (double val);
+        template <class T> bool write(const T& val);
+    };
+
+    // A custom InputStream that can be implemented to read from any arbitrary data structure.
+    // The idea is that it can read from a file, or from memory, or from a temporary buffer that is merged from both.
+    // As opposed to the hardcoded streams that exist in C++, which have a single purpose for their entire lifetime.
+    struct InputStream {
+        virtual bool read(void* buffer, size_t size) = 0;
+        virtual const void* getBuffer() const = 0;
+        virtual size_t size() const = 0;
+
+        template <class T> void read(T& val) { read(&val, sizeof(T)); }
+        template <class T> T read();
+    };
+
+    // A custom OutputStream that writes to memory.
+    struct OutputMemoryStream final : OutputStream {
+
+        OutputMemoryStream(void* data, size_t size);
+        OutputMemoryStream(OutputMemoryStream&& str) noexcept;
+        OutputMemoryStream(const OutputMemoryStream& rhs) noexcept;
+        ~OutputMemoryStream();
+
+        void operator= (const OutputMemoryStream& rhs) noexcept;
+        void operator= (OutputMemoryStream&& rhs) noexcept;
+
+        uint8_t operator[] (size_t index) const;
+        uint8_t& operator[] (size_t index);
+        OutputMemoryStream& operator+= (size_t index);
+        OutputMemoryStream& operator++ ();
+
+        bool write(const void* data, size_t size) override;
+
+        uint8_t* release();
+        void resize(size_t size);
+        void reserve(size_t size);
+        const uint8_t* data() const { return buffer; };
+        uint8_t* dataMut() { return buffer; };
+        size_t size() const { return usage; };
+        void clear();
+        void* skip(size_t size);
+        bool empty() const { return usage == 0; };
+        void free();
+
+        void write(std::string& str);
+        template <class T> void write(const T& val);
+
+    private:
+        uint8_t* buffer;
+        size_t capacity;
+        size_t usage;
+    };
+
+    template <class T> void OutputMemoryStream::write(const T& val){
+        write(&val, sizeof(T));
+    }
+
+
+    template <> inline void OutputMemoryStream::write<bool>(const bool& val) {
+        uint8_t v = val;
+        write(&v, sizeof(v));
+    }
+
+    struct InputMemoryStream final : InputStream {
+        InputMemoryStream(const void* data, size_t size);
+        explicit InputMemoryStream(const OutputMemoryStream& blob);
+
+        void set(const void* data, size_t size);
+        bool read(void* data, size_t size) override;
+        std::string readString();
+        const void* skip(size_t size);
+        const void* getData() const { return data; };
+        const void* getBuffer() const override { return data; };
+        size_t size() const override { return capacity; };
+        size_t pos() const { return position; };
+        void setPos(size_t pos) { position = pos; };
+        void restart() { position = 0; };
+        uint8_t readChar() { position++; return data[position-1]; };
+
+        template<class T>
+        T getAs() const {
+            static_assert(position + sizeof(T) < capacity);
+            return *(T*)(data + position);
+        }
+
+        using InputStream::read;
+
+    private:
+        const uint8_t* data;
+        size_t capacity;
+        size_t position;
+    };
+
+    template <class T>
+    T InputStream::read() {
+        T v;
+        read(&v, sizeof(T));
+        return v;
+    }
+
+    template<> inline bool InputStream::read<bool>() {
+        uint8_t v;
+        read(&v, sizeof(bool));
+        return v;
+    }
+
+    template <class T>
+    bool OutputStream::write(const T &val) {
+        return write(&val, sizeof(T));
+    }
+}

projs/shadow/shadow-engine/shadow-assets/src/fs/path.cpp

@@ -0,0 +1,117 @@
+#include <fs/path.h>
+#include <string.h>
+#include <str/string.h>
+
+namespace ShadowEngine {
+
+    Path::Path() : path {} { }
+
+    Path::Path(const std::string &str) {
+        set(normalise((std::string&) str));
+    }
+
+    void Path::set(const std::string &str) {
+#ifdef _WIN32
+        std::string temp = Str::toLower((std::string&) str);
+        hash = PathHash(temp);
+#else
+        hash = PathHash(str);
+#endif
+        path = str;
+    }
+
+    void Path::operator=(const std::string &rhs) {
+        set(rhs);
+    }
+
+    bool Path::operator==(const std::string &rhs) {
+        return path == rhs;
+    }
+
+    bool Path::operator==(const ShadowEngine::Path &rhs) {
+        return path == rhs.path;
+    }
+
+    bool Path::operator!=(const ShadowEngine::Path &rhs) {
+        return path != rhs.path;
+    }
+
+    std::string Path::normalise(std::string &str) {
+        bool prevSlash = false;
+
+        std::string temp;
+        const char* path = str.c_str();
+        size_t len = str.length();
+        size_t i = 0;
+
+        // Skip initial stuff.
+        size_t ind = str.find_first_of(":");
+        path += ind;
+        if (path[0] == '.' && (path[1] == '\\' || path[1] == '/'))
+            path += 2;
+#ifdef _WIN32
+        if (path[0] == '\\' || path[0] == '/')
+            ++path;
+#endif
+
+        while (*path != '\0' && i < len) {
+            bool slash = *path == '\\' || *path == '/';
+
+            // Skip double slashes.
+            if (slash && prevSlash) {
+                path++; continue;
+            }
+
+            // Convert backslashes to forward slashes.
+            temp.append(std::to_string(*path == '\\' ? '/' : *path));
+
+            path++; i++; prevSlash = slash;
+        }
+
+        return temp;
+    }
+
+    std::string Path::getPrelude(std::string &path) {
+        return path.substr(0, path.find_first_of(":"));
+    }
+
+    std::string Path::getDomain(std::string &path) {
+        return path.substr(path.find_first_of(":"), path.find_first_of("/"));
+    }
+
+    std::string Path::getDirectory(std::string &path) {
+        return path.substr(path.find_first_of(":"), path.find_last_of("/"));
+    }
+
+    std::string Path::getFilename(std::string &path) {
+        return path.substr(path.find_last_of("/"), path.find_last_of("."));
+    }
+
+    std::string Path::getExtension(std::string &path) {
+        return path.substr(path.find_last_of("."), path.length());
+    }
+
+    std::string Path::replaceExtension(std::string &path, std::string &newExt) {
+        return path.substr(0, path.length() - newExt.length()).append(newExt);
+    }
+
+    bool Path::hasExtension(std::string &path, std::string &ext) {
+        return path.find_last_of(ext) == path.length() - ext.length();
+    }
+
+    PathInfo::PathInfo(std::string &str) {
+        std::string normalised = Path::normalise(str);
+
+        std::string preludeS = Path::getPrelude(normalised);
+        memcpy_s(prelude, 10, preludeS.c_str(), preludeS.length());
+        std::string domainS = Path::getDomain(normalised);
+        memcpy_s(domain, 256, domainS.c_str(), domainS.length());
+        std::string directoryS = Path::getDirectory(normalised);
+        memcpy_s(directory, 256, directoryS.c_str(), directoryS.length());
+        std::string filenameS = Path::getFilename(normalised);
+        memcpy_s(baseName, 256, filenameS.c_str(), filenameS.length());
+        std::string extensionS = Path::getExtension(normalised);
+        memcpy_s(extension, 10, extensionS.c_str(), extensionS.length());
+    }
+
+}

projs/shadow/shadow-engine/shadow-assets/src/fs/path.h

@@ -0,0 +1,66 @@
+#pragma once
+
+#include <string>
+#include <fs/hash.h>
+
+namespace ShadowEngine {
+    /**
+     * Stores split data about a path, for easy referencing and decomposition.
+     * Not to be used as a replacement for the Path class.
+     */
+    struct PathInfo {
+        explicit PathInfo(std::string& str);
+
+        char extension[10];
+        char baseName[256];
+        char directory[256];
+        char domain[256];
+        char prelude[10];
+    };
+
+    /**
+     * Stores and handles paths in the VFS.
+     * All operations are copy-instantiated, nothing works in-place.
+     * A typical path is of the form:
+     *  prelude:/domain/directory/filename.extension
+     */
+    struct Path {
+        // Make sure the path is valid.
+        // Always from the root.
+        // One slash separating.
+        static std::string normalise(std::string& path);
+        // Get the prelude of the given path.
+        static std::string getPrelude(std::string& path);
+        // Get the domain of the given path.
+        static std::string getDomain(std::string& path);
+        // Get the directory of the given path.
+        static std::string getDirectory(std::string& path);
+        // Get the name of the file of the given path.
+        static std::string getFilename(std::string& path);
+        // Get the file extension of the given path.
+        static std::string getExtension(std::string& path);
+        // Check if the path has the given extension.
+        static bool hasExtension(std::string& path, std::string& ext);
+        // Replace the extension of the given path.
+        static std::string replaceExtension(std::string& path, std::string& newExt);
+
+        Path();
+        explicit Path(const std::string& str);
+
+        void operator=(const std::string& rhs);
+        bool operator==(const std::string& rhs);
+        bool operator==(const Path& rhs);
+        bool operator!=(const Path& rhs);
+
+        // Use this to set a new value into the path; it handles the hash too.
+        void set(const std::string& path);
+
+        uint32_t length() const { return path.length(); };
+        PathHash getHash() { return hash; }
+        const char* c_str() const { return path.data(); }
+        bool isEmpty() const { return path.length() == 0; }
+    private:
+        std::string path;
+        PathHash hash;
+    };
+}

projs/shadow/shadow-engine/shadow-assets/src/fs/xxhash.h

@@ -0,0 +1,182 @@
+#pragma once
+#include <stdint.h> // for uint32_t and uint64_t
+
+class XXHash64
+{
+public:
+    /// create new XXHash (64 bit)
+    /** @param seed your seed value, even zero is a valid seed **/
+    explicit XXHash64(uint64_t seed)
+    {
+        state[0] = seed + Prime1 + Prime2;
+        state[1] = seed + Prime2;
+        state[2] = seed;
+        state[3] = seed - Prime1;
+        bufferSize  = 0;
+        totalLength = 0;
+    }
+
+    /// add a chunk of bytes
+    /** @param  input  pointer to a continuous block of data
+        @param  length number of bytes
+        @return false if parameters are invalid / zero **/
+    bool add(const void* input, uint64_t length)
+    {
+        // no data ?
+        if (!input || length == 0)
+            return false;
+
+        totalLength += length;
+        // byte-wise access
+        const unsigned char* data = (const unsigned char*)input;
+
+        // unprocessed old data plus new data still fit in temporary buffer ?
+        if (bufferSize + length < MaxBufferSize)
+        {
+            // just add new data
+            while (length-- > 0)
+                buffer[bufferSize++] = *data++;
+            return true;
+        }
+
+        // point beyond last byte
+        const unsigned char* stop      = data + length;
+        const unsigned char* stopBlock = stop - MaxBufferSize;
+
+        // some data left from previous update ?
+        if (bufferSize > 0)
+        {
+            // make sure temporary buffer is full (16 bytes)
+            while (bufferSize < MaxBufferSize)
+                buffer[bufferSize++] = *data++;
+
+            // process these 32 bytes (4x8)
+            process(buffer, state[0], state[1], state[2], state[3]);
+        }
+
+        // copying state to local variables helps optimizer A LOT
+        uint64_t s0 = state[0], s1 = state[1], s2 = state[2], s3 = state[3];
+        // 32 bytes at once
+        while (data <= stopBlock)
+        {
+            // local variables s0..s3 instead of state[0]..state[3] are much faster
+            process(data, s0, s1, s2, s3);
+            data += 32;
+        }
+        // copy back
+        state[0] = s0; state[1] = s1; state[2] = s2; state[3] = s3;
+
+        // copy remainder to temporary buffer
+        bufferSize = stop - data;
+        for (uint64_t i = 0; i < bufferSize; i++)
+            buffer[i] = data[i];
+
+        // done
+        return true;
+    }
+
+    /// get current hash
+    /** @return 64 bit XXHash **/
+    uint64_t hash() const
+    {
+        // fold 256 bit state into one single 64 bit value
+        uint64_t result;
+        if (totalLength >= MaxBufferSize)
+        {
+            result = rotateLeft(state[0],  1) +
+                     rotateLeft(state[1],  7) +
+                     rotateLeft(state[2], 12) +
+                     rotateLeft(state[3], 18);
+            result = (result ^ processSingle(0, state[0])) * Prime1 + Prime4;
+            result = (result ^ processSingle(0, state[1])) * Prime1 + Prime4;
+            result = (result ^ processSingle(0, state[2])) * Prime1 + Prime4;
+            result = (result ^ processSingle(0, state[3])) * Prime1 + Prime4;
+        }
+        else
+        {
+            // internal state wasn't set in add(), therefore original seed is still stored in state2
+            result = state[2] + Prime5;
+        }
+
+        result += totalLength;
+
+        // process remaining bytes in temporary buffer
+        const unsigned char* data = buffer;
+        // point beyond last byte
+        const unsigned char* stop = data + bufferSize;
+
+        // at least 8 bytes left ? => eat 8 bytes per step
+        for (; data + 8 <= stop; data += 8)
+            result = rotateLeft(result ^ processSingle(0, *(uint64_t*)data), 27) * Prime1 + Prime4;
+
+        // 4 bytes left ? => eat those
+        if (data + 4 <= stop)
+        {
+            result = rotateLeft(result ^ (*(uint32_t*)data) * Prime1,   23) * Prime2 + Prime3;
+            data  += 4;
+        }
+
+        // take care of remaining 0..3 bytes, eat 1 byte per step
+        while (data != stop)
+            result = rotateLeft(result ^ (*data++) * Prime5,            11) * Prime1;
+
+        // mix bits
+        result ^= result >> 33;
+        result *= Prime2;
+        result ^= result >> 29;
+        result *= Prime3;
+        result ^= result >> 32;
+        return result;
+    }
+
+
+    /// combine constructor, add() and hash() in one static function (C style)
+    /** @param  input  pointer to a continuous block of data
+        @param  length number of bytes
+        @param  seed your seed value, e.g. zero is a valid seed
+        @return 64 bit XXHash **/
+    static uint64_t hash(const void* input, uint64_t length, uint64_t seed)
+    {
+        XXHash64 hasher(seed);
+        hasher.add(input, length);
+        return hasher.hash();
+    }
+
+private:
+    /// magic constants :-)
+    static const uint64_t Prime1 = 11400714785074694791ULL;
+    static const uint64_t Prime2 = 14029467366897019727ULL;
+    static const uint64_t Prime3 =  1609587929392839161ULL;
+    static const uint64_t Prime4 =  9650029242287828579ULL;
+    static const uint64_t Prime5 =  2870177450012600261ULL;
+
+    /// temporarily store up to 31 bytes between multiple add() calls
+    static const uint64_t MaxBufferSize = 31+1;
+
+    uint64_t      state[4];
+    unsigned char buffer[MaxBufferSize];
+    uint64_t      bufferSize;
+    uint64_t      totalLength;
+
+    /// rotate bits, should compile to a single CPU instruction (ROL)
+    static inline uint64_t rotateLeft(uint64_t x, unsigned char bits)
+    {
+        return (x << bits) | (x >> (64 - bits));
+    }
+
+    /// process a single 64 bit value
+    static inline uint64_t processSingle(uint64_t previous, uint64_t input)
+    {
+        return rotateLeft(previous + input * Prime2, 31) * Prime1;
+    }
+
+    /// process a block of 4x4 bytes, this is the main part of the XXHash32 algorithm
+    static inline void process(const void* data, uint64_t& state0, uint64_t& state1, uint64_t& state2, uint64_t& state3)
+    {
+        const uint64_t* block = (const uint64_t*) data;
+        state0 = processSingle(state0, block[0]);
+        state1 = processSingle(state1, block[1]);
+        state2 = processSingle(state2, block[2]);
+        state3 = processSingle(state3, block[3]);
+    }
+};

projs/shadow/shadow-engine/shadow-assets/src/management/synchronization.h

@@ -0,0 +1,77 @@
+#pragma once
+
+#include <cstdint>
+
+#ifdef __linux__
+#include <pthread.h>
+#endif
+
+namespace ShadowEngine {
+    // A simple synchronization system that allows one "accessing thread" at a time.
+    struct alignas(8) Mutex {
+        friend struct ConditionVariable;
+        Mutex();
+        Mutex(const Mutex&) = delete;
+        ~Mutex();
+
+        void enter();
+        void exit();
+    private:
+#ifdef _WIN32
+        uint8_t data[8];
+#else
+        pthread_mutex_t mutex;
+#endif
+    };
+
+    // A simple synchronization system that allows many threads to wait for one thread to complete an operation.
+    struct Semaphore {
+        Semaphore(int initcount, int maxcount);
+        Semaphore(const Semaphore&) = delete;
+        ~Semaphore();
+
+        void raise();
+        void wait();
+    private:
+#ifdef _WIN32
+        void* id;
+#else
+        struct {
+            pthread_mutex_t mutex;
+            pthread_mutex_cond cond;
+            volatile int32_t count;
+        } id;
+#endif
+    };
+
+    struct ConditionVariable {
+        ConditionVariable();
+        ConditionVariable(const ConditionVariable&) = delete;
+        ~ConditionVariable();
+
+        void sleep(Mutex& mut);
+        void wake();
+
+    private:
+#ifdef _WIN32
+        uint8_t data[64];
+#else
+        pthread_cond_t cond;
+#endif
+    };
+
+    // A simple RAII wrapper for Mutexes, that locks and unlocks as the Guard goes in and out of scope.
+    struct MutexGuard {
+        explicit MutexGuard(Mutex& mut) : mut(mut) {
+            mut.enter();
+        }
+
+        ~MutexGuard() { mut.exit(); }
+
+        MutexGuard(const MutexGuard&) = delete;
+        void operator=(const MutexGuard&) = delete;
+
+    private:
+        Mutex& mut;
+    };
+}

projs/shadow/shadow-engine/shadow-assets/src/str/string.cpp

@@ -0,0 +1,11 @@
+#include <str/string.h>
+
+namespace ShadowEngine::Str {
+    std::string toLower(const std::string& str) {
+        std::string temp;
+        for (auto c : str) {
+            temp.append(std::to_string(tolower(c)));
+        }
+        return temp;
+    }
+}

projs/shadow/shadow-engine/shadow-assets/src/str/string.h

@@ -0,0 +1,12 @@
+#pragma once
+#include <string>
+
+namespace ShadowEngine {
+    // String manipluation utilities.
+    // Because std::string is heavily lacking.
+    namespace Str {
+        // Convert the string to lower case, return a new string.
+        // This only works in ASCII encoding.
+        std::string toLower(std::string& str);
+    }
+}