From 5e60726a2bdbfff6623fbc8f905122520a429c92 Mon Sep 17 00:00:00 2001 From: Curle Date: Fri, 10 Mar 2023 15:54:29 +0000 Subject: [PATCH] File & FileSystem abstractions --- .github/workflows/build.yml | 4 +- imgui.ini | 41 ++-- projs/shadow/shadow-engine/CMakeLists.txt | 2 + .../CMakeLists.txt | 9 +- .../SFFElement.natvis | 0 .../cmake/Catch.cmake | 0 .../cmake/CatchAddTests.cmake | 0 .../src/SFFElement.cpp | 0 .../src/SFFElement.h | 0 .../src/SFFParser.cpp | 0 .../src/SFFParser.h | 0 .../src/SFFVersion.h | 0 .../src/SFFWriter.cpp | 0 .../src/SFFWriter.h | 0 .../src/Shadow.FileFormat.ixx | 0 .../shadow-engine/shadow-assets/src/fs/file.h | 112 +++++++++ .../shadow-assets/src/fs/hash.cpp | 148 ++++++++++++ .../shadow-engine/shadow-assets/src/fs/hash.h | 158 +++++++++++++ .../shadow-assets/src/fs/iostream.cpp | 212 ++++++++++++++++++ .../shadow-assets/src/fs/iostream.h | 129 +++++++++++ .../shadow-assets/src/fs/path.cpp | 117 ++++++++++ .../shadow-engine/shadow-assets/src/fs/path.h | 66 ++++++ .../shadow-assets/src/fs/xxhash.h | 182 +++++++++++++++ .../src/management/synchronization.h | 77 +++++++ .../shadow-assets/src/str/string.cpp | 11 + .../shadow-assets/src/str/string.h | 12 + .../test/Catch2Test.cpp | 0 .../test/catch2/catch.hpp | 0 .../test/sff_writer_tests.ocpp | 0 .../test/test.ocpp | 0 30 files changed, 1253 insertions(+), 27 deletions(-) rename projs/shadow/shadow-engine/{shadow-file-format => shadow-assets}/CMakeLists.txt (72%) rename projs/shadow/shadow-engine/{shadow-file-format => shadow-assets}/SFFElement.natvis (100%) rename projs/shadow/shadow-engine/{shadow-file-format => shadow-assets}/cmake/Catch.cmake (100%) rename projs/shadow/shadow-engine/{shadow-file-format => shadow-assets}/cmake/CatchAddTests.cmake (100%) rename projs/shadow/shadow-engine/{shadow-file-format => shadow-assets}/src/SFFElement.cpp (100%) rename projs/shadow/shadow-engine/{shadow-file-format => shadow-assets}/src/SFFElement.h (100%) rename projs/shadow/shadow-engine/{shadow-file-format => shadow-assets}/src/SFFParser.cpp (100%) rename projs/shadow/shadow-engine/{shadow-file-format => shadow-assets}/src/SFFParser.h (100%) rename projs/shadow/shadow-engine/{shadow-file-format => shadow-assets}/src/SFFVersion.h (100%) rename projs/shadow/shadow-engine/{shadow-file-format => shadow-assets}/src/SFFWriter.cpp (100%) rename projs/shadow/shadow-engine/{shadow-file-format => shadow-assets}/src/SFFWriter.h (100%) rename projs/shadow/shadow-engine/{shadow-file-format => shadow-assets}/src/Shadow.FileFormat.ixx (100%) create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/fs/file.h create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/fs/hash.cpp create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/fs/hash.h create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/fs/iostream.cpp create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/fs/iostream.h create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/fs/path.cpp create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/fs/path.h create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/fs/xxhash.h create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/management/synchronization.h create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/str/string.cpp create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/str/string.h rename projs/shadow/shadow-engine/{shadow-file-format => shadow-assets}/test/Catch2Test.cpp (100%) rename projs/shadow/shadow-engine/{shadow-file-format => shadow-assets}/test/catch2/catch.hpp (100%) rename projs/shadow/shadow-engine/{shadow-file-format => shadow-assets}/test/sff_writer_tests.ocpp (100%) rename projs/shadow/shadow-engine/{shadow-file-format => shadow-assets}/test/test.ocpp (100%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b15244d..ff75e15 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,8 +58,8 @@ jobs: - name: Make output folder run: mkdir ./test-results - - name: Test shadow-file-format - run: ./bazel-bin/projs/shadow-file-format/test.exe -r junit -o ./test-results/shadow-file-format-test.xml + - name: Test shadow-assets + run: ./bazel-bin/projs/shadow-assets/test.exe -r junit -o ./test-results/shadow-assets-test.xml - name: Publish Test Results uses: EnricoMi/publish-unit-test-result-action/composite@v1 diff --git a/imgui.ini b/imgui.ini index 7162e92..70e47c5 100644 --- a/imgui.ini +++ b/imgui.ini @@ -5,37 +5,36 @@ Collapsed=0 DockId=0x00000008,0 [Window][Dear ImGui Demo] -Pos=1193,178 -Size=62,25 +Pos=1193,111 +Size=62,92 Collapsed=0 -DockId=0x00000007,0 +DockId=0x00000003,0 [Window][Game module window] -Pos=1193,8 -Size=62,50 +Pos=956,275 +Size=302,121 Collapsed=0 -DockId=0x00000004,0 [Window][Time] -Pos=1193,60 -Size=62,49 +Pos=1193,8 +Size=62,101 Collapsed=0 -DockId=0x00000005,0 +DockId=0x00000002,0 [Window][Active Modules] -Pos=1193,111 -Size=62,65 +Pos=829,449 +Size=362,187 +Collapsed=0 + +[Window][Game View] +Pos=60,60 +Size=656,515 Collapsed=0 -DockId=0x00000006,0 [Docking][Data] -DockNode ID=0x00000001 Pos=1196,188 Size=379,195 Split=X - DockNode ID=0x00000008 Parent=0x00000001 SizeRef=652,419 HiddenTabBar=1 Selected=0x55954704 - DockNode ID=0x00000009 Parent=0x00000001 SizeRef=129,419 Split=Y - DockNode ID=0x00000002 Parent=0x00000009 SizeRef=219,34 Split=Y Selected=0xFC1D20C0 - DockNode ID=0x00000004 Parent=0x00000002 SizeRef=219,64 Selected=0xFC1D20C0 - DockNode ID=0x00000005 Parent=0x00000002 SizeRef=219,62 Selected=0xE75A179E - DockNode ID=0x00000003 Parent=0x00000009 SizeRef=219,31 Split=Y Selected=0xEE305C78 - DockNode ID=0x00000006 Parent=0x00000003 SizeRef=219,142 Selected=0xEE305C78 - DockNode ID=0x00000007 Parent=0x00000003 SizeRef=219,55 Selected=0xE87781F4 +DockNode ID=0x00000001 Pos=1196,188 Size=379,195 Split=X + DockNode ID=0x00000008 Parent=0x00000001 SizeRef=652,419 HiddenTabBar=1 Selected=0x55954704 + DockNode ID=0x00000009 Parent=0x00000001 SizeRef=129,419 Split=Y + DockNode ID=0x00000002 Parent=0x00000009 SizeRef=219,34 Selected=0xE75A179E + DockNode ID=0x00000003 Parent=0x00000009 SizeRef=219,31 Selected=0xE87781F4 diff --git a/projs/shadow/shadow-engine/CMakeLists.txt b/projs/shadow/shadow-engine/CMakeLists.txt index e5e2434..0738aa1 100644 --- a/projs/shadow/shadow-engine/CMakeLists.txt +++ b/projs/shadow/shadow-engine/CMakeLists.txt @@ -5,6 +5,8 @@ find_package(imgui REQUIRED) set(CMAKE_CXX_STANDARD 20) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) +add_subdirectory(shadow-assets) + FILE(GLOB_RECURSE SOURCES core/src/*.cpp shadow-renderer/src/*.cpp diff --git a/projs/shadow/shadow-engine/shadow-file-format/CMakeLists.txt b/projs/shadow/shadow-engine/shadow-assets/CMakeLists.txt similarity index 72% rename from projs/shadow/shadow-engine/shadow-file-format/CMakeLists.txt rename to projs/shadow/shadow-engine/shadow-assets/CMakeLists.txt index 62a8648..1c81c2c 100644 --- a/projs/shadow/shadow-engine/shadow-file-format/CMakeLists.txt +++ b/projs/shadow/shadow-engine/shadow-assets/CMakeLists.txt @@ -5,9 +5,10 @@ list(APPEND CMAKE_MODULE_PATH "cmake") enable_testing() # Set up asset sourceset -FILE(GLOB_RECURSE SOURCES src/*.cpp src/*.h) +FILE(GLOB_RECURSE SOURCES src/**.cpp src/**.h) FILE(GLOB_RECURSE TESTS test/*.cpp) +include_directories(src/) add_library(shadow-asset ${SOURCES}) # Set up test executable @@ -15,6 +16,6 @@ add_executable(shadow-asset-test ${TESTS}) target_link_libraries(shadow-asset-test PRIVATE Catch2::Catch2 shadow-utils) # Enable testing on the executable -include(CTest) -include(Catch) -catch_discover_tests(shadow-asset-test) \ No newline at end of file +#include(CTest) +#include(Catch2) +#catch_discover_tests(shadow-asset-test) \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-file-format/SFFElement.natvis b/projs/shadow/shadow-engine/shadow-assets/SFFElement.natvis similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/SFFElement.natvis rename to projs/shadow/shadow-engine/shadow-assets/SFFElement.natvis diff --git a/projs/shadow/shadow-engine/shadow-file-format/cmake/Catch.cmake b/projs/shadow/shadow-engine/shadow-assets/cmake/Catch.cmake similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/cmake/Catch.cmake rename to projs/shadow/shadow-engine/shadow-assets/cmake/Catch.cmake diff --git a/projs/shadow/shadow-engine/shadow-file-format/cmake/CatchAddTests.cmake b/projs/shadow/shadow-engine/shadow-assets/cmake/CatchAddTests.cmake similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/cmake/CatchAddTests.cmake rename to projs/shadow/shadow-engine/shadow-assets/cmake/CatchAddTests.cmake diff --git a/projs/shadow/shadow-engine/shadow-file-format/src/SFFElement.cpp b/projs/shadow/shadow-engine/shadow-assets/src/SFFElement.cpp similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/src/SFFElement.cpp rename to projs/shadow/shadow-engine/shadow-assets/src/SFFElement.cpp diff --git a/projs/shadow/shadow-engine/shadow-file-format/src/SFFElement.h b/projs/shadow/shadow-engine/shadow-assets/src/SFFElement.h similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/src/SFFElement.h rename to projs/shadow/shadow-engine/shadow-assets/src/SFFElement.h diff --git a/projs/shadow/shadow-engine/shadow-file-format/src/SFFParser.cpp b/projs/shadow/shadow-engine/shadow-assets/src/SFFParser.cpp similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/src/SFFParser.cpp rename to projs/shadow/shadow-engine/shadow-assets/src/SFFParser.cpp diff --git a/projs/shadow/shadow-engine/shadow-file-format/src/SFFParser.h b/projs/shadow/shadow-engine/shadow-assets/src/SFFParser.h similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/src/SFFParser.h rename to projs/shadow/shadow-engine/shadow-assets/src/SFFParser.h diff --git a/projs/shadow/shadow-engine/shadow-file-format/src/SFFVersion.h b/projs/shadow/shadow-engine/shadow-assets/src/SFFVersion.h similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/src/SFFVersion.h rename to projs/shadow/shadow-engine/shadow-assets/src/SFFVersion.h diff --git a/projs/shadow/shadow-engine/shadow-file-format/src/SFFWriter.cpp b/projs/shadow/shadow-engine/shadow-assets/src/SFFWriter.cpp similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/src/SFFWriter.cpp rename to projs/shadow/shadow-engine/shadow-assets/src/SFFWriter.cpp diff --git a/projs/shadow/shadow-engine/shadow-file-format/src/SFFWriter.h b/projs/shadow/shadow-engine/shadow-assets/src/SFFWriter.h similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/src/SFFWriter.h rename to projs/shadow/shadow-engine/shadow-assets/src/SFFWriter.h diff --git a/projs/shadow/shadow-engine/shadow-file-format/src/Shadow.FileFormat.ixx b/projs/shadow/shadow-engine/shadow-assets/src/Shadow.FileFormat.ixx similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/src/Shadow.FileFormat.ixx rename to projs/shadow/shadow-engine/shadow-assets/src/Shadow.FileFormat.ixx diff --git a/projs/shadow/shadow-engine/shadow-assets/src/fs/file.h b/projs/shadow/shadow-engine/shadow-assets/src/fs/file.h new file mode 100644 index 0000000..d3265cb --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/fs/file.h @@ -0,0 +1,112 @@ +#pragma once +#include +#include + +namespace ShadowEngine { + + // An input stream that can read a file on disk. + struct FileInput final : InputStream { + FileInput(); + ~FileInput() = default; + + [[nodiscard]] bool open(std::string& path); + void close(); + + using InputStream::read; + [[nodiscard]] bool read(void* data, size_t size) override; + const void* getBuffer() const override { return nullptr; } + + size_t size() const override; + size_t pos(); + + [[nodiscard]] bool seek(size_t pos); + + private: + void* handle; + }; + + // An output stream that can write to a file on disk. + struct FileOutput final : OutputStream { + FileOutput(); + ~FileOutput() = default; + + [[nodiscard]] bool open(std::string& path); + void close(); + void flush(); + bool errored() const { return error; } + using OutputStream::write; + [[nodiscard]] bool write(const void* data, size_t size) override; + + private: + FileOutput(const FileOutput&) = delete; + void* handle; + bool error; + }; + + struct FileInfo { + bool directory; + std::string filename; + }; + + template struct Delegate; + + /** + * A generic Filesystem API. + * Allows interacting with files on disk the same as files in our Virtual Package Format. + */ + struct FileSystem { + // A function called when the data of a file is updated, such as when an asynchronous operation completes. + using ContentCallback = Delegate; + // A handle for asynchronous data movement; such as reading or writing a file. + struct AsyncHandle { + static AsyncHandle invalid() { return AsyncHandle(0xffffffff); } + explicit AsyncHandle(uint32_t val) : value(val) {} + + [[nodiscard]] bool valid() const { return value != 0xffffffff; } + + uint32_t value; + }; + + // Create a Filesystem that interacts with files on disk. + static std::unique_ptr& createDiskFS(std::string& basePath); + // Create a Virtual Filesystem based on the given path. + static std::unique_ptr& createVFS(std::string& basePath); + + virtual ~FileSystem() {} + + // Open a file for reading. + virtual bool open(std::string& path, FileInput& input) = 0; + // Open a file for writing. + virtual bool open(std::string& path, FileOutput& output) = 0; + // Check whether a file exists at the given path. + virtual bool fileExists(std::string& path) = 0; + // Get the time a file at the given path was last modified. + virtual size_t getLastModified(std::string& path) = 0; + // Copy a file from one path to another. + virtual bool copyFile(std::string& from, std::string& to) = 0; + // Move a file from one path to another. + virtual bool moveFile(std::string& from, std::string& to) = 0; + // Disassociate any files at the given path (not an immediate delete) + virtual bool deleteFile(std::string& path) = 0; + + // Get the path that this FileSystem originates at. The default is "/" for VFS, and whatever the Executable Path is for Disk FS. + virtual std::string& getBasePath() = 0; + // Set a new base path for the FileSystem. Any operations involving file paths will be relative to this new path. + virtual void setBasePath(std::string& path) = 0; + + // Process all the callbacks for async file operations. + virtual void processCallbacks() = 0; + // Check whether there are any outstanding async operations that need work. + virtual bool hasWork() = 0; + + // Write new content to a file synchronously. The thread will be blocked when doing this. + virtual bool saveSync(const struct Path& file, const uint8_t* content) = 0; + // Read content from a file synchronously. The thread will be blocked when doing this. + virtual bool readSync(const struct Path& file, struct OutputMemoryStream& content) = 0; + + // Read a file asynchronously. The given callback will be called with the file content once it is available. + virtual AsyncHandle readAsync(const Path& file, const ContentCallback& callback) = 0; + // Cancel an asynchronous operation, if it is not already complete. The associated callback will be called with a special flag. + virtual void cancelAsync(AsyncHandle& handle) = 0; + }; +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/fs/hash.cpp b/projs/shadow/shadow-engine/shadow-assets/src/fs/hash.cpp new file mode 100644 index 0000000..67970e9 --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/fs/hash.cpp @@ -0,0 +1,148 @@ +#include +#include "xxhash.h" + +namespace ShadowEngine { + + StableHash::StableHash(const void *data, uint32_t length) { + hash = XXHash64::hash(data, length, 0); + } + + StableHash::StableHash(std::string &str) { + hash = XXHash64::hash(str.data(), str.size(), 0); + } + + HeapHash::HeapHash(const void *data, uint32_t length) { + hash = XXHash64::hash(data, length, 0); + } + + HeapHash::HeapHash(std::string &str) { + hash = XXHash64::hash(str.data(), str.size(), 0); + } + + HeapHash HeapHash::fromLong(size_t hash) { + HeapHash heap; + heap.hash = hash; + return heap; + } + + HeapHash32 HeapHash32::fromInt(uint32_t hash) { + HeapHash32 heap; + heap.hash = hash; + return heap; + } + + StableHash StableHash::fromLong(size_t hash) { + StableHash stable; + stable.hash = hash; + return stable; + } + + StableHash32 StableHash32::fromInt(uint32_t hash) { + StableHash32 stable; + stable.hash = hash; + return stable; + } + + static uint32_t CRC[256] = { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, + 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, + 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, + 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, + 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, + 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, + 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, + 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, + 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, + 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, + 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, + 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, + 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, + 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, + 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, + 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, + 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, + 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d}; + + static uint32_t CRC32(const void* data, uint32_t length) { + const auto* c = static_cast(data); + uint32_t crcTemp = 0xFFFFFFFF; + uint32_t len = length; + while (len) { + crcTemp = (crcTemp >> 8) ^ CRC[(crcTemp & 0xFF) ^ *c]; + --len; ++c; + } + + return ~crcTemp; + } + + StableHash32::StableHash32(const void *data, uint32_t length) { + hash = CRC32(data, length); + } + + StableHash32::StableHash32(std::string &str) { + hash = CRC32(str.data(), str.size()); + } + + static XXHash64 DeferredHashState(0); + + DeferredHash::DeferredHash() { + DeferredHashState = XXHash64(0); + } + + void DeferredHash::insert(const void *data, uint32_t length) { + DeferredHashState.add(data, length); + } + + StableHash32 DeferredHash::submit32() { + const auto result = DeferredHashState.hash(); + return StableHash32::fromInt(uint32_t(result ^ (result >> 32))); + } + + StableHash DeferredHash::submit() { + const auto result = DeferredHashState.hash(); + return StableHash::fromLong(result); + } + + DeferredHeapHash::DeferredHeapHash() { + DeferredHashState = XXHash64(0); + } + + void DeferredHeapHash::insert(const void *data, uint32_t length) { + DeferredHashState.add(data, length); + } + + HeapHash32 DeferredHeapHash::submit32() { + const auto result = DeferredHashState.hash(); + return HeapHash32::fromInt(uint32_t(result ^ (result >> 32))); + } + + HeapHash DeferredHeapHash::submit() { + const auto result = DeferredHashState.hash(); + return HeapHash::fromLong(result); + } +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/fs/hash.h b/projs/shadow/shadow-engine/shadow-assets/src/fs/hash.h new file mode 100644 index 0000000..39e477e --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/fs/hash.h @@ -0,0 +1,158 @@ +#pragma once +#include + +namespace ShadowEngine { + + /** + * A 64-bit hashing algorithm that uses the state of the allocation heap as a "salt". + * Outputs are NOT stable, so do not serialize this. + * However, because it uses the heap, it has a very low collision rate. + */ + struct HeapHash { + // For if you MUST recreate a hash exactly. + // Please only use this for testing. + static HeapHash fromLong(size_t hash); + + HeapHash() = default; + // Hash a string; for paths and such. + explicit HeapHash(std::string& str); + // Hash arbitrary data. + HeapHash(const void* data, uint32_t length); + + bool operator!= (HeapHash& other) const { return hash != other.hash; } + bool operator== (HeapHash& other) const { return hash == other.hash; } + + size_t getHash() const { return hash; } + private: + size_t hash = 0; + }; + + /** + * A 32-bit hashing algorithm that uses the state of the allocation heap as a "salt". + * Outputs are NOT stable, so do not serialize this. + * However, because it uses the heap, it has a very low collision rate. + */ + struct HeapHash32 { + // For if you MUST recreate a hash exactly. + // Please only use this for testing. + static HeapHash32 fromInt(uint32_t hash); + + HeapHash32() = default; + // Hash a string; for paths and such. + explicit HeapHash32(std::string& str); + // Hash arbitrary data. + HeapHash32(const void* data, uint32_t length); + + bool operator!= (HeapHash32& other) const { return hash != other.hash; } + bool operator== (HeapHash32& other) const { return hash == other.hash; } + + uint32_t getHash() const { return hash; } + private: + uint32_t hash = 0; + }; + + /** + * A 64-bit hashing algorithm that generates the same hash value per input every time. + * A little more likely to generate conflicts than the hash that uses the state of the heap as a salt. + * Suitable for serialization. + */ + struct StableHash { + static StableHash fromLong(size_t data); + StableHash() = default; + StableHash(std::string& str); + StableHash(const void* data, uint32_t length); + + bool operator!= (StableHash& other) const { return hash != other.hash; } + bool operator== (StableHash& other) const { return hash == other.hash; } + bool operator< (StableHash& other) const { return hash < other.hash; } + + size_t getHash() const { return hash; } + + private: + size_t hash = 0; + }; + + /** + * A 32-bit hashing algorithm that generates the same hash value per input every time. + * A little more likely to generate conflicts than the hash that uses the state of the heap as a salt. + * Suitable for serialization. + */ + struct StableHash32 { + static StableHash32 fromInt(uint32_t data); + StableHash32() = default; + StableHash32(std::string& str); + StableHash32(const void* data, uint32_t length); + + bool operator!= (StableHash32& other) const { return hash != other.hash; } + bool operator== (StableHash32& other) const { return hash == other.hash; } + bool operator< (StableHash32& other) const { return hash < other.hash; } + + uint32_t getHash() const { return hash; } + + private: + uint32_t hash = 0; + }; + + // File Paths are hashed using the 64-bit StableHash system. + using PathHash = StableHash; + + /** + * A hashing utility that lets you insert data piecemeal before committing to the hash. + * Useful for when you're parsing a file and need to wait for more data to be available before hashing. + * Generates a Stable Hash. + */ + struct DeferredHash { + DeferredHash(); + // Insert new data to be considered for hashing + void insert(const void* data, uint32_t length); + // Submit the data to the hashing algorithm, and return a value in 64-bit StableHash + StableHash submit(); + // Submit the data to the hashing algorithm, and return a value in 32-bit StableHash + StableHash32 submit32(); + }; + + /** + * A hashing utility that lets you insert data piecemeal before committing to the hash. + * Useful for when you're parsing a file and need to wait for more data to be available before hashing. + * Generates a Heap Hash. + */ + struct DeferredHeapHash { + DeferredHeapHash(); + // Insert new data to be considered for hashing + void insert(const void* data, uint32_t length); + // Submit the data to the hashing algorithm, and return a value in 64-bit HeapHash + HeapHash submit(); + // Submit the data to the hashing algorithm, and return a value in 32-bit HeapHash + HeapHash32 submit32(); + }; + + /** The implementations of these hashing algorithms */ + + template struct HashFunc; + + template<> struct HashFunc { + static uint32_t get(const HeapHash& h) { + const size_t hash = h.getHash(); + return uint32_t(hash & (hash >> 16)); + } + }; + + template<> struct HashFunc { + static uint32_t get(const StableHash& h) { + const size_t hash = h.getHash(); + return uint32_t(hash & (hash >> 16)); + } + }; + + template<> struct HashFunc { + static uint32_t get(const HeapHash32& h) { + return h.getHash(); + } + }; + + template<> struct HashFunc { + static uint32_t get(const StableHash& h) { + return h.getHash(); + } + }; +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/fs/iostream.cpp b/projs/shadow/shadow-engine/shadow-assets/src/fs/iostream.cpp new file mode 100644 index 0000000..2c268a4 --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/fs/iostream.cpp @@ -0,0 +1,212 @@ +#include + +namespace ShadowEngine { + + OutputMemoryStream::OutputMemoryStream(void *data, size_t size) + : buffer(static_cast(data)), capacity(size), usage(0) {} + + OutputMemoryStream::OutputMemoryStream(ShadowEngine::OutputMemoryStream &&str) noexcept { + capacity = str.capacity; + buffer = str.buffer; + usage = str.usage; + + str.free(); + } + + void OutputMemoryStream::operator=(ShadowEngine::OutputMemoryStream &&str) noexcept { + capacity = str.capacity; + buffer = str.buffer; + usage = str.usage; + + str.free(); + } + + void OutputMemoryStream::operator=(const ShadowEngine::OutputMemoryStream &rhs) noexcept { + usage = rhs.usage; + + if (rhs.capacity > 0) { + buffer = (uint8_t*)malloc(rhs.capacity); + memcpy(buffer, rhs.buffer, rhs.capacity); + capacity = rhs.capacity; + } else { + buffer = nullptr; + capacity = 0; + } + } + + OutputMemoryStream::OutputMemoryStream(const ShadowEngine::OutputMemoryStream &rhs) noexcept { + usage = rhs.usage; + + if (rhs.capacity > 0) { + buffer = (uint8_t*)malloc(rhs.capacity); + memcpy(buffer, rhs.buffer, rhs.capacity); + capacity = rhs.capacity; + } else { + buffer = nullptr; + capacity = 0; + } + } + + + OutputMemoryStream::~OutputMemoryStream() = default; + + OutputStream &OutputStream::operator<<(std::string &str) { + write(str.data(), str.length()); + return *this; + } + + OutputStream &OutputStream::operator<<(const char* str) { + write(str, strlen(str)); + return *this; + } + + OutputStream &OutputStream::operator<<(uint32_t val) { + std::string str = std::to_string(val); + write(str.c_str(), str.length()); + return *this; + } + + OutputStream &OutputStream::operator<<(int32_t val) { + std::string str = std::to_string(val); + write(str.c_str(), str.length()); + return *this; + } + + OutputStream &OutputStream::operator<<(uint64_t val) { + std::string str = std::to_string(val); + write(str.c_str(), str.length()); + return *this; + } + + OutputStream &OutputStream::operator<<(int64_t val) { + std::string str = std::to_string(val); + write(str.c_str(), str.length()); + return *this; + } + + OutputStream &OutputStream::operator<<(float val) { + std::string str = std::to_string(val); + write(str.c_str(), str.length()); + return *this; + } + + OutputStream &OutputStream::operator<<(double val) { + std::string str = std::to_string(val); + write(str.c_str(), str.length()); + return *this; + } + + void OutputMemoryStream::write(std::string &str) { + write(str.c_str(), str.length()); + } + + void *OutputMemoryStream::skip(size_t size) { + if (size + usage > capacity) { + reserve((size + usage) << 1); + } + + void* ret = (uint8_t*)buffer + usage; + usage += size; + return ret; + } + + OutputMemoryStream& OutputMemoryStream::operator+=(size_t size) { + skip(size); + return *this; + } + + OutputMemoryStream& OutputMemoryStream::operator++() { + skip(1); + return *this; + } + + uint8_t OutputMemoryStream::operator[](size_t index) const { + return buffer[index]; + } + + uint8_t &OutputMemoryStream::operator[](size_t index) { + return buffer[index]; + } + + bool OutputMemoryStream::write(const void *data, size_t size) { + if (!size) return true; + + if (usage + size > capacity) { + reserve((usage + size) << 1); + } + + memcpy((uint8_t*)data + usage, data, size); + usage += size; + return true; + } + + void OutputMemoryStream::clear() { usage = 0; } + + void OutputMemoryStream::free() { + usage = 0; + capacity = 0; + delete[] buffer; + buffer = nullptr; + } + + void OutputMemoryStream::reserve(size_t size) { + if (size < capacity) return; + + auto* temp = static_cast(malloc(size)); + memcpy(temp, buffer, capacity); + delete[] buffer; + buffer = temp; + capacity = size; + } + + uint8_t *OutputMemoryStream::release() { + auto* temp = static_cast(malloc(usage)); + memcpy(temp, buffer, usage); + free(); + return temp; + } + + InputMemoryStream::InputMemoryStream(const void *data, size_t size) + : data(static_cast(data)), capacity(size), position(0) {} + + InputMemoryStream::InputMemoryStream(const ShadowEngine::OutputMemoryStream &blob) + : data(blob.data()), capacity(blob.size()), position(0) {} + + void InputMemoryStream::set(const void *newData, size_t size) { + data = (uint8_t*) newData; capacity = size; position = 0; + } + + const void *InputMemoryStream::skip(size_t size) { + auto* pos = data + position; + position += size; + if (position > capacity) { + position = capacity; + } + + return (const void*) pos; + } + + bool InputMemoryStream::read(void *out, size_t size) { + if (position + (uint32_t) size > capacity) { + for (int32_t i = 0; i < size; i++) + ((unsigned char*)out)[i] = 0; + return false; + } + + if (size) { + memcpy(out, ((char*)data) + position, capacity); + } + + position += size; + + return true; + } + + std::string InputMemoryStream::readString() { + const char* ret = (const char*) data + position; + while (position < capacity && data[position]) ++position; + ++position; + + return { ret }; + } +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/fs/iostream.h b/projs/shadow/shadow-engine/shadow-assets/src/fs/iostream.h new file mode 100644 index 0000000..6ac41a6 --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/fs/iostream.h @@ -0,0 +1,129 @@ +#pragma once +#include + +namespace ShadowEngine { + // A custom OutputStream that can be implemented to output to any arbitrary data structure. + // The idea is that it can write to a file, or into memory, or into a temporary buffer that is copied to both. + // As opposed to the hardcoded streams that exist in C++, which have a single purpose for their entire lifetime. + struct OutputStream { + virtual bool write(const void* data, size_t size) = 0; + + OutputStream& operator<< (std::string& str); + OutputStream& operator<< (const char* str); + OutputStream& operator<< (size_t val); + OutputStream& operator<< (int64_t val); + OutputStream& operator<< (uint32_t val); + OutputStream& operator<< (int32_t val); + OutputStream& operator<< (float val); + OutputStream& operator<< (double val); + template bool write(const T& val); + }; + + // A custom InputStream that can be implemented to read from any arbitrary data structure. + // The idea is that it can read from a file, or from memory, or from a temporary buffer that is merged from both. + // As opposed to the hardcoded streams that exist in C++, which have a single purpose for their entire lifetime. + struct InputStream { + virtual bool read(void* buffer, size_t size) = 0; + virtual const void* getBuffer() const = 0; + virtual size_t size() const = 0; + + template void read(T& val) { read(&val, sizeof(T)); } + template T read(); + }; + + // A custom OutputStream that writes to memory. + struct OutputMemoryStream final : OutputStream { + + OutputMemoryStream(void* data, size_t size); + OutputMemoryStream(OutputMemoryStream&& str) noexcept; + OutputMemoryStream(const OutputMemoryStream& rhs) noexcept; + ~OutputMemoryStream(); + + void operator= (const OutputMemoryStream& rhs) noexcept; + void operator= (OutputMemoryStream&& rhs) noexcept; + + uint8_t operator[] (size_t index) const; + uint8_t& operator[] (size_t index); + OutputMemoryStream& operator+= (size_t index); + OutputMemoryStream& operator++ (); + + bool write(const void* data, size_t size) override; + + uint8_t* release(); + void resize(size_t size); + void reserve(size_t size); + const uint8_t* data() const { return buffer; }; + uint8_t* dataMut() { return buffer; }; + size_t size() const { return usage; }; + void clear(); + void* skip(size_t size); + bool empty() const { return usage == 0; }; + void free(); + + void write(std::string& str); + template void write(const T& val); + + private: + uint8_t* buffer; + size_t capacity; + size_t usage; + }; + + template void OutputMemoryStream::write(const T& val){ + write(&val, sizeof(T)); + } + + + template <> inline void OutputMemoryStream::write(const bool& val) { + uint8_t v = val; + write(&v, sizeof(v)); + } + + struct InputMemoryStream final : InputStream { + InputMemoryStream(const void* data, size_t size); + explicit InputMemoryStream(const OutputMemoryStream& blob); + + void set(const void* data, size_t size); + bool read(void* data, size_t size) override; + std::string readString(); + const void* skip(size_t size); + const void* getData() const { return data; }; + const void* getBuffer() const override { return data; }; + size_t size() const override { return capacity; }; + size_t pos() const { return position; }; + void setPos(size_t pos) { position = pos; }; + void restart() { position = 0; }; + uint8_t readChar() { position++; return data[position-1]; }; + + template + T getAs() const { + static_assert(position + sizeof(T) < capacity); + return *(T*)(data + position); + } + + using InputStream::read; + + private: + const uint8_t* data; + size_t capacity; + size_t position; + }; + + template + T InputStream::read() { + T v; + read(&v, sizeof(T)); + return v; + } + + template<> inline bool InputStream::read() { + uint8_t v; + read(&v, sizeof(bool)); + return v; + } + + template + bool OutputStream::write(const T &val) { + return write(&val, sizeof(T)); + } +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/fs/path.cpp b/projs/shadow/shadow-engine/shadow-assets/src/fs/path.cpp new file mode 100644 index 0000000..01bad0d --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/fs/path.cpp @@ -0,0 +1,117 @@ +#include +#include +#include + +namespace ShadowEngine { + + Path::Path() : path {} { } + + Path::Path(const std::string &str) { + set(normalise((std::string&) str)); + } + + void Path::set(const std::string &str) { +#ifdef _WIN32 + std::string temp = Str::toLower((std::string&) str); + hash = PathHash(temp); +#else + hash = PathHash(str); +#endif + path = str; + } + + void Path::operator=(const std::string &rhs) { + set(rhs); + } + + bool Path::operator==(const std::string &rhs) { + return path == rhs; + } + + bool Path::operator==(const ShadowEngine::Path &rhs) { + return path == rhs.path; + } + + bool Path::operator!=(const ShadowEngine::Path &rhs) { + return path != rhs.path; + } + + std::string Path::normalise(std::string &str) { + bool prevSlash = false; + + std::string temp; + const char* path = str.c_str(); + size_t len = str.length(); + size_t i = 0; + + // Skip initial stuff. + size_t ind = str.find_first_of(":"); + path += ind; + if (path[0] == '.' && (path[1] == '\\' || path[1] == '/')) + path += 2; +#ifdef _WIN32 + if (path[0] == '\\' || path[0] == '/') + ++path; +#endif + + while (*path != '\0' && i < len) { + bool slash = *path == '\\' || *path == '/'; + + // Skip double slashes. + if (slash && prevSlash) { + path++; continue; + } + + // Convert backslashes to forward slashes. + temp.append(std::to_string(*path == '\\' ? '/' : *path)); + + path++; i++; prevSlash = slash; + } + + return temp; + } + + std::string Path::getPrelude(std::string &path) { + return path.substr(0, path.find_first_of(":")); + } + + std::string Path::getDomain(std::string &path) { + return path.substr(path.find_first_of(":"), path.find_first_of("/")); + } + + std::string Path::getDirectory(std::string &path) { + return path.substr(path.find_first_of(":"), path.find_last_of("/")); + } + + std::string Path::getFilename(std::string &path) { + return path.substr(path.find_last_of("/"), path.find_last_of(".")); + } + + std::string Path::getExtension(std::string &path) { + return path.substr(path.find_last_of("."), path.length()); + } + + std::string Path::replaceExtension(std::string &path, std::string &newExt) { + return path.substr(0, path.length() - newExt.length()).append(newExt); + } + + bool Path::hasExtension(std::string &path, std::string &ext) { + return path.find_last_of(ext) == path.length() - ext.length(); + } + + PathInfo::PathInfo(std::string &str) { + std::string normalised = Path::normalise(str); + + std::string preludeS = Path::getPrelude(normalised); + memcpy_s(prelude, 10, preludeS.c_str(), preludeS.length()); + std::string domainS = Path::getDomain(normalised); + memcpy_s(domain, 256, domainS.c_str(), domainS.length()); + std::string directoryS = Path::getDirectory(normalised); + memcpy_s(directory, 256, directoryS.c_str(), directoryS.length()); + std::string filenameS = Path::getFilename(normalised); + memcpy_s(baseName, 256, filenameS.c_str(), filenameS.length()); + std::string extensionS = Path::getExtension(normalised); + memcpy_s(extension, 10, extensionS.c_str(), extensionS.length()); + } + +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/fs/path.h b/projs/shadow/shadow-engine/shadow-assets/src/fs/path.h new file mode 100644 index 0000000..8ce6628 --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/fs/path.h @@ -0,0 +1,66 @@ +#pragma once + +#include +#include + +namespace ShadowEngine { + /** + * Stores split data about a path, for easy referencing and decomposition. + * Not to be used as a replacement for the Path class. + */ + struct PathInfo { + explicit PathInfo(std::string& str); + + char extension[10]; + char baseName[256]; + char directory[256]; + char domain[256]; + char prelude[10]; + }; + + /** + * Stores and handles paths in the VFS. + * All operations are copy-instantiated, nothing works in-place. + * A typical path is of the form: + * prelude:/domain/directory/filename.extension + */ + struct Path { + // Make sure the path is valid. + // Always from the root. + // One slash separating. + static std::string normalise(std::string& path); + // Get the prelude of the given path. + static std::string getPrelude(std::string& path); + // Get the domain of the given path. + static std::string getDomain(std::string& path); + // Get the directory of the given path. + static std::string getDirectory(std::string& path); + // Get the name of the file of the given path. + static std::string getFilename(std::string& path); + // Get the file extension of the given path. + static std::string getExtension(std::string& path); + // Check if the path has the given extension. + static bool hasExtension(std::string& path, std::string& ext); + // Replace the extension of the given path. + static std::string replaceExtension(std::string& path, std::string& newExt); + + Path(); + explicit Path(const std::string& str); + + void operator=(const std::string& rhs); + bool operator==(const std::string& rhs); + bool operator==(const Path& rhs); + bool operator!=(const Path& rhs); + + // Use this to set a new value into the path; it handles the hash too. + void set(const std::string& path); + + uint32_t length() const { return path.length(); }; + PathHash getHash() { return hash; } + const char* c_str() const { return path.data(); } + bool isEmpty() const { return path.length() == 0; } + private: + std::string path; + PathHash hash; + }; +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/fs/xxhash.h b/projs/shadow/shadow-engine/shadow-assets/src/fs/xxhash.h new file mode 100644 index 0000000..e75b9c9 --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/fs/xxhash.h @@ -0,0 +1,182 @@ +#pragma once +#include // for uint32_t and uint64_t + +class XXHash64 +{ +public: + /// create new XXHash (64 bit) + /** @param seed your seed value, even zero is a valid seed **/ + explicit XXHash64(uint64_t seed) + { + state[0] = seed + Prime1 + Prime2; + state[1] = seed + Prime2; + state[2] = seed; + state[3] = seed - Prime1; + bufferSize = 0; + totalLength = 0; + } + + /// add a chunk of bytes + /** @param input pointer to a continuous block of data + @param length number of bytes + @return false if parameters are invalid / zero **/ + bool add(const void* input, uint64_t length) + { + // no data ? + if (!input || length == 0) + return false; + + totalLength += length; + // byte-wise access + const unsigned char* data = (const unsigned char*)input; + + // unprocessed old data plus new data still fit in temporary buffer ? + if (bufferSize + length < MaxBufferSize) + { + // just add new data + while (length-- > 0) + buffer[bufferSize++] = *data++; + return true; + } + + // point beyond last byte + const unsigned char* stop = data + length; + const unsigned char* stopBlock = stop - MaxBufferSize; + + // some data left from previous update ? + if (bufferSize > 0) + { + // make sure temporary buffer is full (16 bytes) + while (bufferSize < MaxBufferSize) + buffer[bufferSize++] = *data++; + + // process these 32 bytes (4x8) + process(buffer, state[0], state[1], state[2], state[3]); + } + + // copying state to local variables helps optimizer A LOT + uint64_t s0 = state[0], s1 = state[1], s2 = state[2], s3 = state[3]; + // 32 bytes at once + while (data <= stopBlock) + { + // local variables s0..s3 instead of state[0]..state[3] are much faster + process(data, s0, s1, s2, s3); + data += 32; + } + // copy back + state[0] = s0; state[1] = s1; state[2] = s2; state[3] = s3; + + // copy remainder to temporary buffer + bufferSize = stop - data; + for (uint64_t i = 0; i < bufferSize; i++) + buffer[i] = data[i]; + + // done + return true; + } + + /// get current hash + /** @return 64 bit XXHash **/ + uint64_t hash() const + { + // fold 256 bit state into one single 64 bit value + uint64_t result; + if (totalLength >= MaxBufferSize) + { + result = rotateLeft(state[0], 1) + + rotateLeft(state[1], 7) + + rotateLeft(state[2], 12) + + rotateLeft(state[3], 18); + result = (result ^ processSingle(0, state[0])) * Prime1 + Prime4; + result = (result ^ processSingle(0, state[1])) * Prime1 + Prime4; + result = (result ^ processSingle(0, state[2])) * Prime1 + Prime4; + result = (result ^ processSingle(0, state[3])) * Prime1 + Prime4; + } + else + { + // internal state wasn't set in add(), therefore original seed is still stored in state2 + result = state[2] + Prime5; + } + + result += totalLength; + + // process remaining bytes in temporary buffer + const unsigned char* data = buffer; + // point beyond last byte + const unsigned char* stop = data + bufferSize; + + // at least 8 bytes left ? => eat 8 bytes per step + for (; data + 8 <= stop; data += 8) + result = rotateLeft(result ^ processSingle(0, *(uint64_t*)data), 27) * Prime1 + Prime4; + + // 4 bytes left ? => eat those + if (data + 4 <= stop) + { + result = rotateLeft(result ^ (*(uint32_t*)data) * Prime1, 23) * Prime2 + Prime3; + data += 4; + } + + // take care of remaining 0..3 bytes, eat 1 byte per step + while (data != stop) + result = rotateLeft(result ^ (*data++) * Prime5, 11) * Prime1; + + // mix bits + result ^= result >> 33; + result *= Prime2; + result ^= result >> 29; + result *= Prime3; + result ^= result >> 32; + return result; + } + + + /// combine constructor, add() and hash() in one static function (C style) + /** @param input pointer to a continuous block of data + @param length number of bytes + @param seed your seed value, e.g. zero is a valid seed + @return 64 bit XXHash **/ + static uint64_t hash(const void* input, uint64_t length, uint64_t seed) + { + XXHash64 hasher(seed); + hasher.add(input, length); + return hasher.hash(); + } + +private: + /// magic constants :-) + static const uint64_t Prime1 = 11400714785074694791ULL; + static const uint64_t Prime2 = 14029467366897019727ULL; + static const uint64_t Prime3 = 1609587929392839161ULL; + static const uint64_t Prime4 = 9650029242287828579ULL; + static const uint64_t Prime5 = 2870177450012600261ULL; + + /// temporarily store up to 31 bytes between multiple add() calls + static const uint64_t MaxBufferSize = 31+1; + + uint64_t state[4]; + unsigned char buffer[MaxBufferSize]; + uint64_t bufferSize; + uint64_t totalLength; + + /// rotate bits, should compile to a single CPU instruction (ROL) + static inline uint64_t rotateLeft(uint64_t x, unsigned char bits) + { + return (x << bits) | (x >> (64 - bits)); + } + + /// process a single 64 bit value + static inline uint64_t processSingle(uint64_t previous, uint64_t input) + { + return rotateLeft(previous + input * Prime2, 31) * Prime1; + } + + /// process a block of 4x4 bytes, this is the main part of the XXHash32 algorithm + static inline void process(const void* data, uint64_t& state0, uint64_t& state1, uint64_t& state2, uint64_t& state3) + { + const uint64_t* block = (const uint64_t*) data; + state0 = processSingle(state0, block[0]); + state1 = processSingle(state1, block[1]); + state2 = processSingle(state2, block[2]); + state3 = processSingle(state3, block[3]); + } +}; \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/management/synchronization.h b/projs/shadow/shadow-engine/shadow-assets/src/management/synchronization.h new file mode 100644 index 0000000..3d3278e --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/management/synchronization.h @@ -0,0 +1,77 @@ +#pragma once + +#include + +#ifdef __linux__ +#include +#endif + +namespace ShadowEngine { + // A simple synchronization system that allows one "accessing thread" at a time. + struct alignas(8) Mutex { + friend struct ConditionVariable; + Mutex(); + Mutex(const Mutex&) = delete; + ~Mutex(); + + void enter(); + void exit(); + private: +#ifdef _WIN32 + uint8_t data[8]; +#else + pthread_mutex_t mutex; +#endif + }; + + // A simple synchronization system that allows many threads to wait for one thread to complete an operation. + struct Semaphore { + Semaphore(int initcount, int maxcount); + Semaphore(const Semaphore&) = delete; + ~Semaphore(); + + void raise(); + void wait(); + private: +#ifdef _WIN32 + void* id; +#else + struct { + pthread_mutex_t mutex; + pthread_mutex_cond cond; + volatile int32_t count; + } id; +#endif + }; + + struct ConditionVariable { + ConditionVariable(); + ConditionVariable(const ConditionVariable&) = delete; + ~ConditionVariable(); + + void sleep(Mutex& mut); + void wake(); + + private: +#ifdef _WIN32 + uint8_t data[64]; +#else + pthread_cond_t cond; +#endif + }; + + // A simple RAII wrapper for Mutexes, that locks and unlocks as the Guard goes in and out of scope. + struct MutexGuard { + explicit MutexGuard(Mutex& mut) : mut(mut) { + mut.enter(); + } + + ~MutexGuard() { mut.exit(); } + + MutexGuard(const MutexGuard&) = delete; + void operator=(const MutexGuard&) = delete; + + private: + Mutex& mut; + }; +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/str/string.cpp b/projs/shadow/shadow-engine/shadow-assets/src/str/string.cpp new file mode 100644 index 0000000..6cb12ee --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/str/string.cpp @@ -0,0 +1,11 @@ +#include + +namespace ShadowEngine::Str { + std::string toLower(const std::string& str) { + std::string temp; + for (auto c : str) { + temp.append(std::to_string(tolower(c))); + } + return temp; + } +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/str/string.h b/projs/shadow/shadow-engine/shadow-assets/src/str/string.h new file mode 100644 index 0000000..0274948 --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/str/string.h @@ -0,0 +1,12 @@ +#pragma once +#include + +namespace ShadowEngine { + // String manipluation utilities. + // Because std::string is heavily lacking. + namespace Str { + // Convert the string to lower case, return a new string. + // This only works in ASCII encoding. + std::string toLower(std::string& str); + } +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-file-format/test/Catch2Test.cpp b/projs/shadow/shadow-engine/shadow-assets/test/Catch2Test.cpp similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/test/Catch2Test.cpp rename to projs/shadow/shadow-engine/shadow-assets/test/Catch2Test.cpp diff --git a/projs/shadow/shadow-engine/shadow-file-format/test/catch2/catch.hpp b/projs/shadow/shadow-engine/shadow-assets/test/catch2/catch.hpp similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/test/catch2/catch.hpp rename to projs/shadow/shadow-engine/shadow-assets/test/catch2/catch.hpp diff --git a/projs/shadow/shadow-engine/shadow-file-format/test/sff_writer_tests.ocpp b/projs/shadow/shadow-engine/shadow-assets/test/sff_writer_tests.ocpp similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/test/sff_writer_tests.ocpp rename to projs/shadow/shadow-engine/shadow-assets/test/sff_writer_tests.ocpp diff --git a/projs/shadow/shadow-engine/shadow-file-format/test/test.ocpp b/projs/shadow/shadow-engine/shadow-assets/test/test.ocpp similarity index 100% rename from projs/shadow/shadow-engine/shadow-file-format/test/test.ocpp rename to projs/shadow/shadow-engine/shadow-assets/test/test.ocpp