From b723a0778f910d10574db41f65823f275f2ef568 Mon Sep 17 00:00:00 2001 From: Curle Date: Sun, 12 Mar 2023 16:44:10 +0000 Subject: [PATCH] FileSystem specializations for Disk & VFS operations. --- .../shadow-assets/src/fs/file.cpp | 244 ++++++++++++++++++ .../shadow-engine/shadow-assets/src/fs/file.h | 15 +- .../shadow-engine/shadow-assets/src/fs/hash.h | 10 +- .../shadow-assets/src/fs/iostream.h | 6 +- .../shadow-assets/src/fs/path.cpp | 3 +- .../shadow-engine/shadow-assets/src/fs/path.h | 11 +- .../shadow-assets/src/management/delegate.h | 92 +++++++ 7 files changed, 362 insertions(+), 19 deletions(-) create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/fs/file.cpp create mode 100644 projs/shadow/shadow-engine/shadow-assets/src/management/delegate.h diff --git a/projs/shadow/shadow-engine/shadow-assets/src/fs/file.cpp b/projs/shadow/shadow-engine/shadow-assets/src/fs/file.cpp new file mode 100644 index 0000000..6af0083 --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/fs/file.cpp @@ -0,0 +1,244 @@ +#include +#include +#include "management/synchronization.h" +#include "../../../../../../cmake-build-debug/_deps/spdlog-src/include/spdlog/spdlog.h" +#include +#include +#include + +namespace ShadowEngine { + + /** + * An async operation to be performed. + * For reading files from disk into memory. + */ + + struct AsyncRead { + enum class Flags : uint32_t { + FAILED = 0, // The read failed due to some error. + CANCELLED // The read was cancelled due to the resource not being needed any more. + }; + + AsyncRead() : data() {} + + bool isFailed() const { return flags == Flags::FAILED; } + bool isCancelled() const { return flags == Flags::CANCELLED; } + + FileSystem::ContentCallback callback; + OutputMemoryStream data; + std::string path; + uint32_t id = 0; + Flags flags; + }; + + // The FileSystem that operates on raw on-disk files. + struct DiskFS; + + struct DiskFS : FileSystem { + + explicit DiskFS(std::string& path) : sem(0, 0xffff) { + setBasePath(path); + } + + bool hasWork() override { + return workCounter != 0; + } + + std::string const& getBasePath() const override { return basePath; } + void setBasePath(std::string& path) final { + basePath = Path::normalise(path); + if (!basePath.ends_with('/') && !basePath.ends_with('\\')) + basePath.append("/"); + } + + bool saveSync(const Path& path, const uint8_t* data, const size_t size) override { + FileOutput file; + std::string fullPath(basePath.append(path.c_str())); + + if (!file.open(fullPath)) return false; + bool res = file.write(data, size); + file.close(); + + return res; + } + + bool readSync(const Path& path, struct OutputMemoryStream& content) override { + FileInput file; + std::string fullPath(basePath.append(path.c_str())); + + if (!file.open(fullPath)) return false; + + content.resize(file.size()); + if (!file.read(content.dataMut(), content.size())) { + file.close(); + return false; + } + + file.close(); + return true; + } + + AsyncHandle readAsync(const Path& file, const ContentCallback& callback) override { + if (!file.isEmpty()) return AsyncHandle::invalid(); + + MutexGuard lock(mutex); + workCounter++; + + AsyncRead& read = queue.emplace_back(); + if (++lastID == 0) lastID++; + + read.id = lastID; + read.path = file.c_str(); + read.callback = callback; + sem.raise(); + + return AsyncHandle(read.id); + } + + void cancelAsync(AsyncHandle& handle) override { + MutexGuard lock(mutex); + + for (AsyncRead& read : queue) { + if (read.id == handle.value) { + read.flags = AsyncRead::Flags::CANCELLED; + workCounter--; + return; + } + } + + for (AsyncRead& read : finished) { + if (read.id == handle.value) { + read.flags = AsyncRead::Flags::CANCELLED; + return; + } + } + } + + bool open(std::string& path, FileInput& file) override { + return file.open(basePath.append(path)); + } + + bool open(std::string& path, FileOutput& file) override { + return file.open(basePath.append(path)); + } + + bool deleteFile(std::string& path) override { + return std::remove((basePath.append(path).c_str())); + } + + bool moveFile(std::string& from, std::string& to) override { + try { + std::rename(basePath.append(from).c_str(), basePath.append(to).c_str()); + } catch (std::filesystem::filesystem_error& e) { + return false; + } + return true; + } + + bool copyFile(std::string& from, std::string& to) override { + try { + std::filesystem::copy(basePath.append(from).c_str(), basePath.append(to).c_str()); + } catch (std::filesystem::filesystem_error& e) { + return false; + } + + return true; + } + + bool fileExists(std::string& path) override { + return std::filesystem::exists(path); + } + + size_t getLastModified(std::string& path) override { + return std::filesystem::last_write_time(path).time_since_epoch().count(); + } + + // TODO: File iterators + + void processCallbacks() override { + // TODO: Timeout this function! + for (;;) { + mutex.enter(); + if (finished.empty() || workCounter == 0) { + mutex.exit(); + break; + } + + AsyncRead item = finished[0]; + finished.erase(finished.begin()); + --workCounter; + + mutex.exit(); + + if (!item.isCancelled()) + item.callback.invoke(item.data.size(), (const uint8_t*) item.data.data(), !item.isFailed()); + + } + } + + // TODO: Task Management + std::string basePath; + std::vector queue; + uint64_t workCounter; + std::vector finished; + Mutex mutex; + Semaphore sem; + + uint32_t lastID; + + }; + + struct VFS : DiskFS { + VFS(std::string& root_pack_path) : DiskFS((std::string &) "vfs:/") { + if (!pack.open(root_pack_path)) { + spdlog::error("Unable to open " + root_pack_path + ", please check paths"); + return; + } + + const auto count = pack.read(); + for (size_t i = 0; i < count; i++) { + const auto hash = pack.read(); + PackFile& file = packFiles[hash]; + file.offset = pack.read(); + file.size = pack.read(); + } + } + + ~VFS() { pack.close(); } + + bool readSync(const Path& path, OutputMemoryStream& content) override { + std::string basename = Path::getFilename(const_cast(path.get())); + PathHash hash = path.getHash(); + + auto i = packFiles.find(hash); + if (i == packFiles.end()) return false; + + content.resize(i->second.size); + MutexGuard lock(mutex); + + const size_t headerSize = sizeof(uint32_t) + packFiles.size() * (3 * sizeof(size_t)); + if (pack.seek(i->second.offset + headerSize) || !pack.read(content.dataMut(), content.size())) { + spdlog::error("Could not read file " + path.get() + " from the pack file."); + return false; + } + + return true; + } + + struct PackFile { + size_t offset; + size_t size; + }; + + std::map packFiles; + FileInput pack; + }; + + std::unique_ptr FileSystem::createDiskFS(std::string &basePath) { + return std::make_unique(basePath); + } + + std::unique_ptr FileSystem::createVFS(std::string& basePath) { + return std::make_unique(basePath); + } +} \ No newline at end of file diff --git a/projs/shadow/shadow-engine/shadow-assets/src/fs/file.h b/projs/shadow/shadow-engine/shadow-assets/src/fs/file.h index d3265cb..9c6750a 100644 --- a/projs/shadow/shadow-engine/shadow-assets/src/fs/file.h +++ b/projs/shadow/shadow-engine/shadow-assets/src/fs/file.h @@ -1,7 +1,11 @@ #pragma once #include +#include +#include #include +template struct Delegate; + namespace ShadowEngine { // An input stream that can read a file on disk. @@ -48,7 +52,6 @@ namespace ShadowEngine { std::string filename; }; - template struct Delegate; /** * A generic Filesystem API. @@ -68,9 +71,9 @@ namespace ShadowEngine { }; // Create a Filesystem that interacts with files on disk. - static std::unique_ptr& createDiskFS(std::string& basePath); + static std::unique_ptr createDiskFS(std::string& basePath); // Create a Virtual Filesystem based on the given path. - static std::unique_ptr& createVFS(std::string& basePath); + static std::unique_ptr createVFS(std::string& basePath); virtual ~FileSystem() {} @@ -90,7 +93,7 @@ namespace ShadowEngine { virtual bool deleteFile(std::string& path) = 0; // Get the path that this FileSystem originates at. The default is "/" for VFS, and whatever the Executable Path is for Disk FS. - virtual std::string& getBasePath() = 0; + virtual std::string const& getBasePath() const = 0; // Set a new base path for the FileSystem. Any operations involving file paths will be relative to this new path. virtual void setBasePath(std::string& path) = 0; @@ -100,9 +103,9 @@ namespace ShadowEngine { virtual bool hasWork() = 0; // Write new content to a file synchronously. The thread will be blocked when doing this. - virtual bool saveSync(const struct Path& file, const uint8_t* content) = 0; + virtual bool saveSync(const Path& file, const uint8_t* content, const size_t size) = 0; // Read content from a file synchronously. The thread will be blocked when doing this. - virtual bool readSync(const struct Path& file, struct OutputMemoryStream& content) = 0; + virtual bool readSync(const Path& file, struct OutputMemoryStream& content) = 0; // Read a file asynchronously. The given callback will be called with the file content once it is available. virtual AsyncHandle readAsync(const Path& file, const ContentCallback& callback) = 0; diff --git a/projs/shadow/shadow-engine/shadow-assets/src/fs/hash.h b/projs/shadow/shadow-engine/shadow-assets/src/fs/hash.h index 39e477e..0b58187 100644 --- a/projs/shadow/shadow-engine/shadow-assets/src/fs/hash.h +++ b/projs/shadow/shadow-engine/shadow-assets/src/fs/hash.h @@ -59,14 +59,14 @@ namespace ShadowEngine { struct StableHash { static StableHash fromLong(size_t data); StableHash() = default; - StableHash(std::string& str); + explicit StableHash(std::string& str); StableHash(const void* data, uint32_t length); - bool operator!= (StableHash& other) const { return hash != other.hash; } - bool operator== (StableHash& other) const { return hash == other.hash; } - bool operator< (StableHash& other) const { return hash < other.hash; } + bool operator!= (const StableHash& other) const { return hash != other.hash; } + bool operator== (const StableHash& other) const { return hash == other.hash; } + bool operator< (const StableHash& other) const { return hash < other.hash; } - size_t getHash() const { return hash; } + [[nodiscard]] size_t getHash() const { return hash; } private: size_t hash = 0; diff --git a/projs/shadow/shadow-engine/shadow-assets/src/fs/iostream.h b/projs/shadow/shadow-engine/shadow-assets/src/fs/iostream.h index 6ac41a6..1c5c7c8 100644 --- a/projs/shadow/shadow-engine/shadow-assets/src/fs/iostream.h +++ b/projs/shadow/shadow-engine/shadow-assets/src/fs/iostream.h @@ -34,13 +34,14 @@ namespace ShadowEngine { // A custom OutputStream that writes to memory. struct OutputMemoryStream final : OutputStream { + OutputMemoryStream(); OutputMemoryStream(void* data, size_t size); OutputMemoryStream(OutputMemoryStream&& str) noexcept; OutputMemoryStream(const OutputMemoryStream& rhs) noexcept; ~OutputMemoryStream(); - void operator= (const OutputMemoryStream& rhs) noexcept; - void operator= (OutputMemoryStream&& rhs) noexcept; + OutputMemoryStream& operator= (const OutputMemoryStream& rhs) noexcept; + OutputMemoryStream& operator= (OutputMemoryStream&& rhs) noexcept; uint8_t operator[] (size_t index) const; uint8_t& operator[] (size_t index); @@ -79,6 +80,7 @@ namespace ShadowEngine { write(&v, sizeof(v)); } + // A custom InputStream that writes from memory. struct InputMemoryStream final : InputStream { InputMemoryStream(const void* data, size_t size); explicit InputMemoryStream(const OutputMemoryStream& blob); diff --git a/projs/shadow/shadow-engine/shadow-assets/src/fs/path.cpp b/projs/shadow/shadow-engine/shadow-assets/src/fs/path.cpp index 01bad0d..8766231 100644 --- a/projs/shadow/shadow-engine/shadow-assets/src/fs/path.cpp +++ b/projs/shadow/shadow-engine/shadow-assets/src/fs/path.cpp @@ -20,8 +20,9 @@ namespace ShadowEngine { path = str; } - void Path::operator=(const std::string &rhs) { + Path& Path::operator=(const std::string &rhs) { set(rhs); + return *this; } bool Path::operator==(const std::string &rhs) { diff --git a/projs/shadow/shadow-engine/shadow-assets/src/fs/path.h b/projs/shadow/shadow-engine/shadow-assets/src/fs/path.h index 8ce6628..5c943a2 100644 --- a/projs/shadow/shadow-engine/shadow-assets/src/fs/path.h +++ b/projs/shadow/shadow-engine/shadow-assets/src/fs/path.h @@ -47,7 +47,7 @@ namespace ShadowEngine { Path(); explicit Path(const std::string& str); - void operator=(const std::string& rhs); + Path& operator=(const std::string& rhs); bool operator==(const std::string& rhs); bool operator==(const Path& rhs); bool operator!=(const Path& rhs); @@ -55,10 +55,11 @@ namespace ShadowEngine { // Use this to set a new value into the path; it handles the hash too. void set(const std::string& path); - uint32_t length() const { return path.length(); }; - PathHash getHash() { return hash; } - const char* c_str() const { return path.data(); } - bool isEmpty() const { return path.length() == 0; } + [[nodiscard]] uint32_t length() const { return path.length(); }; + [[nodiscard]] PathHash getHash() const { return hash; } + [[nodiscard]] const char* c_str() const { return path.data(); } + [[nodiscard]] std::string const& get() const { return path; } + [[nodiscard]] bool isEmpty() const { return path.length() == 0; } private: std::string path; PathHash hash; diff --git a/projs/shadow/shadow-engine/shadow-assets/src/management/delegate.h b/projs/shadow/shadow-engine/shadow-assets/src/management/delegate.h new file mode 100644 index 0000000..5ef30c5 --- /dev/null +++ b/projs/shadow/shadow-engine/shadow-assets/src/management/delegate.h @@ -0,0 +1,92 @@ +#pragma once + +namespace ShadowEngine { + + template struct Delegate; + + /** + * A simple, generic callback template. + * A Delegate takes the form of a std::function that can be called, redirected and bound freely. + * Advantages over std::function includes the ability to reference types that aren't yet fully qualified. + * + * Idea taken from https://blog.molecular-matters.com/2011/09/19/generic-type-safe-delegates-and-events-in-c/ + * + * @tparam R the return type + * @tparam Args the arguments to the function + */ + template struct Delegate { + private: + using InstancePtr = void*; + using InternalFunction = R (*)(InstancePtr, Args...); + struct Stub { + InstancePtr first; + InternalFunction second; + }; + + template static R FunctionStub(InstancePtr, Args... args) { + return (Function)(args...); + } + + template + static R ClassMethodStub(InstancePtr instance, Args... args) { + return (static_cast(instance)->*Function)(args...); + } + + template + static R ClassMethodStub(InstancePtr instance, Args... args) { + return (static_cast(instance)->*Function)(args...); + } + + public: + Delegate() { + m_stub.first = nullptr; + m_stub.second = nullptr; + } + + template + Delegate(const T& obj) { + m_stub.first = (InstancePtr)&obj; + m_stub.second = [](InstancePtr inst, Args... args) -> R { + const T& obj = *(const T*)inst; + return obj(args...); + }; + } + + bool isValid() { return m_stub.second != nullptr; } + + template void bind() { + m_stub.first = nullptr; + m_stub.second = &FunctionStub; + } + + template void bind(C* instance) { + m_stub.first = instance; + m_stub.second = &ClassMethodStub; + } + + R invoke(Args... args) const { + return m_stub.second(m_stub.first, args...); + } + + bool operator==(const Delegate& rhs) { + return m_stub.first == rhs.m_stub.first && m_stub.second == rhs.m_stub.second; + } + + private: + Stub m_stub; + }; + + template struct ToDelegate_T; + template struct ToDelegate_T { + using Type = Delegate; + }; + + template using ToDelegate = typename ToDelegate_T::Type; + + template + auto makeDelegate(C* inst) { + ToDelegate res; + res.template bind(inst); + return res; + }; +} \ No newline at end of file