FileSystem specializations for Disk & VFS operations.
This commit is contained in:
parent
5e60726a2b
commit
b723a0778f
244
projs/shadow/shadow-engine/shadow-assets/src/fs/file.cpp
Normal file
244
projs/shadow/shadow-engine/shadow-assets/src/fs/file.cpp
Normal file
|
|
@ -0,0 +1,244 @@
|
|||
#include <fs/file.h>
|
||||
#include <vector>
|
||||
#include "management/synchronization.h"
|
||||
#include "../../../../../../cmake-build-debug/_deps/spdlog-src/include/spdlog/spdlog.h"
|
||||
#include <filesystem>
|
||||
#include <fs/path.h>
|
||||
#include <map>
|
||||
|
||||
namespace ShadowEngine {
|
||||
|
||||
/**
|
||||
* An async operation to be performed.
|
||||
* For reading files from disk into memory.
|
||||
*/
|
||||
|
||||
struct AsyncRead {
|
||||
enum class Flags : uint32_t {
|
||||
FAILED = 0, // The read failed due to some error.
|
||||
CANCELLED // The read was cancelled due to the resource not being needed any more.
|
||||
};
|
||||
|
||||
AsyncRead() : data() {}
|
||||
|
||||
bool isFailed() const { return flags == Flags::FAILED; }
|
||||
bool isCancelled() const { return flags == Flags::CANCELLED; }
|
||||
|
||||
FileSystem::ContentCallback callback;
|
||||
OutputMemoryStream data;
|
||||
std::string path;
|
||||
uint32_t id = 0;
|
||||
Flags flags;
|
||||
};
|
||||
|
||||
// The FileSystem that operates on raw on-disk files.
|
||||
struct DiskFS;
|
||||
|
||||
struct DiskFS : FileSystem {
|
||||
|
||||
explicit DiskFS(std::string& path) : sem(0, 0xffff) {
|
||||
setBasePath(path);
|
||||
}
|
||||
|
||||
bool hasWork() override {
|
||||
return workCounter != 0;
|
||||
}
|
||||
|
||||
std::string const& getBasePath() const override { return basePath; }
|
||||
void setBasePath(std::string& path) final {
|
||||
basePath = Path::normalise(path);
|
||||
if (!basePath.ends_with('/') && !basePath.ends_with('\\'))
|
||||
basePath.append("/");
|
||||
}
|
||||
|
||||
bool saveSync(const Path& path, const uint8_t* data, const size_t size) override {
|
||||
FileOutput file;
|
||||
std::string fullPath(basePath.append(path.c_str()));
|
||||
|
||||
if (!file.open(fullPath)) return false;
|
||||
bool res = file.write(data, size);
|
||||
file.close();
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
bool readSync(const Path& path, struct OutputMemoryStream& content) override {
|
||||
FileInput file;
|
||||
std::string fullPath(basePath.append(path.c_str()));
|
||||
|
||||
if (!file.open(fullPath)) return false;
|
||||
|
||||
content.resize(file.size());
|
||||
if (!file.read(content.dataMut(), content.size())) {
|
||||
file.close();
|
||||
return false;
|
||||
}
|
||||
|
||||
file.close();
|
||||
return true;
|
||||
}
|
||||
|
||||
AsyncHandle readAsync(const Path& file, const ContentCallback& callback) override {
|
||||
if (!file.isEmpty()) return AsyncHandle::invalid();
|
||||
|
||||
MutexGuard lock(mutex);
|
||||
workCounter++;
|
||||
|
||||
AsyncRead& read = queue.emplace_back();
|
||||
if (++lastID == 0) lastID++;
|
||||
|
||||
read.id = lastID;
|
||||
read.path = file.c_str();
|
||||
read.callback = callback;
|
||||
sem.raise();
|
||||
|
||||
return AsyncHandle(read.id);
|
||||
}
|
||||
|
||||
void cancelAsync(AsyncHandle& handle) override {
|
||||
MutexGuard lock(mutex);
|
||||
|
||||
for (AsyncRead& read : queue) {
|
||||
if (read.id == handle.value) {
|
||||
read.flags = AsyncRead::Flags::CANCELLED;
|
||||
workCounter--;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
for (AsyncRead& read : finished) {
|
||||
if (read.id == handle.value) {
|
||||
read.flags = AsyncRead::Flags::CANCELLED;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool open(std::string& path, FileInput& file) override {
|
||||
return file.open(basePath.append(path));
|
||||
}
|
||||
|
||||
bool open(std::string& path, FileOutput& file) override {
|
||||
return file.open(basePath.append(path));
|
||||
}
|
||||
|
||||
bool deleteFile(std::string& path) override {
|
||||
return std::remove((basePath.append(path).c_str()));
|
||||
}
|
||||
|
||||
bool moveFile(std::string& from, std::string& to) override {
|
||||
try {
|
||||
std::rename(basePath.append(from).c_str(), basePath.append(to).c_str());
|
||||
} catch (std::filesystem::filesystem_error& e) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool copyFile(std::string& from, std::string& to) override {
|
||||
try {
|
||||
std::filesystem::copy(basePath.append(from).c_str(), basePath.append(to).c_str());
|
||||
} catch (std::filesystem::filesystem_error& e) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool fileExists(std::string& path) override {
|
||||
return std::filesystem::exists(path);
|
||||
}
|
||||
|
||||
size_t getLastModified(std::string& path) override {
|
||||
return std::filesystem::last_write_time(path).time_since_epoch().count();
|
||||
}
|
||||
|
||||
// TODO: File iterators
|
||||
|
||||
void processCallbacks() override {
|
||||
// TODO: Timeout this function!
|
||||
for (;;) {
|
||||
mutex.enter();
|
||||
if (finished.empty() || workCounter == 0) {
|
||||
mutex.exit();
|
||||
break;
|
||||
}
|
||||
|
||||
AsyncRead item = finished[0];
|
||||
finished.erase(finished.begin());
|
||||
--workCounter;
|
||||
|
||||
mutex.exit();
|
||||
|
||||
if (!item.isCancelled())
|
||||
item.callback.invoke(item.data.size(), (const uint8_t*) item.data.data(), !item.isFailed());
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Task Management
|
||||
std::string basePath;
|
||||
std::vector<AsyncRead> queue;
|
||||
uint64_t workCounter;
|
||||
std::vector<AsyncRead> finished;
|
||||
Mutex mutex;
|
||||
Semaphore sem;
|
||||
|
||||
uint32_t lastID;
|
||||
|
||||
};
|
||||
|
||||
struct VFS : DiskFS {
|
||||
VFS(std::string& root_pack_path) : DiskFS((std::string &) "vfs:/") {
|
||||
if (!pack.open(root_pack_path)) {
|
||||
spdlog::error("Unable to open " + root_pack_path + ", please check paths");
|
||||
return;
|
||||
}
|
||||
|
||||
const auto count = pack.read<size_t>();
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
const auto hash = pack.read<PathHash>();
|
||||
PackFile& file = packFiles[hash];
|
||||
file.offset = pack.read<size_t>();
|
||||
file.size = pack.read<size_t>();
|
||||
}
|
||||
}
|
||||
|
||||
~VFS() { pack.close(); }
|
||||
|
||||
bool readSync(const Path& path, OutputMemoryStream& content) override {
|
||||
std::string basename = Path::getFilename(const_cast<std::string &>(path.get()));
|
||||
PathHash hash = path.getHash();
|
||||
|
||||
auto i = packFiles.find(hash);
|
||||
if (i == packFiles.end()) return false;
|
||||
|
||||
content.resize(i->second.size);
|
||||
MutexGuard lock(mutex);
|
||||
|
||||
const size_t headerSize = sizeof(uint32_t) + packFiles.size() * (3 * sizeof(size_t));
|
||||
if (pack.seek(i->second.offset + headerSize) || !pack.read(content.dataMut(), content.size())) {
|
||||
spdlog::error("Could not read file " + path.get() + " from the pack file.");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
struct PackFile {
|
||||
size_t offset;
|
||||
size_t size;
|
||||
};
|
||||
|
||||
std::map<PathHash, PackFile> packFiles;
|
||||
FileInput pack;
|
||||
};
|
||||
|
||||
std::unique_ptr<FileSystem> FileSystem::createDiskFS(std::string &basePath) {
|
||||
return std::make_unique<DiskFS>(basePath);
|
||||
}
|
||||
|
||||
std::unique_ptr<FileSystem> FileSystem::createVFS(std::string& basePath) {
|
||||
return std::make_unique<VFS>(basePath);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,11 @@
|
|||
#pragma once
|
||||
#include <fs/iostream.h>
|
||||
#include <fs/path.h>
|
||||
#include <management/delegate.h>
|
||||
#include <memory>
|
||||
|
||||
template <class T> struct Delegate;
|
||||
|
||||
namespace ShadowEngine {
|
||||
|
||||
// An input stream that can read a file on disk.
|
||||
|
|
@ -48,7 +52,6 @@ namespace ShadowEngine {
|
|||
std::string filename;
|
||||
};
|
||||
|
||||
template <class T> struct Delegate;
|
||||
|
||||
/**
|
||||
* A generic Filesystem API.
|
||||
|
|
@ -68,9 +71,9 @@ namespace ShadowEngine {
|
|||
};
|
||||
|
||||
// Create a Filesystem that interacts with files on disk.
|
||||
static std::unique_ptr<FileSystem>& createDiskFS(std::string& basePath);
|
||||
static std::unique_ptr<FileSystem> createDiskFS(std::string& basePath);
|
||||
// Create a Virtual Filesystem based on the given path.
|
||||
static std::unique_ptr<FileSystem>& createVFS(std::string& basePath);
|
||||
static std::unique_ptr<FileSystem> createVFS(std::string& basePath);
|
||||
|
||||
virtual ~FileSystem() {}
|
||||
|
||||
|
|
@ -90,7 +93,7 @@ namespace ShadowEngine {
|
|||
virtual bool deleteFile(std::string& path) = 0;
|
||||
|
||||
// Get the path that this FileSystem originates at. The default is "/" for VFS, and whatever the Executable Path is for Disk FS.
|
||||
virtual std::string& getBasePath() = 0;
|
||||
virtual std::string const& getBasePath() const = 0;
|
||||
// Set a new base path for the FileSystem. Any operations involving file paths will be relative to this new path.
|
||||
virtual void setBasePath(std::string& path) = 0;
|
||||
|
||||
|
|
@ -100,9 +103,9 @@ namespace ShadowEngine {
|
|||
virtual bool hasWork() = 0;
|
||||
|
||||
// Write new content to a file synchronously. The thread will be blocked when doing this.
|
||||
virtual bool saveSync(const struct Path& file, const uint8_t* content) = 0;
|
||||
virtual bool saveSync(const Path& file, const uint8_t* content, const size_t size) = 0;
|
||||
// Read content from a file synchronously. The thread will be blocked when doing this.
|
||||
virtual bool readSync(const struct Path& file, struct OutputMemoryStream& content) = 0;
|
||||
virtual bool readSync(const Path& file, struct OutputMemoryStream& content) = 0;
|
||||
|
||||
// Read a file asynchronously. The given callback will be called with the file content once it is available.
|
||||
virtual AsyncHandle readAsync(const Path& file, const ContentCallback& callback) = 0;
|
||||
|
|
|
|||
|
|
@ -59,14 +59,14 @@ namespace ShadowEngine {
|
|||
struct StableHash {
|
||||
static StableHash fromLong(size_t data);
|
||||
StableHash() = default;
|
||||
StableHash(std::string& str);
|
||||
explicit StableHash(std::string& str);
|
||||
StableHash(const void* data, uint32_t length);
|
||||
|
||||
bool operator!= (StableHash& other) const { return hash != other.hash; }
|
||||
bool operator== (StableHash& other) const { return hash == other.hash; }
|
||||
bool operator< (StableHash& other) const { return hash < other.hash; }
|
||||
bool operator!= (const StableHash& other) const { return hash != other.hash; }
|
||||
bool operator== (const StableHash& other) const { return hash == other.hash; }
|
||||
bool operator< (const StableHash& other) const { return hash < other.hash; }
|
||||
|
||||
size_t getHash() const { return hash; }
|
||||
[[nodiscard]] size_t getHash() const { return hash; }
|
||||
|
||||
private:
|
||||
size_t hash = 0;
|
||||
|
|
|
|||
|
|
@ -34,13 +34,14 @@ namespace ShadowEngine {
|
|||
// A custom OutputStream that writes to memory.
|
||||
struct OutputMemoryStream final : OutputStream {
|
||||
|
||||
OutputMemoryStream();
|
||||
OutputMemoryStream(void* data, size_t size);
|
||||
OutputMemoryStream(OutputMemoryStream&& str) noexcept;
|
||||
OutputMemoryStream(const OutputMemoryStream& rhs) noexcept;
|
||||
~OutputMemoryStream();
|
||||
|
||||
void operator= (const OutputMemoryStream& rhs) noexcept;
|
||||
void operator= (OutputMemoryStream&& rhs) noexcept;
|
||||
OutputMemoryStream& operator= (const OutputMemoryStream& rhs) noexcept;
|
||||
OutputMemoryStream& operator= (OutputMemoryStream&& rhs) noexcept;
|
||||
|
||||
uint8_t operator[] (size_t index) const;
|
||||
uint8_t& operator[] (size_t index);
|
||||
|
|
@ -79,6 +80,7 @@ namespace ShadowEngine {
|
|||
write(&v, sizeof(v));
|
||||
}
|
||||
|
||||
// A custom InputStream that writes from memory.
|
||||
struct InputMemoryStream final : InputStream {
|
||||
InputMemoryStream(const void* data, size_t size);
|
||||
explicit InputMemoryStream(const OutputMemoryStream& blob);
|
||||
|
|
|
|||
|
|
@ -20,8 +20,9 @@ namespace ShadowEngine {
|
|||
path = str;
|
||||
}
|
||||
|
||||
void Path::operator=(const std::string &rhs) {
|
||||
Path& Path::operator=(const std::string &rhs) {
|
||||
set(rhs);
|
||||
return *this;
|
||||
}
|
||||
|
||||
bool Path::operator==(const std::string &rhs) {
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ namespace ShadowEngine {
|
|||
Path();
|
||||
explicit Path(const std::string& str);
|
||||
|
||||
void operator=(const std::string& rhs);
|
||||
Path& operator=(const std::string& rhs);
|
||||
bool operator==(const std::string& rhs);
|
||||
bool operator==(const Path& rhs);
|
||||
bool operator!=(const Path& rhs);
|
||||
|
|
@ -55,10 +55,11 @@ namespace ShadowEngine {
|
|||
// Use this to set a new value into the path; it handles the hash too.
|
||||
void set(const std::string& path);
|
||||
|
||||
uint32_t length() const { return path.length(); };
|
||||
PathHash getHash() { return hash; }
|
||||
const char* c_str() const { return path.data(); }
|
||||
bool isEmpty() const { return path.length() == 0; }
|
||||
[[nodiscard]] uint32_t length() const { return path.length(); };
|
||||
[[nodiscard]] PathHash getHash() const { return hash; }
|
||||
[[nodiscard]] const char* c_str() const { return path.data(); }
|
||||
[[nodiscard]] std::string const& get() const { return path; }
|
||||
[[nodiscard]] bool isEmpty() const { return path.length() == 0; }
|
||||
private:
|
||||
std::string path;
|
||||
PathHash hash;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,92 @@
|
|||
#pragma once
|
||||
|
||||
namespace ShadowEngine {
|
||||
|
||||
template <typename T> struct Delegate;
|
||||
|
||||
/**
|
||||
* A simple, generic callback template.
|
||||
* A Delegate takes the form of a std::function that can be called, redirected and bound freely.
|
||||
* Advantages over std::function includes the ability to reference types that aren't yet fully qualified.
|
||||
*
|
||||
* Idea taken from https://blog.molecular-matters.com/2011/09/19/generic-type-safe-delegates-and-events-in-c/
|
||||
*
|
||||
* @tparam R the return type
|
||||
* @tparam Args the arguments to the function
|
||||
*/
|
||||
template <typename R, typename... Args> struct Delegate<R(Args...)> {
|
||||
private:
|
||||
using InstancePtr = void*;
|
||||
using InternalFunction = R (*)(InstancePtr, Args...);
|
||||
struct Stub {
|
||||
InstancePtr first;
|
||||
InternalFunction second;
|
||||
};
|
||||
|
||||
template <R (*Function)(Args...)> static R FunctionStub(InstancePtr, Args... args) {
|
||||
return (Function)(args...);
|
||||
}
|
||||
|
||||
template <typename C, R(C::*Function)(Args...)>
|
||||
static R ClassMethodStub(InstancePtr instance, Args... args) {
|
||||
return (static_cast<C*>(instance)->*Function)(args...);
|
||||
}
|
||||
|
||||
template <typename C, R(C::*Function)(Args...) const>
|
||||
static R ClassMethodStub(InstancePtr instance, Args... args) {
|
||||
return (static_cast<C*>(instance)->*Function)(args...);
|
||||
}
|
||||
|
||||
public:
|
||||
Delegate() {
|
||||
m_stub.first = nullptr;
|
||||
m_stub.second = nullptr;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
Delegate(const T& obj) {
|
||||
m_stub.first = (InstancePtr)&obj;
|
||||
m_stub.second = [](InstancePtr inst, Args... args) -> R {
|
||||
const T& obj = *(const T*)inst;
|
||||
return obj(args...);
|
||||
};
|
||||
}
|
||||
|
||||
bool isValid() { return m_stub.second != nullptr; }
|
||||
|
||||
template <R (*Function)(Args...)> void bind() {
|
||||
m_stub.first = nullptr;
|
||||
m_stub.second = &FunctionStub<Function>;
|
||||
}
|
||||
|
||||
template <auto F, typename C> void bind(C* instance) {
|
||||
m_stub.first = instance;
|
||||
m_stub.second = &ClassMethodStub<C, F>;
|
||||
}
|
||||
|
||||
R invoke(Args... args) const {
|
||||
return m_stub.second(m_stub.first, args...);
|
||||
}
|
||||
|
||||
bool operator==(const Delegate<R(Args...)>& rhs) {
|
||||
return m_stub.first == rhs.m_stub.first && m_stub.second == rhs.m_stub.second;
|
||||
}
|
||||
|
||||
private:
|
||||
Stub m_stub;
|
||||
};
|
||||
|
||||
template <typename T> struct ToDelegate_T;
|
||||
template <typename R, typename C, typename... Args> struct ToDelegate_T<R (C::*)(Args...)> {
|
||||
using Type = Delegate<R (Args...)>;
|
||||
};
|
||||
|
||||
template <typename T> using ToDelegate = typename ToDelegate_T<T>::Type;
|
||||
|
||||
template <auto M, typename C>
|
||||
auto makeDelegate(C* inst) {
|
||||
ToDelegate<decltype(M)> res;
|
||||
res.template bind<M, C>(inst);
|
||||
return res;
|
||||
};
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user