Skip to content

Commit be728d8

Browse files
Allow file flags on IFileArchive::getFile, which means actually care about the ones ISystem::createFile got.
P.S. Also remove a whole bunch of legacy code and comments that weren't even up to date with latest designs.
1 parent be825d6 commit be728d8

File tree

9 files changed

+162
-129
lines changed

9 files changed

+162
-129
lines changed

include/nbl/system/CFileArchive.h

Lines changed: 45 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -73,27 +73,41 @@ class NBL_API2 CFileArchive : public IFileArchive
7373
static inline constexpr size_t SIZEOF_INNER_ARCHIVE_FILE = std::max(sizeof(CInnerArchiveFile<CPlainHeapAllocator>), sizeof(CInnerArchiveFile<VirtualMemoryAllocator>));
7474
static inline constexpr size_t ALIGNOF_INNER_ARCHIVE_FILE = std::max(alignof(CInnerArchiveFile<CPlainHeapAllocator>), alignof(CInnerArchiveFile<VirtualMemoryAllocator>));
7575

76-
public:
77-
inline core::smart_refctd_ptr<IFile> getFile(const path& pathRelativeToArchive, const std::string_view& password) override
76+
protected:
77+
CFileArchive(path&& _defaultAbsolutePath, system::logger_opt_smart_ptr&& logger, std::shared_ptr<core::vector<SFileList::SEntry>> _items) :
78+
IFileArchive(std::move(_defaultAbsolutePath),std::move(logger))
7879
{
79-
const auto* item = getItemFromPath(pathRelativeToArchive);
80-
if (!item)
81-
return nullptr;
82-
83-
switch (item->allocatorType)
80+
setItemList(_items);
81+
82+
const auto fileCount = _items->size();
83+
m_filesBuffer = (std::byte*)_NBL_ALIGNED_MALLOC(fileCount*SIZEOF_INNER_ARCHIVE_FILE, ALIGNOF_INNER_ARCHIVE_FILE);
84+
m_fileFlags = (std::atomic_flag*)_NBL_ALIGNED_MALLOC(fileCount*sizeof(std::atomic_flag), alignof(std::atomic_flag));
85+
for (size_t i=0u; i<fileCount; i++)
86+
m_fileFlags[i].clear();
87+
memset(m_filesBuffer,0,fileCount*SIZEOF_INNER_ARCHIVE_FILE);
88+
}
89+
~CFileArchive()
90+
{
91+
_NBL_ALIGNED_FREE(m_filesBuffer);
92+
_NBL_ALIGNED_FREE(m_fileFlags);
93+
}
94+
95+
inline core::smart_refctd_ptr<IFile> getFile_impl(const SFileList::found_t& found, const core::bitflag<IFile::E_CREATE_FLAGS> flags, const std::string_view& password) override
96+
{
97+
switch (found->allocatorType)
8498
{
8599
case EAT_NULL:
86-
return getFile_impl<CNullAllocator>(item);
100+
return getFile_impl<CNullAllocator>(found,flags);
87101
break;
88102
case EAT_MALLOC:
89-
return getFile_impl<CPlainHeapAllocator>(item);
103+
return getFile_impl<CPlainHeapAllocator>(found,flags);
90104
break;
91105
case EAT_VIRTUAL_ALLOC:
92-
return getFile_impl<VirtualMemoryAllocator>(item);
106+
return getFile_impl<VirtualMemoryAllocator>(found,flags);
93107
break;
94108
case EAT_APK_ALLOCATOR:
95109
#ifdef _NBL_PLATFORM_ANDROID_
96-
return getFile_impl<CFileViewAPKAllocator>(item);
110+
return getFile_impl<CFileViewAPKAllocator>(found,flags);
97111
#else
98112
assert(false);
99113
#endif
@@ -103,44 +117,35 @@ class NBL_API2 CFileArchive : public IFileArchive
103117
}
104118
return nullptr;
105119
}
106-
107-
protected:
108-
CFileArchive(path&& _defaultAbsolutePath, system::logger_opt_smart_ptr&& logger, std::shared_ptr<core::vector<SFileList::SEntry>> _items) :
109-
IFileArchive(std::move(_defaultAbsolutePath),std::move(logger))
110-
{
111-
setItemList(_items);
112-
113-
const auto fileCount = _items->size();
114-
m_filesBuffer = (std::byte*)_NBL_ALIGNED_MALLOC(fileCount*SIZEOF_INNER_ARCHIVE_FILE, ALIGNOF_INNER_ARCHIVE_FILE);
115-
m_fileFlags = (std::atomic_flag*)_NBL_ALIGNED_MALLOC(fileCount*sizeof(std::atomic_flag), alignof(std::atomic_flag));
116-
for (size_t i=0u; i<fileCount; i++)
117-
m_fileFlags[i].clear();
118-
memset(m_filesBuffer,0,fileCount*SIZEOF_INNER_ARCHIVE_FILE);
119-
}
120-
~CFileArchive()
121-
{
122-
_NBL_ALIGNED_FREE(m_filesBuffer);
123-
_NBL_ALIGNED_FREE(m_fileFlags);
124-
}
125120

126121
template<class Allocator>
127-
inline core::smart_refctd_ptr<CInnerArchiveFile<Allocator>> getFile_impl(const IFileArchive::SFileList::SEntry* item)
122+
inline core::smart_refctd_ptr<CInnerArchiveFile<Allocator>> getFile_impl(const SFileList::found_t& found, core::bitflag<IFile::E_CREATE_FLAGS> flags)
128123
{
129-
auto* file = reinterpret_cast<CInnerArchiveFile<Allocator>*>(m_filesBuffer+item->ID*SIZEOF_INNER_ARCHIVE_FILE);
124+
// TODO: figure out a new system of cached allocations which can handle files being added/removed from an archive,
125+
// which will also allow for changing the flags that a File View is created with.
126+
if (flags.hasFlags(IFile::ECF_MAPPABLE))
127+
{
128+
m_logger.log("Overriding file flags for %s, creating it as mappable anyway.",ILogger::ELL_INFO,found->pathRelativeToArchive.c_str());
129+
flags |= IFile::ECF_MAPPABLE;
130+
}
131+
// IFileArchive should have already checked for this, stay like this until we allow write access to archived files
132+
assert(!flags.hasFlags(IFile::ECF_WRITE));
133+
134+
auto* file = reinterpret_cast<CInnerArchiveFile<Allocator>*>(m_filesBuffer+found->ID*SIZEOF_INNER_ARCHIVE_FILE);
130135
// NOTE: Intentionally calling grab() on maybe-not-existing object!
131136
const auto oldRefcount = file->grab();
132137

133138
if (oldRefcount==0) // need to construct (previous refcount was 0)
134139
{
135-
const auto fileBuffer = getFileBuffer(item);
140+
const auto fileBuffer = getFileBuffer(found);
136141
// Might have barged inbetween a refctr drop and finish of a destructor + delete,
137142
// need to wait for the "alive" flag to become `false` which tells us `operator delete` has finished.
138-
m_fileFlags[item->ID].wait(true);
143+
m_fileFlags[found->ID].wait(true);
139144
// coast is clear, do placement new
140-
new (file, &m_fileFlags[item->ID]) CInnerArchiveFile<Allocator>(
141-
m_fileFlags+item->ID,
142-
getDefaultAbsolutePath()/item->pathRelativeToArchive,
143-
IFile::ECF_READ, // TODO: stay like this until we allow write access to archived files
145+
new (file, &m_fileFlags[found->ID]) CInnerArchiveFile<Allocator>(
146+
m_fileFlags+found->ID,
147+
getDefaultAbsolutePath()/found->pathRelativeToArchive,
148+
flags,
144149
fileBuffer.buffer,
145150
fileBuffer.size,
146151
Allocator(fileBuffer.allocatorState) // no archive uses stateful allocators yet
@@ -150,14 +155,14 @@ class NBL_API2 CFileArchive : public IFileArchive
150155
return core::smart_refctd_ptr<CInnerArchiveFile<Allocator>>(file,core::dont_grab);
151156
}
152157

153-
// this function will return a buffer that needs to be deallocated with an allocator matching `item->allocatorType`
158+
// this function will return a buffer that needs to be deallocated with an allocator matching `found->allocatorType`
154159
struct file_buffer_t
155160
{
156161
void* buffer;
157162
size_t size;
158163
void* allocatorState;
159164
};
160-
virtual file_buffer_t getFileBuffer(const IFileArchive::SFileList::SEntry* item) = 0;
165+
virtual file_buffer_t getFileBuffer(const SFileList::found_t& found) = 0;
161166

162167
std::atomic_flag* m_fileFlags = nullptr;
163168
std::byte* m_filesBuffer = nullptr;

include/nbl/system/CMountDirectoryArchive.h

Lines changed: 11 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,23 +20,9 @@ class CMountDirectoryArchive : public IFileArchive
2020
m_system = system;
2121
}
2222

23-
core::smart_refctd_ptr<IFile> getFile(const path& pathRelativeToArchive, const std::string_view& password) override
24-
{
25-
{
26-
//std::unique_lock(itemMutex); already inside `getItemFromPath`
27-
if (!getItemFromPath(pathRelativeToArchive))
28-
return nullptr;
29-
}
30-
system::ISystem::future_t<core::smart_refctd_ptr<system::IFile>> future;
31-
m_system->createFile(future, m_defaultAbsolutePath / pathRelativeToArchive, system::IFile::ECF_READ);
32-
if (auto file = future.acquire())
33-
return *file;
34-
35-
return nullptr;
36-
}
37-
3823

39-
SFileList listAssets() const override {
24+
SFileList listAssets() const override
25+
{
4026
auto items = m_system->listItemsInDirectory(m_defaultAbsolutePath);
4127
auto new_entries = std::make_shared<core::vector<SFileList::SEntry>>();
4228
for (auto item : items)
@@ -53,8 +39,15 @@ class CMountDirectoryArchive : public IFileArchive
5339
return IFileArchive::listAssets();
5440
}
5541

56-
void populateItemList(const path& p) const
57-
{
42+
protected:
43+
inline core::smart_refctd_ptr<IFile> getFile_impl(const SFileList::found_t& found, const core::bitflag<IFile::E_CREATE_FLAGS> flags, const std::string_view& password) override
44+
{
45+
system::ISystem::future_t<core::smart_refctd_ptr<system::IFile>> future;
46+
m_system->createFile(future,m_defaultAbsolutePath/found->pathRelativeToArchive,flags);
47+
if (auto file=future.acquire())
48+
return *file;
49+
50+
return nullptr;
5851
}
5952
};
6053

include/nbl/system/IFileArchive.h

Lines changed: 94 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
#include <string_view>
1616
#include <algorithm>
17+
#include <span>
1718

1819
namespace nbl::system
1920
{
@@ -33,99 +34,134 @@ class NBL_API2 IFileArchive : public core::IReferenceCounted
3334
EAT_MALLOC // decompress to RAM
3435
};
3536
//! An entry in a list of items, can be a folder or a file.
36-
struct SFileList
37+
class SFileList
3738
{
38-
struct SEntry
39-
{
40-
//same stuff as `SListEntry` right now
41-
//! The name of the file including the path relative to archive root
42-
system::path pathRelativeToArchive;
39+
public:
40+
struct SEntry
41+
{
42+
//same stuff as `SListEntry` right now
43+
//! The name of the file including the path relative to archive root
44+
system::path pathRelativeToArchive;
4345

44-
//! The size of the file in bytes
45-
size_t size;
46+
//! The size of the file in bytes
47+
size_t size;
4648

47-
//! FileOffset inside an archive
48-
size_t offset;
49+
//! FileOffset inside an archive
50+
size_t offset;
4951

50-
//! The ID of the file in an archive, it maps it to a memory pool entry for CFileView
51-
uint32_t ID;
52+
//! The ID of the file in an archive, it maps it to a memory pool entry for CFileView
53+
uint32_t ID;
5254

53-
// `EAT_NONE` for directories
54-
IFileArchive::E_ALLOCATOR_TYPE allocatorType;
55+
// `EAT_NONE` for directories
56+
IFileArchive::E_ALLOCATOR_TYPE allocatorType;
5557

56-
//! The == operator is provided so that CFileList can slowly search the list!
57-
inline bool operator==(const struct SEntry& other) const
58-
{
59-
return pathRelativeToArchive.string() == other.pathRelativeToArchive.string();
60-
}
58+
//! The == operator is provided so that CFileList can slowly search the list!
59+
inline bool operator==(const struct SEntry& other) const
60+
{
61+
return pathRelativeToArchive.string() == other.pathRelativeToArchive.string();
62+
}
63+
64+
//! The < operator is provided so that CFileList can sort and quickly search the list.
65+
inline bool operator<(const struct SEntry& other) const
66+
{
67+
return pathRelativeToArchive < other.pathRelativeToArchive;
68+
}
69+
};
70+
using refctd_storage_t = std::shared_ptr<const core::vector<SEntry>>;
6171

62-
//! The < operator is provided so that CFileList can sort and quickly search the list.
63-
inline bool operator<(const struct SEntry& other) const
72+
class found_t final
6473
{
65-
return pathRelativeToArchive < other.pathRelativeToArchive;
66-
}
67-
};
68-
using refctd_storage_t = std::shared_ptr<const core::vector<SEntry>>;
69-
using range_t = core::SRange<const SEntry>;
70-
71-
inline operator range_t() const { return m_range; }
72-
73-
SFileList(const SFileList&) = default;
74-
SFileList(SFileList&&) = default;
75-
SFileList& operator=(const SFileList&) = default;
76-
SFileList& operator=(SFileList&&) = default;
77-
78-
private:
79-
// default ctor full range
80-
SFileList(refctd_storage_t _data) : m_data(_data), m_range({ _data->data(),_data->data() + _data->size() }) {}
81-
82-
friend class IFileArchive;
83-
refctd_storage_t m_data;
84-
range_t m_range;
74+
refctd_storage_t m_backingStorage = nullptr;
75+
76+
public:
77+
using type = refctd_storage_t::element_type::const_pointer;
78+
79+
found_t() = default;
80+
found_t(refctd_storage_t&& _storage, type _iter) : m_backingStorage(_storage), m_iter(_iter) {}
81+
82+
explicit inline operator bool() const {return m_iter;}
83+
84+
inline const SEntry& operator*() const {return *m_iter;}
85+
inline const SEntry* operator->() const {return m_iter;}
86+
87+
type m_iter = nullptr;
88+
};
89+
90+
/*deprecated*/ using range_t = std::span<const SEntry>;
91+
using span_t = std::span<const SEntry>;
92+
inline operator span_t() const {return m_span;}
93+
94+
SFileList(const SFileList&) = default;
95+
SFileList(SFileList&&) = default;
96+
SFileList& operator=(const SFileList&) = default;
97+
SFileList& operator=(SFileList&&) = default;
98+
99+
private:
100+
// default ctor full range
101+
SFileList(refctd_storage_t _data) : m_data(_data), m_span(m_data->data(),m_data->data()+m_data->size()) {}
102+
103+
friend class IFileArchive;
104+
refctd_storage_t m_data;
105+
span_t m_span;
85106
};
86107

87108
//
88-
virtual inline SFileList listAssets() const {
109+
virtual inline SFileList listAssets() const
110+
{
89111
return { m_items.load() };
90112
}
91113

92114
// List all files and directories in a specific dir of the archive
93115
SFileList listAssets(path pathRelativeToArchive) const;
94116

95117
//
96-
virtual core::smart_refctd_ptr<IFile> getFile(const path& pathRelativeToArchive, const std::string_view& password) = 0;
118+
inline core::smart_refctd_ptr<IFile> getFile(const path& pathRelativeToArchive, const core::bitflag<IFileBase::E_CREATE_FLAGS> flags, const std::string_view& password)
119+
{
120+
const auto item = getItemFromPath(pathRelativeToArchive);
121+
if (!item)
122+
return nullptr;
123+
124+
if (flags.hasFlags(IFileBase::ECF_WRITE))
125+
{
126+
m_logger.log("Cannot open file %s with WRITE flag, we don't support writing to archives yet!",ILogger::ELL_ERROR,pathRelativeToArchive.c_str());
127+
return nullptr;
128+
}
129+
return getFile_impl(item,flags,password);
130+
}
97131

98132
//
99133
const path& getDefaultAbsolutePath() const {return m_defaultAbsolutePath;}
100134

101135
protected:
102136
IFileArchive(path&& _defaultAbsolutePath, system::logger_opt_smart_ptr&& logger) :
103-
m_defaultAbsolutePath(std::move(_defaultAbsolutePath.make_preferred())), m_logger(std::move(logger))
104-
{
105-
}
137+
m_defaultAbsolutePath(std::move(_defaultAbsolutePath.make_preferred())), m_logger(std::move(logger)) {}
138+
139+
//
140+
virtual core::smart_refctd_ptr<IFile> getFile_impl(const SFileList::found_t& found, const core::bitflag<IFileBase::E_CREATE_FLAGS> flags, const std::string_view& password) = 0;
106141

107-
inline const SFileList::SEntry* getItemFromPath(const system::path& pathRelativeToArchive) const
142+
inline const SFileList::found_t getItemFromPath(const system::path& pathRelativeToArchive) const
108143
{
109-
const SFileList::SEntry itemToFind = { pathRelativeToArchive };
110-
auto items = m_items.load();
111-
const auto found = std::lower_bound(items->begin(), items->end(),itemToFind);
112-
if (found==items->end() || found->pathRelativeToArchive != pathRelativeToArchive)
113-
return nullptr;
114-
return &(*found);
144+
const SFileList::SEntry itemToFind = { pathRelativeToArchive };
145+
// calling `listAssets` makes sure any "update list" overload can kick in
146+
auto items = listAssets();
147+
const auto span = SFileList::span_t(items);
148+
const auto found = std::lower_bound(span.begin(),span.end(),itemToFind);
149+
if (found==span.end() || found->pathRelativeToArchive!=pathRelativeToArchive)
150+
return {};
151+
return SFileList::found_t(std::move(items.m_data),&(*found));
115152
}
116153

117154
const path m_defaultAbsolutePath;
118-
// files and directories
119-
//
120155
system::logger_opt_smart_ptr m_logger;
121156

122-
inline void setItemList(std::shared_ptr<core::vector<SFileList::SEntry>> _items) const {
123-
124-
std::sort(_items->begin(), _items->end());
157+
inline void setItemList(std::shared_ptr<core::vector<SFileList::SEntry>> _items) const
158+
{
159+
std::sort(_items->begin(),_items->end());
125160
m_items.store(_items);
126161
}
127162

128163
private:
164+
// files and directories
129165
mutable std::atomic<SFileList::refctd_storage_t> m_items;
130166
};
131167

src/nbl/system/CArchiveLoaderTar.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,10 @@ struct STarHeader
4141
using namespace nbl;
4242
using namespace nbl::system;
4343

44-
CFileArchive::file_buffer_t CArchiveLoaderTar::CArchive::getFileBuffer(const IFileArchive::SFileList::SEntry* item)
44+
CFileArchive::file_buffer_t CArchiveLoaderTar::CArchive::getFileBuffer(const IFileArchive::SFileList::found_t& found)
4545
{
46-
assert(item->allocatorType==EAT_NULL);
47-
return {reinterpret_cast<uint8_t*>(m_file->getMappedPointer())+item->offset,item->size,nullptr};
46+
assert(found->allocatorType==EAT_NULL);
47+
return {reinterpret_cast<uint8_t*>(m_file->getMappedPointer())+found->offset,found->size,nullptr};
4848
}
4949

5050

0 commit comments

Comments
 (0)