Skip to content

Commit fe15d3d

Browse files
AnastaZIukAnastaZIuk
committed
fix builtin resource targets creation for GENERATED mix case - move archive source & header creation to build step instead of configure time which lead to bugs (obvsly GENERATED file doesn't exist at configure time for very first run on clean build but do when any rule/target generate it), remove their CMake templates & generate them with python scripts on fly with proper data
1 parent e9855a5 commit fe15d3d

File tree

5 files changed

+113
-117
lines changed

5 files changed

+113
-117
lines changed

‎src/nbl/builtin/builtinDataGen.py

Lines changed: 44 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,27 @@
11
# Creates a c++ file for builtin resources that contains binary data of all resources
22

3-
# parameters are
4-
# 0 - path to the .py file
5-
# 1 - output file path
6-
# 2 - cmake source dir
7-
# 3 - list of paths to resource files
3+
# TODO: use argparse not this by-hand-shit
84

95
import sys, os, subprocess, json
106
from datetime import datetime, timezone
117

12-
if len(sys.argv) < 4 :
8+
if len(sys.argv) < 8 :
139
print(sys.argv[0] + " - Incorrect argument count")
1410
else:
15-
outputFilename = sys.argv[1]
16-
cmakeSourceDir = sys.argv[2]
17-
resourcesFile = sys.argv[3]
18-
resourcesNamespace = sys.argv[4]
19-
correspondingHeaderFile = sys.argv[5]
20-
xxHash256Exe = sys.argv[6]
11+
outputBuiltinPath = sys.argv[1]
12+
outputArchivePath = sys.argv[2]
13+
bundleAbsoluteEntryPath = sys.argv[3]
14+
resourcesFile = sys.argv[4]
15+
resourcesNamespace = sys.argv[5]
16+
correspondingHeaderFile = sys.argv[6]
17+
xxHash256Exe = sys.argv[7]
2118

2219
forceConstexprHash = True if not xxHash256Exe else False
2320

2421
file = open(resourcesFile, 'r')
2522
resourcePaths = file.readlines()
2623

27-
outp = open(outputFilename, "w+")
24+
outp = open(outputBuiltinPath, "w+")
2825

2926
outp.write(f"""
3027
#include "{correspondingHeaderFile}"
@@ -38,12 +35,14 @@
3835
template<nbl::core::StringLiteral Path>
3936
const nbl::system::SBuiltinFile& get_resource();
4037
""")
41-
42-
# writing binary data of all files
43-
for z in resourcePaths:
38+
39+
resourcesInitList = ""
40+
41+
# writing binary data of all files + archive source in-place
42+
for id, z in enumerate(resourcePaths):
4443
itemData = z.split(',')
4544
x = itemData[0].rstrip()
46-
inputBuiltinResource = cmakeSourceDir+'/'+x
45+
inputBuiltinResource = bundleAbsoluteEntryPath+'/'+x
4746

4847
outp.write(f"""
4948
template<> const nbl::system::SBuiltinFile& get_resource<NBL_CORE_UNIQUE_STRING_LITERAL_TYPE("{x}")>()
@@ -70,6 +69,12 @@
7069
outp.write(f"""
7170
Error: BuiltinResources - file with the following path not found: {x}
7271
""")
72+
raise(IOError) # must throw back and fail the script
73+
74+
fileSize = os.path.getsize(inputBuiltinResource)
75+
76+
for item in itemData:
77+
resourcesInitList += f"\t\t\t{{\"{item.rstrip()}\", {fileSize}, 0xdeadbeefu, {id}, nbl::system::IFileArchive::E_ALLOCATOR_TYPE::EAT_NULL}},\n"
7378

7479
modificationDateT = datetime.fromtimestamp(os.path.getmtime(inputBuiltinResource), timezone.utc) # since the Unix epoch (00:00:00 UTC on 1 January 1970).
7580

@@ -170,4 +175,26 @@
170175
}}
171176
""")
172177

178+
outp.close()
179+
180+
archiveSource = f"""
181+
#include "CArchive.h"
182+
183+
using namespace {resourcesNamespace};
184+
185+
static const std::shared_ptr<nbl::core::vector<nbl::system::IFileArchive::SFileList::SEntry>> k_builtinArchiveFileList = std::make_shared<nbl::core::vector<nbl::system::IFileArchive::SFileList::SEntry>>(
186+
nbl::core::vector<nbl::system::IFileArchive::SFileList::SEntry>{{
187+
{resourcesInitList}
188+
}});
189+
190+
CArchive::CArchive(nbl::system::logger_opt_smart_ptr&& logger)
191+
: nbl::system::CFileArchive(nbl::system::path(pathPrefix.data()),std::move(logger), k_builtinArchiveFileList)
192+
{{
193+
bool ok = k_builtinArchiveFileList->size() > 0;
194+
assert(ok); // build system bug, it seems you have empty archive entries!
195+
}}
196+
"""
197+
198+
outp = open(outputArchivePath, "w+")
199+
outp.write(archiveSource)
173200
outp.close()

‎src/nbl/builtin/builtinHeaderGen.py

Lines changed: 51 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,26 @@
11
# Creates a header file for builtin resources
22

3-
# parameters are
4-
# 0 - path to the .py file
5-
# 1 - output file path
6-
# 2 - cmake source dir
7-
# 3 - list of paths to resource files
3+
# TODO: use argparse not this by-hand-shit
84

95
import sys, os
106

11-
if len(sys.argv) < 4 :
7+
if len(sys.argv) < 8 :
128
print(sys.argv[0] + " - Incorrect argument count")
139
else:
14-
outputFilename = sys.argv[1]
15-
cmakeSourceDir = sys.argv[2]
16-
resourcesFile = sys.argv[3]
17-
resourcesNamespace = sys.argv[4]
18-
guardSuffix = sys.argv[5]
19-
isSharedLibrary = True if sys.argv[6] == "True" else False
10+
outputBuiltinPath = sys.argv[1]
11+
outputArchivePath = sys.argv[2]
12+
archiveBundlePath = sys.argv[3]
13+
resourcesFile = sys.argv[4]
14+
resourcesNamespace = sys.argv[5]
15+
guardSuffix = sys.argv[6]
16+
isSharedLibrary = True if sys.argv[7] == "True" else False
2017

2118
NBL_BR_API = "NBL_BR_API" if isSharedLibrary else ""
2219

2320
file = open(resourcesFile, 'r')
2421
resourcePaths = file.readlines()
2522

26-
outp = open(outputFilename, "w+")
23+
outp = open(outputBuiltinPath, "w+")
2724

2825
outp.write(f"""
2926
#ifndef _{guardSuffix}_BUILTINRESOURCEDATA_H_
@@ -96,3 +93,44 @@
9693
""")
9794

9895
outp.close()
96+
97+
archiveHeader = f"""
98+
#ifndef _{guardSuffix}_C_ARCHIVE_H_
99+
#define _{guardSuffix}_C_ARCHIVE_H_
100+
101+
#include "nbl/system/CFileArchive.h"
102+
#include "nbl/core/def/smart_refctd_ptr.h"
103+
#include "{os.path.basename(outputBuiltinPath)}"
104+
#include <memory>
105+
106+
namespace {resourcesNamespace}
107+
{{
108+
constexpr std::string_view pathPrefix = "{archiveBundlePath}";
109+
110+
inline bool hasPathPrefix(nbl::system::path _path)
111+
{{
112+
_path.make_preferred();
113+
const auto prefix = nbl::system::path(pathPrefix).make_preferred();
114+
return _path.string().find(prefix.string())==0ull;
115+
}}
116+
117+
class {NBL_BR_API} CArchive final : public nbl::system::CFileArchive
118+
{{
119+
public:
120+
CArchive(nbl::system::logger_opt_smart_ptr&& logger);
121+
122+
protected:
123+
file_buffer_t getFileBuffer(const nbl::system::IFileArchive::SFileList::found_t& found) override
124+
{{
125+
auto resource = get_resource_runtime(found->pathRelativeToArchive.string());
126+
return {{const_cast<uint8_t*>(resource.contents),resource.size,nullptr}};
127+
}}
128+
}};
129+
}}
130+
131+
#endif // _{guardSuffix}_C_ARCHIVE_H_
132+
"""
133+
134+
outp = open(outputArchivePath, "w+")
135+
outp.write(archiveHeader)
136+
outp.close()

‎src/nbl/builtin/template/CArchive.cpp.in

Lines changed: 0 additions & 16 deletions
This file was deleted.

‎src/nbl/builtin/template/CArchive.h.in

Lines changed: 0 additions & 35 deletions
This file was deleted.

‎src/nbl/builtin/utils.cmake

Lines changed: 18 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Assigns builtin resources to a bundle a target library will be created with
22
# _BUNDLE_NAME_ is a bundle name, must be a valid CMake list variable
33
# _LBR_PATH_ is a path to builtin resource
4+
# optional aliases may be preset after the _LBR_PATH_
45

56
macro(LIST_BUILTIN_RESOURCE _BUNDLE_NAME_ _LBR_PATH_)
67
math(EXPR _ALIAS_C_ "${ARGC} - 2" OUTPUT_FORMAT DECIMAL)
@@ -69,7 +70,6 @@ function(ADD_CUSTOM_BUILTIN_RESOURCES _TARGET_NAME_ _BUNDLE_NAME_ _BUNDLE_SEARCH
6970
endif()
7071

7172
set(NBL_TEMPLATE_RESOURCES_ARCHIVE_HEADER "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/template/CArchive.h.in")
72-
set(NBL_TEMPLATE_RESOURCES_ARCHIVE_SOURCE "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/template/CArchive.cpp.in")
7373
set(NBL_BUILTIN_HEADER_GEN_PY "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/builtinHeaderGen.py")
7474
set(NBL_BUILTIN_DATA_GEN_PY "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/builtinDataGen.py")
7575
set(NBL_BS_HEADER_FILENAME "builtinResources.h")
@@ -90,36 +90,19 @@ function(ADD_CUSTOM_BUILTIN_RESOURCES _TARGET_NAME_ _BUNDLE_NAME_ _BUNDLE_SEARCH
9090
set(_ITR_ 0)
9191
foreach(X IN LISTS _LBR_${_BUNDLE_NAME_}_) # iterate over builtin resources bundle list given bundle name
9292
set(_CURRENT_ITEM_ "${X}")
93-
string(FIND "${_CURRENT_ITEM_}" "," _FOUND_ REVERSE)
9493

9594
string(REPLACE "," ";" _ITEM_DATA_ "${_CURRENT_ITEM_}")
9695
list(LENGTH _ITEM_DATA_ _ITEM_D_SIZE_)
9796
list(GET _ITEM_DATA_ 0 _CURRENT_PATH_) # _LBR_PATH_ path for given bundle
9897

99-
if(_ITEM_D_SIZE_ GREATER 1)
100-
list(SUBLIST _ITEM_DATA_ "1" "${_ITEM_D_SIZE_}" _ITEM_ALIASES_) # optional aliases for given builtin resource
101-
else()
102-
unset(_ITEM_ALIASES_)
103-
endif()
104-
10598
set(_BUNDLE_ARCHIVE_ABSOLUTE_PATH_ "${_BUNDLE_ARCHIVE_ABSOLUTE_PATH_}")
10699
set(NBL_BUILTIN_RESOURCE_ABS_PATH "${_BUNDLE_SEARCH_DIRECTORY_}/${_BUNDLE_ARCHIVE_ABSOLUTE_PATH_}/${_CURRENT_PATH_}") # an absolute path to a resource a builtin resource will be created as
100+
107101
list(APPEND NBL_BUILTIN_RESOURCES "${NBL_BUILTIN_RESOURCE_ABS_PATH}")
108102
list(APPEND NBL_DEPENDENCY_FILES "${NBL_BUILTIN_RESOURCE_ABS_PATH}")
109103

110-
if(EXISTS "${NBL_BUILTIN_RESOURCE_ABS_PATH}")
111-
file(SIZE "${NBL_BUILTIN_RESOURCE_ABS_PATH}" _FILE_SIZE_) # determine size of builtin resource in bytes
112-
113-
macro(LIST_RESOURCE_FOR_ARCHIVER _LBR_PATH_ _LBR_FILE_SIZE_ _LBR_ID_)
114-
string(APPEND _RESOURCES_INIT_LIST_ "\t\t\t\t\t{\"${_LBR_PATH_}\", ${_LBR_FILE_SIZE_}, 0xdeadbeefu, ${_LBR_ID_}, nbl::system::IFileArchive::E_ALLOCATOR_TYPE::EAT_NULL},\n") # initializer list
115-
endmacro()
116-
117-
LIST_RESOURCE_FOR_ARCHIVER("${_CURRENT_PATH_}" "${_FILE_SIZE_}" "${_ITR_}") # pass builtin resource path to an archive without _BUNDLE_ARCHIVE_ABSOLUTE_PATH_
118-
119-
foreach(_CURRENT_ALIAS_ IN LISTS _ITEM_ALIASES_)
120-
LIST_RESOURCE_FOR_ARCHIVER("${_CURRENT_ALIAS_}" "${_FILE_SIZE_}" "${_ITR_}")
121-
endforeach()
122-
else()
104+
# validate
105+
if(NOT EXISTS "${NBL_BUILTIN_RESOURCE_ABS_PATH}")
123106
get_source_file_property(NBL_BUILTIN_IS_GENERATED "${NBL_BUILTIN_RESOURCE_ABS_PATH}" GENERATED)
124107

125108
if(NOT NBL_BUILTIN_IS_GENERATED)
@@ -130,8 +113,8 @@ function(ADD_CUSTOM_BUILTIN_RESOURCES _TARGET_NAME_ _BUNDLE_NAME_ _BUNDLE_SEARCH
130113
math(EXPR _ITR_ "${_ITR_} + 1")
131114
endforeach()
132115

133-
configure_file("${NBL_TEMPLATE_RESOURCES_ARCHIVE_HEADER}" "${_OUTPUT_HEADER_DIRECTORY_}/CArchive.h")
134-
configure_file("${NBL_TEMPLATE_RESOURCES_ARCHIVE_SOURCE}" "${_OUTPUT_SOURCE_DIRECTORY_}/CArchive.cpp")
116+
set(NBL_BUILTIN_DATA_ARCHIVE_H "${_OUTPUT_HEADER_DIRECTORY_}/CArchive.h")
117+
set(NBL_BUILTIN_DATA_ARCHIVE_CPP "${_OUTPUT_SOURCE_DIRECTORY_}/CArchive.cpp")
135118

136119
list(APPEND NBL_DEPENDENCY_FILES "${NBL_BUILTIN_HEADER_GEN_PY}")
137120
list(APPEND NBL_DEPENDENCY_FILES "${NBL_BUILTIN_DATA_GEN_PY}")
@@ -141,29 +124,28 @@ function(ADD_CUSTOM_BUILTIN_RESOURCES _TARGET_NAME_ _BUNDLE_NAME_ _BUNDLE_SEARCH
141124
string(REPLACE ";" "\n" RESOURCES_ARGS "${_LBR_${_BUNDLE_NAME_}_}")
142125
file(WRITE "${NBL_RESOURCES_LIST_FILE}" "${RESOURCES_ARGS}")
143126

144-
set(NBL_BUILTIN_RESOURCES_HEADER "${_OUTPUT_HEADER_DIRECTORY_}/${NBL_BS_HEADER_FILENAME}")
145-
set(NBL_BUILTIN_RESOURCE_DATA_SOURCE "${_OUTPUT_SOURCE_DIRECTORY_}/${NBL_BS_DATA_SOURCE_FILENAME}")
127+
set(NBL_BUILTIN_RESOURCES_H "${_OUTPUT_HEADER_DIRECTORY_}/${NBL_BS_HEADER_FILENAME}")
128+
set(NBL_BUILTIN_RESOURCE_DATA_CPP "${_OUTPUT_SOURCE_DIRECTORY_}/${NBL_BS_DATA_SOURCE_FILENAME}")
146129

147130
if(NBL_BR_FORCE_CONSTEXPR_HASH)
148131
set(_NBL_BR_RUNTIME_HASH_ 0)
149132
else()
150133
set(_NBL_BR_RUNTIME_HASH_ 1)
151134
endif()
152135

153-
add_custom_command(
154-
OUTPUT "${NBL_BUILTIN_RESOURCES_HEADER}" "${NBL_BUILTIN_RESOURCE_DATA_SOURCE}"
155-
COMMAND "${_Python3_EXECUTABLE}" "${NBL_BUILTIN_HEADER_GEN_PY}" "${NBL_BUILTIN_RESOURCES_HEADER}" "${_BUNDLE_SEARCH_DIRECTORY_}/${_BUNDLE_ARCHIVE_ABSOLUTE_PATH_}" "${NBL_RESOURCES_LIST_FILE}" "${_NAMESPACE_}" "${_GUARD_SUFFIX_}" "${_SHARED_}"
156-
COMMAND "${_Python3_EXECUTABLE}" "${NBL_BUILTIN_DATA_GEN_PY}" "${NBL_BUILTIN_RESOURCE_DATA_SOURCE}" "${_BUNDLE_SEARCH_DIRECTORY_}/${_BUNDLE_ARCHIVE_ABSOLUTE_PATH_}" "${NBL_RESOURCES_LIST_FILE}" "${_NAMESPACE_}" "${NBL_BS_HEADER_FILENAME}" "$<${_NBL_BR_RUNTIME_HASH_}:$<TARGET_FILE:xxHash256>>"
157-
COMMENT "Generating builtin resources for ${_TARGET_NAME_} target"
136+
add_custom_command(OUTPUT "${NBL_BUILTIN_RESOURCES_H}" "${NBL_BUILTIN_RESOURCE_DATA_CPP}" "${NBL_BUILTIN_DATA_ARCHIVE_H}" "${NBL_BUILTIN_DATA_ARCHIVE_CPP}"
137+
COMMAND "${_Python3_EXECUTABLE}" "${NBL_BUILTIN_HEADER_GEN_PY}" "${NBL_BUILTIN_RESOURCES_H}" "${NBL_BUILTIN_DATA_ARCHIVE_H}" "${_BUNDLE_ARCHIVE_ABSOLUTE_PATH_}" "${NBL_RESOURCES_LIST_FILE}" "${_NAMESPACE_}" "${_GUARD_SUFFIX_}" "${_SHARED_}"
138+
COMMAND "${_Python3_EXECUTABLE}" "${NBL_BUILTIN_DATA_GEN_PY}" "${NBL_BUILTIN_RESOURCE_DATA_CPP}" "${NBL_BUILTIN_DATA_ARCHIVE_CPP}" "${_BUNDLE_SEARCH_DIRECTORY_}/${_BUNDLE_ARCHIVE_ABSOLUTE_PATH_}" "${NBL_RESOURCES_LIST_FILE}" "${_NAMESPACE_}" "${NBL_BS_HEADER_FILENAME}" "$<${_NBL_BR_RUNTIME_HASH_}:$<TARGET_FILE:xxHash256>>"
139+
COMMENT "Generating \"${_TARGET_NAME_}\"'s sources & headers"
158140
DEPENDS ${NBL_DEPENDENCY_FILES}
159141
VERBATIM
160142
)
161143

162144
add_library(${_TARGET_NAME_} ${_LIB_TYPE_}
163-
"${NBL_BUILTIN_RESOURCES_HEADER}"
164-
"${NBL_BUILTIN_RESOURCE_DATA_SOURCE}"
165-
"${_OUTPUT_SOURCE_DIRECTORY_}/CArchive.cpp"
166-
"${_OUTPUT_HEADER_DIRECTORY_}/CArchive.h"
145+
"${NBL_BUILTIN_RESOURCES_H}"
146+
"${NBL_BUILTIN_RESOURCE_DATA_CPP}"
147+
"${NBL_BUILTIN_DATA_ARCHIVE_H}"
148+
"${NBL_BUILTIN_DATA_ARCHIVE_CPP}"
167149
)
168150

169151
if(NBL_FORCE_RELEASE_3RDPARTY) # priority over RWDI
@@ -229,8 +211,8 @@ function(ADD_CUSTOM_BUILTIN_RESOURCES _TARGET_NAME_ _BUNDLE_NAME_ _BUNDLE_SEARCH
229211
set(NBL_BUILTIN_RESOURCES ${NBL_BUILTIN_RESOURCES}) # turn builtin resources paths list into variable
230212

231213
set(NBL_BUILTIN_RESOURCES_HEADERS
232-
"${NBL_BUILTIN_RESOURCES_HEADER}"
233-
"${_OUTPUT_HEADER_DIRECTORY_}/CArchive.h"
214+
"${NBL_BUILTIN_RESOURCES_H}"
215+
"${NBL_BUILTIN_DATA_ARCHIVE_H}"
234216
)
235217

236218
macro(_ADD_PROPERTY_ _BR_PROPERTY_ _BR_PROXY_VAR_)

0 commit comments

Comments
 (0)