Skip to content

Commit 9c5f0d5

Browse files
laanwjlaanwj
committed
Merge #13878: utils: Add fstream wrapper to allow to pass unicode filename on Windows
43c7fbb Make MSVC compiler read the source code using utf-8 (Chun Kuan Lee) f86a571 tests: Add test case for std::ios_base::ate (Chun Kuan Lee) a554cc9 Move boost/std fstream to fsbridge (Chun Kuan Lee) 86eb3b3 utils: Add fsbridge fstream function wrapper (Chun Kuan Lee) Pull request description: If compiled with mingw, use glibc++ extension `stdio_filebuf` to open the file by `FILE*` instead of filename. In other condition, we can use boost::fstream. Tree-SHA512: b5dbd83e347fb9b2a0c8b1c2c7bd71a272e839ec0617883b2a0ec12506ae9e825373cf6e95b9bcc91d7edc85bf51580a7716b56a9ecaad776bc3ae61638cb3da
2 parents 041224a + 43c7fbb commit 9c5f0d5

File tree

9 files changed

+228
-16
lines changed

9 files changed

+228
-16
lines changed

‎build_msvc/common.vcxproj

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,9 @@
1212
Outputs="$(MSBuildThisFileDirectory)..\src\config\bitcoin-config.h">
1313
<Copy SourceFiles="$(MSBuildThisFileDirectory)bitcoin_config.h" DestinationFiles="$(MSBuildThisFileDirectory)..\src\config\bitcoin-config.h" />
1414
</Target>
15+
<ItemDefinitionGroup>
16+
<ClCompile>
17+
<AdditionalOptions>/utf-8 %(AdditionalOptions)</AdditionalOptions>
18+
</ClCompile>
19+
</ItemDefinitionGroup>
1520
</Project>

‎src/Makefile.test.include

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ BITCOIN_TESTS =\
5151
test/cuckoocache_tests.cpp \
5252
test/denialofservice_tests.cpp \
5353
test/descriptor_tests.cpp \
54+
test/fs_tests.cpp \
5455
test/getarg_tests.cpp \
5556
test/hash_tests.cpp \
5657
test/key_io_tests.cpp \

‎src/fs.cpp

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,4 +113,106 @@ std::string get_filesystem_error_message(const fs::filesystem_error& e)
113113
#endif
114114
}
115115

116+
#ifdef WIN32
117+
#ifdef __GLIBCXX__
118+
119+
// reference: https://github.com/gcc-mirror/gcc/blob/gcc-7_3_0-release/libstdc%2B%2B-v3/include/std/fstream#L270
120+
121+
static std::string openmodeToStr(std::ios_base::openmode mode)
122+
{
123+
switch (mode & ~std::ios_base::ate) {
124+
case std::ios_base::out:
125+
case std::ios_base::out | std::ios_base::trunc:
126+
return "w";
127+
case std::ios_base::out | std::ios_base::app:
128+
case std::ios_base::app:
129+
return "a";
130+
case std::ios_base::in:
131+
return "r";
132+
case std::ios_base::in | std::ios_base::out:
133+
return "r+";
134+
case std::ios_base::in | std::ios_base::out | std::ios_base::trunc:
135+
return "w+";
136+
case std::ios_base::in | std::ios_base::out | std::ios_base::app:
137+
case std::ios_base::in | std::ios_base::app:
138+
return "a+";
139+
case std::ios_base::out | std::ios_base::binary:
140+
case std::ios_base::out | std::ios_base::trunc | std::ios_base::binary:
141+
return "wb";
142+
case std::ios_base::out | std::ios_base::app | std::ios_base::binary:
143+
case std::ios_base::app | std::ios_base::binary:
144+
return "ab";
145+
case std::ios_base::in | std::ios_base::binary:
146+
return "rb";
147+
case std::ios_base::in | std::ios_base::out | std::ios_base::binary:
148+
return "r+b";
149+
case std::ios_base::in | std::ios_base::out | std::ios_base::trunc | std::ios_base::binary:
150+
return "w+b";
151+
case std::ios_base::in | std::ios_base::out | std::ios_base::app | std::ios_base::binary:
152+
case std::ios_base::in | std::ios_base::app | std::ios_base::binary:
153+
return "a+b";
154+
default:
155+
return std::string();
156+
}
157+
}
158+
159+
void ifstream::open(const fs::path& p, std::ios_base::openmode mode)
160+
{
161+
close();
162+
m_file = fsbridge::fopen(p, openmodeToStr(mode).c_str());
163+
if (m_file == nullptr) {
164+
return;
165+
}
166+
m_filebuf = __gnu_cxx::stdio_filebuf<char>(m_file, mode);
167+
rdbuf(&m_filebuf);
168+
if (mode & std::ios_base::ate) {
169+
seekg(0, std::ios_base::end);
170+
}
171+
}
172+
173+
void ifstream::close()
174+
{
175+
if (m_file != nullptr) {
176+
m_filebuf.close();
177+
fclose(m_file);
178+
}
179+
m_file = nullptr;
180+
}
181+
182+
void ofstream::open(const fs::path& p, std::ios_base::openmode mode)
183+
{
184+
close();
185+
m_file = fsbridge::fopen(p, openmodeToStr(mode).c_str());
186+
if (m_file == nullptr) {
187+
return;
188+
}
189+
m_filebuf = __gnu_cxx::stdio_filebuf<char>(m_file, mode);
190+
rdbuf(&m_filebuf);
191+
if (mode & std::ios_base::ate) {
192+
seekp(0, std::ios_base::end);
193+
}
194+
}
195+
196+
void ofstream::close()
197+
{
198+
if (m_file != nullptr) {
199+
m_filebuf.close();
200+
fclose(m_file);
201+
}
202+
m_file = nullptr;
203+
}
204+
#else // __GLIBCXX__
205+
206+
static_assert(sizeof(*fs::path().BOOST_FILESYSTEM_C_STR) == sizeof(wchar_t),
207+
"Warning: This build is using boost::filesystem ofstream and ifstream "
208+
"implementations which will fail to open paths containing multibyte "
209+
"characters. You should delete this static_assert to ignore this warning, "
210+
"or switch to a different C++ standard library like the Microsoft C++ "
211+
"Standard Library (where boost uses non-standard extensions to construct "
212+
"stream objects with wide filenames), or the GNU libstdc++ library (where "
213+
"a more complicated workaround has been implemented above).");
214+
215+
#endif // __GLIBCXX__
216+
#endif // WIN32
217+
116218
} // fsbridge

‎src/fs.h

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77

88
#include <stdio.h>
99
#include <string>
10+
#if defined WIN32 && defined __GLIBCXX__
11+
#include <ext/stdio_filebuf.h>
12+
#endif
1013

1114
#include <boost/filesystem.hpp>
1215
#include <boost/filesystem/fstream.hpp>
@@ -39,6 +42,54 @@ namespace fsbridge {
3942
};
4043

4144
std::string get_filesystem_error_message(const fs::filesystem_error& e);
45+
46+
// GNU libstdc++ specific workaround for opening UTF-8 paths on Windows.
47+
//
48+
// On Windows, it is only possible to reliably access multibyte file paths through
49+
// `wchar_t` APIs, not `char` APIs. But because the C++ standard doesn't
50+
// require ifstream/ofstream `wchar_t` constructors, and the GNU library doesn't
51+
// provide them (in contrast to the Microsoft C++ library, see
52+
// https://stackoverflow.com/questions/821873/how-to-open-an-stdfstream-ofstream-or-ifstream-with-a-unicode-filename/822032#822032),
53+
// Boost is forced to fall back to `char` constructors which may not work properly.
54+
//
55+
// Work around this issue by creating stream objects with `_wfopen` in
56+
// combination with `__gnu_cxx::stdio_filebuf`. This workaround can be removed
57+
// with an upgrade to C++17, where streams can be constructed directly from
58+
// `std::filesystem::path` objects.
59+
60+
#if defined WIN32 && defined __GLIBCXX__
61+
class ifstream : public std::istream
62+
{
63+
public:
64+
ifstream() = default;
65+
explicit ifstream(const fs::path& p, std::ios_base::openmode mode = std::ios_base::in) { open(p, mode); }
66+
~ifstream() { close(); }
67+
void open(const fs::path& p, std::ios_base::openmode mode = std::ios_base::in);
68+
bool is_open() { return m_filebuf.is_open(); }
69+
void close();
70+
71+
private:
72+
__gnu_cxx::stdio_filebuf<char> m_filebuf;
73+
FILE* m_file = nullptr;
74+
};
75+
class ofstream : public std::ostream
76+
{
77+
public:
78+
ofstream() = default;
79+
explicit ofstream(const fs::path& p, std::ios_base::openmode mode = std::ios_base::out) { open(p, mode); }
80+
~ofstream() { close(); }
81+
void open(const fs::path& p, std::ios_base::openmode mode = std::ios_base::out);
82+
bool is_open() { return m_filebuf.is_open(); }
83+
void close();
84+
85+
private:
86+
__gnu_cxx::stdio_filebuf<char> m_filebuf;
87+
FILE* m_file = nullptr;
88+
};
89+
#else // !(WIN32 && __GLIBCXX__)
90+
typedef fs::ifstream ifstream;
91+
typedef fs::ofstream ofstream;
92+
#endif // WIN32 && __GLIBCXX__
4293
};
4394

4495
#endif // BITCOIN_FS_H

‎src/qt/guiutil.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -367,7 +367,7 @@ bool openBitcoinConf()
367367
fs::path pathConfig = GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME));
368368

369369
/* Create the file */
370-
fs::ofstream configFile(pathConfig, std::ios_base::app);
370+
fsbridge::ofstream configFile(pathConfig, std::ios_base::app);
371371

372372
if (!configFile.good())
373373
return false;
@@ -611,7 +611,7 @@ fs::path static GetAutostartFilePath()
611611

612612
bool GetStartOnSystemStartup()
613613
{
614-
fs::ifstream optionFile(GetAutostartFilePath());
614+
fsbridge::ifstream optionFile(GetAutostartFilePath());
615615
if (!optionFile.good())
616616
return false;
617617
// Scan through file for "Hidden=true":
@@ -642,7 +642,7 @@ bool SetStartOnSystemStartup(bool fAutoStart)
642642

643643
fs::create_directories(GetAutostartDir());
644644

645-
fs::ofstream optionFile(GetAutostartFilePath(), std::ios_base::out|std::ios_base::trunc);
645+
fsbridge::ofstream optionFile(GetAutostartFilePath(), std::ios_base::out | std::ios_base::trunc);
646646
if (!optionFile.good())
647647
return false;
648648
std::string chain = gArgs.GetChainName();

‎src/rpc/protocol.cpp

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@
1212
#include <utiltime.h>
1313
#include <version.h>
1414

15-
#include <fstream>
16-
1715
/**
1816
* JSON-RPC protocol. Bitcoin speaks version 1.0 for maximum compatibility,
1917
* but uses JSON-RPC 1.1/2.0 standards for parts of the 1.0 standard that were
@@ -85,9 +83,9 @@ bool GenerateAuthCookie(std::string *cookie_out)
8583
/** the umask determines what permissions are used to create this file -
8684
* these are set to 077 in init.cpp unless overridden with -sysperms.
8785
*/
88-
std::ofstream file;
86+
fsbridge::ofstream file;
8987
fs::path filepath_tmp = GetAuthCookieFile(true);
90-
file.open(filepath_tmp.string().c_str());
88+
file.open(filepath_tmp);
9189
if (!file.is_open()) {
9290
LogPrintf("Unable to open cookie authentication file %s for writing\n", filepath_tmp.string());
9391
return false;
@@ -109,10 +107,10 @@ bool GenerateAuthCookie(std::string *cookie_out)
109107

110108
bool GetAuthCookie(std::string *cookie_out)
111109
{
112-
std::ifstream file;
110+
fsbridge::ifstream file;
113111
std::string cookie;
114112
fs::path filepath = GetAuthCookieFile();
115-
file.open(filepath.string().c_str());
113+
file.open(filepath);
116114
if (!file.is_open())
117115
return false;
118116
std::getline(file, cookie);

‎src/test/fs_tests.cpp

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// Copyright (c) 2011-2018 The Bitcoin Core developers
2+
// Distributed under the MIT software license, see the accompanying
3+
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
4+
//
5+
#include <fs.h>
6+
#include <test/test_bitcoin.h>
7+
8+
#include <boost/test/unit_test.hpp>
9+
10+
BOOST_FIXTURE_TEST_SUITE(fs_tests, BasicTestingSetup)
11+
12+
BOOST_AUTO_TEST_CASE(fsbridge_fstream)
13+
{
14+
fs::path tmpfolder = SetDataDir("fsbridge_fstream");
15+
// tmpfile1 should be the same as tmpfile2
16+
fs::path tmpfile1 = tmpfolder / "fs_tests_₿_🏃";
17+
fs::path tmpfile2 = tmpfolder / L"fs_tests_₿_🏃";
18+
{
19+
fsbridge::ofstream file(tmpfile1);
20+
file << "bitcoin";
21+
}
22+
{
23+
fsbridge::ifstream file(tmpfile2);
24+
std::string input_buffer;
25+
file >> input_buffer;
26+
BOOST_CHECK_EQUAL(input_buffer, "bitcoin");
27+
}
28+
{
29+
fsbridge::ifstream file(tmpfile1, std::ios_base::in | std::ios_base::ate);
30+
std::string input_buffer;
31+
file >> input_buffer;
32+
BOOST_CHECK_EQUAL(input_buffer, "");
33+
}
34+
{
35+
fsbridge::ofstream file(tmpfile2, std::ios_base::out | std::ios_base::app);
36+
file << "tests";
37+
}
38+
{
39+
fsbridge::ifstream file(tmpfile1);
40+
std::string input_buffer;
41+
file >> input_buffer;
42+
BOOST_CHECK_EQUAL(input_buffer, "bitcointests");
43+
}
44+
{
45+
fsbridge::ofstream file(tmpfile2, std::ios_base::out | std::ios_base::trunc);
46+
file << "bitcoin";
47+
}
48+
{
49+
fsbridge::ifstream file(tmpfile1);
50+
std::string input_buffer;
51+
file >> input_buffer;
52+
BOOST_CHECK_EQUAL(input_buffer, "bitcoin");
53+
}
54+
}
55+
56+
BOOST_AUTO_TEST_SUITE_END()

‎src/util.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -892,7 +892,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys)
892892
}
893893

894894
const std::string confPath = GetArg("-conf", BITCOIN_CONF_FILENAME);
895-
fs::ifstream stream(GetConfigFile(confPath));
895+
fsbridge::ifstream stream(GetConfigFile(confPath));
896896

897897
// ok to not have a config file
898898
if (stream.good()) {
@@ -925,7 +925,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys)
925925
}
926926

927927
for (const std::string& to_include : includeconf) {
928-
fs::ifstream include_config(GetConfigFile(to_include));
928+
fsbridge::ifstream include_config(GetConfigFile(to_include));
929929
if (include_config.good()) {
930930
if (!ReadConfigStream(include_config, error, ignore_invalid_keys)) {
931931
return false;

‎src/wallet/rpcdump.cpp

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717

1818
#include <wallet/rpcwallet.h>
1919

20-
#include <fstream>
2120
#include <stdint.h>
2221

2322
#include <boost/algorithm/string.hpp>
@@ -540,8 +539,8 @@ UniValue importwallet(const JSONRPCRequest& request)
540539

541540
EnsureWalletIsUnlocked(pwallet);
542541

543-
std::ifstream file;
544-
file.open(request.params[0].get_str().c_str(), std::ios::in | std::ios::ate);
542+
fsbridge::ifstream file;
543+
file.open(request.params[0].get_str(), std::ios::in | std::ios::ate);
545544
if (!file.is_open()) {
546545
throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open wallet dump file");
547546
}
@@ -717,8 +716,8 @@ UniValue dumpwallet(const JSONRPCRequest& request)
717716
throw JSONRPCError(RPC_INVALID_PARAMETER, filepath.string() + " already exists. If you are sure this is what you want, move it out of the way first");
718717
}
719718

720-
std::ofstream file;
721-
file.open(filepath.string().c_str());
719+
fsbridge::ofstream file;
720+
file.open(filepath);
722721
if (!file.is_open())
723722
throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open wallet dump file");
724723

0 commit comments

Comments
 (0)