Skip to content

Commit 6fb2879

Browse files
dzhwinterdzhwinter
authored
memory (#13143)
1 parent e722f68 commit 6fb2879

File tree

3 files changed

+126
-69
lines changed

3 files changed

+126
-69
lines changed

‎paddle/fluid/operators/save_combine_op.cc

Lines changed: 1 addition & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ See the License for the specific language governing permissions and
1313
limitations under the License. */
1414

1515
#include <stdint.h>
16-
#include <sys/stat.h>
1716
#include <fstream>
1817
#include <numeric>
1918
#include <sstream>
@@ -23,40 +22,11 @@ limitations under the License. */
2322
#include "paddle/fluid/framework/lod_tensor.h"
2423
#include "paddle/fluid/framework/op_registry.h"
2524
#include "paddle/fluid/platform/device_context.h"
25+
#include "paddle/fluid/platform/port.h"
2626

2727
namespace paddle {
2828
namespace operators {
2929

30-
// TODO(sidgoyal78): These function are needed by other files (save_op), move
31-
// them to paddle::filesystem namespace. (as noted by yuyang18 in save_op).
32-
constexpr char kSEP = '/';
33-
static bool FileExists(const std::string &filepath) {
34-
struct stat buffer;
35-
return (stat(filepath.c_str(), &buffer) == 0);
36-
}
37-
38-
static std::string DirName(const std::string &filepath) {
39-
auto pos = filepath.rfind(kSEP);
40-
if (pos == std::string::npos) {
41-
return "";
42-
}
43-
return filepath.substr(0, pos);
44-
}
45-
46-
static void MkDir(const char *path) {
47-
if (mkdir(path, 0755)) {
48-
PADDLE_ENFORCE_EQ(errno, EEXIST, "%s mkdir failed!", path);
49-
}
50-
}
51-
52-
static void MkDirRecursively(const char *fullpath) {
53-
if (*fullpath == '\0') return; // empty string
54-
if (FileExists(fullpath)) return;
55-
56-
MkDirRecursively(DirName(fullpath).c_str());
57-
MkDir(fullpath);
58-
}
59-
6030
class SaveCombineOp : public framework::OperatorBase {
6131
public:
6232
SaveCombineOp(const std::string &type,

‎paddle/fluid/operators/save_op.cc

Lines changed: 1 addition & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ See the License for the specific language governing permissions and
1313
limitations under the License. */
1414

1515
#include <stdint.h>
16-
#include <sys/stat.h>
1716
#include <fstream>
1817
#include <numeric>
1918

@@ -25,6 +24,7 @@ limitations under the License. */
2524
#include "paddle/fluid/framework/selected_rows.h"
2625
#include "paddle/fluid/framework/variable.h"
2726
#include "paddle/fluid/platform/device_context.h"
27+
#include "paddle/fluid/platform/port.h"
2828

2929
namespace paddle {
3030
namespace operators {
@@ -33,36 +33,6 @@ namespace operators {
3333
// to directory specified.
3434
constexpr char LOOKUP_TABLE_PATH[] = "kLookupTablePath";
3535

36-
// TODO(yuyang18): If the functions below are needed by other files, move them
37-
// to paddle::filesystem namespace.
38-
constexpr char kSEP = '/';
39-
static bool FileExists(const std::string &filepath) {
40-
struct stat buffer;
41-
return (stat(filepath.c_str(), &buffer) == 0);
42-
}
43-
44-
static std::string DirName(const std::string &filepath) {
45-
auto pos = filepath.rfind(kSEP);
46-
if (pos == std::string::npos) {
47-
return "";
48-
}
49-
return filepath.substr(0, pos);
50-
}
51-
52-
static void MkDir(const char *path) {
53-
if (mkdir(path, 0755)) {
54-
PADDLE_ENFORCE_EQ(errno, EEXIST, "%s mkdir failed!", path);
55-
}
56-
}
57-
58-
static void MkDirRecursively(const char *fullpath) {
59-
if (*fullpath == '\0') return; // empty string
60-
if (FileExists(fullpath)) return;
61-
62-
MkDirRecursively(DirName(fullpath).c_str());
63-
MkDir(fullpath);
64-
}
65-
6636
class SaveOp : public framework::OperatorBase {
6737
public:
6838
SaveOp(const std::string &type, const framework::VariableNameMap &inputs,

‎paddle/fluid/platform/port.h

Lines changed: 124 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,24 +14,141 @@
1414

1515
#pragma once
1616

17+
#include <cstdio>
1718
#include <stdexcept>
19+
20+
#include <memory>
1821
#include <string>
1922

23+
#define GLOG_NO_ABBREVIATED_SEVERITIES // msvc conflict logging with windows.h
24+
#include "glog/logging.h"
25+
2026
#if !defined(_WIN32)
21-
#include <dlfcn.h> // for dladdr
22-
#include <execinfo.h> // for backtrace
27+
#define UNUSED __attribute__((unused))
28+
#include <dlfcn.h> // dladdr
29+
#include <execinfo.h> // backtrace
30+
#include <sys/stat.h>
31+
#include <algorithm> // std::accumulate
2332
#else
24-
#include <Shlwapi.h>
25-
#include <Windows.h>
33+
#include <io.h> // _popen, _pclose
34+
#include <windows.h>
35+
#if defined(_WIN32)
36+
#include <numeric> // std::accumulate in msvc
37+
#endif
38+
// windows version of __attribute__((unused))
39+
#define UNUSED __pragma(warning(suppress : 4100))
2640

27-
static void* dlsym(void* handle, const char* symbol_name) {
41+
#ifndef S_ISDIR // windows port for sys/stat.h
42+
#define S_ISDIR(mode) (((mode)&S_IFMT) == S_IFDIR)
43+
#endif // S_ISDIR
44+
45+
static void *dlsym(void *handle, const char *symbol_name) {
2846
FARPROC found_symbol;
2947
found_symbol = GetProcAddress((HMODULE)handle, symbol_name);
3048

3149
if (found_symbol == NULL) {
3250
throw std::runtime_error(std::string(symbol_name) + " not found.");
3351
}
34-
return reinterpret_cast<void*>(found_symbol);
52+
return reinterpret_cast<void *>(found_symbol);
3553
}
3654

37-
#endif
55+
static void *dlopen(const char *filename, int flag) {
56+
std::string file_name(filename);
57+
file_name.replace(0, file_name.size() - 1, '/', '\\');
58+
HMODULE hModule = LoadLibrary(file_name.c_str());
59+
if (!hModule) {
60+
throw std::runtime_error(file_name + " not found.");
61+
}
62+
return reinterpret_cast<void *>(hModule);
63+
}
64+
65+
#endif // !_WIN32
66+
67+
static void ExecShellCommand(const std::string &cmd, std::string *message) {
68+
char buffer[128];
69+
#if !defined(_WIN32)
70+
std::shared_ptr<FILE> pipe(popen(cmd.c_str(), "r"), pclose);
71+
#else
72+
std::shared_ptr<FILE> pipe(_popen(cmd.c_str(), "r"), _pclose);
73+
#endif // _WIN32
74+
if (!pipe) {
75+
LOG(ERROR) << "error running command: " << cmd;
76+
return;
77+
}
78+
while (!feof(pipe.get())) {
79+
if (fgets(buffer, 128, pipe.get()) != nullptr) {
80+
*message += buffer;
81+
}
82+
}
83+
}
84+
85+
static bool PathExists(const std::string &path) {
86+
#if !defined(_WIN32)
87+
struct stat statbuf;
88+
if (stat(path.c_str(), &statbuf) != -1) {
89+
if (S_ISDIR(statbuf.st_mode)) {
90+
return true;
91+
}
92+
}
93+
#else
94+
struct _stat statbuf;
95+
if (_stat(path.c_str(), &statbuf) != -1) {
96+
if (S_ISDIR(statbuf.st_mode)) {
97+
return true;
98+
}
99+
}
100+
#endif // !_WIN32
101+
return false;
102+
}
103+
104+
// TODO(yuyang18): If the functions below are needed by other files, move them
105+
// to paddle::filesystem namespace.
106+
#if !defined(_WIN32)
107+
constexpr char kSEP = '/';
108+
#else
109+
constexpr char kSEP = '\\';
110+
#endif // _WIN32
111+
112+
static bool FileExists(const std::string &filepath) {
113+
#if !defined(_WIN32)
114+
struct stat buffer;
115+
return (stat(filepath.c_str(), &buffer) == 0);
116+
#else
117+
struct _stat buffer;
118+
return (_stat(filepath.c_str(), &buffer) == 0);
119+
#endif // !_WIN32
120+
}
121+
122+
static std::string DirName(const std::string &filepath) {
123+
auto pos = filepath.rfind(kSEP);
124+
if (pos == std::string::npos) {
125+
return "";
126+
}
127+
return filepath.substr(0, pos);
128+
}
129+
130+
static void MkDir(const char *path) {
131+
std::string path_error(path);
132+
path_error += " mkdir failed!";
133+
#if !defined(_WIN32)
134+
if (mkdir(path, 0755)) {
135+
if (errno != EEXIST) {
136+
throw std::runtime_error(path_error);
137+
}
138+
}
139+
#else
140+
CreateDirectory(path, NULL);
141+
auto errorno = GetLastError();
142+
if (errorno != ERROR_ALREADY_EXISTS) {
143+
throw std::runtime_error(path_error);
144+
}
145+
#endif // !_WIN32
146+
}
147+
148+
static void MkDirRecursively(const char *fullpath) {
149+
if (*fullpath == '\0') return; // empty string
150+
if (FileExists(fullpath)) return;
151+
152+
MkDirRecursively(DirName(fullpath).c_str());
153+
MkDir(fullpath);
154+
}

0 commit comments

Comments
 (0)