Skip to content

Commit 2ff34c0

Browse files
author
lind
committed
add file descriptor support to file_data_loader
Summary: Test Plan: Reviewers: Subscribers: Tasks: Tags:
1 parent 97a4600 commit 2ff34c0

File tree

5 files changed

+204
-1
lines changed

5 files changed

+204
-1
lines changed

‎examples/portable/executor_runner/executor_runner.cpp‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@
2222

2323
#include <gflags/gflags.h>
2424

25+
#include <fcntl.h>
26+
#include <unistd.h>
27+
2528
#include <executorch/extension/data_loader/file_data_loader.h>
2629
#include <executorch/extension/evalue_util/print_evalue.h>
2730
#include <executorch/extension/runner_util/inputs.h>
@@ -36,6 +39,10 @@ DEFINE_string(
3639
model_path,
3740
"model.pte",
3841
"Model serialized in flatbuffer format.");
42+
DEFINE_bool(
43+
is_fd_uri,
44+
false,
45+
"True if the model_path passed is a file descriptor with the prefix \"fd:///\".");
3946

4047
using executorch::extension::FileDataLoader;
4148
using executorch::runtime::Error;
@@ -66,7 +73,12 @@ int main(int argc, char** argv) {
6673
// DataLoaders that use mmap() or point to data that's already in memory, and
6774
// users can create their own DataLoaders to load from arbitrary sources.
6875
const char* model_path = FLAGS_model_path.c_str();
69-
Result<FileDataLoader> loader = FileDataLoader::from(model_path);
76+
const bool is_fd_uri = FLAGS_is_fd_uri;
77+
78+
Result<FileDataLoader> loader = is_fd_uri
79+
? FileDataLoader::fromFileDescriptorUri(model_path)
80+
: FileDataLoader::from(model_path);
81+
7082
ET_CHECK_MSG(
7183
loader.ok(),
7284
"FileDataLoader::from() failed: 0x%" PRIx32,

‎examples/portable/executor_runner/open_file_by_fd.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
FILENAME="$1"
3+
exec {FD}<${FILENAME} # open file for read, assign descriptor
4+
echo "Opened ${FILENAME} for read using descriptor ${FD}"
5+
out/host/linux-x86/bin/executor_runner --model_path fd:///${FD} --is_fd_uri=true
6+
exec {FD}<&- # close file

‎extension/data_loader/file_data_loader.cpp‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ namespace extension {
4343

4444
namespace {
4545

46+
static constexpr char kFdFilesystemPrefix[] = "fd:///";
47+
4648
/**
4749
* Returns true if the value is an integer power of 2.
4850
*/
@@ -74,6 +76,74 @@ FileDataLoader::~FileDataLoader() {
7476
::close(fd_);
7577
}
7678

79+
Result<int> getFDFromUri(const char* file_descriptor_uri) {
80+
// check if the uri starts with the prefix "fd://"
81+
ET_CHECK_OR_RETURN_ERROR(
82+
strncmp(
83+
file_descriptor_uri,
84+
kFdFilesystemPrefix,
85+
strlen(kFdFilesystemPrefix)) == 0,
86+
InvalidArgument,
87+
"File descriptor uri (%s) does not start with %s",
88+
file_descriptor_uri,
89+
kFdFilesystemPrefix);
90+
91+
// strip "fd:///" from the uri
92+
int fd_len = strlen(file_descriptor_uri) - strlen(kFdFilesystemPrefix);
93+
char fd_without_prefix[fd_len + 1];
94+
memcpy(
95+
fd_without_prefix,
96+
&file_descriptor_uri[strlen(kFdFilesystemPrefix)],
97+
fd_len);
98+
fd_without_prefix[fd_len] = '\0';
99+
100+
// check if remaining fd string is a valid integer
101+
int fd = ::atoi(fd_without_prefix);
102+
return fd;
103+
}
104+
105+
Result<FileDataLoader> FileDataLoader::fromFileDescriptorUri(
106+
const char* file_descriptor_uri,
107+
size_t alignment) {
108+
ET_CHECK_OR_RETURN_ERROR(
109+
is_power_of_2(alignment),
110+
InvalidArgument,
111+
"Alignment %zu is not a power of 2",
112+
alignment);
113+
114+
auto parsed_fd = getFDFromUri(file_descriptor_uri);
115+
if (!parsed_fd.ok()) {
116+
return parsed_fd.error();
117+
}
118+
119+
int fd = parsed_fd.get();
120+
121+
// Cache the file size.
122+
struct stat st;
123+
int err = ::fstat(fd, &st);
124+
if (err < 0) {
125+
ET_LOG(
126+
Error,
127+
"Could not get length of %s: %s (%d)",
128+
file_descriptor_uri,
129+
::strerror(errno),
130+
errno);
131+
::close(fd);
132+
return Error::AccessFailed;
133+
}
134+
size_t file_size = st.st_size;
135+
136+
// Copy the filename so we can print better debug messages if reads fail.
137+
const char* file_name_copy = ::strdup(file_descriptor_uri);
138+
if (file_name_copy == nullptr) {
139+
ET_LOG(Error, "strdup(%s) failed", file_descriptor_uri);
140+
::close(fd);
141+
return Error::MemoryAllocationFailed;
142+
}
143+
144+
return FileDataLoader(fd, file_size, alignment, file_name_copy);
145+
}
146+
77147
Result<FileDataLoader> FileDataLoader::from(
78148
const char* file_name,
79149
size_t alignment) {

‎extension/data_loader/file_data_loader.h‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,24 @@ namespace extension {
2626
*/
2727
class FileDataLoader final : public executorch::runtime::DataLoader {
2828
public:
29+
/**
30+
* Creates a new FileDataLoader that wraps the named file descriptor.
31+
*
32+
* @param[in] file_descriptor_uri File descriptor with the prefix "fd:///",
33+
* followed by the file descriptor number.
34+
* @param[in] alignment Alignment in bytes of pointers returned by this
35+
* instance. Must be a power of two.
36+
*
37+
* @returns A new FileDataLoader on success.
38+
* @retval Error::InvalidArgument `alignment` is not a power of two.
39+
* @retval Error::AccessFailed `file_name` could not be opened, or its size
40+
* could not be found.
41+
* @retval Error::MemoryAllocationFailed Internal memory allocation failure.
42+
*/
43+
static executorch::runtime::Result<FileDataLoader> fromFileDescriptorUri(
44+
const char* file_descriptor_uri,
45+
size_t alignment = alignof(std::max_align_t));
46+
2947
/**
3048
* Creates a new FileDataLoader that wraps the named file.
3149
*

‎extension/data_loader/test/file_data_loader_test.cpp‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,103 @@ class FileDataLoaderTest : public ::testing::TestWithParam<size_t> {
4040
}
4141
};
4242

43+
TEST_P(FileDataLoaderTest, InBoundsFileDescriptorLoadsSucceed) {
44+
// Write some heterogeneous data to a file.
45+
uint8_t data[256];
46+
for (int i = 0; i < sizeof(data); ++i) {
47+
data[i] = i;
48+
}
49+
TempFile tf(data, sizeof(data));
50+
51+
int fd = ::open(tf.path().c_str(), O_RDONLY);
52+
53+
// Wrap it in a loader.
54+
Result<FileDataLoader> fdl = FileDataLoader::fromFileDescriptorUri(
55+
("fd:///" + std::to_string(fd)).c_str(), alignment());
56+
ASSERT_EQ(fdl.error(), Error::Ok);
57+
58+
// size() should succeed and reflect the total size.
59+
Result<size_t> size = fdl->size();
60+
ASSERT_EQ(size.error(), Error::Ok);
61+
EXPECT_EQ(*size, sizeof(data));
62+
63+
// Load the first bytes of the data.
64+
{
65+
Result<FreeableBuffer> fb = fdl->load(
66+
/*offset=*/0,
67+
/*size=*/8,
68+
DataLoader::SegmentInfo(DataLoader::SegmentInfo::Type::Program));
69+
ASSERT_EQ(fb.error(), Error::Ok);
70+
EXPECT_ALIGNED(fb->data(), alignment());
71+
EXPECT_EQ(fb->size(), 8);
72+
EXPECT_EQ(
73+
0,
74+
std::memcmp(
75+
fb->data(),
76+
"\x00\x01\x02\x03"
77+
"\x04\x05\x06\x07",
78+
fb->size()));
79+
80+
// Freeing should release the buffer and clear out the segment.
81+
fb->Free();
82+
EXPECT_EQ(fb->size(), 0);
83+
EXPECT_EQ(fb->data(), nullptr);
84+
85+
// Safe to call multiple times.
86+
fb->Free();
87+
}
88+
89+
// Load the last few bytes of the data, a different size than the first time.
90+
{
91+
Result<FreeableBuffer> fb = fdl->load(
92+
/*offset=*/sizeof(data) - 3,
93+
/*size=*/3,
94+
DataLoader::SegmentInfo(DataLoader::SegmentInfo::Type::Program));
95+
ASSERT_EQ(fb.error(), Error::Ok);
96+
EXPECT_ALIGNED(fb->data(), alignment());
97+
EXPECT_EQ(fb->size(), 3);
98+
EXPECT_EQ(0, std::memcmp(fb->data(), "\xfd\xfe\xff", fb->size()));
99+
}
100+
101+
// Loading all of the data succeeds.
102+
{
103+
Result<FreeableBuffer> fb = fdl->load(
104+
/*offset=*/0,
105+
/*size=*/sizeof(data),
106+
DataLoader::SegmentInfo(DataLoader::SegmentInfo::Type::Program));
107+
ASSERT_EQ(fb.error(), Error::Ok);
108+
EXPECT_ALIGNED(fb->data(), alignment());
109+
EXPECT_EQ(fb->size(), sizeof(data));
110+
EXPECT_EQ(0, std::memcmp(fb->data(), data, fb->size()));
111+
}
112+
113+
// Loading zero-sized data succeeds, even at the end of the data.
114+
{
115+
Result<FreeableBuffer> fb = fdl->load(
116+
/*offset=*/sizeof(data),
117+
/*size=*/0,
118+
DataLoader::SegmentInfo(DataLoader::SegmentInfo::Type::Program));
119+
ASSERT_EQ(fb.error(), Error::Ok);
120+
EXPECT_EQ(fb->size(), 0);
121+
}
122+
}
123+
124+
TEST_P(FileDataLoaderTest, FileDescriptorLoadPrefixFail) {
125+
// Write some heterogeneous data to a file.
126+
uint8_t data[256];
127+
for (int i = 0; i < sizeof(data); ++i) {
128+
data[i] = i;
129+
}
130+
TempFile tf(data, sizeof(data));
131+
132+
int fd = ::open(tf.path().c_str(), O_RDONLY);
133+
134+
// Wrap it in a loader.
135+
Result<FileDataLoader> fdl = FileDataLoader::fromFileDescriptorUri(
136+
std::to_string(fd).c_str(), alignment());
137+
ASSERT_EQ(fdl.error(), Error::InvalidArgument);
138+
}
139+
43140
TEST_P(FileDataLoaderTest, InBoundsLoadsSucceed) {
44141
// Write some heterogeneous data to a file.
45142
uint8_t data[256];

0 commit comments

Comments
 (0)