Skip to content

feat: open/close dl without self-defined reference count #3401

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

feat: open/close dl without self-defined reference count #3401

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
9b9ee3a
dl_without_mutex_lock_handler_commit1
zhangziheng01233 Jul 26, 2023
d2a36ea
dl_without_mutex_lock_handler_commit2
zhangziheng01233 Jul 26, 2023
ac2143c
dl_without_mutex_lock_handler_commit3
zhangziheng01233 Aug 1, 2023
d84ce1d
dl_without_mutex_lock_handler_commit4
zhangziheng01233 Aug 1, 2023
8d690c8
dl_without_mutex_lock_handler_commit5
zhangziheng01233 Aug 1, 2023
24cfb8c
dl_without_mutex_lock_handler_commit6
zhangziheng01233 Aug 1, 2023
b8d262b
resolve cpplint
dl239 Aug 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 28 additions & 16 deletions hybridse/src/udf/dynamic_lib_manager.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,10 @@ namespace udf {
DynamicLibManager::~DynamicLibManager() {
for (const auto& kv : handle_map_) {
auto so_handle = kv.second;
if (so_handle) {
dlclose(so_handle->handle);
// if (so_handle) {
// dlclose(so_handle->handle);
if(so_handle != nullptr) {
dlclose(so_handle);
}
}
handle_map_.clear();
Expand All @@ -34,47 +36,54 @@ DynamicLibManager::~DynamicLibManager() {
base::Status DynamicLibManager::ExtractFunction(const std::string& name, bool is_aggregate, const std::string& file,
std::vector<void*>* funs) {
CHECK_TRUE(funs != nullptr, common::kExternalUDFError, "funs is nullptr")
std::shared_ptr<DynamicLibHandle> so_handle;
// std::shared_ptr<DynamicLibHandle> so_handle;
void* so_handle;
{
std::lock_guard<std::mutex> lock(mu_);
auto iter = handle_map_.find(file);
if (iter != handle_map_.end()) {
so_handle = iter->second;
so_handle->ref_cnt++;
// so_handle->ref_cnt++;
}
}
if (!so_handle) {
// if (!so_handle) {
if (so_handle == nullptr) {
void* handle = dlopen(file.c_str(), RTLD_LAZY);
CHECK_TRUE(handle != nullptr, common::kExternalUDFError,
"can not open the dynamic library: " + file + ", error: " + dlerror())
so_handle = std::make_shared<DynamicLibHandle>(handle);
// so_handle = std::make_shared<DynamicLibHandle>(handle);
so_handle = handle;
std::lock_guard<std::mutex> lock(mu_);
handle_map_.emplace(file, so_handle);
}
if (is_aggregate) {
std::string init_fun_name = name + "_init";
auto init_fun = dlsym(so_handle->handle, init_fun_name.c_str());
// auto init_fun = dlsym(so_handle->handle, init_fun_name.c_str());
auto init_fun = dlsym(so_handle, init_fun_name.c_str());
if (init_fun == nullptr) {
RemoveHandler(file);
return {common::kExternalUDFError, "can not find the init function: " + init_fun_name};
}
funs->emplace_back(init_fun);
std::string update_fun_name = name + "_update";
auto update_fun = dlsym(so_handle->handle, update_fun_name.c_str());
// auto update_fun = dlsym(so_handle->handle, update_fun_name.c_str());
auto update_fun = dlsym(so_handle, update_fun_name.c_str());
if (update_fun == nullptr) {
RemoveHandler(file);
return {common::kExternalUDFError, "can not find the update function: " + update_fun_name};
}
funs->emplace_back(update_fun);
std::string output_fun_name = name + "_output";
auto output_fun = dlsym(so_handle->handle, output_fun_name.c_str());
// auto output_fun = dlsym(so_handle->handle, output_fun_name.c_str());
auto output_fun = dlsym(so_handle, output_fun_name.c_str());
if (output_fun == nullptr) {
RemoveHandler(file);
return {common::kExternalUDFError, "can not find the output function: " + output_fun_name};
}
funs->emplace_back(output_fun);
} else {
auto fun = dlsym(so_handle->handle, name.c_str());
// auto fun = dlsym(so_handle->handle, name.c_str());
auto fun = dlsym(so_handle, name.c_str());
if (fun == nullptr) {
RemoveHandler(file);
return {common::kExternalUDFError, "can not find the function: " + name};
Expand All @@ -85,20 +94,23 @@ base::Status DynamicLibManager::ExtractFunction(const std::string& name, bool is
}

base::Status DynamicLibManager::RemoveHandler(const std::string& file) {
std::shared_ptr<DynamicLibHandle> so_handle;
// std::shared_ptr<DynamicLibHandle> so_handle;
void* so_handle;
{
std::lock_guard<std::mutex> lock(mu_);
if (auto iter = handle_map_.find(file); iter != handle_map_.end()) {
iter->second->ref_cnt--;
if (iter->second->ref_cnt == 0) {
// iter->second->ref_cnt--;
// if (iter->second->ref_cnt == 0) {
if (iter->second != nullptr && dlclose(iter->second) == 0) {
so_handle = iter->second;
handle_map_.erase(iter);
}
}
}
if (so_handle) {
CHECK_TRUE(dlclose(so_handle->handle) == 0, common::kExternalUDFError, "dlclose run error. file is " + file)
}
// if (so_handle) {
// CHECK_TRUE(dlclose(so_handle->handle) == 0, common::kExternalUDFError, "dlclose run error. file is " + file)
// }
CHECK_TRUE(dlclose(so_handle) == 0, common::kExternalUDFError, "dlclose run error. file is " + file)
return {};
}

Expand Down
7 changes: 4 additions & 3 deletions hybridse/src/udf/dynamic_lib_manager.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,14 @@
namespace hybridse {
namespace udf {

struct DynamicLibHandle {
/*struct DynamicLibHandle {
explicit DynamicLibHandle(void* ptr) {
handle = ptr;
ref_cnt = 1;
}
void* handle = nullptr;
uint32_t ref_cnt = 0;
};
};*/

class DynamicLibManager {
public:
Expand All @@ -50,7 +50,8 @@ class DynamicLibManager {

private:
std::mutex mu_;
std::map<std::string, std::shared_ptr<DynamicLibHandle>> handle_map_;
// std::map<std::string, std::shared_ptr<DynamicLibHandle>> handle_map_;
std::map<std::string, void*> handle_map_;
};

} // namespace udf
Expand Down