Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

Commit aaf8287

Browse files
namchuainamchuai
committed
feat: add proxy support
1 parent 24bebed commit aaf8287

File tree

11 files changed

+230
-57
lines changed

11 files changed

+230
-57
lines changed

‎engine/cli/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ add_executable(${TARGET_NAME} main.cc
7373
${CMAKE_CURRENT_SOURCE_DIR}/../utils/cpuid/cpu_info.cc
7474
${CMAKE_CURRENT_SOURCE_DIR}/../utils/file_logger.cc
7575
${CMAKE_CURRENT_SOURCE_DIR}/command_line_parser.cc
76+
${CMAKE_CURRENT_SOURCE_DIR}/../services/config_service.cc
7677
${CMAKE_CURRENT_SOURCE_DIR}/../services/download_service.cc
7778
${CMAKE_CURRENT_SOURCE_DIR}/../services/engine_service.cc
7879
${CMAKE_CURRENT_SOURCE_DIR}/../services/model_service.cc

‎engine/common/api_server_configuration.h‎

Lines changed: 93 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,68 @@
55
#include <unordered_map>
66
#include <vector>
77

8+
// current only support basic auth
9+
enum class ProxyAuthMethod {
10+
Basic,
11+
Digest,
12+
DigestIe,
13+
Bearer,
14+
Negotiate,
15+
Ntlm,
16+
NtlmWb,
17+
Any,
18+
AnySafe,
19+
AuthOnly,
20+
AwsSigV4
21+
};
22+
823
class ApiServerConfiguration {
924
public:
1025
ApiServerConfiguration(bool cors = true,
11-
std::vector<std::string> allowed_origins = {})
12-
: cors{cors}, allowed_origins{allowed_origins} {}
26+
std::vector<std::string> allowed_origins = {},
27+
bool verify_proxy_ssl = true,
28+
const std::string& proxy_url = "",
29+
const std::string& proxy_username = "",
30+
const std::string& proxy_password = "",
31+
const std::string& no_proxy = "")
32+
: cors{cors},
33+
allowed_origins{allowed_origins},
34+
verify_proxy_ssl{verify_proxy_ssl},
35+
proxy_url{proxy_url},
36+
proxy_username{proxy_username},
37+
proxy_password{proxy_password},
38+
no_proxy{no_proxy} {}
1339

40+
// cors
1441
bool cors{true};
1542
std::vector<std::string> allowed_origins;
1643

44+
// proxy
45+
bool verify_proxy_ssl{true};
46+
ProxyAuthMethod proxy_auth_method{ProxyAuthMethod::Basic};
47+
std::string proxy_url{""};
48+
std::string proxy_username{""};
49+
std::string proxy_password{""};
50+
std::string no_proxy{""};
51+
52+
// just placeholder here
53+
bool verify_peer_ssl{true};
54+
bool verify_host_ssl{true};
55+
1756
Json::Value ToJson() const {
1857
Json::Value root;
1958
root["cors"] = cors;
2059
root["allowed_origins"] = Json::Value(Json::arrayValue);
2160
for (const auto& origin : allowed_origins) {
2261
root["allowed_origins"].append(origin);
2362
}
63+
root["verify_proxy_ssl"] = verify_proxy_ssl;
64+
// TODO: current only support basic auth
65+
root["proxy_url"] = proxy_url;
66+
root["proxy_username"] = proxy_username;
67+
root["proxy_password"] = proxy_password;
68+
root["no_proxy"] = no_proxy;
69+
2470
return root;
2571
}
2672

@@ -31,6 +77,51 @@ class ApiServerConfiguration {
3177
const std::unordered_map<std::string,
3278
std::function<bool(const Json::Value&)>>
3379
field_updater{
80+
{"verify_proxy_ssl",
81+
[this](const Json::Value& value) -> bool {
82+
if (!value.isBool()) {
83+
return false;
84+
}
85+
verify_proxy_ssl = value.asBool();
86+
return true;
87+
}},
88+
89+
{"no_proxy",
90+
[this](const Json::Value& value) -> bool {
91+
if (!value.isString()) {
92+
return false;
93+
}
94+
no_proxy = value.asString();
95+
return true;
96+
}},
97+
98+
{"proxy_url",
99+
[this](const Json::Value& value) -> bool {
100+
if (!value.isString()) {
101+
return false;
102+
}
103+
proxy_url = value.asString();
104+
return true;
105+
}},
106+
107+
{"proxy_username",
108+
[this](const Json::Value& value) -> bool {
109+
if (!value.isString()) {
110+
return false;
111+
}
112+
proxy_username = value.asString();
113+
return true;
114+
}},
115+
116+
{"proxy_password",
117+
[this](const Json::Value& value) -> bool {
118+
if (!value.isString()) {
119+
return false;
120+
}
121+
proxy_password = value.asString();
122+
return true;
123+
}},
124+
34125
{"cors",
35126
[this](const Json::Value& value) -> bool {
36127
if (!value.isBool()) {

‎engine/config/yaml_config.cc‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#include "utils/file_manager_utils.h"
99
#include "utils/format_utils.h"
1010
#include "yaml_config.h"
11+
1112
namespace config {
1213
// Method to read YAML file
1314
void YamlHandler::Reset() {
@@ -44,6 +45,7 @@ void YamlHandler::ReadYamlFile(const std::string& file_path) {
4445
throw;
4546
}
4647
}
48+
4749
void YamlHandler::SplitPromptTemplate(ModelConfig& mc) {
4850
if (mc.prompt_template.size() > 0) {
4951
auto& pt = mc.prompt_template;
@@ -220,7 +222,7 @@ void YamlHandler::UpdateModelConfig(ModelConfig new_model_config) {
220222
yaml_node_["ngl"] = model_config_.ngl;
221223
if (!std::isnan(static_cast<double>(model_config_.ctx_len)))
222224
yaml_node_["ctx_len"] = model_config_.ctx_len;
223-
if (!std::isnan(static_cast<double>(model_config_.n_parallel)))
225+
if (!std::isnan(static_cast<double>(model_config_.n_parallel)))
224226
yaml_node_["n_parallel"] = model_config_.n_parallel;
225227
if (!std::isnan(static_cast<double>(model_config_.tp)))
226228
yaml_node_["tp"] = model_config_.tp;
@@ -377,7 +379,8 @@ void YamlHandler::WriteYamlFile(const std::string& file_path) const {
377379
outFile << format_utils::writeKeyValue(
378380
"ctx_len", yaml_node_["ctx_len"],
379381
"llama.context_length | 0 or undefined = loaded from model");
380-
outFile << format_utils::writeKeyValue("n_parallel", yaml_node_["n_parallel"]);
382+
outFile << format_utils::writeKeyValue("n_parallel",
383+
yaml_node_["n_parallel"]);
381384
outFile << format_utils::writeKeyValue("ngl", yaml_node_["ngl"],
382385
"Undefined = loaded from model");
383386
outFile << "# END OPTIONAL\n";

‎engine/main.cc‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,13 +106,14 @@ void RunServer(std::optional<int> port, bool ignore_cout) {
106106
auto event_queue_ptr = std::make_shared<EventQueue>();
107107
cortex::event::EventProcessor event_processor(event_queue_ptr);
108108

109-
auto download_service = std::make_shared<DownloadService>(event_queue_ptr);
109+
auto config_service = std::make_shared<ConfigService>();
110+
auto download_service =
111+
std::make_shared<DownloadService>(event_queue_ptr, config_service);
110112
auto engine_service = std::make_shared<EngineService>(download_service);
111113
auto inference_svc =
112114
std::make_shared<services::InferenceService>(engine_service);
113115
auto model_service = std::make_shared<ModelService>(
114116
download_service, inference_svc, engine_service);
115-
auto config_service = std::make_shared<ConfigService>();
116117

117118
// initialize custom controllers
118119
auto engine_ctl = std::make_shared<Engines>(engine_service);

‎engine/services/config_service.cc‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,11 @@
55
cpp::result<ApiServerConfiguration, std::string>
66
ConfigService::UpdateApiServerConfiguration(const Json::Value& json) {
77
auto config = file_manager_utils::GetCortexConfig();
8-
ApiServerConfiguration api_server_config{config.enableCors,
9-
config.allowedOrigins};
8+
ApiServerConfiguration api_server_config{
9+
config.enableCors, config.allowedOrigins, config.verifyProxySsl,
10+
config.proxyUrl, config.proxyUsername, config.proxyPassword,
11+
config.noProxy};
12+
1013
std::vector<std::string> updated_fields;
1114
std::vector<std::string> invalid_fields;
1215
std::vector<std::string> unknown_fields;
@@ -20,13 +23,21 @@ ConfigService::UpdateApiServerConfiguration(const Json::Value& json) {
2023

2124
config.enableCors = api_server_config.cors;
2225
config.allowedOrigins = api_server_config.allowed_origins;
26+
config.proxyUrl = api_server_config.proxy_url;
27+
config.verifyProxySsl = api_server_config.verify_proxy_ssl;
28+
config.proxyUsername = api_server_config.proxy_username;
29+
config.proxyPassword = api_server_config.proxy_password;
30+
config.noProxy = api_server_config.no_proxy;
2331

2432
auto result = file_manager_utils::UpdateCortexConfig(config);
2533
return api_server_config;
2634
}
2735

2836
cpp::result<ApiServerConfiguration, std::string>
29-
ConfigService::GetApiServerConfiguration() const {
37+
ConfigService::GetApiServerConfiguration() {
3038
auto config = file_manager_utils::GetCortexConfig();
31-
return ApiServerConfiguration{config.enableCors, config.allowedOrigins};
39+
return ApiServerConfiguration{config.enableCors, config.allowedOrigins,
40+
config.verifyProxySsl, config.proxyUrl,
41+
config.proxyUsername, config.proxyPassword,
42+
config.noProxy};
3243
}

‎engine/services/config_service.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,5 @@ class ConfigService {
88
cpp::result<ApiServerConfiguration, std::string> UpdateApiServerConfiguration(
99
const Json::Value& json);
1010

11-
cpp::result<ApiServerConfiguration, std::string> GetApiServerConfiguration()
12-
const;
11+
cpp::result<ApiServerConfiguration, std::string> GetApiServerConfiguration();
1312
};

‎engine/services/download_service.cc‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ cpp::result<uint64_t, std::string> DownloadService::GetFileSize(
8787
return cpp::fail(static_cast<std::string>("Failed to init CURL"));
8888
}
8989

90+
// TODO: namh add header here
9091
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
9192
curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
9293
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
@@ -189,6 +190,7 @@ cpp::result<bool, std::string> DownloadService::Download(
189190

190191
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers);
191192
}
193+
// TODO: namh add proxy setting here
192194
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WriteCallback);
193195
curl_easy_setopt(curl, CURLOPT_WRITEDATA, file);
194196
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
@@ -337,6 +339,7 @@ void DownloadService::ProcessTask(DownloadTask& task, int worker_id) {
337339
});
338340
worker_data->downloading_data_map[item.id] = dl_data_ptr;
339341

342+
CTL_ERR("Namh Setup curl");
340343
SetUpCurlHandle(handle, item, file, dl_data_ptr.get());
341344
curl_multi_add_handle(worker_data->multi_handle, handle);
342345
task_handles.push_back(std::make_pair(handle, file));
@@ -407,13 +410,46 @@ cpp::result<void, ProcessDownloadFailed> DownloadService::ProcessMultiDownload(
407410

408411
void DownloadService::SetUpCurlHandle(CURL* handle, const DownloadItem& item,
409412
FILE* file, DownloadingData* dl_data) {
413+
auto configuration = config_service_->GetApiServerConfiguration();
414+
if (configuration.has_value()) {
415+
if (!configuration->proxy_url.empty()) {
416+
auto proxy_url = configuration->proxy_url;
417+
auto verify_proxy_ssl = configuration->verify_proxy_ssl;
418+
auto proxy_username = configuration->proxy_username;
419+
auto proxy_password = configuration->proxy_password;
420+
421+
CTL_ERR("=== Proxy configuration ===");
422+
CTL_ERR("Proxy url: " << proxy_url);
423+
CTL_ERR("Verify proxy ssl: " << verify_proxy_ssl);
424+
425+
curl_easy_setopt(handle, CURLOPT_PROXY, proxy_url.c_str());
426+
curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L);
427+
curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L);
428+
// todo: allow this later on
429+
curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYPEER, 0L);
430+
curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYHOST, 0L);
431+
432+
if (!proxy_username.empty()) {
433+
std::string proxy_auth = proxy_username + ":" + proxy_password;
434+
CTL_ERR("Proxy auth: " << proxy_auth);
435+
curl_easy_setopt(handle, CURLOPT_PROXYUSERPWD, proxy_auth.c_str());
436+
}
437+
438+
curl_easy_setopt(handle, CURLOPT_NOPROXY,
439+
configuration->no_proxy.c_str());
440+
}
441+
} else {
442+
CTL_ERR("Failed to get configuration");
443+
}
444+
410445
curl_easy_setopt(handle, CURLOPT_URL, item.downloadUrl.c_str());
411446
curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, WriteCallback);
412447
curl_easy_setopt(handle, CURLOPT_WRITEDATA, file);
413448
curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L);
414449
curl_easy_setopt(handle, CURLOPT_NOPROGRESS, 0L);
415450
curl_easy_setopt(handle, CURLOPT_XFERINFOFUNCTION, ProgressCallback);
416451
curl_easy_setopt(handle, CURLOPT_XFERINFODATA, dl_data);
452+
curl_easy_setopt(handle, CURLOPT_VERBOSE, 1L);
417453

418454
auto headers = curl_utils::GetHeaders(item.downloadUrl);
419455
if (headers) {

‎engine/services/download_service.h‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#include <unordered_set>
99
#include "common/download_task_queue.h"
1010
#include "common/event.h"
11+
#include "services/config_service.h"
1112
#include "utils/result.hpp"
1213

1314
struct ProcessDownloadFailed {
@@ -20,6 +21,8 @@ class DownloadService {
2021
private:
2122
static constexpr int MAX_CONCURRENT_TASKS = 4;
2223

24+
std::shared_ptr<ConfigService> config_service_;
25+
2326
struct DownloadingData {
2427
std::string task_id;
2528
std::string item_id;
@@ -82,8 +85,9 @@ class DownloadService {
8285

8386
explicit DownloadService() = default;
8487

85-
explicit DownloadService(std::shared_ptr<EventQueue> event_queue)
86-
: event_queue_{event_queue} {
88+
explicit DownloadService(std::shared_ptr<EventQueue> event_queue,
89+
std::shared_ptr<ConfigService> config_service)
90+
: event_queue_{event_queue}, config_service_{config_service} {
8791
InitializeWorkers();
8892
};
8993

‎engine/test/components/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ add_executable(${PROJECT_NAME}
99
${CMAKE_CURRENT_SOURCE_DIR}/../../config/gguf_parser.cc
1010
${CMAKE_CURRENT_SOURCE_DIR}/../../cli/commands/cortex_upd_cmd.cc
1111
${CMAKE_CURRENT_SOURCE_DIR}/../../cli/commands/server_stop_cmd.cc
12+
${CMAKE_CURRENT_SOURCE_DIR}/../../services/config_service.cc
1213
${CMAKE_CURRENT_SOURCE_DIR}/../../services/download_service.cc
1314
${CMAKE_CURRENT_SOURCE_DIR}/../../database/models.cc
1415
)

0 commit comments

Comments
 (0)