From b3f6b8ed649df27a826eea6ce4b6d86448c84542 Mon Sep 17 00:00:00 2001 From: Arash Hatami Date: Sun, 18 May 2025 13:42:01 +0330 Subject: [PATCH 1/6] feat: use a global cURL handler --- src/functions/get_tranco.cpp | 24 +++++------- src/utils/utils.cpp | 74 +++++++++++++++++++++++++++++------- src/utils/utils.hpp | 5 +++ 3 files changed, 74 insertions(+), 29 deletions(-) diff --git a/src/functions/get_tranco.cpp b/src/functions/get_tranco.cpp index ce0651e..0caa6f3 100644 --- a/src/functions/get_tranco.cpp +++ b/src/functions/get_tranco.cpp @@ -14,7 +14,7 @@ namespace duckdb // Function to get the download code for the Tranco list std::string GetTrancoDownloadCode (char *date) { - CURL *curl; + CURL *curl = GetCurlHandler (); CURLcode res; std::string readBuffer; @@ -23,21 +23,15 @@ namespace duckdb LogMessage ("INFO", "Get Tranco download code for date: " + std::string (date)); - curl = curl_easy_init (); - if (curl) + curl_easy_setopt (curl, CURLOPT_URL, url.c_str ()); + curl_easy_setopt (curl, CURLOPT_WRITEDATA, &readBuffer); + res = curl_easy_perform (curl); + curl_easy_cleanup (curl); + + if (res != CURLE_OK) { - curl_easy_setopt (curl, CURLOPT_URL, url.c_str ()); - curl_easy_setopt (curl, CURLOPT_FOLLOWLOCATION, 1L); // Follow redirects - curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, WriteCallback); - curl_easy_setopt (curl, CURLOPT_WRITEDATA, &readBuffer); - res = curl_easy_perform (curl); - curl_easy_cleanup (curl); - - if (res != CURLE_OK) - { - LogMessage ("ERROR", "Failed to fetch Tranco download code: " + std::string (curl_easy_strerror (res))); - throw std::runtime_error ("Failed to fetch Tranco download code."); - } + LogMessage ("ERROR", "Failed to fetch Tranco download code: " + std::string (curl_easy_strerror (res))); + throw std::runtime_error ("Failed to fetch Tranco download code."); } // Extract the download code from the URL diff --git a/src/utils/utils.cpp b/src/utils/utils.cpp index 5e7dcf9..2258a5f 100644 --- a/src/utils/utils.cpp +++ b/src/utils/utils.cpp @@ -12,6 +12,57 @@ namespace duckdb { namespace netquack { + CURL *GetCurlHandler () + { + CURL *curl = curl_easy_init (); + if (!curl) + { + throw std::runtime_error ("Failed to initialize CURL"); + } + + const char *ca_info = std::getenv ("CURL_CA_INFO"); +#if !defined(_WIN32) && !defined(__APPLE__) + if (!ca_info) + { + // Check for common CA certificate bundle locations on Linux + for (const auto *path : { + "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo etc. + "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6 + "/etc/ssl/ca-bundle.pem", // OpenSUSE + "/etc/pki/tls/cacert.pem", // OpenELEC + "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7 + "/etc/ssl/cert.pem" // Alpine Linux + }) + { + if (FILE *f = fopen (path, "r")) + { + fclose (f); + ca_info = path; + break; + } + } + } +#endif + curl_easy_setopt (curl, CURLOPT_FOLLOWLOCATION, 1L); // Follow redirects + curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, WriteCallback); + if (ca_info) + { + // Set the custom CA certificate bundle file + // https://github.com/hatamiarash7/duckdb-netquack/issues/6 + LogMessage ("DEBUG", "Using custom CA certificate bundle: " + std::string (ca_info)); + curl_easy_setopt (curl, CURLOPT_CAINFO, ca_info); + } + const char *ca_path = std::getenv ("CURL_CA_PATH"); + if (ca_path) + { + // Set the custom CA certificate directory + LogMessage ("DEBUG", "Using custom CA certificate directory: " + std::string (ca_path)); + curl_easy_setopt (curl, CURLOPT_CAPATH, ca_path); + } + + return curl; + } + void LogMessage (const std::string &level, const std::string &message) { std::ofstream log_file ("netquack.log", std::ios_base::app); @@ -31,24 +82,19 @@ namespace duckdb std::string DownloadPublicSuffixList () { - CURL *curl; + CURL *curl = GetCurlHandler (); CURLcode res; std::string readBuffer; - curl = curl_easy_init (); - if (curl) - { - curl_easy_setopt (curl, CURLOPT_URL, "https://publicsuffix.org/list/public_suffix_list.dat"); - curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, WriteCallback); - curl_easy_setopt (curl, CURLOPT_WRITEDATA, &readBuffer); - res = curl_easy_perform (curl); - curl_easy_cleanup (curl); + curl_easy_setopt (curl, CURLOPT_URL, "https://publicsuffix.org/list/public_suffix_list.dat"); + curl_easy_setopt (curl, CURLOPT_WRITEDATA, &readBuffer); + res = curl_easy_perform (curl); + curl_easy_cleanup (curl); - if (res != CURLE_OK) - { - LogMessage ("ERROR", "Failed to download public suffix list: " + std::string (curl_easy_strerror (res))); - throw std::runtime_error ("Failed to download public suffix list. Check logs for details."); - } + if (res != CURLE_OK) + { + LogMessage ("ERROR", "Failed to download public suffix list: " + std::string (curl_easy_strerror (res))); + throw std::runtime_error ("Failed to download public suffix list. Check logs for details."); } return readBuffer; diff --git a/src/utils/utils.hpp b/src/utils/utils.hpp index 59691ed..0a0f328 100644 --- a/src/utils/utils.hpp +++ b/src/utils/utils.hpp @@ -1,11 +1,16 @@ #pragma once +#include + #include "duckdb.hpp" namespace duckdb { namespace netquack { + // Function to get a CURL handler + CURL *GetCurlHandler (); + // Function to log messages with a specified log level void LogMessage (const std::string &level, const std::string &message); From b4fe2a0ceb0f13b13faa818e25276c464732d583 Mon Sep 17 00:00:00 2001 From: Arash Hatami Date: Sun, 18 May 2025 13:45:35 +0330 Subject: [PATCH 2/6] docs: move experimental warning to top section --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 14392d8..0113c46 100644 --- a/README.md +++ b/README.md @@ -334,6 +334,9 @@ This extension provides various functions for manipulating and analyzing IP addr #### IP Calculator +> [!WARNING] +> It's an experimental function. + The `ipcalc` function takes an IP address and netmask and calculates the resulting broadcast, network, wildcard mask, and host range. ![ipcalc-sc](./.github/ipcalc-sc.png) @@ -369,9 +372,6 @@ D SELECT i.IP, └────────────────┴───────┘ ``` -> [!WARNING] -> It's an experimental function. - ### Get Extension Version You can use the `netquack_version` function to get the extension version. From edfabea060e9c91e6c5dcacea3db4d9dc3953742 Mon Sep 17 00:00:00 2001 From: Arash Hatami Date: Sun, 18 May 2025 13:47:40 +0330 Subject: [PATCH 3/6] chore: update extension version --- README.md | 2 +- src/functions/get_version.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0113c46..8bfabb8 100644 --- a/README.md +++ b/README.md @@ -382,7 +382,7 @@ D select * from netquack_version(); │ version │ │ varchar │ ├─────────┤ -│ v1.2.0 │ +│ v1.4.0 │ └─────────┘ ``` diff --git a/src/functions/get_version.cpp b/src/functions/get_version.cpp index e00270f..0d69542 100644 --- a/src/functions/get_version.cpp +++ b/src/functions/get_version.cpp @@ -43,7 +43,7 @@ namespace duckdb output.SetCardinality (1); // Set version - output.data[0].SetValue (0, "v1.1.0"); + output.data[0].SetValue (0, "v1.4.0"); // Set done auto &local_state = (VersionLocalState &)*data_p.local_state; local_state.done = true; From 9ac30ad77df73041a5c6ce320630a3b2aa690995 Mon Sep 17 00:00:00 2001 From: Arash Hatami Date: Sun, 18 May 2025 21:27:12 +0330 Subject: [PATCH 4/6] chore: bump DuckDB to v1.2.2 --- .github/workflows/MainDistributionPipeline.yml | 6 +++--- duckdb | 2 +- extension-ci-tools | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 91cb74c..4bfb95c 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -25,8 +25,8 @@ jobs: duckdb-stable-build: name: Build extension binaries - uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.2.1 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.2.2 with: - duckdb_version: v1.2.1 - ci_tools_version: v1.2.1 + duckdb_version: v1.2.2 + ci_tools_version: v1.2.2 extension_name: netquack diff --git a/duckdb b/duckdb index 8e52ec4..7c03946 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit 8e52ec43959ab363643d63cb78ee214577111da4 +Subproject commit 7c039464e452ddc3330e2691d3fa6d305521d09b diff --git a/extension-ci-tools b/extension-ci-tools index 58970c5..5ae521a 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit 58970c538d35919db875096460c05806056f4de0 +Subproject commit 5ae521a4c9709bb3cb35dc6d79e044f67bc58bf0 From f14d6d496735f60724d565068120c6b4a6679823 Mon Sep 17 00:00:00 2001 From: Arash Hatami Date: Mon, 19 May 2025 11:03:35 +0330 Subject: [PATCH 5/6] chore: use stat instead of fopen [skip ci] --- src/utils/utils.cpp | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/utils/utils.cpp b/src/utils/utils.cpp index 2258a5f..2f422db 100644 --- a/src/utils/utils.cpp +++ b/src/utils/utils.cpp @@ -8,10 +8,27 @@ #include #include +#ifdef _WIN32 +#include +#else // POSIX +#include +#endif + namespace duckdb { namespace netquack { + bool file_exists (const char *file_path) + { +#ifdef _WIN32 + DWORD attributes = GetFileAttributesA (file_path); + return (attributes != INVALID_FILE_ATTRIBUTES); +#else // POSIX + struct stat buffer; + return (stat (file_path, &buffer) == 0); +#endif + } + CURL *GetCurlHandler () { CURL *curl = curl_easy_init (); @@ -34,9 +51,8 @@ namespace duckdb "/etc/ssl/cert.pem" // Alpine Linux }) { - if (FILE *f = fopen (path, "r")) + if (file_exists (path)) { - fclose (f); ca_info = path; break; } From 74744060764afb089e5b847ef11e3ca0130fb7ba Mon Sep 17 00:00:00 2001 From: Arash Hatami Date: Mon, 19 May 2025 11:18:44 +0330 Subject: [PATCH 6/6] chore: rename curl handler function [skip ci] --- src/functions/get_tranco.cpp | 2 +- src/utils/utils.cpp | 4 ++-- src/utils/utils.hpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/functions/get_tranco.cpp b/src/functions/get_tranco.cpp index 0caa6f3..eb537ab 100644 --- a/src/functions/get_tranco.cpp +++ b/src/functions/get_tranco.cpp @@ -14,7 +14,7 @@ namespace duckdb // Function to get the download code for the Tranco list std::string GetTrancoDownloadCode (char *date) { - CURL *curl = GetCurlHandler (); + CURL *curl = CreateCurlHandler (); CURLcode res; std::string readBuffer; diff --git a/src/utils/utils.cpp b/src/utils/utils.cpp index 2f422db..9e73e5b 100644 --- a/src/utils/utils.cpp +++ b/src/utils/utils.cpp @@ -29,7 +29,7 @@ namespace duckdb #endif } - CURL *GetCurlHandler () + CURL *CreateCurlHandler () { CURL *curl = curl_easy_init (); if (!curl) @@ -98,7 +98,7 @@ namespace duckdb std::string DownloadPublicSuffixList () { - CURL *curl = GetCurlHandler (); + CURL *curl = CreateCurlHandler (); CURLcode res; std::string readBuffer; diff --git a/src/utils/utils.hpp b/src/utils/utils.hpp index 0a0f328..380e81c 100644 --- a/src/utils/utils.hpp +++ b/src/utils/utils.hpp @@ -9,7 +9,7 @@ namespace duckdb namespace netquack { // Function to get a CURL handler - CURL *GetCurlHandler (); + CURL *CreateCurlHandler (); // Function to log messages with a specified log level void LogMessage (const std::string &level, const std::string &message);