Merge pull request #96 from Tmonster/use_curl_as_optional_client_v1.4

Use curl as optional client v1.4

Author: samansmink

.github/workflows/MinioTests.yml

@@ -18,7 +18,9 @@ jobs:
       AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password
       DUCKDB_S3_ENDPOINT: duckdb-minio.com:9000
       DUCKDB_S3_USE_SSL: false
+      CORE_EXTENSIONS: 'parquet;json'
       GEN: ninja
+      VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
       VCPKG_TARGET_TRIPLET: x64-linux
 
     steps:

CMakeLists.txt

@@ -10,7 +10,7 @@ include_directories(extension/httpfs/include
                     ${DUCKDB_MODULE_BASE_DIR}/third_party/httplib)
 
 if (NOT EMSCRIPTEN)
-  set(EXTRA_SOURCES extension/httpfs/crypto.cpp extension/httpfs/httpfs_client.cpp)
+  set(EXTRA_SOURCES extension/httpfs/crypto.cpp extension/httpfs/httpfs_httplib_client.cpp extension/httpfs/httpfs_curl_client.cpp)
   add_definitions(-DOVERRIDE_ENCRYPTION_UTILS=1)
 else()
   set(EXTRA_SOURCES extension/httpfs/httpfs_client_wasm.cpp)
@@ -48,21 +48,29 @@ if(MINGW)
 endif()
 
 find_package(OpenSSL REQUIRED)
+find_package(CURL REQUIRED)
 include_directories(${OPENSSL_INCLUDE_DIR})
+include_directories(${CURL_INCLUDE_DIRS})
 if(EMSCRIPTEN)
   target_link_libraries(httpfs_loadable_extension duckdb_mbedtls)
 else()
   target_link_libraries(httpfs_loadable_extension duckdb_mbedtls
                         ${OPENSSL_LIBRARIES})
   target_link_libraries(httpfs_extension duckdb_mbedtls ${OPENSSL_LIBRARIES})
 
+  # Link dependencies into extension
+  target_link_libraries(httpfs_loadable_extension ${CURL_LIBRARIES})
+  target_link_libraries(httpfs_extension ${CURL_LIBRARIES})
+
+
   if(MINGW)
     find_package(ZLIB)
     target_link_libraries(httpfs_loadable_extension ZLIB::ZLIB -lcrypt32)
     target_link_libraries(httpfs_extension ZLIB::ZLIB -lcrypt32)
   endif()
 endif()
 
+
 install(
   TARGETS httpfs_extension
   EXPORT "${DUCKDB_EXPORT_SET}"

duckdb

@@ -6,7 +6,16 @@
 #include <stdio.h>
 
 #define CPPHTTPLIB_OPENSSL_SUPPORT
-#include "httplib.hpp"
+
+#include <openssl/err.h>
+#include <openssl/evp.h>
+#include <openssl/ssl.h>
+#include <openssl/x509v3.h>
+#include <openssl/rand.h>
+
+#if defined(_WIN32) && defined(OPENSSL_USE_APPLINK)
+#include <openssl/applink.c>
+#endif
 
 namespace duckdb {