Merge pull request #135 from Mytherin/merge

Merge v.14 into main

Author: carlopi

.github/workflows/MinioTests.yml renamed to .github/workflows/IntegrationTests.yml

@@ -1,4 +1,4 @@
-name: Minio Tests
+name: Integration Tests
 on: [push, pull_request,repository_dispatch]
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
@@ -8,17 +8,19 @@ defaults:
     shell: bash
 
 jobs:
-  minio-tests:
-    name: Minio Tests
-    runs-on: ubuntu-24.04
+  linux-tests-httpfs:
+    name: MinIO Tests
+    runs-on: ubuntu-latest
     env:
       S3_TEST_SERVER_AVAILABLE: 1
       AWS_DEFAULT_REGION: eu-west-1
       AWS_ACCESS_KEY_ID: minio_duckdb_user
       AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password
       DUCKDB_S3_ENDPOINT: duckdb-minio.com:9000
       DUCKDB_S3_USE_SSL: false
-      CORE_EXTENSIONS: 'parquet;json'
+      HTTP_PROXY_PUBLIC: localhost:3128
+      TEST_PERSISTENT_SECRETS_AVAILABLE: true
+      CORE_EXTENSIONS: "parquet;json;tpch"
       GEN: ninja
       VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
       VCPKG_TARGET_TRIPLET: x64-linux
@@ -43,37 +45,44 @@ jobs:
         uses: hendrikmuhs/ccache-action@main
         with:
           key: ${{ github.job }}
-          save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}
+          save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb-httpfs' }}
 
       - name: Setup vcpkg
         uses: lukka/[email protected]
         with:
           vcpkgGitCommitId: 5e5d0e1cd7785623065e77eff011afdeec1a3574
 
-      - name: Build
+      - name: Fix permissions of test secrets
         shell: bash
-        run: |
-          echo -e "\nduckdb_extension_load(tpch)\n" >> extension_config.cmake
-          make
+        run: chmod -R 700 data/secrets
 
-      - name: Start S3/HTTP test server
+      # TODO: fix the authenticated proxy here
+      - name: Install and run http proxy squid
         shell: bash
         run: |
-          cd duckdb
-          mkdir data/attach_test
-          touch data/attach_test/attach.db
-          sudo ./scripts/install_s3_test_server.sh
-          source ./scripts/run_s3_test_server.sh
-          sleep 30
+          sudo apt-get install squid
+          ./scripts/run_squid.sh --port 3128 --log_dir squid_logs &
 
-      - name: Run & Populate test server
+      - name: Run & Populate Python test server
         shell: bash
         run: |
           mkdir -p $PYTHON_HTTP_SERVER_DIR
           cd $PYTHON_HTTP_SERVER_DIR
           python3 -m http.server 8008 &
 
-      - name: Test
+      - name: Build
+        shell: bash
+        run: make
+
+      - name: Install test server
+        shell: bash
+        run: |
+          sudo ./scripts/install_s3_test_server.sh
+          ./scripts/generate_presigned_url.sh
+
+      - name: Start test server & run tests
         shell: bash
         run: |
-          make test
+          source ./scripts/run_s3_test_server.sh
+          source ./scripts/set_s3_test_server_variables.sh
+          make test

.github/workflows/MainDistributionPipeline.yml

@@ -17,7 +17,7 @@ jobs:
     uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
     with:
       extension_name: httpfs
-      duckdb_version: v1.4-andium
+      duckdb_version: v1.4.0
       ci_tools_version: main
 
 
@@ -28,6 +28,7 @@ jobs:
     secrets: inherit
     with:
       extension_name: httpfs
-      duckdb_version: v1.4-andium
+      duckdb_version: v1.4.0
       ci_tools_version: main
-      deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
+      deploy_latest: ${{ startsWith(github.ref, 'refs/heads/v') }}
+      deploy_versioned: ${{ startsWith(github.ref, 'refs/heads/v') || github.ref == 'refs/heads/main' }}

.gitignore

@@ -355,3 +355,4 @@ extension_external
 
 test/sql/pragma/output.json
 tools/pythonpkg/duckdb_build/
+test/test_data

CMakeLists.txt

@@ -1,48 +1,15 @@
 cmake_minimum_required(VERSION 2.8.12...3.29)
 
-set(HTTPFS_BASE_FOLDER "extension/httpfs")
-
 project(HTTPFsExtension)
 
 add_extension_definitions()
 
-include_directories(extension/httpfs/include
-                    ${DUCKDB_MODULE_BASE_DIR}/third_party/httplib)
-
 if (NOT EMSCRIPTEN)
-  set(EXTRA_SOURCES extension/httpfs/crypto.cpp extension/httpfs/httpfs_httplib_client.cpp extension/httpfs/httpfs_curl_client.cpp)
   add_definitions(-DOVERRIDE_ENCRYPTION_UTILS=1)
 else()
-  set(EXTRA_SOURCES extension/httpfs/httpfs_client_wasm.cpp)
-  set (DUCKDB_EXTENSION_HTTPFS_LINKED_LIBS "../../third_party/mbedtls/libduckdb_mbedtls.a")
+  set(DUCKDB_EXTENSION_HTTPFS_LINKED_LIBS "../../third_party/mbedtls/libduckdb_mbedtls.a")
 endif()
 
-build_static_extension(
-  httpfs
-  extension/httpfs/hffs.cpp
-  extension/httpfs/s3fs.cpp
-  extension/httpfs/httpfs.cpp
-  extension/httpfs/http_state.cpp
-  extension/httpfs/crypto.cpp
-  extension/httpfs/hash_functions.cpp
-  extension/httpfs/create_secret_functions.cpp
-  extension/httpfs/httpfs_extension.cpp
-  ${EXTRA_SOURCES} )
-
-set(PARAMETERS "-warnings")
-build_loadable_extension(
-  httpfs
-  ${PARAMETERS}
-  extension/httpfs/hffs.cpp
-  extension/httpfs/s3fs.cpp
-  extension/httpfs/httpfs.cpp
-  extension/httpfs/http_state.cpp
-  extension/httpfs/crypto.cpp
-  extension/httpfs/hash_functions.cpp
-  extension/httpfs/create_secret_functions.cpp
-  extension/httpfs/httpfs_extension.cpp
-  ${EXTRA_SOURCES} )
-
 if(MINGW)
   set(OPENSSL_USE_STATIC_LIBS TRUE)
 endif()
@@ -51,6 +18,17 @@ find_package(OpenSSL REQUIRED)
 find_package(CURL REQUIRED)
 include_directories(${OPENSSL_INCLUDE_DIR})
 include_directories(${CURL_INCLUDE_DIRS})
+
+include_directories(src/include
+                    ${DUCKDB_MODULE_BASE_DIR}/third_party/httplib)
+add_subdirectory(src)
+set(EXTENSION_SOURCES ${ALL_OBJECT_FILES})
+
+build_static_extension(httpfs ${EXTENSION_SOURCES})
+
+set(PARAMETERS "-warnings")
+build_loadable_extension(httpfs ${PARAMETERS} ${EXTENSION_SOURCES})
+
 if(EMSCRIPTEN)
   target_link_libraries(httpfs_loadable_extension duckdb_mbedtls)
 else()

Makefile

@@ -4,21 +4,5 @@ PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 EXT_NAME=httpfs
 EXT_CONFIG=${PROJ_DIR}extension_config.cmake
 
-CORE_EXTENSIONS=''
-
 # Include the Makefile from extension-ci-tools
 include extension-ci-tools/makefiles/duckdb_extension.Makefile
-
-
-## Add some more extra tests
-test_release_internal:
-	./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*"
-	./build/release/$(TEST_PATH) --test-dir duckdb --test-config test/configs/duckdb-tests.json
-
-test_debug_internal:
-	./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*"
-	./build/debug/$(TEST_PATH) --test-dir duckdb --test-config test/configs/duckdb-tests.json
-
-test_reldebug_internal:
-	./build/reldebug/$(TEST_PATH) "$(PROJ_DIR)test/*"
-	./build/reldebug/$(TEST_PATH) --test-dir duckdb --test-config test/configs/duckdb-tests.json

data/secrets/README.md

@@ -0,0 +1,10 @@
+# Test secrets
+DuckDB only allows persistent secrets with the x00 permission (e.g. 600 or 700). Therefore to use these 
+secrets, the permissions need to be set before running any tests that uses them.
+
+The recommended way to add tests that touch these persistent secret files is to put them behind a
+```shell
+require-env TEST_PERSISTENT_SECRETS_AVAILABLE
+```
+statement, which ensures the tests only run in CI jobs where the permissions are set correctly.
+