Merge pull request #132 from Mytherin/addextensions

Move over httpfs tests from main repo to duckdb-httpfs + add CI run

Author: Mytherin

.github/workflows/MinioTests.yml renamed to .github/workflows/IntegrationTests.yml

@@ -1,4 +1,4 @@
-name: Minio Tests
+name: Integration Tests
 on: [push, pull_request,repository_dispatch]
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
@@ -8,17 +8,19 @@ defaults:
     shell: bash
 
 jobs:
-  minio-tests:
-    name: Minio Tests
-    runs-on: ubuntu-24.04
+  linux-tests-httpfs:
+    name: MinIO Tests
+    runs-on: ubuntu-latest
     env:
       S3_TEST_SERVER_AVAILABLE: 1
       AWS_DEFAULT_REGION: eu-west-1
       AWS_ACCESS_KEY_ID: minio_duckdb_user
       AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password
       DUCKDB_S3_ENDPOINT: duckdb-minio.com:9000
       DUCKDB_S3_USE_SSL: false
-      CORE_EXTENSIONS: 'parquet;json'
+      HTTP_PROXY_PUBLIC: localhost:3128
+      TEST_PERSISTENT_SECRETS_AVAILABLE: true
+      CORE_EXTENSIONS: "parquet;json;tpch"
       GEN: ninja
       VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
       VCPKG_TARGET_TRIPLET: x64-linux
@@ -43,37 +45,44 @@ jobs:
         uses: hendrikmuhs/ccache-action@main
         with:
           key: ${{ github.job }}
-          save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}
+          save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb-httpfs' }}
 
       - name: Setup vcpkg
         uses: lukka/[email protected]
         with:
           vcpkgGitCommitId: 5e5d0e1cd7785623065e77eff011afdeec1a3574
 
-      - name: Build
+      - name: Fix permissions of test secrets
         shell: bash
-        run: |
-          echo -e "\nduckdb_extension_load(tpch)\n" >> extension_config.cmake
-          make
+        run: chmod -R 700 data/secrets
 
-      - name: Start S3/HTTP test server
+      # TODO: fix the authenticated proxy here
+      - name: Install and run http proxy squid
         shell: bash
         run: |
-          cd duckdb
-          mkdir data/attach_test
-          touch data/attach_test/attach.db
-          sudo ./scripts/install_s3_test_server.sh
-          source ./scripts/run_s3_test_server.sh
-          sleep 30
+          sudo apt-get install squid
+          ./scripts/run_squid.sh --port 3128 --log_dir squid_logs &
 
-      - name: Run & Populate test server
+      - name: Run & Populate Python test server
         shell: bash
         run: |
           mkdir -p $PYTHON_HTTP_SERVER_DIR
           cd $PYTHON_HTTP_SERVER_DIR
           python3 -m http.server 8008 &
 
-      - name: Test
+      - name: Build
+        shell: bash
+        run: make
+
+      - name: Install test server
+        shell: bash
+        run: |
+          sudo ./scripts/install_s3_test_server.sh
+          ./scripts/generate_presigned_url.sh
+
+      - name: Start test server & run tests
         shell: bash
         run: |
-          make test
+          source ./scripts/run_s3_test_server.sh
+          source ./scripts/set_s3_test_server_variables.sh
+          make test

.gitignore

@@ -355,3 +355,4 @@ extension_external
 
 test/sql/pragma/output.json
 tools/pythonpkg/duckdb_build/
+test/test_data

Makefile

@@ -4,21 +4,5 @@ PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 EXT_NAME=httpfs
 EXT_CONFIG=${PROJ_DIR}extension_config.cmake
 
-CORE_EXTENSIONS=''
-
 # Include the Makefile from extension-ci-tools
 include extension-ci-tools/makefiles/duckdb_extension.Makefile
-
-
-## Add some more extra tests
-test_release_internal:
-	./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*"
-	./build/release/$(TEST_PATH) --test-dir duckdb --test-config test/configs/duckdb-tests.json
-
-test_debug_internal:
-	./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*"
-	./build/debug/$(TEST_PATH) --test-dir duckdb --test-config test/configs/duckdb-tests.json
-
-test_reldebug_internal:
-	./build/reldebug/$(TEST_PATH) "$(PROJ_DIR)test/*"
-	./build/reldebug/$(TEST_PATH) --test-dir duckdb --test-config test/configs/duckdb-tests.json

data/secrets/README.md

@@ -0,0 +1,10 @@
+# Test secrets
+DuckDB only allows persistent secrets with the x00 permission (e.g. 600 or 700). Therefore to use these 
+secrets, the permissions need to be set before running any tests that uses them.
+
+The recommended way to add tests that touch these persistent secret files is to put them behind a
+```shell
+require-env TEST_PERSISTENT_SECRETS_AVAILABLE
+```
+statement, which ensures the tests only run in CI jobs where the permissions are set correctly.
+