diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index cf79c241..b70366fe 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -39,6 +39,7 @@ jobs: # mingw arch below is not opt-in, included to check that it is allowed in the list opt_in_archs: 'windows_arm64;windows_amd64_mingw;' save_cache: ${{ github.event_name != 'pull_request' }} + vcpkg_triplets_override: 'windows_arm64:arm64-windows-static-md-release-vs2019comp;' extension-template-main: name: Extension template diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml index 08811ca2..32676142 100644 --- a/.github/workflows/_extension_distribution.yml +++ b/.github/workflows/_extension_distribution.yml @@ -72,6 +72,11 @@ on: required: false type: string default: '' + # format: 'arch_name1:triplet1;arch_name2:triplet2' + vcpkg_triplets_override: + required: false + type: string + default: '' # Override the default script producing the matrices. Allows specifying custom matrices. matrix_parse_script: required: false @@ -200,10 +205,42 @@ jobs: - id: parse-matrices run: | mkdir build - python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os linux --output build/linux_matrix.json --exclude "${{ inputs.exclude_archs }}" --opt_in "${{ inputs.opt_in_archs }}" --pretty --reduced_ci_mode ${{ inputs.reduced_ci_mode }} - python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os osx --output build/osx_matrix.json --exclude "${{ inputs.exclude_archs }}" --opt_in "${{ inputs.opt_in_archs }}" --pretty --reduced_ci_mode ${{ inputs.reduced_ci_mode }} - python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os windows --output build/windows_matrix.json --exclude "${{ inputs.exclude_archs }}" --opt_in "${{ inputs.opt_in_archs }}" --pretty --reduced_ci_mode ${{ inputs.reduced_ci_mode }} - python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os wasm --output build/wasm_matrix.json --exclude "${{ inputs.exclude_archs }}" --opt_in "${{ inputs.opt_in_archs }}" --pretty --reduced_ci_mode ${{ inputs.reduced_ci_mode }} + python3 ${{ inputs.matrix_parse_script }} \ + --input extension-ci-tools/config/distribution_matrix.json \ + --select_os linux \ + --output build/linux_matrix.json \ + --exclude "${{ inputs.exclude_archs }}" \ + --opt_in "${{ inputs.opt_in_archs }}" \ + --pretty \ + --reduced_ci_mode ${{ inputs.reduced_ci_mode }} \ + --vcpkg_triplets_override "${{ inputs.vcpkg_triplets_override }}" + python3 ${{ inputs.matrix_parse_script }} \ + --input extension-ci-tools/config/distribution_matrix.json \ + --select_os osx \ + --output build/osx_matrix.json \ + --exclude "${{ inputs.exclude_archs }}" \ + --opt_in "${{ inputs.opt_in_archs }}" \ + --pretty \ + --reduced_ci_mode ${{ inputs.reduced_ci_mode }} \ + --vcpkg_triplets_override "${{ inputs.vcpkg_triplets_override }}" + python3 ${{ inputs.matrix_parse_script }} \ + --input extension-ci-tools/config/distribution_matrix.json \ + --select_os windows \ + --output build/windows_matrix.json \ + --exclude "${{ inputs.exclude_archs }}" \ + --opt_in "${{ inputs.opt_in_archs }}" \ + --pretty \ + --reduced_ci_mode ${{ inputs.reduced_ci_mode }} \ + --vcpkg_triplets_override "${{ inputs.vcpkg_triplets_override }}" + python3 ${{ inputs.matrix_parse_script }} \ + --input extension-ci-tools/config/distribution_matrix.json \ + --select_os wasm \ + --output build/wasm_matrix.json \ + --exclude "${{ inputs.exclude_archs }}" \ + --opt_in "${{ inputs.opt_in_archs }}" \ + --pretty \ + --reduced_ci_mode ${{ inputs.reduced_ci_mode }} \ + --vcpkg_triplets_override "${{ inputs.vcpkg_triplets_override }}" - id: set-matrix-linux run: | diff --git a/makefiles/c_api_extensions/base.Makefile b/makefiles/c_api_extensions/base.Makefile index cd4e9508..e952217e 100644 --- a/makefiles/c_api_extensions/base.Makefile +++ b/makefiles/c_api_extensions/base.Makefile @@ -39,6 +39,13 @@ ifeq ($(DUCKDB_PLATFORM),windows_amd64_mingw) EXTENSION_LIB_FILENAME=lib$(EXTENSION_NAME).dll endif +# Propagate toolchains +ifeq ($(DUCKDB_PLATFORM),windows_amd64) + TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_OVERLAY_TRIPLETS=${PROJ_DIR}extension-ci-tools/toolchains/ +else ifeq ($(DUCKDB_PLATFORM),windows_arm64) + TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_OVERLAY_TRIPLETS=${PROJ_DIR}extension-ci-tools/toolchains/ +endif + ############################################# ### Main extension parameters ############################################# diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index b65eb6a6..6f921aaf 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -97,6 +97,12 @@ ifneq ("${VCPKG_HOST_TRIPLET}", "") TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_HOST_TRIPLET='${VCPKG_HOST_TRIPLET}' endif +ifeq ($(DUCKDB_PLATFORM),windows_amd64) + TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_OVERLAY_TRIPLETS=${PROJ_DIR}extension-ci-tools/toolchains/ +else ifeq ($(DUCKDB_PLATFORM),windows_arm64) + TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_OVERLAY_TRIPLETS=${PROJ_DIR}extension-ci-tools/toolchains/ +endif + #### Enable Ninja as generator ifeq ($(GEN),ninja) GENERATOR=-G "Ninja" -DFORCE_COLORED_OUTPUT=1 diff --git a/scripts/modify_distribution_matrix.py b/scripts/modify_distribution_matrix.py index 09c2c2cb..1003c23a 100644 --- a/scripts/modify_distribution_matrix.py +++ b/scripts/modify_distribution_matrix.py @@ -15,6 +15,7 @@ parser.add_argument("--reduced_ci_mode", required=True, help="Set to default/enabled/disabled, when enabled, filters out redundant archs for testing") parser.add_argument("--select_os", help="Select an OS to include in the output JSON") parser.add_argument("--deploy_matrix", action="store_true", help="Create a merged list used in deploy step") +parser.add_argument("--vcpkg_triplets_override", required=False, default="", help="Format: 'arch_name1:triplet1;arch_name2:triplet2'") args = parser.parse_args() @@ -39,6 +40,19 @@ else: raise Exception("Unknown reduced_ci_mode value: " + reduced_ci_mode + " - must be auto/enabled/disabled.") +# Parse VCPKG triplets override +triplet_pairs = args.vcpkg_triplets_override.split(";") +triplet_dict = {} +for pair in triplet_pairs: + stripped = pair.strip() + if len(stripped) == 0: + continue + parts = stripped.split(":") + if len(parts) != 2: + raise Exception(f"Invalid 'vcpkg_triplets_override' entry: {stripped},\ + must be in format: 'arch_name1:triplet1'") + triplet_dict[parts[0]] = parts[1] + # Read the input JSON file with open(input_json_file_path, "r") as json_file: data = json.load(json_file) @@ -58,6 +72,14 @@ def filter_entries(data, excluded_arch_values, opt_in_arch_values): for os, config in data.items(): if "include" in config: config["include"] = [entry for entry in config["include"] if should_run(entry, reduced_ci_mode, excluded_arch_values, opt_in_arch_values)] + + # Override VCPKG triplets + for entry in config["include"]: + override = triplet_dict.get(entry["duckdb_arch"]) + if override is not None: + entry["vcpkg_target_triplet"] = override + entry["vcpkg_host_triplet"] = override + if not config["include"]: del config["include"] @@ -67,6 +89,7 @@ def filter_entries(data, excluded_arch_values, opt_in_arch_values): # Filter the JSON data filtered_data = filter_entries(data, excluded_arch_values, opt_in_arch_values) + # Select an OS if specified if select_os: found = False diff --git a/toolchains/arm64-windows-static-md-release-vs2019comp.cmake b/toolchains/arm64-windows-static-md-release-vs2019comp.cmake new file mode 100644 index 00000000..5343db8b --- /dev/null +++ b/toolchains/arm64-windows-static-md-release-vs2019comp.cmake @@ -0,0 +1,11 @@ +# It is necessary to pass this flag to AWS C++ SDK to enable compatibility with VS2019 C++ stdlib +set(VCPKG_CXX_FLAGS "/D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR") +# If VCPKG_CXX_FLAGS is set, VCPKG_C_FLAGS must be set +set(VCPKG_C_FLAGS "") + +# The following is copied from arm64-windows-static-md-release.cmake +set(VCPKG_TARGET_ARCHITECTURE arm64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_BUILD_TYPE release) diff --git a/toolchains/x64-windows-static-md-release-vs2019comp.cmake b/toolchains/x64-windows-static-md-release-vs2019comp.cmake new file mode 100644 index 00000000..7970c1f7 --- /dev/null +++ b/toolchains/x64-windows-static-md-release-vs2019comp.cmake @@ -0,0 +1,11 @@ +# It is necessary to pass this flag to AWS C++ SDK to enable compatibility with VS2019 C++ stdlib +set(VCPKG_CXX_FLAGS "/D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR") +# If VCPKG_CXX_FLAGS is set, VCPKG_C_FLAGS must be set +set(VCPKG_C_FLAGS "") + +# The following is copied from x64-windows-static-md-release.cmake +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_BUILD_TYPE release)