Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/TestCITools.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ jobs:
# mingw arch below is not opt-in, included to check that it is allowed in the list
opt_in_archs: 'windows_arm64;windows_amd64_mingw;'
save_cache: ${{ github.event_name != 'pull_request' }}
vcpkg_triplets_override: 'windows_arm64:arm64-windows-static-md-release-vs2019comp;'

extension-template-main:
name: Extension template
Expand Down
45 changes: 41 additions & 4 deletions .github/workflows/_extension_distribution.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,11 @@ on:
required: false
type: string
default: ''
# format: 'arch_name1:triplet1;arch_name2:triplet2'
vcpkg_triplets_override:
required: false
type: string
default: ''
# Override the default script producing the matrices. Allows specifying custom matrices.
matrix_parse_script:
required: false
Expand Down Expand Up @@ -200,10 +205,42 @@ jobs:
- id: parse-matrices
run: |
mkdir build
python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os linux --output build/linux_matrix.json --exclude "${{ inputs.exclude_archs }}" --opt_in "${{ inputs.opt_in_archs }}" --pretty --reduced_ci_mode ${{ inputs.reduced_ci_mode }}
python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os osx --output build/osx_matrix.json --exclude "${{ inputs.exclude_archs }}" --opt_in "${{ inputs.opt_in_archs }}" --pretty --reduced_ci_mode ${{ inputs.reduced_ci_mode }}
python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os windows --output build/windows_matrix.json --exclude "${{ inputs.exclude_archs }}" --opt_in "${{ inputs.opt_in_archs }}" --pretty --reduced_ci_mode ${{ inputs.reduced_ci_mode }}
python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os wasm --output build/wasm_matrix.json --exclude "${{ inputs.exclude_archs }}" --opt_in "${{ inputs.opt_in_archs }}" --pretty --reduced_ci_mode ${{ inputs.reduced_ci_mode }}
python3 ${{ inputs.matrix_parse_script }} \
--input extension-ci-tools/config/distribution_matrix.json \
--select_os linux \
--output build/linux_matrix.json \
--exclude "${{ inputs.exclude_archs }}" \
--opt_in "${{ inputs.opt_in_archs }}" \
--pretty \
--reduced_ci_mode ${{ inputs.reduced_ci_mode }} \
--vcpkg_triplets_override "${{ inputs.vcpkg_triplets_override }}"
python3 ${{ inputs.matrix_parse_script }} \
--input extension-ci-tools/config/distribution_matrix.json \
--select_os osx \
--output build/osx_matrix.json \
--exclude "${{ inputs.exclude_archs }}" \
--opt_in "${{ inputs.opt_in_archs }}" \
--pretty \
--reduced_ci_mode ${{ inputs.reduced_ci_mode }} \
--vcpkg_triplets_override "${{ inputs.vcpkg_triplets_override }}"
python3 ${{ inputs.matrix_parse_script }} \
--input extension-ci-tools/config/distribution_matrix.json \
--select_os windows \
--output build/windows_matrix.json \
--exclude "${{ inputs.exclude_archs }}" \
--opt_in "${{ inputs.opt_in_archs }}" \
--pretty \
--reduced_ci_mode ${{ inputs.reduced_ci_mode }} \
--vcpkg_triplets_override "${{ inputs.vcpkg_triplets_override }}"
python3 ${{ inputs.matrix_parse_script }} \
--input extension-ci-tools/config/distribution_matrix.json \
--select_os wasm \
--output build/wasm_matrix.json \
--exclude "${{ inputs.exclude_archs }}" \
--opt_in "${{ inputs.opt_in_archs }}" \
--pretty \
--reduced_ci_mode ${{ inputs.reduced_ci_mode }} \
--vcpkg_triplets_override "${{ inputs.vcpkg_triplets_override }}"

- id: set-matrix-linux
run: |
Expand Down
7 changes: 7 additions & 0 deletions makefiles/c_api_extensions/base.Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,13 @@ ifeq ($(DUCKDB_PLATFORM),windows_amd64_mingw)
EXTENSION_LIB_FILENAME=lib$(EXTENSION_NAME).dll
endif

# Propagate toolchains
ifeq ($(DUCKDB_PLATFORM),windows_amd64)
TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_OVERLAY_TRIPLETS=${PROJ_DIR}extension-ci-tools/toolchains/
else ifeq ($(DUCKDB_PLATFORM),windows_arm64)
TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_OVERLAY_TRIPLETS=${PROJ_DIR}extension-ci-tools/toolchains/
endif

#############################################
### Main extension parameters
#############################################
Expand Down
6 changes: 6 additions & 0 deletions makefiles/duckdb_extension.Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,12 @@ ifneq ("${VCPKG_HOST_TRIPLET}", "")
TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_HOST_TRIPLET='${VCPKG_HOST_TRIPLET}'
endif

ifeq ($(DUCKDB_PLATFORM),windows_amd64)
TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_OVERLAY_TRIPLETS=${PROJ_DIR}extension-ci-tools/toolchains/
else ifeq ($(DUCKDB_PLATFORM),windows_arm64)
TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_OVERLAY_TRIPLETS=${PROJ_DIR}extension-ci-tools/toolchains/
endif

#### Enable Ninja as generator
ifeq ($(GEN),ninja)
GENERATOR=-G "Ninja" -DFORCE_COLORED_OUTPUT=1
Expand Down
23 changes: 23 additions & 0 deletions scripts/modify_distribution_matrix.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
parser.add_argument("--reduced_ci_mode", required=True, help="Set to default/enabled/disabled, when enabled, filters out redundant archs for testing")
parser.add_argument("--select_os", help="Select an OS to include in the output JSON")
parser.add_argument("--deploy_matrix", action="store_true", help="Create a merged list used in deploy step")
parser.add_argument("--vcpkg_triplets_override", required=False, default="", help="Format: 'arch_name1:triplet1;arch_name2:triplet2'")
args = parser.parse_args()


Expand All @@ -39,6 +40,19 @@
else:
raise Exception("Unknown reduced_ci_mode value: " + reduced_ci_mode + " - must be auto/enabled/disabled.")

# Parse VCPKG triplets override
triplet_pairs = args.vcpkg_triplets_override.split(";")
triplet_dict = {}
for pair in triplet_pairs:
stripped = pair.strip()
if len(stripped) == 0:
continue
parts = stripped.split(":")
if len(parts) != 2:
raise Exception(f"Invalid 'vcpkg_triplets_override' entry: {stripped},\
must be in format: 'arch_name1:triplet1'")
triplet_dict[parts[0]] = parts[1]

# Read the input JSON file
with open(input_json_file_path, "r") as json_file:
data = json.load(json_file)
Expand All @@ -58,6 +72,14 @@ def filter_entries(data, excluded_arch_values, opt_in_arch_values):
for os, config in data.items():
if "include" in config:
config["include"] = [entry for entry in config["include"] if should_run(entry, reduced_ci_mode, excluded_arch_values, opt_in_arch_values)]

# Override VCPKG triplets
for entry in config["include"]:
override = triplet_dict.get(entry["duckdb_arch"])
if override is not None:
entry["vcpkg_target_triplet"] = override
entry["vcpkg_host_triplet"] = override

if not config["include"]:
del config["include"]

Expand All @@ -67,6 +89,7 @@ def filter_entries(data, excluded_arch_values, opt_in_arch_values):
# Filter the JSON data
filtered_data = filter_entries(data, excluded_arch_values, opt_in_arch_values)


# Select an OS if specified
if select_os:
found = False
Expand Down
11 changes: 11 additions & 0 deletions toolchains/arm64-windows-static-md-release-vs2019comp.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# It is necessary to pass this flag to AWS C++ SDK to enable compatibility with VS2019 C++ stdlib
set(VCPKG_CXX_FLAGS "/D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR")
# If VCPKG_CXX_FLAGS is set, VCPKG_C_FLAGS must be set
set(VCPKG_C_FLAGS "")

# The following is copied from arm64-windows-static-md-release.cmake
set(VCPKG_TARGET_ARCHITECTURE arm64)
set(VCPKG_CRT_LINKAGE dynamic)
set(VCPKG_LIBRARY_LINKAGE static)

set(VCPKG_BUILD_TYPE release)
11 changes: 11 additions & 0 deletions toolchains/x64-windows-static-md-release-vs2019comp.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# It is necessary to pass this flag to AWS C++ SDK to enable compatibility with VS2019 C++ stdlib
set(VCPKG_CXX_FLAGS "/D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR")
# If VCPKG_CXX_FLAGS is set, VCPKG_C_FLAGS must be set
set(VCPKG_C_FLAGS "")

# The following is copied from x64-windows-static-md-release.cmake
set(VCPKG_TARGET_ARCHITECTURE x64)
set(VCPKG_CRT_LINKAGE dynamic)
set(VCPKG_LIBRARY_LINKAGE static)

set(VCPKG_BUILD_TYPE release)
Loading