Merge pull request #4752 from kinke/standalone-runtime2

Better support building the runtime separately [mod]

Author: kinke

.github/actions/helper-build-ldc/action.yml

@@ -30,11 +30,14 @@ runs:
         installDir="$PWD/install"
         mkdir '${{ inputs.build_dir }}'
         cd '${{ inputs.build_dir }}'
+        # wrt. CMAKE_C[XX]_COMPILER for Apple: https://github.com/ldc-developers/ldc/issues/3901
         cmake -G Ninja ../ldc \
           -DCMAKE_BUILD_TYPE=Release \
           -DLLVM_ROOT_DIR="$PWD/../${{ inputs.llvm_dir }}" \
           -DD_COMPILER='${{ inputs.host_dc }}' \
           -DLDC_LINK_MANUALLY=OFF \
+          ${{ runner.os == 'macOS' && '-DCMAKE_C_COMPILER=/usr/bin/cc' || '' }} \
+          ${{ runner.os == 'macOS' && '-DCMAKE_CXX_COMPILER=/usr/bin/c++' || '' }} \
           ${{ inputs.specify_install_dir == 'true' && '-DCMAKE_INSTALL_PREFIX="$installDir"' || '' }} \
           ${{ inputs.specify_install_dir == 'true' && '-DINCLUDE_INSTALL_DIR="$installDir/import"' || '' }} \
           ${{ inputs.cmake_flags }}

.github/workflows/main.yml

@@ -41,13 +41,12 @@ jobs:
             arch: x86_64
             bootstrap_cmake_flags: >-
               -DBUILD_LTO_LIBS=ON
-              -DD_COMPILER_FLAGS=-gcc=/usr/bin/c++
             # https://github.com/ldc-developers/ldc/issues/4462:
             # When using LTO, we need to explicitly export ~all symbols for plugin support via `ld64 -exported_symbol '__*'`.
             # Additionally `-w` to suppress resulting linker warnings.
             extra_cmake_flags: >-
               -DBUILD_LTO_LIBS=ON
-              -DD_COMPILER_FLAGS="-gcc=/usr/bin/c++ -O -flto=full -defaultlib=phobos2-ldc-lto,druntime-ldc-lto -L-exported_symbol '-L__*' -L-w"
+              -DD_COMPILER_FLAGS="-O -flto=full -defaultlib=phobos2-ldc-lto,druntime-ldc-lto -L-exported_symbol '-L__*' -L-w"
               -DEXTRA_CXXFLAGS=-flto=full
             with_pgo: true
 
@@ -56,10 +55,9 @@ jobs:
             arch: arm64
             bootstrap_cmake_flags: >-
               -DBUILD_LTO_LIBS=ON
-              -DD_COMPILER_FLAGS=-gcc=/usr/bin/c++
             extra_cmake_flags: >-
               -DBUILD_LTO_LIBS=ON
-              -DD_COMPILER_FLAGS="-gcc=/usr/bin/c++ -O -flto=full -defaultlib=phobos2-ldc-lto,druntime-ldc-lto -L-exported_symbol '-L__*' -L-w"
+              -DD_COMPILER_FLAGS="-O -flto=full -defaultlib=phobos2-ldc-lto,druntime-ldc-lto -L-exported_symbol '-L__*' -L-w"
               -DEXTRA_CXXFLAGS=-flto=full
             with_pgo: true
 
@@ -77,8 +75,10 @@ jobs:
             os: windows-2022
             arch: x86
             bootstrap_cmake_flags: -DBUILD_LTO_LIBS=ON
+            # `RT_CFLAGS=-m32` needed to make 64-bit clang-cl output 32-bit code for druntime integration tests
             extra_cmake_flags: >-
               -DBUILD_LTO_LIBS=ON
+              -DRT_CFLAGS=-m32
             # Undefined symbol errors with FullLTO; ThinLTO used to work, but apparently miscompiles a lexer.d:138 assertion since v1.33.
             #  "-DD_COMPILER_FLAGS=-O -flto=thin -defaultlib=phobos2-ldc-lto,druntime-ldc-lto"
             #  -DEXTRA_CXXFLAGS=-flto=thin
@@ -94,9 +94,9 @@ jobs:
         with:
           submodules: true
           fetch-depth: 50
-      - name: 'macOS 14: Switch to Xcode 16 Beta 6'
+      - name: 'macOS 14: Switch to Xcode 16'
         if: matrix.os == 'macos-14'
-        run: sudo xcode-select -switch /Applications/Xcode_16_beta_6.app
+        run: sudo xcode-select -switch /Applications/Xcode_16.app
       - name: Install prerequisites
         uses: ./.github/actions/1-setup
         with:

.github/workflows/supported_llvm_versions.yml

@@ -25,12 +25,12 @@ jobs:
             os: macos-14
             host_dc: ldc-beta
             llvm_version: 17.0.5
-            cmake_flags: -DBUILD_SHARED_LIBS=ON -DRT_SUPPORT_SANITIZERS=ON -DD_COMPILER_FLAGS=-gcc=/usr/bin/c++ -DCMAKE_EXE_LINKER_FLAGS=-L/opt/homebrew/opt/zstd/lib
+            cmake_flags: -DBUILD_SHARED_LIBS=ON -DRT_SUPPORT_SANITIZERS=ON -DCMAKE_EXE_LINKER_FLAGS=-L/opt/homebrew/opt/zstd/lib -DCMAKE_C_COMPILER=/usr/bin/cc -DCMAKE_CXX_COMPILER=/usr/bin/c++
           - job_name: macOS 14, LLVM 16, latest LDC beta
             os: macos-14
             host_dc: ldc-beta
             llvm_version: 16.0.5
-            cmake_flags: -DBUILD_SHARED_LIBS=OFF -DD_COMPILER_FLAGS=-gcc=/usr/bin/c++ -DCMAKE_EXE_LINKER_FLAGS=-L/opt/homebrew/opt/zstd/lib
+            cmake_flags: -DBUILD_SHARED_LIBS=OFF -DCMAKE_EXE_LINKER_FLAGS=-L/opt/homebrew/opt/zstd/lib -DCMAKE_C_COMPILER=/usr/bin/cc -DCMAKE_CXX_COMPILER=/usr/bin/c++
           - job_name: Ubuntu 20.04, LLVM 15, latest DMD beta
             os: ubuntu-20.04
             host_dc: dmd-beta

runtime/CMakeLists.txt

@@ -1,6 +1,5 @@
-project(runtime C ASM)
-
 cmake_minimum_required(VERSION 3.4.3)
+project(runtime C ASM)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR})
 
@@ -337,7 +336,7 @@ if(LDC_EXE)
                     "$<$<BOOL:${SHARED_LIBS_SUPPORTED}>:${MULTILIB_DIR}>"
                     ${conf_path}.in
                     ${conf_path}
-            DEPENDS ${LDC_EXE} ${LDC_EXE_FULL}
+            DEPENDS ${LDC_EXE_FULL}
         )
         add_custom_command(OUTPUT ${install_conf_path} VERBATIM
             COMMAND ${PROJECT_PARENT_DIR}/tools/add-multilib-section.sh
@@ -347,7 +346,7 @@ if(LDC_EXE)
                     "$<$<BOOL:${SHARED_LIBS_SUPPORTED}>:${MULTILIB_INSTALL_DIR}>"
                     ${install_conf_path}.in
                     ${install_conf_path}
-            DEPENDS ${LDC_EXE} ${LDC_EXE_FULL}
+            DEPENDS ${LDC_EXE_FULL}
         )
         add_custom_target(add-multilib-section ALL
             DEPENDS ${conf_path} ${install_conf_path}
@@ -437,7 +436,9 @@ macro(dc src_files src_basedir d_flags output_basedir emit_bc all_at_once single
         list(APPEND dc_flags -flto=thin --output-bc)
     endif()
 
-    set(dc_deps ${LDC_EXE} ${LDC_EXE_FULL} ${GCCBUILTINS})
+    # dc_deps can only contain paths, otherwise cmake will ignore the dependency.
+    # See: https://github.com/ldc-developers/ldc/pull/4743#issuecomment-2323156173
+    set(dc_deps ${LDC_EXE_FULL} ${GCCBUILTINS})
     if(TARGET add-multilib-section)
         # Make sure the config files are available before invoking LDC.
         set(dc_deps ${dc_deps} add-multilib-section)
@@ -905,6 +906,7 @@ install(FILES ${GCCBUILTINS} DESTINATION ${INCLUDE_INSTALL_DIR}/ldc)
 #
 # Test targets.
 #
+enable_testing()
 
 # Build the "test runner" executables containing the druntime and Phobos unit
 # tests. They are invoked with the modules to test later.

runtime/DRuntimeIntegrationTests.cmake

@@ -1,3 +1,5 @@
+enable_language(CXX) # for CMAKE_CXX_COMPILER
+
 # Try to find GNU make, use specific version first (BSD) and fall back to default 'make' (Linux)
 find_program(GNU_MAKE_BIN NAMES gmake gnumake make)
 if(NOT GNU_MAKE_BIN)
@@ -34,8 +36,13 @@ else()
     endif()
 endif()
 
-if(NOT "${TARGET_SYSTEM}" MATCHES "MSVC")
-    set(cflags_base "CFLAGS_BASE=-Wall -Wl,-rpath,${CMAKE_BINARY_DIR}/lib${LIB_SUFFIX}")
+string(REPLACE ";" " " dflags_base "${D_EXTRA_FLAGS}")
+
+string(REPLACE ";" " " cflags_base "${RT_CFLAGS}")
+if("${TARGET_SYSTEM}" MATCHES "MSVC")
+    set(cflags_base "${cflags_base} /Wall")
+else()
+    set(cflags_base "${cflags_base} -Wall -Wl,-rpath,${CMAKE_BINARY_DIR}/lib${LIB_SUFFIX}")
 endif()
 
 set(linkdl "")
@@ -72,9 +79,10 @@ foreach(name ${testnames})
         )
         add_test(NAME ${fullname}
             COMMAND ${GNU_MAKE_BIN} -C ${PROJECT_SOURCE_DIR}/druntime/test/${name}
-                ROOT=${outdir} DMD=${LDMD_EXE_FULL} BUILD=${build}
+                ROOT=${outdir} DMD=${LDMD_EXE_FULL} BUILD=${build} SHARED=1
+                CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER}
                 DRUNTIME=${druntime_path_build} DRUNTIMESO=${shared_druntime_path_build}
-                SHARED=1 ${cflags_base} ${linkdl}
+                CFLAGS_BASE=${cflags_base} DFLAGS_BASE=${dflags_base} ${linkdl}
         )
         set_tests_properties(${fullname} PROPERTIES DEPENDS clean-${fullname})
     endforeach()

runtime/druntime/test/common.mak

@@ -37,8 +37,10 @@ ifeq (,$(findstring ldmd2,$(DMD)))
         CFLAGS_BASE+=--target=x86_64-darwin-apple  # ARM cpu is not supported by dmd
     endif
 endif
-# LDC: use `-defaultlib=druntime-ldc [-link-defaultlib-shared]` instead of `-defaultlib= -L$(DRUNTIME[_IMPLIB])`
-DFLAGS:=$(MODEL_FLAG) $(PIC) -w -I../../src -I../../import -I$(SRC) -defaultlib=$(if $(findstring ldmd2,$(DMD)),druntime-ldc,) -preview=dip1000 $(if $(findstring $(OS),windows),,-L-lpthread -L-lm $(LINKDL))
+# LDC:
+# * include optional DFLAGS_BASE
+# * use `-defaultlib=druntime-ldc [-link-defaultlib-shared]` instead of `-defaultlib= -L$(DRUNTIME[_IMPLIB])`
+DFLAGS:=$(MODEL_FLAG) $(PIC) $(if $(findstring ldmd2,$(DMD)),$(DFLAGS_BASE),) -w -I../../src -I../../import -I$(SRC) -defaultlib=$(if $(findstring ldmd2,$(DMD)),druntime-ldc,) -preview=dip1000 $(if $(findstring $(OS),windows),,-L-lpthread -L-lm $(LINKDL))
 # LINK_SHARED may be set by importing makefile
 ifeq (,$(findstring ldmd2,$(DMD)))
     DFLAGS+=$(if $(LINK_SHARED),-L$(DRUNTIME_IMPLIB) $(if $(findstring $(OS),windows),-dllimport=all),-L$(DRUNTIME))

runtime/druntime/test/shared/Makefile

@@ -131,24 +131,24 @@ $(ROOT)/dynamiccast$(DOTDLL): $(SRC)/dynamiccast.d  $(if $(LINK_SHARED),$(DRUNTI
 ifeq (windows,$(OS))
     CC:=cl
     CC_OUTFLAG:=/Fe
-    # we additionally specify the .obj output path (/Fo) to prevent collisions
-    CC_EXTRAS:=
+    # additionally specify the .obj output directory to prevent collisions
+    CC_EXTRAS:=/Fo$(ROOT)/
 else
     CC_OUTFLAG:=-o
     CC_EXTRAS:=$(LDL) -pthread
 endif
 
 $(ROOT)/linkD$(DOTEXE): $(SRC)/linkD.c $(ROOT)/lib$(DOTDLL) $(DRUNTIMESO)
-	$(QUIET)$(CC) $(CFLAGS) $(CC_OUTFLAG)$@ $(if $(findstring $(OS),windows),/Fo$@.obj,) $< $(ROOT)/lib$(DOTIMPLIB) $(CC_EXTRAS)
+	$(QUIET)$(CC) $(CFLAGS) $(CC_OUTFLAG)$@ $< $(ROOT)/lib$(DOTIMPLIB) $(CC_EXTRAS)
 
 $(ROOT)/linkDR$(DOTEXE): $(SRC)/linkDR.c $(ROOT)/lib$(DOTDLL) $(DRUNTIMESO)
-	$(QUIET)$(CC) $(CFLAGS) $(CC_OUTFLAG)$@ $(if $(findstring $(OS),windows),/Fo$@.obj,) $< $(DRUNTIME_IMPLIB) $(CC_EXTRAS)
+	$(QUIET)$(CC) $(CFLAGS) $(CC_OUTFLAG)$@ $< $(DRUNTIME_IMPLIB) $(CC_EXTRAS)
 
 $(ROOT)/loadDR$(DOTEXE): $(SRC)/loadDR.c $(ROOT)/lib$(DOTDLL) $(DRUNTIMESO)
-	$(QUIET)$(CC) $(CFLAGS) $(CC_OUTFLAG)$@ $(if $(findstring $(OS),windows),/Fo$@.obj,) $< $(CC_EXTRAS)
+	$(QUIET)$(CC) $(CFLAGS) $(CC_OUTFLAG)$@ $< $(CC_EXTRAS)
 
 $(ROOT)/host$(DOTEXE): $(SRC)/host.c $(ROOT)/plugin1$(DOTDLL) $(ROOT)/plugin2$(DOTDLL)
-	$(QUIET)$(CC) $(CFLAGS) $(CC_OUTFLAG)$@ $(if $(findstring $(OS),windows),/Fo$@.obj,) $< $(CC_EXTRAS)
+	$(QUIET)$(CC) $(CFLAGS) $(CC_OUTFLAG)$@ $< $(CC_EXTRAS)
 
 $(ROOT)/liblinkdep$(DOTDLL): $(ROOT)/lib$(DOTDLL)
 $(ROOT)/liblinkdep$(DOTDLL): DFLAGS+=-L$(ROOT)/lib$(DOTIMPLIB)

runtime/druntime/test/stdcpp/Makefile

@@ -4,7 +4,7 @@ include ../common.mak
 
 ifeq (windows,$(OS))
 
-CC:=cl
+CXX:=cl
 EXTRA_CXXFLAGS:=/nologo /EHsc
 EXTRA_DFLAGS:=
 
@@ -23,19 +23,19 @@ $(ROOT)/%: $(SRC)/%.cpp $(SRC)/%_test.d
 	@echo Testing $*
 	@mkdir -p $(dir $@)
 
-	$(QUIET)$(CC) /MT $(EXTRA_CXXFLAGS) -c /Fo$@_cpp$(DOTOBJ) $<
+	$(QUIET)$(CXX) /MT $(CXXFLAGS_BASE) $(EXTRA_CXXFLAGS) -c /Fo$@_cpp$(DOTOBJ) $<
 	$(QUIET)$(DMD) -mscrtlib=libcmt $(DFLAGS) $(EXTRA_DFLAGS) -main -unittest -version=CoreUnittest -version=_MSC_VER_$(MSC_VER) -of$@$(DOTEXE) $@_cpp$(DOTOBJ) $(SRC)/$*_test.d
 	$(QUIET)$(TIMELIMIT)$@ $(RUN_ARGS)
 
-	$(QUIET)$(CC) /MD $(EXTRA_CXXFLAGS) -c /Fo$@_cpp$(DOTOBJ) $<
+	$(QUIET)$(CXX) /MD $(CXXFLAGS_BASE) $(EXTRA_CXXFLAGS) -c /Fo$@_cpp$(DOTOBJ) $<
 	$(QUIET)$(DMD) -mscrtlib=msvcrt $(DFLAGS) $(EXTRA_DFLAGS) -main -unittest -version=CoreUnittest -version=_MSC_VER_$(MSC_VER) -of$@$(DOTEXE) $@_cpp$(DOTOBJ) $(SRC)/$*_test.d
 	$(QUIET)$(TIMELIMIT)$@ $(RUN_ARGS)
 
-	$(QUIET)$(CC) /MTd $(EXTRA_CXXFLAGS) -c /Fo$@_cpp$(DOTOBJ) $<
+	$(QUIET)$(CXX) /MTd $(CXXFLAGS_BASE) $(EXTRA_CXXFLAGS) -c /Fo$@_cpp$(DOTOBJ) $<
 	$(QUIET)$(DMD) -mscrtlib=libcmtd $(DFLAGS) $(EXTRA_DFLAGS) -main -unittest -version=CoreUnittest -version=_MSC_VER_$(MSC_VER) -of$@$(DOTEXE) $@_cpp$(DOTOBJ) $(SRC)/$*_test.d
 	$(QUIET)$(TIMELIMIT)$@ $(RUN_ARGS)
 
-	$(QUIET)$(CC) /MDd $(EXTRA_CXXFLAGS) -c /Fo$@_cpp$(DOTOBJ) $<
+	$(QUIET)$(CXX) /MDd $(CXXFLAGS_BASE) $(EXTRA_CXXFLAGS) -c /Fo$@_cpp$(DOTOBJ) $<
 	$(QUIET)$(DMD) -mscrtlib=msvcrtd $(DFLAGS) $(EXTRA_DFLAGS) -main -unittest -version=CoreUnittest -version=_MSC_VER_$(MSC_VER) -of$@$(DOTEXE) $@_cpp$(DOTOBJ) $(SRC)/$*_test.d
 	$(QUIET)$(TIMELIMIT)$@ $(RUN_ARGS)
 

runtime/ldc-build-runtime.d.in

@@ -150,9 +150,12 @@ void prepareLdcSource() {
 void runCMake() {
     const wd = WorkingDirScope(config.buildDir);
 
+    const ldmdExecutable = buildPath(config.ldcExecutable.dirName, "ldmd2" ~ exeSuffix);
+
     string[] args = [
         "cmake",
         "-DLDC_EXE_FULL=" ~ config.ldcExecutable,
+        "-DLDMD_EXE_FULL=" ~ ldmdExecutable,
         "-DDMDFE_MINOR_VERSION=@DMDFE_MINOR_VERSION@",
         "-DDMDFE_PATCH_VERSION=@DMDFE_PATCH_VERSION@",
     ];