all: switch to LLVM 11 for static builds

This commit switches to LLVM 11 for builds with LLVM linked statically
(e.g. `make`). It does not yet switch the default for builds dynamically
linked to LLVM, that should be done in a later change.

This commit also changes to use the default host toolchain (probably
GCC) instead of Clang as the default compiler in CI. There were some
issues with Clang 3.8 in CI and hopefully this will fix it.

Additionally it updates the way LLVM is built on Windows, with
-DLLVM_ENABLE_PIC=OFF (which should have been used all along). This
change makes it possible to revert a hack to build libclang manually and
instead uses the libclang static library like on all other operating
systems, simplifying the Makefile.

Author: aykevl

.circleci/config.yml

@@ -68,12 +68,12 @@ commands:
     steps:
       - restore_cache:
           keys:
-            - llvm-source-10-v1
+            - llvm-source-11-v1
       - run:
           name: "Fetch LLVM source"
           command: make llvm-source
       - save_cache:
-          key: llvm-source-10-v1
+          key: llvm-source-11-v1
           paths:
             - llvm-project
   build-wasi-libc:
@@ -153,25 +153,22 @@ commands:
       - llvm-source-linux
       - restore_cache:
           keys:
-            - llvm-build-10-linux-v1-assert
+            - llvm-build-11-linux-v1-assert
       - run:
           name: "Build LLVM"
           command: |
             if [ ! -f llvm-build/lib/liblldELF.a ]
             then
               # install dependencies
-              sudo apt-get install cmake clang ninja-build
-              # make build faster
-              export CC=clang
-              export CXX=clang++
+              sudo apt-get install cmake ninja-build
               # hack ninja to use less jobs
               echo -e '#!/bin/sh\n/usr/bin/ninja -j3 "$@"' > /go/bin/ninja
               chmod +x /go/bin/ninja
               # build!
               make ASSERT=1 llvm-build
             fi
       - save_cache:
-          key: llvm-build-10-linux-v1-assert
+          key: llvm-build-11-linux-v1-assert
           paths:
             llvm-build
       - run: make ASSERT=1
@@ -214,25 +211,22 @@ commands:
       - llvm-source-linux
       - restore_cache:
           keys:
-            - llvm-build-10-linux-v1
+            - llvm-build-11-linux-v1-noassert
       - run:
           name: "Build LLVM"
           command: |
             if [ ! -f llvm-build/lib/liblldELF.a ]
             then
               # install dependencies
-              sudo apt-get install cmake clang ninja-build
-              # make build faster
-              export CC=clang
-              export CXX=clang++
+              sudo apt-get install cmake ninja-build
               # hack ninja to use less jobs
               echo -e '#!/bin/sh\n/usr/bin/ninja -j3 "$@"' > /go/bin/ninja
               chmod +x /go/bin/ninja
               # build!
               make llvm-build
             fi
       - save_cache:
-          key: llvm-build-10-linux-v1
+          key: llvm-build-11-linux-v1-noassert
           paths:
             llvm-build
       - build-wasi-libc
@@ -286,17 +280,17 @@ commands:
             - go-cache-macos-v2-{{ checksum "go.mod" }}
       - restore_cache:
           keys:
-            - llvm-source-10-macos-v1
+            - llvm-source-11-macos-v1
       - run:
           name: "Fetch LLVM source"
           command: make llvm-source
       - save_cache:
-          key: llvm-source-10-macos-v1
+          key: llvm-source-11-macos-v1
           paths:
             - llvm-project
       - restore_cache:
           keys:
-            - llvm-build-10-macos-v1
+            - llvm-build-11-macos-v1
       - run:
           name: "Build LLVM"
           command: |
@@ -308,7 +302,7 @@ commands:
               make llvm-build
             fi
       - save_cache:
-          key: llvm-build-10-macos-v1
+          key: llvm-build-11-macos-v1
           paths:
             llvm-build
       - restore_cache:

Makefile

@@ -48,23 +48,13 @@ ifeq ($(OS),Windows_NT)
     # LLVM compiled using MinGW on Windows appears to have problems with threads.
     # Without this flag, linking results in errors like these:
     #     libLLVMSupport.a(Threading.cpp.obj):Threading.cpp:(.text+0x55): undefined reference to `std::thread::hardware_concurrency()'
-    LLVM_OPTION += -DLLVM_ENABLE_THREADS=OFF
+    LLVM_OPTION += -DLLVM_ENABLE_THREADS=OFF -DLLVM_ENABLE_PIC=OFF
 
+    CGO_CPPFLAGS += -DCINDEX_NO_EXPORTS
     CGO_LDFLAGS += -static -static-libgcc -static-libstdc++
     CGO_LDFLAGS_EXTRA += -lversion
 
-    # Build libclang manually because the CMake-based build system on Windows
-    # doesn't allow building libclang as a static library.
-    LIBCLANG_PATH = $(abspath build/libclang-custom.a)
-    LIBCLANG_FILES = $(abspath $(wildcard $(LLVM_BUILDDIR)/tools/clang/tools/libclang/CMakeFiles/libclang.dir/*.cpp.obj))
-
-    # Add the libclang dependency to the tinygo binary target.
-tinygo: $(LIBCLANG_PATH)
-test: $(LIBCLANG_PATH)
-    # Build libclang.
-$(LIBCLANG_PATH): $(LIBCLANG_FILES)
-	@mkdir -p build
-	ar rcs $(LIBCLANG_PATH) $^
+    LIBCLANG_PATH = $(abspath $(LLVM_BUILDDIR))/lib/liblibclang.a
 
 else ifeq ($(shell uname -s),Darwin)
     MD5SUM = md5
@@ -87,7 +77,7 @@ LLD_LIBS = $(START_GROUP) -llldCOFF -llldCommon -llldCore -llldDriver -llldELF -
 
 # For static linking.
 ifneq ("$(wildcard $(LLVM_BUILDDIR)/bin/llvm-config*)","")
-    CGO_CPPFLAGS=$(shell $(LLVM_BUILDDIR)/bin/llvm-config --cppflags) -I$(abspath $(LLVM_BUILDDIR))/tools/clang/include -I$(abspath $(CLANG_SRC))/include -I$(abspath $(LLD_SRC))/include
+    CGO_CPPFLAGS+=$(shell $(LLVM_BUILDDIR)/bin/llvm-config --cppflags) -I$(abspath $(LLVM_BUILDDIR))/tools/clang/include -I$(abspath $(CLANG_SRC))/include -I$(abspath $(LLD_SRC))/include
     CGO_CXXFLAGS=-std=c++14
     CGO_LDFLAGS+=$(LIBCLANG_PATH) -L$(abspath $(LLVM_BUILDDIR)/lib) $(CLANG_LIBS) $(LLD_LIBS) $(shell $(LLVM_BUILDDIR)/bin/llvm-config --ldflags --libs --system-libs $(LLVM_COMPONENTS)) -lstdc++ $(CGO_LDFLAGS_EXTRA)
 endif
@@ -146,7 +136,7 @@ gen-device-stm32: build/gen-device-svd
 
 # Get LLVM sources.
 $(LLVM_PROJECTDIR)/README.md:
-	git clone -b xtensa_release_10.0.1 --depth=1 https://github.com/tinygo-org/llvm-project $(LLVM_PROJECTDIR)
+	git clone -b xtensa_release_11.0.0 --depth=1 https://github.com/tinygo-org/llvm-project $(LLVM_PROJECTDIR)
 llvm-source: $(LLVM_PROJECTDIR)/README.md
 
 # Configure LLVM.

README.md

@@ -148,6 +148,6 @@ The original reasoning was: if [Python](https://micropython.org/) can run on mic
 
 This project is licensed under the BSD 3-clause license, just like the [Go project](https://golang.org/LICENSE) itself.
 
-Some code has been copied from the LLVM project and is therefore licensed under [a variant of the Apache 2.0 license](http://releases.llvm.org/10.0.0/LICENSE.TXT). This has been clearly indicated in the header of these files.
+Some code has been copied from the LLVM project and is therefore licensed under [a variant of the Apache 2.0 license](http://releases.llvm.org/11.0.0/LICENSE.TXT). This has been clearly indicated in the header of these files.
 
 Some code has been copied and/or ported from Paul Stoffregen's Teensy libraries and is therefore licensed under PJRC's license. This has been clearly indicated in the header of these files.

azure-pipelines.yml

@@ -18,7 +18,7 @@ jobs:
     - task: Cache@2
       displayName: Cache LLVM source
       inputs:
-        key: llvm-source-10-windows-v1
+        key: llvm-source-11-windows-v1
         path: llvm-project
     - task: Bash@3
       displayName: Download LLVM source
@@ -32,7 +32,7 @@ jobs:
     - task: CacheBeta@0
       displayName: Cache LLVM build
       inputs:
-        key: llvm-build-10-windows-v1
+        key: llvm-build-11-windows-v3
         path: llvm-build
     - task: Bash@3
       displayName: Build LLVM
@@ -41,7 +41,11 @@ jobs:
         script: |
           if [ ! -f llvm-build/lib/liblldELF.a ]
           then
+            # install dependencies
             choco install ninja
+            # hack ninja to use fewer jobs
+            echo -e 'C:\\ProgramData\\Chocolatey\\bin\\ninja -j4 %*' > /usr/bin/ninja.bat
+            # build!
             make llvm-build
           fi
     - task: Bash@3

builder/cc1as.cpp

@@ -101,7 +101,7 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
 
   // Target Options
   Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple));
-  Opts.CPU = Args.getLastArgValue(OPT_target_cpu);
+  Opts.CPU = std::string(Args.getLastArgValue(OPT_target_cpu));
   Opts.Features = Args.getAllArgValues(OPT_target_feature);
 
   // Use the default target triple if unspecified.
@@ -132,13 +132,19 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
 
   Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations);
   Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags);
-  Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags);
-  Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer);
-  Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir);
-  Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name);
-
-  for (const auto &Arg : Args.getAllArgValues(OPT_fdebug_prefix_map_EQ))
-    Opts.DebugPrefixMap.insert(StringRef(Arg).split('='));
+  Opts.DwarfDebugFlags =
+      std::string(Args.getLastArgValue(OPT_dwarf_debug_flags));
+  Opts.DwarfDebugProducer =
+      std::string(Args.getLastArgValue(OPT_dwarf_debug_producer));
+  Opts.DebugCompilationDir =
+      std::string(Args.getLastArgValue(OPT_fdebug_compilation_dir));
+  Opts.MainFileName = std::string(Args.getLastArgValue(OPT_main_file_name));
+
+  for (const auto &Arg : Args.getAllArgValues(OPT_fdebug_prefix_map_EQ)) {
+    auto Split = StringRef(Arg).split('=');
+    Opts.DebugPrefixMap.insert(
+        {std::string(Split.first), std::string(Split.second)});
+  }
 
   // Frontend Options
   if (Args.hasArg(OPT_INPUT)) {
@@ -154,8 +160,9 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
     }
   }
   Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm);
-  Opts.OutputPath = Args.getLastArgValue(OPT_o);
-  Opts.SplitDwarfOutput = Args.getLastArgValue(OPT_split_dwarf_output);
+  Opts.OutputPath = std::string(Args.getLastArgValue(OPT_o));
+  Opts.SplitDwarfOutput =
+      std::string(Args.getLastArgValue(OPT_split_dwarf_output));
   if (Arg *A = Args.getLastArg(OPT_filetype)) {
     StringRef Name = A->getValue();
     unsigned OutputType = StringSwitch<unsigned>(Name)
@@ -183,8 +190,9 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
   Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack);
   Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings);
   Opts.NoWarn = Args.hasArg(OPT_massembler_no_warn);
-  Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic");
-  Opts.TargetABI = Args.getLastArgValue(OPT_target_abi);
+  Opts.RelocationModel =
+      std::string(Args.getLastArgValue(OPT_mrelocation_model, "pic"));
+  Opts.TargetABI = std::string(Args.getLastArgValue(OPT_target_abi));
   Opts.IncrementalLinkerCompatible =
       Args.hasArg(OPT_mincremental_linker_compatible);
   Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym);
@@ -314,12 +322,7 @@ bool ExecuteAssembler(AssemblerInvocation &Opts, DiagnosticsEngine &Diags) {
                             SrcMgr.getMemoryBuffer(BufferIndex)->getBuffer());
 
   // Build up the feature string from the target feature list.
-  std::string FS;
-  if (!Opts.Features.empty()) {
-    FS = Opts.Features[0];
-    for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i)
-      FS += "," + Opts.Features[i];
-  }
+  std::string FS = llvm::join(Opts.Features, ",");
 
   std::unique_ptr<MCStreamer> Str;
 
@@ -383,7 +386,7 @@ bool ExecuteAssembler(AssemblerInvocation &Opts, DiagnosticsEngine &Diags) {
     MCSection *AsmLabel = Ctx.getMachOSection(
         "__LLVM", "__asm", MachO::S_REGULAR, 4, SectionKind::getReadOnly());
     Str.get()->SwitchSection(AsmLabel);
-    Str.get()->EmitZeros(1);
+    Str.get()->emitZeros(1);
   }
 
   // Assembly to object compilation should leverage assembly info.

builder/clang.cpp

@@ -11,6 +11,7 @@
 #include <clang/FrontendTool/Utils.h>
 #include <llvm/ADT/IntrusiveRefCntPtr.h>
 #include <llvm/Option/Option.h>
+#include <llvm/Support/Host.h>
 
 using namespace llvm;
 using namespace clang;