Merge pull request #6784 from xmake-io/opti

Continue to optimize building targets speed

Author: waruqi

tests/apis/custom_toolchain/xmake/modules/core/tools/cl6x/has_flags.lua

@@ -60,7 +60,6 @@ function _check_from_arglist(flags, opt, islinker)
             end
         end
         detectcache:set2(key, flagskey, allflags)
-        detectcache:save()
     end
     local flag = flags[1]
     return allflags[flag]

tests/benchmarks/build_targets/test.lua

@@ -7,10 +7,13 @@ function test_build(t)
         return
     end
 
+    local jobs = tostring(os.default_njob())
+
     -- xmake
     os.tryrm("build")
+    os.tryrm(".xmake")
     local xmake_dt = os.mclock()
-    os.run("xmake")
+    os.runv("xmake", {"-j" .. jobs})
     xmake_dt = os.mclock() - xmake_dt
     print("build targets/30: xmake: %d ms", xmake_dt)
 
@@ -21,10 +24,10 @@ function test_build(t)
         os.mkdir("build")
         local cmake_default_dt = os.mclock()
         os.runv(cmake.program, {".."}, {curdir = "build"})
-        os.runv(cmake.program, {"--build", "."}, {curdir = "build"})
+        os.runv(cmake.program, {"--build", ".", "-j" .. jobs}, {curdir = "build"})
         cmake_default_dt = os.mclock() - cmake_default_dt
         print("build targets/30: cmake/default: %d ms", cmake_default_dt)
-        --t:require((cmake_default_dt > xmake_dt) or (cmake_default_dt + 2000 > xmake_dt))
+        t:require((cmake_default_dt > xmake_dt) or (cmake_default_dt + 2000 > xmake_dt))
 
         local ninja = find_tool("ninja")
         if ninja then
@@ -43,11 +46,10 @@ function test_build(t)
             end
             local cmake_ninja_dt = os.mclock()
             os.runv(cmake.program, table.join("..", "-G", "Ninja", configs), {curdir = "build", envs = envs})
-            os.runv(cmake.program, {"--build", "."}, {curdir = "build", envs = envs})
+            os.runv(cmake.program, {"--build", ".", "-j" .. jobs}, {curdir = "build", envs = envs})
             cmake_ninja_dt = os.mclock() - cmake_ninja_dt
             print("build targets/30: cmake/ninja: %d ms", cmake_ninja_dt)
-            -- TODO we need to improve it
-            --t:require((cmake_ninja_dt > xmake_dt) or (cmake_ninja_dt + 2000 > xmake_dt))
+            t:require((cmake_ninja_dt > xmake_dt) or (cmake_ninja_dt + 2000 > xmake_dt))
         end
 
         local make = find_tool("make")
@@ -56,7 +58,7 @@ function test_build(t)
             os.mkdir("build")
             local cmake_makefile_dt = os.mclock()
             os.runv(cmake.program, {"..", "-G", "Unix Makefiles"}, {curdir = "build"})
-            os.runv(cmake.program, {"--build", "."}, {curdir = "build"})
+            os.runv(cmake.program, {"--build", ".", "-j" .. jobs}, {curdir = "build"})
             cmake_makefile_dt = os.mclock() - cmake_makefile_dt
             print("build targets/30: cmake/makefile: %d ms", cmake_makefile_dt)
             t:require((cmake_makefile_dt > xmake_dt) or (cmake_makefile_dt + 2000 > xmake_dt))
@@ -67,12 +69,19 @@ function test_build(t)
     local meson = find_tool("meson")
     if meson then
         os.tryrm("build")
-        local meson_dt = os.mclock()
+        local meson_setup_dt = os.mclock()
         os.runv(meson.program, {"setup", "build"})
-        os.runv(meson.program, {"compile", "-C", "build"})
-        meson_dt = os.mclock() - meson_dt
+        meson_setup_dt = os.mclock() - meson_setup_dt
+
+        -- ccache will cache object files globally, which may affect the results of the second run.
+        io.replace("build/build.ninja", "ccache", "")
+
+        local meson_build_dt = os.mclock()
+        os.runv(meson.program, {"compile", "-j", jobs, "-C", "build"})
+        meson_build_dt = os.mclock() - meson_build_dt
+        local meson_dt = meson_setup_dt + meson_build_dt
         print("build targets/30: meson: %d ms", meson_dt)
-        --t:require((meson_dt > xmake_dt) or (meson_dt + 2000 > xmake_dt))
+        t:require((meson_dt > xmake_dt) or (meson_dt + 2000 > xmake_dt))
     end
 end
 

tests/benchmarks/config_targets/test.lua

@@ -9,6 +9,7 @@ function test_config(t)
 
     -- xmake
     os.tryrm("build")
+    os.tryrm(".xmake")
     local xmake_dt = os.mclock()
     os.runv("xmake", {"config", "-c"})
     xmake_dt = os.mclock() - xmake_dt

xmake/actions/build/main.lua

@@ -23,6 +23,7 @@ import("core.base.option")
 import("core.base.global")
 import("core.base.task")
 import("core.tool.toolchain")
+import("core.cache.detectcache")
 import("core.project.rule")
 import("core.project.config")
 import("core.project.project")
@@ -130,6 +131,9 @@ function build_targets(targetnames, opt)
             -- save toolchain configs
             toolchain.save()
 
+            -- save detect cache
+            detectcache:save()
+
             -- dump cache stats
             if option.get("diagnosis") then
                 build_cache.dump_stats()
@@ -148,6 +152,9 @@ function build_targets(targetnames, opt)
                 -- save toolchain configs
                 toolchain.save()
 
+                -- save detect cache
+                detectcache:save()
+
                 -- raise
                 if errors then
                     raise(errors)

xmake/actions/config/main.lua

@@ -28,6 +28,7 @@ import("core.project.project")
 import("core.platform.platform")
 import("private.detect.find_platform")
 import("core.cache.localcache")
+import("core.cache.detectcache")
 import("scangen")
 import("menuconf", {alias = "menuconf_show"})
 import("configfiles", {alias = "generate_configfiles"})
@@ -448,6 +449,9 @@ force to build in current directory via run `xmake -P .`]], os.projectdir())
     -- save toolchain cache
     toolchain.save()
 
+    -- save detect cache
+    detectcache:save()
+
     -- unlock the whole project
     project.unlock()
 end

xmake/core/sandbox/modules/import/lib/detect/find_program.lua

@@ -338,7 +338,6 @@ function sandbox_lib_detect_find_program.main(name, opt)
 
     -- cache result
     detectcache:set2(cachekey, name, result and result or false)
-    detectcache:save()
 
     -- trace
     if option.get("verbose") or opt.verbose then

xmake/core/sandbox/modules/import/lib/detect/find_programver.lua

@@ -109,7 +109,6 @@ function sandbox_lib_detect_find_programver.main(program, opt)
 
     -- save result
     detectcache:set2(cachekey, program, result and result or false)
-    detectcache:save()
     scheduler.co_unlock(lockname)
     return result
 end

xmake/core/tool/compiler.lua

@@ -122,17 +122,18 @@ function compiler.load(sourcekind, target)
         return nil, "unknown source kind!"
     end
 
-    -- load compiler tool first (with cache)
-    local compiler_tool, program_or_errors = compiler._load_tool(sourcekind, target)
-    if not compiler_tool then
-        return nil, program_or_errors
-    end
-
     -- init cache key
     -- @note we need plat/arch,
     -- because it is possible for the compiler to do cross-compilation with the -target parameter
-    local plat = compiler_tool:plat() or config.plat() or os.host()
-    local arch = compiler_tool:arch() or config.arch() or os.arch()
+    local plat = config.plat() or os.host()
+    local arch = config.arch() or os.arch()
+    if target and target.tool then
+        local _, _, toolchain_info = target:tool(sourcekind)
+        if toolchain_info then
+            plat = toolchain_info.plat
+            arch = toolchain_info.arch
+        end
+    end
     local cachekey = sourcekind .. (program_or_errors or "") .. plat .. arch
     if target then
         cachekey = cachekey .. tostring(target)
@@ -143,6 +144,13 @@ function compiler.load(sourcekind, target)
     local instance = compiler._INSTANCES[cachekey]
     if not instance then
         instance = table.inherit(compiler, builder)
+
+        -- load compiler tool
+        -- @NOTE We cannot cache the tool, otherwise it may cause duplicate toolchain flags to be added
+        local compiler_tool, program_or_errors = compiler._load_tool(sourcekind, target)
+        if not compiler_tool then
+            return nil, program_or_errors
+        end
         instance._TOOL = compiler_tool
 
         -- load the compiler language from the source kind
@@ -188,7 +196,7 @@ function compiler.load(sourcekind, target)
 
     -- we need to load it at the end because in tool.load().
     -- because we may need to call has_flags, which requires the full platform toolchain flags
-    local ok, errors = compiler_tool:_load_once()
+    local ok, errors = instance:_tool():_load_once()
     if not ok then
         return nil, errors
     end

xmake/core/tool/linker.lua

@@ -130,7 +130,8 @@ function linker.load(targetkind, sourcekinds, target)
         end
     end
 
-    -- load linker tool first (with cache)
+    -- load linker tool first, we need to get the correct plat/arch
+    -- @NOTE We cannot cache the tool, otherwise it may cause duplicate toolchain flags to be added
     local linkertool, linkerinfo_or_errors = linker._load_tool(targetkind, sourcekinds, target)
     if not linkertool then
         return nil, linkerinfo_or_errors
@@ -218,7 +219,7 @@ function linker.load(targetkind, sourcekinds, target)
 
     -- we need to load it at the end because in tool.load().
     -- because we may need to call has_flags, which requires the full platform toolchain flags
-    local ok, errors = linkertool:_load_once()
+    local ok, errors = instance:_tool():_load_once()
     if not ok then
         return nil, errors
     end

xmake/core/tool/tool.lua

@@ -241,22 +241,8 @@ function tool.load(kind, opt)
     local toolchain_info = opt.toolchain_info or {}
 
     -- get platform and architecture
-    local plat = toolchain_info.plat or config.get("plat") or os.host()
-    local arch = toolchain_info.arch or config.get("arch") or os.arch()
-
-    -- init cachekey
-    local cachekey = kind .. (program or "") .. plat .. arch .. (opt.host and "host" or "")
-    if toolchain_info and toolchain_info.cachekey then
-        -- it maybe contains target key
-        -- @see https://github.com/xmake-io/xmake/issues/6672
-        cachekey = cachekey .. toolchain_info.cachekey
-    end
-
-    -- get it directly from cache dirst
-    tool._TOOLS = tool._TOOLS or {}
-    if tool._TOOLS[cachekey] then
-        return tool._TOOLS[cachekey]
-    end
+    local plat = toolchain_info.plat or config.plat() or os.host()
+    local arch = toolchain_info.arch or config.arch() or os.arch()
 
     -- contain toolname? parse it, e.g. '[email protected]'
     if program then
@@ -310,7 +296,6 @@ function tool.load(kind, opt)
     if not instance then
         return nil, errors
     end
-    tool._TOOLS[cachekey] = instance
     return instance
 end