Skip to content

Commit 6b5c342

Browse files
Neko-Box-CoderNeko-Box-Coder
authored
Merge pull request #43 from Neko-Box-Coder/MultiThreadProcessing
Multi thread processing
2 parents c8cc424 + 04b67ba commit 6b5c342

File tree

14 files changed

+555
-187
lines changed

14 files changed

+555
-187
lines changed

‎CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@ target_link_libraries(runcpp2 PRIVATE ssLogger System2 CppOverride dylib)
157157
target_link_libraries(runcpp2 PUBLIC ghc_filesystem ryml::ryml mpark_variant)
158158

159159
if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
160+
# TODO: Try to change to /Wall
160161
set(STANDARD_COMPILE_FLAGS "/utf-8;/W1;/DGHC_WIN_DISABLE_WSTRING_STORAGE_TYPE=1")
161162
else()
162163
set(STANDARD_COMPILE_FLAGS "-Wall"

‎Include/runcpp2/CompilingLinking.hpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ namespace runcpp2
2222
const Data::ScriptInfo& scriptInfo,
2323
const std::vector<Data::DependencyInfo*>& availableDependencies,
2424
const Data::Profile& profile,
25-
bool buildExecutable);
25+
bool buildExecutable,
26+
const int maxThreads);
2627

2728
//TODO: Convert string paths to filesystem paths
2829
bool CompileAndLinkScript( const ghc::filesystem::path& buildDir,
@@ -35,7 +36,8 @@ namespace runcpp2
3536
const std::vector<Data::DependencyInfo*>& availableDependencies,
3637
const Data::Profile& profile,
3738
const std::vector<std::string>& compiledObjectsPaths,
38-
bool buildExecutable);
39+
bool buildExecutable,
40+
const int maxThreads);
3941
}
4042

4143
#endif

‎Include/runcpp2/Data/CmdOptions.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ namespace runcpp2
2121
CONFIG_FILE,
2222
CLEANUP,
2323
BUILD_SOURCE_ONLY,
24+
THREADS,
2425
COUNT
2526
};
2627
}

‎Include/runcpp2/DependenciesHelper.hpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,14 @@ namespace runcpp2
3333
const Data::ScriptInfo& scriptInfo,
3434
std::vector<Data::DependencyInfo*>& availableDependencies,
3535
const std::vector<std::string>& dependenciesLocalCopiesPaths,
36-
const std::vector<std::string>& dependenciesSourcePaths);
36+
const std::vector<std::string>& dependenciesSourcePaths,
37+
const int maxThreads);
3738

3839
bool BuildDependencies( const runcpp2::Data::Profile& profile,
3940
const Data::ScriptInfo& scriptInfo,
4041
const std::vector<Data::DependencyInfo*>& availableDependencies,
41-
const std::vector<std::string>& dependenciesLocalCopiesPaths);
42+
const std::vector<std::string>& dependenciesLocalCopiesPaths,
43+
const int maxThreads);
4244

4345
bool GatherDependenciesBinaries(const std::vector<Data::DependencyInfo*>& availableDependencies,
4446
const std::vector<std::string>& dependenciesCopiesPaths,

‎Include/runcpp2/PipelineSteps.hpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ namespace runcpp2
6666
const Data::Profile& profile,
6767
const ghc::filesystem::path& absoluteScriptPath,
6868
const Data::ScriptInfo* lastScriptInfo,
69+
const int maxThreads,
6970
bool& outRecompileNeeded,
7071
bool& outRelinkNeeded,
7172
std::vector<std::string>& outChangedDependencies);
@@ -77,6 +78,7 @@ namespace runcpp2
7778
const ghc::filesystem::path& buildDir,
7879
const std::unordered_map<CmdOptions, std::string>& currentOptions,
7980
const std::vector<std::string>& changedDependencies,
81+
const int maxThreads,
8082
std::vector<Data::DependencyInfo*>& outAvailableDependencies,
8183
std::vector<std::string>& outGatheredBinariesPaths);
8284

‎Include/runcpp2/runcpp2.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ namespace runcpp2
3939
const std::unordered_map<CmdOptions, std::string> currentOptions,
4040
const std::vector<std::string>& runArgs,
4141
const Data::ScriptInfo* lastScriptInfo,
42-
Data::ScriptInfo& outScriptInfo,
4342
const std::string& buildOutputDir,
43+
Data::ScriptInfo& outScriptInfo,
4444
int& returnStatus);
4545

4646
std::string PipelineResultToString(PipelineResult result);

‎Src/runcpp2/CompilingLinking.cpp‎

Lines changed: 201 additions & 109 deletions
Large diffs are not rendered by default.

‎Src/runcpp2/DependenciesHelper.cpp‎

Lines changed: 131 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
#include "ssLogger/ssLog.hpp"
88

99
#include <unordered_set>
10+
#include <future>
11+
#include <chrono>
1012

1113
namespace
1214
{
@@ -489,7 +491,8 @@ bool runcpp2::SetupDependenciesIfNeeded(const runcpp2::Data::Profile& profile,
489491
const Data::ScriptInfo& scriptInfo,
490492
std::vector<Data::DependencyInfo*>& availableDependencies,
491493
const std::vector<std::string>& dependenciesLocalCopiesPaths,
492-
const std::vector<std::string>& dependenciesSourcePaths)
494+
const std::vector<std::string>& dependenciesSourcePaths,
495+
const int maxThreads)
493496
{
494497
ssLOG_FUNC_INFO();
495498

@@ -510,9 +513,13 @@ bool runcpp2::SetupDependenciesIfNeeded(const runcpp2::Data::Profile& profile,
510513
}
511514

512515
if(!PopulateAbsoluteIncludePaths(availableDependencies, dependenciesLocalCopiesPaths))
513-
{
514516
return false;
515-
}
517+
518+
std::vector<std::future<bool>> actions;
519+
520+
//Cache logs for worker threads
521+
ssLOG_ENABLE_CACHE_OUTPUT_FOR_NEW_THREADS();
522+
int logLevel = ssLOG_GET_CURRENT_THREAD_TARGET_LEVEL();
516523

517524
//Run setup steps
518525
for(int i = 0; i < availableDependencies.size(); ++i)
@@ -524,46 +531,150 @@ bool runcpp2::SetupDependenciesIfNeeded(const runcpp2::Data::Profile& profile,
524531
continue;
525532
}
526533

527-
ssLOG_INFO("Running setup commands for " << availableDependencies.at(i)->Name);
528-
if(!RunDependenciesSteps( profile,
529-
availableDependencies.at(i)->Setup,
530-
dependenciesLocalCopiesPaths.at(i),
531-
true))
534+
actions.emplace_back
535+
(
536+
std::async
537+
(
538+
std::launch::async,
539+
[i, &profile, &availableDependencies, &dependenciesLocalCopiesPaths, logLevel]()
540+
{
541+
ssLOG_SET_CURRENT_THREAD_TARGET_LEVEL(logLevel);
542+
543+
ssLOG_INFO("Running setup commands for " << availableDependencies.at(i)->Name);
544+
if(!RunDependenciesSteps( profile,
545+
availableDependencies.at(i)->Setup,
546+
dependenciesLocalCopiesPaths.at(i),
547+
true))
548+
{
549+
ssLOG_ERROR("Failed to setup dependency " <<
550+
availableDependencies.at(i)->Name);
551+
return false;
552+
}
553+
return true;
554+
}
555+
)
556+
);
557+
558+
//Evaluate the setup results for each batch
559+
if(actions.size() >= maxThreads || i == availableDependencies.size() - 1)
532560
{
533-
ssLOG_ERROR("Failed to setup dependency " << availableDependencies.at(i)->Name);
534-
return false;
561+
std::chrono::system_clock::time_point deadline =
562+
std::chrono::system_clock::now() + std::chrono::seconds(60);
563+
for(int j = 0; j < actions.size(); ++j)
564+
{
565+
if(!actions.at(j).valid())
566+
{
567+
ssLOG_ERROR("Failed to construct actions for setup");
568+
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
569+
return false;
570+
}
571+
572+
std::future_status actionStatus = actions.at(j).wait_until(deadline);
573+
if(actionStatus == std::future_status::ready)
574+
{
575+
if(!actions.at(j).get())
576+
{
577+
ssLOG_ERROR("Setup failed for dependencies");
578+
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
579+
return false;
580+
}
581+
}
582+
else
583+
{
584+
ssLOG_ERROR("Dependencies setup timeout");
585+
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
586+
return false;
587+
}
588+
}
589+
actions.clear();
535590
}
536591
}
537-
592+
593+
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
538594
return true;
539595
}
540596

541597
bool runcpp2::BuildDependencies(const runcpp2::Data::Profile& profile,
542598
const Data::ScriptInfo& scriptInfo,
543599
const std::vector<Data::DependencyInfo*>& availableDependencies,
544-
const std::vector<std::string>& dependenciesLocalCopiesPaths)
600+
const std::vector<std::string>& dependenciesLocalCopiesPaths,
601+
const int maxThreads)
545602
{
546603
ssLOG_FUNC_INFO();
547604

548605
//If the script info is not populated (i.e. empty script info), don't do anything
549606
if(!scriptInfo.Populated)
550607
return true;
551-
608+
609+
std::vector<std::future<bool>> actions;
610+
611+
//Cache logs for worker threads
612+
ssLOG_ENABLE_CACHE_OUTPUT_FOR_NEW_THREADS();
613+
int logLevel = ssLOG_GET_CURRENT_THREAD_TARGET_LEVEL();
614+
552615
//Run build steps
553616
for(int i = 0; i < availableDependencies.size(); ++i)
554617
{
555618
ssLOG_INFO("Running build commands for " << availableDependencies.at(i)->Name);
556619

557-
if(!RunDependenciesSteps( profile,
558-
availableDependencies.at(i)->Build,
559-
dependenciesLocalCopiesPaths.at(i),
560-
true))
620+
actions.emplace_back
621+
(
622+
std::async
623+
(
624+
std::launch::async,
625+
[i, &profile, &availableDependencies, &dependenciesLocalCopiesPaths, logLevel]()
626+
{
627+
ssLOG_SET_CURRENT_THREAD_TARGET_LEVEL(logLevel);
628+
629+
if(!RunDependenciesSteps( profile,
630+
availableDependencies.at(i)->Build,
631+
dependenciesLocalCopiesPaths.at(i),
632+
true))
633+
{
634+
ssLOG_ERROR("Failed to build dependency " << availableDependencies.at(i)->Name);
635+
return false;
636+
}
637+
return true;
638+
}
639+
)
640+
);
641+
642+
//Evaluate the setup results for each batch
643+
if(actions.size() >= maxThreads || i == availableDependencies.size() - 1)
561644
{
562-
ssLOG_ERROR("Failed to build dependency " << availableDependencies.at(i)->Name);
563-
return false;
645+
std::chrono::system_clock::time_point deadline =
646+
std::chrono::system_clock::now() + std::chrono::seconds(60);
647+
for(int j = 0; j < actions.size(); ++j)
648+
{
649+
if(!actions.at(j).valid())
650+
{
651+
ssLOG_ERROR("Failed to construct actions for building dependencies");
652+
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
653+
return false;
654+
}
655+
656+
std::future_status actionStatus = actions.at(j).wait_until(deadline);
657+
if(actionStatus == std::future_status::ready)
658+
{
659+
if(!actions.at(j).get())
660+
{
661+
ssLOG_ERROR("Build failed for dependencies");
662+
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
663+
return false;
664+
}
665+
}
666+
else
667+
{
668+
ssLOG_ERROR("Dependencies build timeout");
669+
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
670+
return false;
671+
}
672+
}
673+
actions.clear();
564674
}
565675
}
566676

677+
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
567678
return true;
568679
}
569680

@@ -875,7 +986,6 @@ bool runcpp2::ResolveImports( Data::ScriptInfo& scriptInfo,
875986
INTERNAL_RUNCPP2_SAFE_START();
876987

877988
//For each dependency, check if import path exists
878-
//for(Data::DependencyInfo& dependency : scriptInfo.Dependencies)
879989
for(int i = 0; i < scriptInfo.Dependencies.size(); ++i)
880990
{
881991
Data::DependencyInfo& dependency = scriptInfo.Dependencies.at(i);
@@ -884,7 +994,7 @@ bool runcpp2::ResolveImports( Data::ScriptInfo& scriptInfo,
884994
Data::DependencySource& source = dependency.Source;
885995
if(source.ImportPath.empty())
886996
continue;
887-
997+
888998
if(!source.ImportPath.is_relative())
889999
{
8901000
ssLOG_ERROR("Import path is not relative: " << source.ImportPath.string());

‎Src/runcpp2/PipelineSteps.cpp‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -524,6 +524,7 @@ runcpp2::CheckScriptInfoChanges(const ghc::filesystem::path& buildDir,
524524
const Data::Profile& profile,
525525
const ghc::filesystem::path& absoluteScriptPath,
526526
const Data::ScriptInfo* lastScriptInfo,
527+
const int maxThreads,
527528
bool& outRecompileNeeded,
528529
bool& outRelinkNeeded,
529530
std::vector<std::string>& outChangedDependencies)
@@ -731,6 +732,7 @@ runcpp2::ProcessDependencies( Data::ScriptInfo& scriptInfo,
731732
const ghc::filesystem::path& buildDir,
732733
const std::unordered_map<CmdOptions, std::string>& currentOptions,
733734
const std::vector<std::string>& changedDependencies,
735+
const int maxThreads,
734736
std::vector<Data::DependencyInfo*>& outAvailableDependencies,
735737
std::vector<std::string>& outGatheredBinariesPaths)
736738
{
@@ -796,7 +798,8 @@ runcpp2::ProcessDependencies( Data::ScriptInfo& scriptInfo,
796798
scriptInfo,
797799
outAvailableDependencies,
798800
dependenciesLocalCopiesPaths,
799-
dependenciesSourcePaths))
801+
dependenciesSourcePaths,
802+
maxThreads))
800803
{
801804
ssLOG_ERROR("Failed to setup script dependencies");
802805
return PipelineResult::DEPENDENCIES_FAILED;
@@ -816,7 +819,8 @@ runcpp2::ProcessDependencies( Data::ScriptInfo& scriptInfo,
816819
if(!BuildDependencies( profile,
817820
scriptInfo,
818821
outAvailableDependencies,
819-
dependenciesLocalCopiesPaths))
822+
dependenciesLocalCopiesPaths,
823+
maxThreads))
820824
{
821825
ssLOG_ERROR("Failed to build script dependencies");
822826
return PipelineResult::DEPENDENCIES_FAILED;

0 commit comments

Comments
 (0)