Skip to content

Commit f26cdf8

Browse files
Neko-Box-CoderNeko-Box-Coder
committed
Adding multi-thread support for compiling
1 parent c8cc424 commit f26cdf8

File tree

7 files changed

+290
-115
lines changed

7 files changed

+290
-115
lines changed

‎Include/runcpp2/CompilingLinking.hpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ namespace runcpp2
2222
const Data::ScriptInfo& scriptInfo,
2323
const std::vector<Data::DependencyInfo*>& availableDependencies,
2424
const Data::Profile& profile,
25-
bool buildExecutable);
25+
bool buildExecutable,
26+
const int maxThreads);
2627

2728
//TODO: Convert string paths to filesystem paths
2829
bool CompileAndLinkScript( const ghc::filesystem::path& buildDir,
@@ -35,7 +36,8 @@ namespace runcpp2
3536
const std::vector<Data::DependencyInfo*>& availableDependencies,
3637
const Data::Profile& profile,
3738
const std::vector<std::string>& compiledObjectsPaths,
38-
bool buildExecutable);
39+
bool buildExecutable,
40+
const int maxThreads);
3941
}
4042

4143
#endif

‎Include/runcpp2/DependenciesHelper.hpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,8 @@ namespace runcpp2
5050

5151
bool ResolveImports(Data::ScriptInfo& scriptInfo,
5252
const ghc::filesystem::path& scriptPath,
53-
const ghc::filesystem::path& buildDir);
53+
const ghc::filesystem::path& buildDir,
54+
const int maxThreads);
5455

5556
bool SyncLocalDependency( const Data::DependencyInfo& dependency,
5657
const ghc::filesystem::path& sourcePath,

‎Include/runcpp2/PipelineSteps.hpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,15 @@ namespace runcpp2
5959

6060
PipelineResult ResolveScriptImports(Data::ScriptInfo& scriptInfo,
6161
const ghc::filesystem::path& scriptPath,
62-
const ghc::filesystem::path& buildDir);
62+
const ghc::filesystem::path& buildDir,
63+
const int maxThreads);
6364

6465
PipelineResult CheckScriptInfoChanges( const ghc::filesystem::path& buildDir,
6566
const Data::ScriptInfo& scriptInfo,
6667
const Data::Profile& profile,
6768
const ghc::filesystem::path& absoluteScriptPath,
6869
const Data::ScriptInfo* lastScriptInfo,
70+
const int maxThreads,
6971
bool& outRecompileNeeded,
7072
bool& outRelinkNeeded,
7173
std::vector<std::string>& outChangedDependencies);

‎Src/runcpp2/CompilingLinking.cpp‎

Lines changed: 197 additions & 103 deletions
Large diffs are not rendered by default.

‎Src/runcpp2/DependenciesHelper.cpp‎

Lines changed: 70 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,14 @@ namespace
157157

158158
outPrePopulated.resize(dependencies.size());
159159

160+
//std::vector<std::thread> compileActions;
161+
//std::vector<bool> runResults(sourceFiles.size(), false);
162+
//int doneCount = 0;
163+
//std::mutex resultMutex;
164+
//std::condition_variable resultCV;
165+
166+
167+
//TODO(NOW): Multi-thread?
160168
for(int i = 0; i < dependencies.size(); ++i)
161169
{
162170
if(!dependencies.at(i)->Source.ImportPath.empty())
@@ -514,6 +522,7 @@ bool runcpp2::SetupDependenciesIfNeeded(const runcpp2::Data::Profile& profile,
514522
return false;
515523
}
516524

525+
//TODO(NOW): Multi-thread?
517526
//Run setup steps
518527
for(int i = 0; i < availableDependencies.size(); ++i)
519528
{
@@ -549,6 +558,7 @@ bool runcpp2::BuildDependencies(const runcpp2::Data::Profile& profile,
549558
if(!scriptInfo.Populated)
550559
return true;
551560

561+
//TODO(NOW): Multi-thread?
552562
//Run build steps
553563
for(int i = 0; i < availableDependencies.size(); ++i)
554564
{
@@ -869,13 +879,29 @@ bool runcpp2::HandleImport( Data::DependencyInfo& dependency,
869879

870880
bool runcpp2::ResolveImports( Data::ScriptInfo& scriptInfo,
871881
const ghc::filesystem::path& scriptPath,
872-
const ghc::filesystem::path& buildDir)
882+
const ghc::filesystem::path& buildDir,
883+
const int maxThreads)
873884
{
874885
ssLOG_FUNC_INFO();
875886
INTERNAL_RUNCPP2_SAFE_START();
876887

888+
//TODO(NOW): Multi-thread this
889+
890+
891+
//std::vector<std::thread> importActions;
892+
//std::vector<bool> dispatched(scriptInfo.Dependencies.size(), false);
893+
//std::vector<bool> runResults(scriptInfo.Dependencies.size(), false);
894+
//int doneCount = 0;
895+
//std::mutex resultMutex;
896+
//std::condition_variable resultCV;
897+
(void)maxThreads;
898+
899+
900+
901+
902+
903+
877904
//For each dependency, check if import path exists
878-
//for(Data::DependencyInfo& dependency : scriptInfo.Dependencies)
879905
for(int i = 0; i < scriptInfo.Dependencies.size(); ++i)
880906
{
881907
Data::DependencyInfo& dependency = scriptInfo.Dependencies.at(i);
@@ -885,6 +911,10 @@ bool runcpp2::ResolveImports( Data::ScriptInfo& scriptInfo,
885911
if(source.ImportPath.empty())
886912
continue;
887913

914+
915+
916+
917+
888918
if(!source.ImportPath.is_relative())
889919
{
890920
ssLOG_ERROR("Import path is not relative: " << source.ImportPath.string());
@@ -909,6 +939,44 @@ bool runcpp2::ResolveImports( Data::ScriptInfo& scriptInfo,
909939
if(!dependency.Source.ImportPath.empty())
910940
--i;
911941
}
942+
943+
944+
//Evaluate the compile results for each batch for compilations
945+
946+
//if(i - startIndex >= maxThreads || i == sourceFiles.size() - 1)
947+
//{
948+
// std::unique_lock<std::mutex> lk(resultMutex);
949+
// resultCV.wait_for
950+
// (
951+
// lk,
952+
// std::chrono::seconds(maxThreads)
953+
// [&doneCount, &sourceFiles, &startIndex, &maxThreads]()
954+
// {
955+
// return doneCount - startIndex >= maxThreads ||
956+
// doneCount == sourceFiles.size();
957+
// }
958+
// );
959+
//
960+
// ssLOG_OUTPUT_ALL_CACHE_GROUPED();
961+
//
962+
// //Check if all threads have finished the work
963+
// if(doneCount - startIndex < maxThreads && doneCount != sourceFiles.size())
964+
// {
965+
// ssLOG_ERROR("Compilation workers timed out...");
966+
// return false;
967+
// }
968+
//
969+
// //Check if all the compilations are successful or not
970+
// for(int j = 0; j < runResults.size(); ++j)
971+
// {
972+
// if(!runResults.at(j + startIndex))
973+
// return false;
974+
// }
975+
//
976+
// //Update the start index
977+
// startIndex = i + 1;
978+
//}
979+
912980

913981
return true;
914982

‎Src/runcpp2/PipelineSteps.cpp‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -501,13 +501,14 @@ runcpp2::InitializeBuildDirectory( const ghc::filesystem::path& configDir,
501501
runcpp2::PipelineResult
502502
runcpp2::ResolveScriptImports( Data::ScriptInfo& scriptInfo,
503503
const ghc::filesystem::path& scriptPath,
504-
const ghc::filesystem::path& buildDir)
504+
const ghc::filesystem::path& buildDir,
505+
const int maxThreads)
505506
{
506507
ssLOG_FUNC_INFO();
507508
INTERNAL_RUNCPP2_SAFE_START();
508509

509510
//Resolve all the script info imports first before evaluating it
510-
if(!ResolveImports(scriptInfo, scriptPath, buildDir))
511+
if(!ResolveImports(scriptInfo, scriptPath, buildDir, maxThreads))
511512
{
512513
ssLOG_ERROR("Failed to resolve imports");
513514
return PipelineResult::UNEXPECTED_FAILURE;
@@ -524,6 +525,7 @@ runcpp2::CheckScriptInfoChanges(const ghc::filesystem::path& buildDir,
524525
const Data::Profile& profile,
525526
const ghc::filesystem::path& absoluteScriptPath,
526527
const Data::ScriptInfo* lastScriptInfo,
528+
const int maxThreads,
527529
bool& outRecompileNeeded,
528530
bool& outRelinkNeeded,
529531
std::vector<std::string>& outChangedDependencies)
@@ -578,7 +580,8 @@ runcpp2::CheckScriptInfoChanges(const ghc::filesystem::path& buildDir,
578580
//Resolve imports for last script info
579581
runcpp2::PipelineResult result = ResolveScriptImports( lastScriptInfoFromDisk,
580582
absoluteScriptPath,
581-
buildDir);
583+
buildDir,
584+
maxThreads);
582585
if(result != PipelineResult::SUCCESS)
583586
break;
584587

‎Src/runcpp2/runcpp2.cpp‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -464,6 +464,8 @@ runcpp2::StartPipeline( const std::string& scriptPath,
464464
//Parsing the script, setting up dependencies, compiling and linking
465465
std::vector<std::string> filesToCopyPaths;
466466
{
467+
const int tempMaxThreads = 16;
468+
467469
BuildsManager buildsManager("/tmp");
468470
IncludeManager includeManager;
469471
PipelineResult result =
@@ -489,7 +491,7 @@ runcpp2::StartPipeline( const std::string& scriptPath,
489491
}
490492

491493
//Resolve imports
492-
result = ResolveScriptImports(scriptInfo, absoluteScriptPath, buildDir);
494+
result = ResolveScriptImports(scriptInfo, absoluteScriptPath, buildDir, tempMaxThreads);
493495
if(result != PipelineResult::SUCCESS)
494496
return result;
495497

@@ -503,6 +505,7 @@ runcpp2::StartPipeline( const std::string& scriptPath,
503505
profiles.at(profileIndex),
504506
absoluteScriptPath,
505507
lastScriptInfo,
508+
tempMaxThreads,
506509
recompileNeeded,
507510
relinkNeeded,
508511
changedDependencies);
@@ -662,7 +665,8 @@ runcpp2::StartPipeline( const std::string& scriptPath,
662665
scriptInfo,
663666
availableDependencies,
664667
profiles.at(profileIndex),
665-
currentOptions.count(CmdOptions::EXECUTABLE) > 0))
668+
currentOptions.count(CmdOptions::EXECUTABLE) > 0,
669+
tempMaxThreads))
666670
{
667671
return PipelineResult::COMPILE_LINK_FAILED;
668672
}
@@ -679,7 +683,8 @@ runcpp2::StartPipeline( const std::string& scriptPath,
679683
availableDependencies,
680684
profiles.at(profileIndex),
681685
linkFilesPaths,
682-
currentOptions.count(CmdOptions::EXECUTABLE) > 0))
686+
currentOptions.count(CmdOptions::EXECUTABLE) > 0,
687+
tempMaxThreads))
683688
{
684689
ssLOG_ERROR("Failed to compile or link script");
685690
return PipelineResult::COMPILE_LINK_FAILED;

0 commit comments

Comments
 (0)