Skip to content

Commit d951cfe

Browse files
Neko-Box-CoderNeko-Box-Coder
committed
Adding dependencies setup/build/cleanup to be multithreaded
1 parent f26cdf8 commit d951cfe

File tree

7 files changed

+137
-118
lines changed

7 files changed

+137
-118
lines changed

‎CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@ target_link_libraries(runcpp2 PRIVATE ssLogger System2 CppOverride dylib)
157157
target_link_libraries(runcpp2 PUBLIC ghc_filesystem ryml::ryml mpark_variant)
158158

159159
if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
160+
# TODO: Try to change to /Wall
160161
set(STANDARD_COMPILE_FLAGS "/utf-8;/W1;/DGHC_WIN_DISABLE_WSTRING_STORAGE_TYPE=1")
161162
else()
162163
set(STANDARD_COMPILE_FLAGS "-Wall"

‎Include/runcpp2/DependenciesHelper.hpp‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,14 @@ namespace runcpp2
3333
const Data::ScriptInfo& scriptInfo,
3434
std::vector<Data::DependencyInfo*>& availableDependencies,
3535
const std::vector<std::string>& dependenciesLocalCopiesPaths,
36-
const std::vector<std::string>& dependenciesSourcePaths);
36+
const std::vector<std::string>& dependenciesSourcePaths,
37+
const int maxThreads);
3738

3839
bool BuildDependencies( const runcpp2::Data::Profile& profile,
3940
const Data::ScriptInfo& scriptInfo,
4041
const std::vector<Data::DependencyInfo*>& availableDependencies,
41-
const std::vector<std::string>& dependenciesLocalCopiesPaths);
42+
const std::vector<std::string>& dependenciesLocalCopiesPaths,
43+
const int maxThreads);
4244

4345
bool GatherDependenciesBinaries(const std::vector<Data::DependencyInfo*>& availableDependencies,
4446
const std::vector<std::string>& dependenciesCopiesPaths,
@@ -50,8 +52,7 @@ namespace runcpp2
5052

5153
bool ResolveImports(Data::ScriptInfo& scriptInfo,
5254
const ghc::filesystem::path& scriptPath,
53-
const ghc::filesystem::path& buildDir,
54-
const int maxThreads);
55+
const ghc::filesystem::path& buildDir);
5556

5657
bool SyncLocalDependency( const Data::DependencyInfo& dependency,
5758
const ghc::filesystem::path& sourcePath,

‎Include/runcpp2/PipelineSteps.hpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,7 @@ namespace runcpp2
5959

6060
PipelineResult ResolveScriptImports(Data::ScriptInfo& scriptInfo,
6161
const ghc::filesystem::path& scriptPath,
62-
const ghc::filesystem::path& buildDir,
63-
const int maxThreads);
62+
const ghc::filesystem::path& buildDir);
6463

6564
PipelineResult CheckScriptInfoChanges( const ghc::filesystem::path& buildDir,
6665
const Data::ScriptInfo& scriptInfo,
@@ -79,6 +78,7 @@ namespace runcpp2
7978
const ghc::filesystem::path& buildDir,
8079
const std::unordered_map<CmdOptions, std::string>& currentOptions,
8180
const std::vector<std::string>& changedDependencies,
81+
const int maxThreads,
8282
std::vector<Data::DependencyInfo*>& outAvailableDependencies,
8383
std::vector<std::string>& outGatheredBinariesPaths);
8484

‎Src/runcpp2/CompilingLinking.cpp‎

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,6 @@
1212

1313
#include <future>
1414
#include <chrono>
15-
//#include <mutex>
16-
//#include <condition_variable>
17-
18-
1915

2016
namespace
2117
{
@@ -74,10 +70,9 @@ namespace
7470
runcpp2::TrimRight(inOutFlags);
7571
}
7672

77-
bool
78-
PopulateFilesTypesMap( const runcpp2::Data::FilesTypesInfo& fileTypesInfo,
79-
std::unordered_map< std::string,
80-
std::vector<std::string>>& outSubstitutionMap)
73+
bool PopulateFilesTypesMap( const runcpp2::Data::FilesTypesInfo& fileTypesInfo,
74+
std::unordered_map< std::string,
75+
std::vector<std::string>>& outSubstitutionMap)
8176
{
8277
ssLOG_FUNC_DEBUG();
8378

@@ -455,12 +450,6 @@ namespace
455450
{
456451
result = actions.at(j).get();
457452
}
458-
else
459-
{
460-
ssLOG_ERROR("Failed to construct actions for importing");
461-
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
462-
return false;
463-
}
464453

465454
if(!result)
466455
{

‎Src/runcpp2/DependenciesHelper.cpp‎

Lines changed: 116 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
#include "ssLogger/ssLog.hpp"
88

99
#include <unordered_set>
10+
#include <future>
11+
#include <chrono>
1012

1113
namespace
1214
{
@@ -157,14 +159,6 @@ namespace
157159

158160
outPrePopulated.resize(dependencies.size());
159161

160-
//std::vector<std::thread> compileActions;
161-
//std::vector<bool> runResults(sourceFiles.size(), false);
162-
//int doneCount = 0;
163-
//std::mutex resultMutex;
164-
//std::condition_variable resultCV;
165-
166-
167-
//TODO(NOW): Multi-thread?
168162
for(int i = 0; i < dependencies.size(); ++i)
169163
{
170164
if(!dependencies.at(i)->Source.ImportPath.empty())
@@ -497,7 +491,8 @@ bool runcpp2::SetupDependenciesIfNeeded(const runcpp2::Data::Profile& profile,
497491
const Data::ScriptInfo& scriptInfo,
498492
std::vector<Data::DependencyInfo*>& availableDependencies,
499493
const std::vector<std::string>& dependenciesLocalCopiesPaths,
500-
const std::vector<std::string>& dependenciesSourcePaths)
494+
const std::vector<std::string>& dependenciesSourcePaths,
495+
const int maxThreads)
501496
{
502497
ssLOG_FUNC_INFO();
503498

@@ -518,11 +513,13 @@ bool runcpp2::SetupDependenciesIfNeeded(const runcpp2::Data::Profile& profile,
518513
}
519514

520515
if(!PopulateAbsoluteIncludePaths(availableDependencies, dependenciesLocalCopiesPaths))
521-
{
522516
return false;
523-
}
524517

525-
//TODO(NOW): Multi-thread?
518+
std::vector<std::future<bool>> actions;
519+
520+
//Cache logs for worker threads
521+
ssLOG_ENABLE_CACHE_OUTPUT_FOR_NEW_THREADS();
522+
526523
//Run setup steps
527524
for(int i = 0; i < availableDependencies.size(); ++i)
528525
{
@@ -533,47 +530,135 @@ bool runcpp2::SetupDependenciesIfNeeded(const runcpp2::Data::Profile& profile,
533530
continue;
534531
}
535532

536-
ssLOG_INFO("Running setup commands for " << availableDependencies.at(i)->Name);
537-
if(!RunDependenciesSteps( profile,
538-
availableDependencies.at(i)->Setup,
539-
dependenciesLocalCopiesPaths.at(i),
540-
true))
533+
actions.emplace_back
534+
(
535+
std::async
536+
(
537+
std::launch::async,
538+
[i, &profile, &availableDependencies, &dependenciesLocalCopiesPaths]()
539+
{
540+
ssLOG_INFO("Running setup commands for " << availableDependencies.at(i)->Name);
541+
if(!RunDependenciesSteps( profile,
542+
availableDependencies.at(i)->Setup,
543+
dependenciesLocalCopiesPaths.at(i),
544+
true))
545+
{
546+
ssLOG_ERROR("Failed to setup dependency " <<
547+
availableDependencies.at(i)->Name);
548+
return false;
549+
}
550+
return true;
551+
}
552+
)
553+
);
554+
555+
//Evaluate the setup results for each batch
556+
if(actions.size() >= maxThreads || i == availableDependencies.size() - 1)
541557
{
542-
ssLOG_ERROR("Failed to setup dependency " << availableDependencies.at(i)->Name);
543-
return false;
558+
std::chrono::system_clock::time_point deadline =
559+
std::chrono::system_clock::now() + std::chrono::seconds(maxThreads);
560+
561+
for(int j = 0; j < actions.size(); ++j)
562+
{
563+
if(!actions.at(j).valid())
564+
{
565+
ssLOG_ERROR("Failed to construct actions for setup");
566+
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
567+
return false;
568+
}
569+
570+
std::future_status actionStatus = actions.at(j).wait_until(deadline);
571+
if(actionStatus == std::future_status::ready)
572+
{
573+
if(!actions.at(j).get())
574+
{
575+
ssLOG_ERROR("Setup failed for dependencies");
576+
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
577+
return false;
578+
}
579+
}
580+
}
581+
actions.clear();
544582
}
545583
}
546-
584+
585+
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
547586
return true;
548587
}
549588

550589
bool runcpp2::BuildDependencies(const runcpp2::Data::Profile& profile,
551590
const Data::ScriptInfo& scriptInfo,
552591
const std::vector<Data::DependencyInfo*>& availableDependencies,
553-
const std::vector<std::string>& dependenciesLocalCopiesPaths)
592+
const std::vector<std::string>& dependenciesLocalCopiesPaths,
593+
const int maxThreads)
554594
{
555595
ssLOG_FUNC_INFO();
556596

557597
//If the script info is not populated (i.e. empty script info), don't do anything
558598
if(!scriptInfo.Populated)
559599
return true;
560-
561-
//TODO(NOW): Multi-thread?
600+
601+
std::vector<std::future<bool>> actions;
602+
603+
//Cache logs for worker threads
604+
ssLOG_ENABLE_CACHE_OUTPUT_FOR_NEW_THREADS();
605+
562606
//Run build steps
563607
for(int i = 0; i < availableDependencies.size(); ++i)
564608
{
565609
ssLOG_INFO("Running build commands for " << availableDependencies.at(i)->Name);
566610

567-
if(!RunDependenciesSteps( profile,
568-
availableDependencies.at(i)->Build,
569-
dependenciesLocalCopiesPaths.at(i),
570-
true))
611+
actions.emplace_back
612+
(
613+
std::async
614+
(
615+
std::launch::async,
616+
[i, &profile, &availableDependencies, &dependenciesLocalCopiesPaths]()
617+
{
618+
if(!RunDependenciesSteps( profile,
619+
availableDependencies.at(i)->Build,
620+
dependenciesLocalCopiesPaths.at(i),
621+
true))
622+
{
623+
ssLOG_ERROR("Failed to build dependency " << availableDependencies.at(i)->Name);
624+
return false;
625+
}
626+
return true;
627+
}
628+
)
629+
);
630+
631+
//Evaluate the setup results for each batch
632+
if(actions.size() >= maxThreads || i == availableDependencies.size() - 1)
571633
{
572-
ssLOG_ERROR("Failed to build dependency " << availableDependencies.at(i)->Name);
573-
return false;
634+
std::chrono::system_clock::time_point deadline =
635+
std::chrono::system_clock::now() + std::chrono::seconds(maxThreads);
636+
637+
for(int j = 0; j < actions.size(); ++j)
638+
{
639+
if(!actions.at(j).valid())
640+
{
641+
ssLOG_ERROR("Failed to construct actions for building dependencies");
642+
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
643+
return false;
644+
}
645+
646+
std::future_status actionStatus = actions.at(j).wait_until(deadline);
647+
if(actionStatus == std::future_status::ready)
648+
{
649+
if(!actions.at(j).get())
650+
{
651+
ssLOG_ERROR("Build failed for dependencies");
652+
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
653+
return false;
654+
}
655+
}
656+
}
657+
actions.clear();
574658
}
575659
}
576660

661+
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
577662
return true;
578663
}
579664

@@ -879,28 +964,11 @@ bool runcpp2::HandleImport( Data::DependencyInfo& dependency,
879964

880965
bool runcpp2::ResolveImports( Data::ScriptInfo& scriptInfo,
881966
const ghc::filesystem::path& scriptPath,
882-
const ghc::filesystem::path& buildDir,
883-
const int maxThreads)
967+
const ghc::filesystem::path& buildDir)
884968
{
885969
ssLOG_FUNC_INFO();
886970
INTERNAL_RUNCPP2_SAFE_START();
887971

888-
//TODO(NOW): Multi-thread this
889-
890-
891-
//std::vector<std::thread> importActions;
892-
//std::vector<bool> dispatched(scriptInfo.Dependencies.size(), false);
893-
//std::vector<bool> runResults(scriptInfo.Dependencies.size(), false);
894-
//int doneCount = 0;
895-
//std::mutex resultMutex;
896-
//std::condition_variable resultCV;
897-
(void)maxThreads;
898-
899-
900-
901-
902-
903-
904972
//For each dependency, check if import path exists
905973
for(int i = 0; i < scriptInfo.Dependencies.size(); ++i)
906974
{
@@ -910,11 +978,7 @@ bool runcpp2::ResolveImports( Data::ScriptInfo& scriptInfo,
910978
Data::DependencySource& source = dependency.Source;
911979
if(source.ImportPath.empty())
912980
continue;
913-
914-
915-
916-
917-
981+
918982
if(!source.ImportPath.is_relative())
919983
{
920984
ssLOG_ERROR("Import path is not relative: " << source.ImportPath.string());
@@ -939,44 +1003,6 @@ bool runcpp2::ResolveImports( Data::ScriptInfo& scriptInfo,
9391003
if(!dependency.Source.ImportPath.empty())
9401004
--i;
9411005
}
942-
943-
944-
//Evaluate the compile results for each batch for compilations
945-
946-
//if(i - startIndex >= maxThreads || i == sourceFiles.size() - 1)
947-
//{
948-
// std::unique_lock<std::mutex> lk(resultMutex);
949-
// resultCV.wait_for
950-
// (
951-
// lk,
952-
// std::chrono::seconds(maxThreads)
953-
// [&doneCount, &sourceFiles, &startIndex, &maxThreads]()
954-
// {
955-
// return doneCount - startIndex >= maxThreads ||
956-
// doneCount == sourceFiles.size();
957-
// }
958-
// );
959-
//
960-
// ssLOG_OUTPUT_ALL_CACHE_GROUPED();
961-
//
962-
// //Check if all threads have finished the work
963-
// if(doneCount - startIndex < maxThreads && doneCount != sourceFiles.size())
964-
// {
965-
// ssLOG_ERROR("Compilation workers timed out...");
966-
// return false;
967-
// }
968-
//
969-
// //Check if all the compilations are successful or not
970-
// for(int j = 0; j < runResults.size(); ++j)
971-
// {
972-
// if(!runResults.at(j + startIndex))
973-
// return false;
974-
// }
975-
//
976-
// //Update the start index
977-
// startIndex = i + 1;
978-
//}
979-
9801006

9811007
return true;
9821008

0 commit comments

Comments
 (0)