Skip to content

Commit 71dccac

Browse files
Neko-Box-CoderNeko-Box-Coder
authored
Merge pull request #46 from Neko-Box-Coder/UpdatedParallelProcess
Updated parallel process for dependencies and compiling
2 parents 3ece141 + f210c68 commit 71dccac

File tree

2 files changed

+136
-70
lines changed

2 files changed

+136
-70
lines changed

‎Src/runcpp2/CompilingLinking.cpp‎

Lines changed: 48 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,7 @@ namespace
216216
std::unordered_map<std::string, std::vector<std::string>> substitutionMap;
217217
substitutionMap = substitutionMapTemplate;
218218
std::vector<std::future<bool>> actions;
219+
std::vector<bool> finished;
219220

220221
//Cache logs for worker threads
221222
ssLOG_ENABLE_CACHE_OUTPUT_FOR_NEW_THREADS();
@@ -235,6 +236,7 @@ namespace
235236
ssLOG_ERROR("Failed to get relative path for " << currentSource);
236237
ssLOG_ERROR("Failed with error: " << e.message());
237238
actions.emplace_back(std::async(std::launch::deferred, []{return false;}));
239+
finished.emplace_back(false);
238240
continue;
239241
}
240242

@@ -260,6 +262,7 @@ namespace
260262
ssLOG_ERROR("profile " << profile.Name << " missing extension for " <<
261263
"object link file");
262264
actions.emplace_back(std::async(std::launch::deferred, []{return false;}));
265+
finished.emplace_back(false);
263266
continue;
264267
}
265268

@@ -273,6 +276,7 @@ namespace
273276
{
274277
ssLOG_ERROR("Failed to substitute \"" << currentPath << "\"");
275278
actions.emplace_back(std::async(std::launch::deferred, []{return false;}));
279+
finished.emplace_back(false);
276280
continue;
277281
}
278282

@@ -283,6 +287,7 @@ namespace
283287
ssLOG_ERROR("Failed to create directory structure for " << currentPath);
284288
ssLOG_ERROR("Failed with error: " << e.message());
285289
actions.emplace_back(std::async(std::launch::deferred, []{return false;}));
290+
finished.emplace_back(false);
286291
continue;
287292
}
288293

@@ -434,44 +439,58 @@ namespace
434439
}
435440
)
436441
); //actions.emplace_back
442+
finished.emplace_back(false);
437443

438444
//Evaluate the compile results for each batch for compilations
439445
if(actions.size() >= maxThreads || i == sourceFiles.size() - 1)
440446
{
441-
std::chrono::system_clock::time_point deadline =
442-
std::chrono::system_clock::now() + std::chrono::seconds(60);
443-
for(int j = 0; j < actions.size(); ++j)
447+
bool needsWaiting = false;
448+
do
444449
{
445-
if(!actions.at(j).valid())
450+
needsWaiting = false;
451+
std::chrono::system_clock::time_point deadline =
452+
std::chrono::system_clock::now() + std::chrono::seconds(60);
453+
for(int j = 0; j < actions.size(); ++j)
446454
{
447-
ssLOG_ERROR("Failed to construct actions for compiling");
448-
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
449-
return false;
450-
}
451-
452-
std::future_status actionStatus = actions.at(j).wait_until(deadline);
453-
bool result = false;
454-
455-
if( actionStatus == std::future_status::deferred ||
456-
actionStatus == std::future_status::ready)
457-
{
458-
result = actions.at(j).get();
459-
}
460-
else
461-
{
462-
ssLOG_ERROR("Compiling timeout");
463-
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
464-
failedAny = true;
465-
}
466-
467-
if(!result)
468-
{
469-
ssLOG_ERROR("Compiling Failed");
470-
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
471-
failedAny = true;
455+
if(finished.at(j))
456+
continue;
457+
458+
if(!actions.at(j).valid())
459+
{
460+
ssLOG_ERROR("Failed to construct actions for compiling");
461+
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
462+
return false;
463+
}
464+
465+
std::future_status actionStatus = actions.at(j).wait_until(deadline);
466+
bool result = false;
467+
468+
if( actionStatus == std::future_status::deferred ||
469+
actionStatus == std::future_status::ready)
470+
{
471+
result = actions.at(j).get();
472+
finished.at(j) = true;
473+
474+
if(!result)
475+
{
476+
ssLOG_ERROR("Compiling Failed");
477+
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
478+
failedAny = true;
479+
}
480+
}
481+
else
482+
{
483+
ssLOG_WARNING("Manual interrupt might be needed");
484+
ssLOG_WARNING("Waited 30 seconds, compiling still going...");
485+
needsWaiting = true;
486+
result = true;
487+
}
472488
}
473489
}
490+
while(needsWaiting);
491+
474492
actions.clear();
493+
finished.clear();
475494
}
476495
}
477496

‎Src/runcpp2/DependenciesHelper.cpp‎

Lines changed: 88 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -521,6 +521,8 @@ bool runcpp2::CleanupDependencies( const runcpp2::Data::Profile& profile,
521521
return true;
522522
}
523523

524+
//#define RUNCPP2_USE_PARALLEL_FOR_DEP 0
525+
524526
bool runcpp2::SetupDependenciesIfNeeded(const runcpp2::Data::Profile& profile,
525527
const ghc::filesystem::path& buildDir,
526528
const Data::ScriptInfo& scriptInfo,
@@ -550,11 +552,14 @@ bool runcpp2::SetupDependenciesIfNeeded(const runcpp2::Data::Profile& profile,
550552
if(!PopulateAbsoluteIncludePaths(availableDependencies, dependenciesLocalCopiesPaths))
551553
return false;
552554

555+
#if RUNCPP2_USE_PARALLEL_FOR_DEP
553556
std::vector<std::future<bool>> actions;
557+
std::vector<bool> finished;
554558

555559
//Cache logs for worker threads
556560
ssLOG_ENABLE_CACHE_OUTPUT_FOR_NEW_THREADS();
557561
int logLevel = ssLOG_GET_CURRENT_THREAD_TARGET_LEVEL();
562+
#endif
558563

559564
//Run setup steps
560565
for(int i = 0; i < availableDependencies.size(); ++i)
@@ -566,6 +571,7 @@ bool runcpp2::SetupDependenciesIfNeeded(const runcpp2::Data::Profile& profile,
566571
continue;
567572
}
568573

574+
#if RUNCPP2_USE_PARALLEL_FOR_DEP
569575
actions.emplace_back
570576
(
571577
std::async
@@ -574,7 +580,8 @@ bool runcpp2::SetupDependenciesIfNeeded(const runcpp2::Data::Profile& profile,
574580
[i, &profile, &availableDependencies, &dependenciesLocalCopiesPaths, logLevel]()
575581
{
576582
ssLOG_SET_CURRENT_THREAD_TARGET_LEVEL(logLevel);
577-
583+
#endif
584+
578585
ssLOG_INFO("Running setup commands for " << availableDependencies.at(i)->Name);
579586
if(!RunDependenciesSteps( profile,
580587
availableDependencies.at(i)->Setup,
@@ -585,44 +592,62 @@ bool runcpp2::SetupDependenciesIfNeeded(const runcpp2::Data::Profile& profile,
585592
availableDependencies.at(i)->Name);
586593
return false;
587594
}
595+
596+
#if RUNCPP2_USE_PARALLEL_FOR_DEP
588597
return true;
589598
}
590599
)
591600
);
592601

602+
finished.emplace_back(false);
603+
593604
//Evaluate the setup results for each batch
594605
if(actions.size() >= maxThreads || i == availableDependencies.size() - 1)
595606
{
596-
std::chrono::system_clock::time_point deadline =
597-
std::chrono::system_clock::now() + std::chrono::seconds(60);
598-
for(int j = 0; j < actions.size(); ++j)
607+
bool needsWaiting = false;
608+
do
599609
{
600-
if(!actions.at(j).valid())
601-
{
602-
ssLOG_ERROR("Failed to construct actions for setup");
603-
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
604-
return false;
605-
}
606-
607-
std::future_status actionStatus = actions.at(j).wait_until(deadline);
608-
if(actionStatus == std::future_status::ready)
610+
std::chrono::system_clock::time_point deadline =
611+
std::chrono::system_clock::now() + std::chrono::seconds(30);
612+
needsWaiting = false;
613+
for(int j = 0; j < actions.size(); ++j)
609614
{
610-
if(!actions.at(j).get())
615+
if(finished.at(j))
616+
continue;
617+
618+
if(!actions.at(j).valid())
611619
{
612-
ssLOG_ERROR("Setup failed for dependencies");
620+
ssLOG_ERROR("Failed to construct actions for setup");
613621
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
614622
return false;
615623
}
616-
}
617-
else
618-
{
619-
ssLOG_ERROR("Dependencies setup timeout");
620-
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
621-
return false;
624+
625+
std::future_status actionStatus = actions.at(j).wait_until(deadline);
626+
if(actionStatus == std::future_status::ready)
627+
{
628+
if(!actions.at(j).get())
629+
{
630+
ssLOG_ERROR("Setup failed for dependencies");
631+
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
632+
return false;
633+
}
634+
finished.at(j) = true;
635+
}
636+
else
637+
{
638+
ssLOG_WARNING("Manual interrupt might be needed");
639+
ssLOG_WARNING("Waited 30 seconds, dependencies setup still going...");
640+
needsWaiting = true;
641+
}
622642
}
623643
}
644+
while(needsWaiting);
645+
624646
actions.clear();
647+
finished.clear();
625648
}
649+
650+
#endif //#if RUNCPP2_USE_PARALLEL_FOR_DEP
626651
}
627652

628653
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
@@ -641,17 +666,21 @@ bool runcpp2::BuildDependencies(const runcpp2::Data::Profile& profile,
641666
if(!scriptInfo.Populated)
642667
return true;
643668

669+
#if RUNCPP2_USE_PARALLEL_FOR_DEP
644670
std::vector<std::future<bool>> actions;
671+
std::vector<bool> finished;
645672

646673
//Cache logs for worker threads
647674
ssLOG_ENABLE_CACHE_OUTPUT_FOR_NEW_THREADS();
648675
int logLevel = ssLOG_GET_CURRENT_THREAD_TARGET_LEVEL();
676+
#endif
649677

650678
//Run build steps
651679
for(int i = 0; i < availableDependencies.size(); ++i)
652680
{
653681
ssLOG_INFO("Running build commands for " << availableDependencies.at(i)->Name);
654682

683+
#if RUNCPP2_USE_PARALLEL_FOR_DEP
655684
actions.emplace_back
656685
(
657686
std::async
@@ -660,6 +689,7 @@ bool runcpp2::BuildDependencies(const runcpp2::Data::Profile& profile,
660689
[i, &profile, &availableDependencies, &dependenciesLocalCopiesPaths, logLevel]()
661690
{
662691
ssLOG_SET_CURRENT_THREAD_TARGET_LEVEL(logLevel);
692+
#endif
663693

664694
if(!RunDependenciesSteps( profile,
665695
availableDependencies.at(i)->Build,
@@ -669,44 +699,61 @@ bool runcpp2::BuildDependencies(const runcpp2::Data::Profile& profile,
669699
ssLOG_ERROR("Failed to build dependency " << availableDependencies.at(i)->Name);
670700
return false;
671701
}
702+
703+
#if RUNCPP2_USE_PARALLEL_FOR_DEP
672704
return true;
673705
}
674706
)
675707
);
676708

709+
finished.emplace_back(false);
710+
677711
//Evaluate the setup results for each batch
678712
if(actions.size() >= maxThreads || i == availableDependencies.size() - 1)
679713
{
680-
std::chrono::system_clock::time_point deadline =
681-
std::chrono::system_clock::now() + std::chrono::seconds(60);
682-
for(int j = 0; j < actions.size(); ++j)
714+
bool needsWaiting = false;
715+
do
683716
{
684-
if(!actions.at(j).valid())
685-
{
686-
ssLOG_ERROR("Failed to construct actions for building dependencies");
687-
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
688-
return false;
689-
}
690-
691-
std::future_status actionStatus = actions.at(j).wait_until(deadline);
692-
if(actionStatus == std::future_status::ready)
717+
std::chrono::system_clock::time_point deadline =
718+
std::chrono::system_clock::now() + std::chrono::seconds(30);
719+
needsWaiting = false;
720+
for(int j = 0; j < actions.size(); ++j)
693721
{
694-
if(!actions.at(j).get())
722+
if(finished.at(j))
723+
continue;
724+
725+
if(!actions.at(j).valid())
695726
{
696-
ssLOG_ERROR("Build failed for dependencies");
727+
ssLOG_ERROR("Failed to construct actions for building dependencies");
697728
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
698729
return false;
699730
}
700-
}
701-
else
702-
{
703-
ssLOG_ERROR("Dependencies build timeout");
704-
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
705-
return false;
731+
732+
std::future_status actionStatus = actions.at(j).wait_until(deadline);
733+
if(actionStatus == std::future_status::ready)
734+
{
735+
if(!actions.at(j).get())
736+
{
737+
ssLOG_ERROR("Build failed for dependencies");
738+
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
739+
return false;
740+
}
741+
finished.at(j) = true;
742+
}
743+
else
744+
{
745+
ssLOG_WARNING("Manual interrupt might be needed");
746+
ssLOG_WARNING("Waited 30 seconds, dependencies build still going...");
747+
needsWaiting = true;
748+
}
706749
}
707750
}
751+
while(needsWaiting);
752+
708753
actions.clear();
754+
finished.clear();
709755
}
756+
#endif ////#if RUNCPP2_USE_PARALLEL_FOR_DEP
710757
}
711758

712759
ssLOG_OUTPUT_ALL_CACHE_GROUPED();

0 commit comments

Comments
 (0)