Skip to content

Commit 14d72aa

Browse files
loks0nloks0n
committed
refactor: extract maintenance
1 parent eb0ba96 commit 14d72aa

File tree

5 files changed

+137
-58
lines changed

5 files changed

+137
-58
lines changed

‎app/http.php‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
require_once __DIR__ . '/controllers.php';
99

1010
use OpenRuntimes\Executor\Runner\Docker;
11+
use OpenRuntimes\Executor\Runner\Maintenance;
1112
use OpenRuntimes\Executor\Runner\Repository\Runtimes;
1213
use OpenRuntimes\Executor\Runner\NetworkManager;
1314
use Swoole\Runtime;
@@ -45,6 +46,7 @@
4546
System::getEnv('OPR_EXECUTOR_DOCKER_HUB_USERNAME', ''),
4647
System::getEnv('OPR_EXECUTOR_DOCKER_HUB_PASSWORD', '')
4748
));
49+
$runtimes = new Runtimes();
4850

4951
/* Create desired networks if they don't exist */
5052
$networks = explode(',', System::getEnv('OPR_EXECUTOR_NETWORK') ?: 'openruntimes-runtimes');
@@ -55,8 +57,14 @@
5557
$selfContainer = $orchestration->list(['name' => $hostname])[0] ?? throw new \RuntimeException('Own container not found');
5658
$networkManager->connectAll($selfContainer);
5759

60+
/* Start maintenance task */
61+
$maintenance = new Maintenance($orchestration, $runtimes);
62+
$maintenance->start(
63+
(int)System::getEnv('OPR_EXECUTOR_MAINTENANCE_INTERVAL', '3600'),
64+
(int)System::getEnv('OPR_EXECUTOR_INACTIVE_THRESHOLD', '60')
65+
);
66+
5867
/* Runner service, used to manage runtimes */
59-
$runtimes = new Runtimes();
6068
$runner = new Docker($orchestration, $runtimes, $networkManager);
6169
Http::setResource('runner', fn () => $runner);
6270

‎src/Executor/Runner/Docker.php‎

Lines changed: 0 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -76,57 +76,6 @@ private function init(): void
7676

7777
Console::success("Image pulling finished.");
7878

79-
/**
80-
* Run a maintenance worker every X seconds to remove inactive runtimes
81-
*/
82-
Console::info('Starting maintenance interval...');
83-
$interval = (int)System::getEnv('OPR_EXECUTOR_MAINTENANCE_INTERVAL', '3600'); // In seconds
84-
Timer::tick($interval * 1000, function () {
85-
Console::info("Running maintenance task ...");
86-
// Stop idling runtimes
87-
foreach ($this->runtimes as $runtimeName => $runtime) {
88-
$inactiveThreshold = \time() - \intval(System::getEnv('OPR_EXECUTOR_INACTIVE_THRESHOLD', '60'));
89-
if ($runtime->updated < $inactiveThreshold) {
90-
go(function () use ($runtimeName, $runtime) {
91-
try {
92-
$this->orchestration->remove($runtime->name, true);
93-
Console::success("Successfully removed {$runtime->name}");
94-
} catch (Throwable $th) {
95-
Console::error('Inactive Runtime deletion failed: ' . $th->getMessage());
96-
} finally {
97-
$this->runtimes->remove($runtimeName);
98-
}
99-
});
100-
}
101-
}
102-
103-
// Clear leftover build folders
104-
$localDevice = new Local();
105-
$tmpPath = DIRECTORY_SEPARATOR . 'tmp' . DIRECTORY_SEPARATOR;
106-
$entries = $localDevice->getFiles($tmpPath);
107-
$prefix = $tmpPath . System::getHostname() . '-';
108-
foreach ($entries as $entry) {
109-
if (\str_starts_with($entry, $prefix)) {
110-
$isActive = false;
111-
112-
foreach ($this->runtimes as $runtimeName => $runtime) {
113-
if (\str_ends_with($entry, $runtimeName)) {
114-
$isActive = true;
115-
break;
116-
}
117-
}
118-
119-
if (!$isActive) {
120-
$localDevice->deletePath($entry);
121-
}
122-
}
123-
}
124-
125-
Console::success("Maintanance task finished.");
126-
});
127-
128-
Console::success('Maintenance interval started.');
129-
13079
Process::signal(SIGINT, fn () => $this->cleanUp());
13180
Process::signal(SIGQUIT, fn () => $this->cleanUp());
13281
Process::signal(SIGKILL, fn () => $this->cleanUp());

‎src/Executor/Runner/Maintenance.php‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
<?php
2+
3+
namespace OpenRuntimes\Executor\Runner;
4+
5+
use OpenRuntimes\Executor\Runner\Repository\Runtimes;
6+
use Swoole\Timer;
7+
use Utopia\Console;
8+
use Utopia\Orchestration\Orchestration;
9+
use Utopia\Storage\Device\Local;
10+
use Utopia\System\System;
11+
12+
use function Swoole\Coroutine\batch;
13+
14+
/**
15+
* Handles periodic cleanup of inactive runtimes and orphaned temp directories.
16+
*/
17+
class Maintenance
18+
{
19+
private int|false $timerId = false;
20+
21+
public function __construct(
22+
private Orchestration $orchestration,
23+
private Runtimes $runtimes
24+
) {
25+
}
26+
27+
/**
28+
* Starts the maintenance loop. No-op if already running.
29+
*/
30+
public function start(int $intervalSeconds, int $inactiveSeconds): void
31+
{
32+
if ($this->timerId !== false) {
33+
return;
34+
}
35+
36+
$intervalMs = $intervalSeconds * 1000;
37+
$this->timerId = Timer::tick($intervalMs, fn () => $this->doMaintenance($inactiveSeconds));
38+
Console::info("[Maintenance] Started task on interval $intervalSeconds seconds.");
39+
}
40+
41+
/**
42+
* Stops the maintenance loop. No-op if already stopped.
43+
*/
44+
public function stop(): void
45+
{
46+
if ($this->timerId === false) {
47+
return;
48+
}
49+
50+
Timer::clear($this->timerId);
51+
$this->timerId = false;
52+
Console::info("[Maintenance] Stopped task.");
53+
}
54+
55+
/**
56+
* Removes runtimes inactive beyond the threshold and cleans up temporary files.
57+
*/
58+
private function doMaintenance(int $inactiveSeconds): void
59+
{
60+
Console::info("[Maintenance] Running task with threshold $inactiveSeconds seconds.");
61+
62+
$threshold = \time() - $inactiveSeconds;
63+
$candidates = array_filter(
64+
$this->runtimes->list(),
65+
fn ($runtime) => $runtime->updated < $threshold
66+
);
67+
68+
// Remove from in-memory state before removing the container.
69+
// Ensures availability, otherwise we would route requests to terminating runtimes.
70+
$keys = array_keys($candidates);
71+
foreach ($keys as $key) {
72+
$this->runtimes->remove($key);
73+
}
74+
// Then, remove forcefully terminate the associated running container.
75+
$jobs = array_map(
76+
fn ($candidate) => fn () => $this->orchestration->remove($candidate->name, force: true),
77+
$candidates
78+
);
79+
$results = batch($jobs);
80+
$removed = \count(array_filter($results));
81+
82+
Console::info("[Maintenance] Removed {$removed}/" . \count($candidates) . " inactive runtimes.");
83+
84+
$this->cleanupTmp();
85+
}
86+
87+
private function cleanupTmp(): void
88+
{
89+
$localDevice = new Local();
90+
$tmpPath = DIRECTORY_SEPARATOR . 'tmp' . DIRECTORY_SEPARATOR;
91+
$prefix = $tmpPath . System::getHostname() . '-';
92+
93+
foreach ($localDevice->getFiles($tmpPath) as $entry) {
94+
if (!\str_starts_with($entry, $prefix)) {
95+
continue;
96+
}
97+
98+
$runtimeName = substr($entry, \strlen($prefix));
99+
if ($this->runtimes->exists($runtimeName)) {
100+
continue;
101+
}
102+
103+
if ($localDevice->deletePath($entry)) {
104+
Console::info("[Maintenance] Removed {$entry}.");
105+
}
106+
}
107+
}
108+
}

‎src/Executor/Runner/NetworkManager.php‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,31 +67,31 @@ public function removeAll(): void
6767
private function remove(string $network): void
6868
{
6969
if (!$this->orchestration->networkExists($network)) {
70-
Console::error("Network {$network} does not exist");
70+
Console::error("[NetworkManager] Network {$network} does not exist");
7171
return;
7272
}
7373

7474
try {
7575
$this->orchestration->removeNetwork($network);
76-
Console::success("Removed network: {$network}");
76+
Console::success("[NetworkManager] Removed network: {$network}");
7777
} catch (\Throwable $e) {
78-
Console::error("Failed to remove network {$network}: {$e->getMessage()}");
78+
Console::error("[NetworkManager] Failed to remove network {$network}: {$e->getMessage()}");
7979
}
8080
}
8181

8282
private function ensure(string $network): ?string
8383
{
8484
if ($this->orchestration->networkExists($network)) {
85-
Console::info("Network {$network} already exists");
85+
Console::info("[NetworkManager] Network {$network} already exists");
8686
return $network;
8787
}
8888

8989
try {
9090
$this->orchestration->createNetwork($network, false);
91-
Console::success("Created network: {$network}");
91+
Console::success("[NetworkManager] Created network: {$network}");
9292
return $network;
9393
} catch (\Throwable $e) {
94-
Console::error("Failed to create network {$network}: {$e->getMessage()}");
94+
Console::error("[NetworkManager] Failed to create network {$network}: {$e->getMessage()}");
9595
return null;
9696
}
9797
}

‎src/Executor/Runner/Repository/Runtimes.php‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,20 @@ public function remove(string $id): bool
8686
return $this->runtimes->del($id);
8787
}
8888

89+
/**
90+
* List all runtimes.
91+
*
92+
* @return array<Runtime> An array of Runtime objects.
93+
*/
94+
public function list(): array
95+
{
96+
$runtimes = [];
97+
foreach ($this->runtimes as $runtimeKey => $runtime) {
98+
$runtimes[$runtimeKey] = Runtime::fromArray($runtime);
99+
}
100+
return $runtimes;
101+
}
102+
89103
// Iterator traits
90104
public function current(): Runtime
91105
{

0 commit comments

Comments
 (0)