Skip to content

Commit 64c5200

Browse files
kesselbkesselb
authored
Merge pull request #54272 from nextcloud/enh/noid/taskprocessing-task-add-cleanup-flag
feat(taskprocessing): add cleanup flag to tasks
2 parents 2fc58bf + aa2ca86 commit 64c5200

23 files changed

+345
-83
lines changed

‎core/Command/TaskProcessing/Cleanup.php‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
namespace OC\Core\Command\TaskProcessing;
10+
11+
use OC\Core\Command\Base;
12+
use OC\TaskProcessing\Db\TaskMapper;
13+
use OC\TaskProcessing\Manager;
14+
use OCP\Files\AppData\IAppDataFactory;
15+
use Psr\Log\LoggerInterface;
16+
use Symfony\Component\Console\Input\InputArgument;
17+
use Symfony\Component\Console\Input\InputInterface;
18+
use Symfony\Component\Console\Output\OutputInterface;
19+
20+
class Cleanup extends Base {
21+
private \OCP\Files\IAppData $appData;
22+
23+
public function __construct(
24+
protected Manager $taskProcessingManager,
25+
private TaskMapper $taskMapper,
26+
private LoggerInterface $logger,
27+
IAppDataFactory $appDataFactory,
28+
) {
29+
parent::__construct();
30+
$this->appData = $appDataFactory->get('core');
31+
}
32+
33+
protected function configure() {
34+
$this
35+
->setName('taskprocessing:task:cleanup')
36+
->setDescription('cleanup old tasks')
37+
->addArgument(
38+
'maxAgeSeconds',
39+
InputArgument::OPTIONAL,
40+
// default is not defined as an argument default value because we want to show a nice "4 months" value
41+
'delete tasks that are older than this number of seconds, defaults to ' . Manager::MAX_TASK_AGE_SECONDS . ' (4 months)',
42+
);
43+
parent::configure();
44+
}
45+
46+
protected function execute(InputInterface $input, OutputInterface $output): int {
47+
$maxAgeSeconds = $input->getArgument('maxAgeSeconds') ?? Manager::MAX_TASK_AGE_SECONDS;
48+
$output->writeln('<comment>Cleanup up tasks older than ' . $maxAgeSeconds . ' seconds and the related output files</comment>');
49+
50+
$taskIdsToCleanup = [];
51+
try {
52+
$fileCleanupGenerator = $this->taskProcessingManager->cleanupTaskProcessingTaskFiles($maxAgeSeconds);
53+
foreach ($fileCleanupGenerator as $cleanedUpEntry) {
54+
$output->writeln(
55+
"<info>\t - " . 'Deleted appData/core/TaskProcessing/' . $cleanedUpEntry['file_name']
56+
. ' (fileId: ' . $cleanedUpEntry['file_id'] . ', taskId: ' . $cleanedUpEntry['task_id'] . ')</info>'
57+
);
58+
}
59+
$taskIdsToCleanup = $fileCleanupGenerator->getReturn();
60+
} catch (\Exception $e) {
61+
$this->logger->warning('Failed to delete stale task processing tasks files', ['exception' => $e]);
62+
$output->writeln('<warning>Failed to delete stale task processing tasks files</warning>');
63+
}
64+
try {
65+
$deletedTaskCount = $this->taskMapper->deleteOlderThan($maxAgeSeconds);
66+
foreach ($taskIdsToCleanup as $taskId) {
67+
$output->writeln("<info>\t - " . 'Deleted task ' . $taskId . ' from the database</info>');
68+
}
69+
$output->writeln("<comment>\t - " . 'Deleted ' . $deletedTaskCount . ' tasks from the database</comment>');
70+
} catch (\OCP\DB\Exception $e) {
71+
$this->logger->warning('Failed to delete stale task processing tasks', ['exception' => $e]);
72+
$output->writeln('<warning>Failed to delete stale task processing tasks</warning>');
73+
}
74+
try {
75+
$textToImageDeletedFileNames = $this->taskProcessingManager->clearFilesOlderThan($this->appData->getFolder('text2image'), $maxAgeSeconds);
76+
foreach ($textToImageDeletedFileNames as $entry) {
77+
$output->writeln("<info>\t - " . 'Deleted appData/core/text2image/' . $entry . '</info>');
78+
}
79+
} catch (\OCP\Files\NotFoundException $e) {
80+
// noop
81+
}
82+
try {
83+
$audioToTextDeletedFileNames = $this->taskProcessingManager->clearFilesOlderThan($this->appData->getFolder('audio2text'), $maxAgeSeconds);
84+
foreach ($audioToTextDeletedFileNames as $entry) {
85+
$output->writeln("<info>\t - " . 'Deleted appData/core/audio2text/' . $entry . '</info>');
86+
}
87+
} catch (\OCP\Files\NotFoundException $e) {
88+
// noop
89+
}
90+
91+
return 0;
92+
}
93+
}

‎core/Command/TaskProcessing/EnabledCommand.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
<?php
22

3+
declare(strict_types=1);
4+
35
/**
46
* SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors
57
* SPDX-License-Identifier: AGPL-3.0-or-later

‎core/Command/TaskProcessing/GetCommand.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
<?php
22

3+
declare(strict_types=1);
4+
35
/**
46
* SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors
57
* SPDX-License-Identifier: AGPL-3.0-or-later

‎core/Command/TaskProcessing/ListCommand.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
<?php
22

3+
declare(strict_types=1);
4+
35
/**
46
* SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors
57
* SPDX-License-Identifier: AGPL-3.0-or-later

‎core/Command/TaskProcessing/Statistics.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
<?php
22

3+
declare(strict_types=1);
4+
35
/**
46
* SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors
57
* SPDX-License-Identifier: AGPL-3.0-or-later

‎core/Controller/TaskProcessingApiController.php‎

Lines changed: 1 addition & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
use OCP\IL10N;
3232
use OCP\IRequest;
3333
use OCP\Lock\LockedException;
34-
use OCP\TaskProcessing\EShapeType;
3534
use OCP\TaskProcessing\Exception\Exception;
3635
use OCP\TaskProcessing\Exception\NotFoundException;
3736
use OCP\TaskProcessing\Exception\PreConditionNotMetException;
@@ -391,7 +390,7 @@ public function setFileContentsExApp(int $taskId): DataResponse {
391390
* @return StreamResponse<Http::STATUS_OK, array{}>|DataResponse<Http::STATUS_INTERNAL_SERVER_ERROR|Http::STATUS_NOT_FOUND, array{message: string}, array{}>
392391
*/
393392
private function getFileContentsInternal(Task $task, int $fileId): StreamResponse|DataResponse {
394-
$ids = $this->extractFileIdsFromTask($task);
393+
$ids = $this->taskProcessingManager->extractFileIdsFromTask($task);
395394
if (!in_array($fileId, $ids)) {
396395
return new DataResponse(['message' => $this->l->t('Not found')], Http::STATUS_NOT_FOUND);
397396
}
@@ -428,45 +427,6 @@ private function getFileContentsInternal(Task $task, int $fileId): StreamRespons
428427
return $response;
429428
}
430429

431-
/**
432-
* @param Task $task
433-
* @return list<int>
434-
* @throws NotFoundException
435-
*/
436-
private function extractFileIdsFromTask(Task $task): array {
437-
$ids = [];
438-
$taskTypes = $this->taskProcessingManager->getAvailableTaskTypes();
439-
if (!isset($taskTypes[$task->getTaskTypeId()])) {
440-
throw new NotFoundException('Could not find task type');
441-
}
442-
$taskType = $taskTypes[$task->getTaskTypeId()];
443-
foreach ($taskType['inputShape'] + $taskType['optionalInputShape'] as $key => $descriptor) {
444-
if (in_array(EShapeType::getScalarType($descriptor->getShapeType()), [EShapeType::File, EShapeType::Image, EShapeType::Audio, EShapeType::Video], true)) {
445-
/** @var int|list<int> $inputSlot */
446-
$inputSlot = $task->getInput()[$key];
447-
if (is_array($inputSlot)) {
448-
$ids = array_merge($inputSlot, $ids);
449-
} else {
450-
$ids[] = $inputSlot;
451-
}
452-
}
453-
}
454-
if ($task->getOutput() !== null) {
455-
foreach ($taskType['outputShape'] + $taskType['optionalOutputShape'] as $key => $descriptor) {
456-
if (in_array(EShapeType::getScalarType($descriptor->getShapeType()), [EShapeType::File, EShapeType::Image, EShapeType::Audio, EShapeType::Video], true)) {
457-
/** @var int|list<int> $outputSlot */
458-
$outputSlot = $task->getOutput()[$key];
459-
if (is_array($outputSlot)) {
460-
$ids = array_merge($outputSlot, $ids);
461-
} else {
462-
$ids[] = $outputSlot;
463-
}
464-
}
465-
}
466-
}
467-
return $ids;
468-
}
469-
470430
/**
471431
* Sets the task progress
472432
*

‎core/Migrations/Version32000Date20250806110519.php‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
namespace OC\Core\Migrations;
10+
11+
use Closure;
12+
use OCP\DB\ISchemaWrapper;
13+
use OCP\DB\Types;
14+
use OCP\Migration\Attributes\AddColumn;
15+
use OCP\Migration\Attributes\ColumnType;
16+
use OCP\Migration\IOutput;
17+
use OCP\Migration\SimpleMigrationStep;
18+
19+
/**
20+
*
21+
*/
22+
#[AddColumn(table: 'taskprocessing_tasks', name: 'allow_cleanup', type: ColumnType::SMALLINT)]
23+
class Version32000Date20250806110519 extends SimpleMigrationStep {
24+
25+
/**
26+
* @param IOutput $output
27+
* @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
28+
* @param array $options
29+
* @return null|ISchemaWrapper
30+
*/
31+
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
32+
/** @var ISchemaWrapper $schema */
33+
$schema = $schemaClosure();
34+
35+
if ($schema->hasTable('taskprocessing_tasks')) {
36+
$table = $schema->getTable('taskprocessing_tasks');
37+
if (!$table->hasColumn('allow_cleanup')) {
38+
$table->addColumn('allow_cleanup', Types::SMALLINT, [
39+
'notnull' => true,
40+
'default' => 1,
41+
'unsigned' => true,
42+
]);
43+
return $schema;
44+
}
45+
}
46+
47+
return null;
48+
}
49+
}

‎core/ResponseDefinitions.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,7 @@
201201
* scheduledAt: ?int,
202202
* startedAt: ?int,
203203
* endedAt: ?int,
204+
* allowCleanup: bool,
204205
* }
205206
*
206207
* @psalm-type CoreProfileAction = array{

‎core/openapi-ex_app.json‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,8 @@
145145
"progress",
146146
"scheduledAt",
147147
"startedAt",
148-
"endedAt"
148+
"endedAt",
149+
"allowCleanup"
149150
],
150151
"properties": {
151152
"id": {
@@ -216,6 +217,9 @@
216217
"type": "integer",
217218
"format": "int64",
218219
"nullable": true
220+
},
221+
"allowCleanup": {
222+
"type": "boolean"
219223
}
220224
}
221225
}

‎core/openapi-full.json‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -639,7 +639,8 @@
639639
"progress",
640640
"scheduledAt",
641641
"startedAt",
642-
"endedAt"
642+
"endedAt",
643+
"allowCleanup"
643644
],
644645
"properties": {
645646
"id": {
@@ -710,6 +711,9 @@
710711
"type": "integer",
711712
"format": "int64",
712713
"nullable": true
714+
},
715+
"allowCleanup": {
716+
"type": "boolean"
713717
}
714718
}
715719
},

0 commit comments

Comments
 (0)