Skip to content

Commit c93666e

Browse files
CarlSchwanCarlSchwan
committed
refactor(preview): Cleanup the implementation of the new preview backend
Signed-off-by: Carl Schwan <[email protected]>
1 parent 8179214 commit c93666e

30 files changed

+575
-791
lines changed

‎build/psalm-baseline.xml‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2997,17 +2997,6 @@
29972997
<code><![CDATA[$this->timeFactory->getTime()]]></code>
29982998
</InvalidScalarArgument>
29992999
</file>
3000-
<file src="core/Command/Preview/Repair.php">
3001-
<UndefinedInterfaceMethod>
3002-
<code><![CDATA[section]]></code>
3003-
<code><![CDATA[section]]></code>
3004-
</UndefinedInterfaceMethod>
3005-
</file>
3006-
<file src="core/Command/Preview/ResetRenderedTexts.php">
3007-
<InvalidReturnStatement>
3008-
<code><![CDATA[[]]]></code>
3009-
</InvalidReturnStatement>
3010-
</file>
30113000
<file src="core/Command/Security/BruteforceAttempts.php">
30123001
<DeprecatedMethod>
30133002
<code><![CDATA[getAttempts]]></code>
@@ -3158,7 +3147,6 @@
31583147
</DeprecatedClass>
31593148
<DeprecatedInterface>
31603149
<code><![CDATA[$object]]></code>
3161-
<code><![CDATA[Task]]></code>
31623150
<code><![CDATA[private]]></code>
31633151
</DeprecatedInterface>
31643152
</file>

‎build/stubs/php-polyfill.php‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
<?php
2+
3+
// PHP 8.4
4+
function array_find(array $array, callable $callback) {}
5+

‎core/BackgroundJobs/MovePreviewJob.php‎

Lines changed: 108 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
<?php
22

33
declare(strict_types=1);
4-
/**
5-
* SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
4+
5+
/*
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH
7+
* SPDX-FileContributor: Carl Schwan
68
* SPDX-License-Identifier: AGPL-3.0-or-later
79
*/
810

@@ -17,7 +19,8 @@
1719
use OCP\DB\Exception;
1820
use OCP\Files\AppData\IAppDataFactory;
1921
use OCP\Files\IAppData;
20-
use OCP\Files\SimpleFS\ISimpleFile;
22+
use OCP\Files\IRootFolder;
23+
use OCP\Files\NotFoundException;
2124
use OCP\Files\SimpleFS\ISimpleFolder;
2225
use OCP\IAppConfig;
2326
use OCP\IDBConnection;
@@ -28,10 +31,11 @@ class MovePreviewJob extends TimedJob {
2831

2932
public function __construct(
3033
ITimeFactory $time,
31-
private IAppConfig $appConfig,
32-
private PreviewMapper $previewMapper,
33-
private StorageFactory $storageFactory,
34-
private IDBConnection $connection,
34+
private readonly IAppConfig $appConfig,
35+
private readonly PreviewMapper $previewMapper,
36+
private readonly StorageFactory $storageFactory,
37+
private readonly IDBConnection $connection,
38+
private readonly IRootFolder $rootFolder,
3539
IAppDataFactory $appDataFactory,
3640
) {
3741
parent::__construct($time);
@@ -42,15 +46,6 @@ public function __construct(
4246
}
4347

4448
protected function run(mixed $argument): void {
45-
try {
46-
$this->doRun($argument);
47-
} catch (\Throwable $exception) {
48-
echo $exception->getMessage();
49-
throw $exception;
50-
}
51-
}
52-
53-
private function doRun($argument): void {
5449
if ($this->appConfig->getValueBool('core', 'previewMovedDone')) {
5550
return;
5651
}
@@ -59,27 +54,21 @@ private function doRun($argument): void {
5954

6055
$startTime = time();
6156
while (true) {
62-
$previewFolders = [];
63-
6457
// Check new hierarchical preview folders first
6558
if (!$emptyHierarchicalPreviewFolders) {
6659
$qb = $this->connection->getQueryBuilder();
6760
$qb->select('*')
6861
->from('filecache')
6962
->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%/%/%/%/%/%/%/%')))
63+
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
7064
->setMaxResults(100);
7165

7266
$result = $qb->executeQuery();
7367
while ($row = $result->fetch()) {
7468
$pathSplit = explode('/', $row['path']);
7569
assert(count($pathSplit) >= 2);
7670
$fileId = $pathSplit[count($pathSplit) - 2];
77-
$previewFolders[$fileId][] = $row['path'];
78-
}
79-
80-
if (!empty($previewFolders)) {
81-
$this->processPreviews($previewFolders, false);
82-
continue;
71+
$this->processPreviews($fileId, false);
8372
}
8473
}
8574

@@ -89,6 +78,7 @@ private function doRun($argument): void {
8978
$qb->select('*')
9079
->from('filecache')
9180
->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%.%')))
81+
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
9282
->setMaxResults(100);
9383

9484
$result = $qb->executeQuery();
@@ -98,18 +88,7 @@ private function doRun($argument): void {
9888
$fileId = $pathSplit[count($pathSplit) - 2];
9989
array_pop($pathSplit);
10090
$path = implode('/', $pathSplit);
101-
if (!isset($previewFolders[$fileId])) {
102-
$previewFolders[$fileId] = [];
103-
}
104-
if (!in_array($path, $previewFolders[$fileId])) {
105-
$previewFolders[$fileId][] = $path;
106-
}
107-
}
108-
109-
if (empty($previewFolders)) {
110-
break;
111-
} else {
112-
$this->processPreviews($previewFolders, true);
91+
$this->processPreviews($fileId, true);
11392
}
11493

11594
// Stop if execution time is more than one hour.
@@ -118,97 +97,114 @@ private function doRun($argument): void {
11897
}
11998
}
12099

121-
// Delete any leftover preview directory
122-
$this->appData->getFolder('.')->delete();
100+
try {
101+
// Delete any leftover preview directory
102+
$this->appData->getFolder('.')->delete();
103+
} catch (NotFoundException) {
104+
// ignore
105+
}
123106
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
124107
}
125108

126109
/**
127110
* @param array<string|int, string[]> $previewFolders
128111
*/
129-
private function processPreviews(array $previewFolders, bool $simplePaths): void {
130-
foreach ($previewFolders as $fileId => $previewFolder) {
131-
$internalPath = $this->getInternalFolder((string)$fileId, $simplePaths);
132-
$folder = $this->appData->getFolder($internalPath);
133-
134-
/**
135-
* @var list<array{file: SimpleFile, width: int, height: int, crop: bool, max: bool, extension: string, mtime: int, size: int}> $previewFiles
136-
*/
137-
$previewFiles = [];
138-
139-
foreach ($folder->getDirectoryListing() as $previewFile) {
140-
/** @var SimpleFile $previewFile */
141-
[0 => $baseName, 1 => $extension] = explode('.', $previewFile->getName());
142-
$nameSplit = explode('-', $baseName);
143-
144-
// TODO VERSION/PREFIX extraction
145-
146-
$width = $nameSplit[0];
147-
$height = $nameSplit[1];
112+
private function processPreviews(int|string $fileId, bool $simplePaths): void {
113+
$internalPath = $this->getInternalFolder((string)$fileId, $simplePaths);
114+
$folder = $this->appData->getFolder($internalPath);
115+
116+
/**
117+
* @var list<array{
118+
* file: SimpleFile, width: int, height: int, crop: bool, max: bool, extension: string, mtime: int, size: int, version: ?int
119+
* }> $previewFiles
120+
*/
121+
$previewFiles = [];
122+
123+
foreach ($folder->getDirectoryListing() as $previewFile) {
124+
/** @var SimpleFile $previewFile */
125+
[0 => $baseName, 1 => $extension] = explode('.', $previewFile->getName());
126+
$nameSplit = explode('-', $baseName);
127+
128+
$offset = 0;
129+
$version = null;
130+
if (count($nameSplit) === 4 || (count($nameSplit) === 3 && is_numeric($nameSplit[2]))) {
131+
$offset = 1;
132+
$version = (int)$nameSplit[0];
133+
}
148134

149-
if (isset($nameSplit[2])) {
150-
$crop = $nameSplit[2] === 'crop';
151-
$max = $nameSplit[2] === 'max';
152-
}
135+
$width = (int)$nameSplit[$offset + 0];
136+
$height = (int)$nameSplit[$offset + 1];
153137

154-
$previewFiles[] = [
155-
'file' => $previewFile,
156-
'width' => $width,
157-
'height' => $height,
158-
'crop' => $crop,
159-
'max' => $max,
160-
'extension' => $extension,
161-
'size' => $previewFile->getSize(),
162-
'mtime' => $previewFile->getMTime(),
163-
];
138+
$crop = false;
139+
$max = false;
140+
if (isset($nameSplit[$offset + 2])) {
141+
$crop = $nameSplit[$offset + 2] === 'crop';
142+
$max = $nameSplit[$offset + 2] === 'max';
164143
}
165144

166-
$qb = $this->connection->getQueryBuilder();
167-
$qb->select('*')
168-
->from('filecache')
169-
->where($qb->expr()->like('fileid', $qb->createNamedParameter($fileId)));
145+
$previewFiles[] = [
146+
'file' => $previewFile,
147+
'width' => $width,
148+
'height' => $height,
149+
'crop' => $crop,
150+
'version' => $version,
151+
'max' => $max,
152+
'extension' => $extension,
153+
'size' => $previewFile->getSize(),
154+
'mtime' => $previewFile->getMTime(),
155+
];
156+
}
170157

171-
$result = $qb->executeQuery();
172-
$result = $result->fetchAll();
173-
174-
if (count($result) > 0) {
175-
foreach ($previewFiles as $previewFile) {
176-
$preview = new Preview();
177-
$preview->setFileId((int)$fileId);
178-
$preview->setOldFileId($previewFile['file']->getId());
179-
$preview->setEtag($result[0]['etag']);
180-
$preview->setMtime($previewFile['mtime']);
181-
$preview->setWidth($previewFile['width']);
182-
$preview->setHeight($previewFile['height']);
183-
$preview->setCrop($previewFile['crop']);
184-
$preview->setIsMax($previewFile['max']);
185-
$preview->setMimetype(match ($previewFile['extension']) {
186-
'png' => IPreview::MIMETYPE_PNG,
187-
'webp' => IPreview::MIMETYPE_WEBP,
188-
'gif' => IPreview::MIMETYPE_GIF,
189-
default => IPreview::MIMETYPE_JPEG,
190-
});
191-
$preview->setSize($previewFile['size']);
192-
try {
193-
$preview = $this->previewMapper->insert($preview);
194-
} catch (Exception $e) {
195-
// We already have this preview in the preview table, skip
196-
continue;
197-
}
198-
199-
try {
200-
$this->storageFactory->migratePreview($preview, $previewFile['file']);
201-
$previewFile['file']->delete();
202-
} catch (\Exception $e) {
203-
$this->previewMapper->delete($preview);
204-
throw $e;
205-
}
158+
$qb = $this->connection->getQueryBuilder();
159+
$qb->select('*')
160+
->from('filecache')
161+
->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId)))
162+
->runAcrossAllShards(); // Unavoidable because we can just extract the file_id in the preview name
163+
164+
$result = $qb->executeQuery();
165+
$result = $result->fetchAll();
166+
167+
if (count($result) > 0) {
168+
foreach ($previewFiles as $previewFile) {
169+
$preview = new Preview();
170+
$preview->setFileId((int)$fileId);
171+
/** @var SimpleFile $file */
172+
$file = $previewFile['file'];
173+
$preview->setOldFileId($file->getId());
174+
$preview->setStorageId($result[0]['storage']);
175+
$preview->setEtag($result[0]['etag']);
176+
$preview->setMtime($previewFile['mtime']);
177+
$preview->setWidth($previewFile['width']);
178+
$preview->setHeight($previewFile['height']);
179+
$preview->setCropped($previewFile['crop']);
180+
$preview->setVersion($previewFile['version']);
181+
$preview->setMax($previewFile['max']);
182+
$preview->setEncrypted(false);
183+
$preview->setMimetype(match ($previewFile['extension']) {
184+
'png' => IPreview::MIMETYPE_PNG,
185+
'webp' => IPreview::MIMETYPE_WEBP,
186+
'gif' => IPreview::MIMETYPE_GIF,
187+
default => IPreview::MIMETYPE_JPEG,
188+
});
189+
$preview->setSize($previewFile['size']);
190+
try {
191+
$preview = $this->previewMapper->insert($preview);
192+
} catch (Exception $e) {
193+
// We already have this preview in the preview table, skip
194+
continue;
195+
}
206196

197+
try {
198+
$this->storageFactory->migratePreview($preview, $previewFile['file']);
199+
$previewFile['file']->delete();
200+
} catch (\Exception $e) {
201+
$this->previewMapper->delete($preview);
202+
throw $e;
207203
}
208204
}
209-
210-
$this->deleteFolder($internalPath, $folder);
211205
}
206+
207+
$this->deleteFolder($internalPath, $folder);
212208
}
213209

214210
public static function getInternalFolder(string $name, bool $simplePaths): string {

0 commit comments

Comments
 (0)