Skip to content

Commit b8c5d86

Browse files
CarlSchwanCarlSchwan
committed
fix(preview): Make version column a string
And move it to a different table so that we don't have to pay the storage cost when not using it (most of the times). Signed-off-by: Carl Schwan <[email protected]>
1 parent 66f50bd commit b8c5d86

23 files changed

+404
-246
lines changed

‎core/BackgroundJobs/MovePreviewJob.php‎

Lines changed: 81 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -17,24 +17,26 @@
1717
use OCP\AppFramework\Utility\ITimeFactory;
1818
use OCP\BackgroundJob\TimedJob;
1919
use OCP\DB\Exception;
20+
use OCP\DB\IResult;
2021
use OCP\Files\AppData\IAppDataFactory;
2122
use OCP\Files\IAppData;
2223
use OCP\Files\IMimeTypeDetector;
2324
use OCP\Files\IMimeTypeLoader;
2425
use OCP\Files\IRootFolder;
25-
use OCP\Files\NotFoundException;
26-
use OCP\Files\SimpleFS\ISimpleFolder;
2726
use OCP\IAppConfig;
27+
use OCP\IConfig;
2828
use OCP\IDBConnection;
2929
use Override;
3030
use Psr\Log\LoggerInterface;
3131

3232
class MovePreviewJob extends TimedJob {
3333
private IAppData $appData;
34+
private string $previewRootPath;
3435

3536
public function __construct(
3637
ITimeFactory $time,
3738
private readonly IAppConfig $appConfig,
39+
private readonly IConfig $config,
3840
private readonly PreviewMapper $previewMapper,
3941
private readonly StorageFactory $storageFactory,
4042
private readonly IDBConnection $connection,
@@ -49,6 +51,7 @@ public function __construct(
4951
$this->appData = $appDataFactory->get('preview');
5052
$this->setTimeSensitivity(self::TIME_INSENSITIVE);
5153
$this->setInterval(24 * 60 * 60);
54+
$this->previewRootPath = 'appdata_' . $this->config->getSystemValueString('instanceid') . '/preview/';
5255
}
5356

5457
#[Override]
@@ -57,49 +60,22 @@ protected function run(mixed $argument): void {
5760
return;
5861
}
5962

60-
$emptyHierarchicalPreviewFolders = false;
61-
6263
$startTime = time();
6364
while (true) {
64-
// Check new hierarchical preview folders first
65-
if (!$emptyHierarchicalPreviewFolders) {
66-
$qb = $this->connection->getQueryBuilder();
67-
$qb->select('*')
68-
->from('filecache')
69-
->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%/%/%/%/%/%/%/%')))
70-
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
71-
->setMaxResults(100);
72-
73-
$result = $qb->executeQuery();
74-
while ($row = $result->fetch()) {
75-
$pathSplit = explode('/', $row['path']);
76-
assert(count($pathSplit) >= 2);
77-
$fileId = $pathSplit[count($pathSplit) - 2];
78-
$this->processPreviews($fileId, false);
79-
}
80-
}
81-
82-
// And then the flat preview folder (legacy)
83-
$emptyHierarchicalPreviewFolders = true;
8465
$qb = $this->connection->getQueryBuilder();
85-
$qb->select('*')
66+
$qb->select('path')
8667
->from('filecache')
87-
->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%.%')))
68+
// Hierarchical preview folder structure
69+
->where($qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%/%/%/%/%/%/%/%')))
70+
// Legacy flat preview folder structure
71+
->orWhere($qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%.%')))
8872
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
8973
->setMaxResults(100);
9074

9175
$result = $qb->executeQuery();
92-
$foundOldPreview = false;
93-
while ($row = $result->fetch()) {
94-
$pathSplit = explode('/', $row['path']);
95-
assert(count($pathSplit) >= 2);
96-
$fileId = $pathSplit[count($pathSplit) - 2];
97-
array_pop($pathSplit);
98-
$this->processPreviews($fileId, true);
99-
$foundOldPreview = true;
100-
}
76+
$foundPreviews = $this->processQueryResult($result);
10177

102-
if (!$foundOldPreview) {
78+
if (!$foundPreviews) {
10379
break;
10480
}
10581

@@ -109,20 +85,46 @@ protected function run(mixed $argument): void {
10985
}
11086
}
11187

112-
try {
113-
// Delete any leftover preview directory
114-
$this->appData->getFolder('.')->delete();
115-
} catch (NotFoundException) {
116-
// ignore
117-
}
11888
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
11989
}
12090

91+
private function processQueryResult(IResult $result): bool {
92+
$foundPreview = false;
93+
$fileIds = [];
94+
$flatFileIds = [];
95+
while ($row = $result->fetch()) {
96+
$pathSplit = explode('/', $row['path']);
97+
assert(count($pathSplit) >= 2);
98+
$fileId = (int)$pathSplit[count($pathSplit) - 2];
99+
if (count($pathSplit) === 11) {
100+
// Hierarchical structure
101+
if (!in_array($fileId, $fileIds)) {
102+
$fileIds[] = $fileId;
103+
}
104+
} else {
105+
// Flat structure
106+
if (!in_array($fileId, $flatFileIds)) {
107+
$flatFileIds[] = $fileId;
108+
}
109+
}
110+
$foundPreview = true;
111+
}
112+
113+
foreach ($fileIds as $fileId) {
114+
$this->processPreviews($fileId, flatPath: false);
115+
}
116+
117+
foreach ($flatFileIds as $fileId) {
118+
$this->processPreviews($fileId, flatPath: true);
119+
}
120+
return $foundPreview;
121+
}
122+
121123
/**
122124
* @param array<string|int, string[]> $previewFolders
123125
*/
124-
private function processPreviews(int|string $fileId, bool $simplePaths): void {
125-
$internalPath = $this->getInternalFolder((string)$fileId, $simplePaths);
126+
private function processPreviews(int $fileId, bool $flatPath): void {
127+
$internalPath = $this->getInternalFolder((string)$fileId, $flatPath);
126128
$folder = $this->appData->getFolder($internalPath);
127129

128130
/**
@@ -133,7 +135,7 @@ private function processPreviews(int|string $fileId, bool $simplePaths): void {
133135
foreach ($folder->getDirectoryListing() as $previewFile) {
134136
$path = $fileId . '/' . $previewFile->getName();
135137
/** @var SimpleFile $previewFile */
136-
$preview = Preview::fromPath($path, $this->mimeTypeDetector, $this->mimeTypeLoader);
138+
$preview = Preview::fromPath($path, $this->mimeTypeDetector);
137139
if (!$preview) {
138140
$this->logger->error('Unable to import old preview at path.');
139141
continue;
@@ -160,59 +162,82 @@ private function processPreviews(int|string $fileId, bool $simplePaths): void {
160162

161163
if (count($result) > 0) {
162164
foreach ($previewFiles as $previewFile) {
165+
/** @var Preview $preview */
163166
$preview = $previewFile['preview'];
164167
/** @var SimpleFile $file */
165168
$file = $previewFile['file'];
166169
$preview->setStorageId($result[0]['storage']);
167170
$preview->setEtag($result[0]['etag']);
168-
$preview->setSourceMimetype($result[0]['mimetype']);
171+
$preview->setSourceMimeType($this->mimeTypeLoader->getMimetypeById($result[0]['mimetype']));
169172
try {
170173
$preview = $this->previewMapper->insert($preview);
171-
} catch (Exception $e) {
174+
} catch (Exception) {
172175
// We already have this preview in the preview table, skip
176+
$qb->delete('filecache')
177+
->where($qb->expr()->eq('fileid', $qb->createNamedParameter($file->getId())))
178+
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
179+
->executeStatement();
173180
continue;
174181
}
175182

176183
try {
177184
$this->storageFactory->migratePreview($preview, $file);
185+
$qb = $this->connection->getQueryBuilder();
178186
$qb->delete('filecache')
179187
->where($qb->expr()->eq('fileid', $qb->createNamedParameter($file->getId())))
188+
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
180189
->executeStatement();
181190
// Do not call $file->delete() as this will also delete the file from the file system
182191
} catch (\Exception $e) {
183192
$this->previewMapper->delete($preview);
184193
throw $e;
185194
}
186195
}
196+
} else {
197+
// No matching fileId, delete preview
198+
try {
199+
$this->connection->beginTransaction();
200+
foreach ($previewFiles as $previewFile) {
201+
/** @var SimpleFile $file */
202+
$file = $previewFile['file'];
203+
$file->delete();
204+
}
205+
$this->connection->commit();
206+
} catch (Exception) {
207+
$this->connection->rollback();
208+
}
187209
}
188210

189-
$this->deleteFolder($internalPath, $folder);
211+
$this->deleteFolder($internalPath);
190212
}
191213

192-
public static function getInternalFolder(string $name, bool $simplePaths): string {
193-
if ($simplePaths) {
194-
return '/' . $name;
214+
public static function getInternalFolder(string $name, bool $flatPath): string {
215+
if ($flatPath) {
216+
return $name;
195217
}
196218
return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name;
197219
}
198220

199-
private function deleteFolder(string $path, ISimpleFolder $folder): void {
200-
$folder->delete();
201-
221+
private function deleteFolder(string $path): void {
202222
$current = $path;
203223

204224
while (true) {
225+
$appDataPath = $this->previewRootPath . $current;
226+
$qb = $this->connection->getQueryBuilder();
227+
$qb->delete('filecache')
228+
->where($qb->expr()->eq('path_hash', $qb->createNamedParameter(md5($appDataPath))))
229+
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
230+
->executeStatement();
231+
205232
$current = dirname($current);
206233
if ($current === '/' || $current === '.' || $current === '') {
207234
break;
208235
}
209236

210-
211237
$folder = $this->appData->getFolder($current);
212238
if (count($folder->getDirectoryListing()) !== 0) {
213239
break;
214240
}
215-
$folder->delete();
216241
}
217242
}
218243
}

‎core/Command/Preview/ResetRenderedTexts.php‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010

1111
use OC\Preview\Db\Preview;
1212
use OC\Preview\PreviewService;
13-
use OCP\Files\IMimeTypeLoader;
1413
use OCP\Files\NotFoundException;
1514
use OCP\Files\NotPermittedException;
1615
use OCP\IAvatarManager;
@@ -28,7 +27,6 @@ public function __construct(
2827
protected readonly IUserManager $userManager,
2928
protected readonly IAvatarManager $avatarManager,
3029
private readonly PreviewService $previewService,
31-
private readonly IMimeTypeLoader $mimeTypeLoader,
3230
) {
3331
parent::__construct();
3432
}
@@ -93,7 +91,7 @@ private function deletePreviews(OutputInterface $output, bool $dryMode): void {
9391
$previewsToDeleteCount = 0;
9492

9593
foreach ($this->getPreviewsToDelete() as $preview) {
96-
$output->writeln('Deleting preview ' . $preview->getName($this->mimeTypeLoader) . ' for fileId ' . $preview->getFileId(), OutputInterface::VERBOSITY_VERBOSE);
94+
$output->writeln('Deleting preview ' . $preview->getName() . ' for fileId ' . $preview->getFileId(), OutputInterface::VERBOSITY_VERBOSE);
9795

9896
$previewsToDeleteCount++;
9997

@@ -112,9 +110,9 @@ private function deletePreviews(OutputInterface $output, bool $dryMode): void {
112110
*/
113111
private function getPreviewsToDelete(): \Generator {
114112
return $this->previewService->getPreviewsForMimeTypes([
115-
$this->mimeTypeLoader->getId('text/plain'),
116-
$this->mimeTypeLoader->getId('text/markdown'),
117-
$this->mimeTypeLoader->getId('text/x-markdown'),
113+
'text/plain',
114+
'text/markdown',
115+
'text/x-markdown'
118116
]);
119117
}
120118
}

‎core/Migrations/Version33000Date20250819110529.php‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,14 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt
3737
$table->setPrimaryKey(['id']);
3838
}
3939

40+
if (!$schema->hasTable('preview_versions')) {
41+
$table = $schema->createTable('preview_versions');
42+
$table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]);
43+
$table->addColumn('file_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]);
44+
$table->addColumn('version', Types::STRING, ['notnull' => true, 'default' => '', 'length' => 1024]);
45+
$table->setPrimaryKey(['id']);
46+
}
47+
4048
if (!$schema->hasTable('previews')) {
4149
$table = $schema->createTable('previews');
4250
$table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]);
@@ -46,18 +54,18 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt
4654
$table->addColumn('location_id', Types::BIGINT, ['notnull' => false, 'length' => 20, 'unsigned' => true]);
4755
$table->addColumn('width', Types::INTEGER, ['notnull' => true, 'unsigned' => true]);
4856
$table->addColumn('height', Types::INTEGER, ['notnull' => true, 'unsigned' => true]);
49-
$table->addColumn('mimetype', Types::INTEGER, ['notnull' => true]);
50-
$table->addColumn('source_mimetype', Types::INTEGER, ['notnull' => true]);
57+
$table->addColumn('mimetype_id', Types::INTEGER, ['notnull' => true]);
58+
$table->addColumn('source_mimetype_id', Types::INTEGER, ['notnull' => true]);
5159
$table->addColumn('max', Types::BOOLEAN, ['notnull' => true, 'default' => false]);
5260
$table->addColumn('cropped', Types::BOOLEAN, ['notnull' => true, 'default' => false]);
5361
$table->addColumn('encrypted', Types::BOOLEAN, ['notnull' => true, 'default' => false]);
5462
$table->addColumn('etag', Types::STRING, ['notnull' => true, 'length' => 40, 'fixed' => true]);
5563
$table->addColumn('mtime', Types::INTEGER, ['notnull' => true, 'unsigned' => true]);
5664
$table->addColumn('size', Types::INTEGER, ['notnull' => true, 'unsigned' => true]);
57-
$table->addColumn('version', Types::BIGINT, ['notnull' => true, 'default' => -1]); // can not be null otherwise unique index doesn't work
65+
$table->addColumn('version_id', Types::BIGINT, ['notnull' => true, 'default' => -1]);
5866
$table->setPrimaryKey(['id']);
5967
$table->addIndex(['file_id']);
60-
$table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype', 'cropped', 'version'], 'previews_file_uniq_idx');
68+
$table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype_id', 'cropped', 'version_id'], 'previews_file_uniq_idx');
6169
}
6270

6371
return $schema;

‎lib/private/Files/Cache/LocalRootScanner.php‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
use OCP\IConfig;
1212
use OCP\Server;
13+
use Override;
1314

1415
class LocalRootScanner extends Scanner {
1516
private string $previewFolder;
@@ -20,6 +21,7 @@ public function __construct(\OC\Files\Storage\Storage $storage) {
2021
$this->previewFolder = 'appdata_' . $config->getSystemValueString('instanceid', '') . '/preview';
2122
}
2223

24+
#[Override]
2325
public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
2426
if ($this->shouldScanPath($file)) {
2527
return parent::scanFile($file, $reuseExisting, $parentId, $cacheData, $lock, $data);
@@ -28,6 +30,7 @@ public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData =
2830
}
2931
}
3032

33+
#[Override]
3134
public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) {
3235
if ($this->shouldScanPath($path)) {
3336
return parent::scan($path, $recursive, $reuse, $lock);
@@ -36,11 +39,16 @@ public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $loc
3639
}
3740
}
3841

39-
private function shouldScanPath(string $path): bool {
40-
$path = trim($path, '/');
42+
#[Override]
43+
protected function scanChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float $oldSize, &$etagChanged = false) {
4144
if (str_starts_with($path, $this->previewFolder)) {
42-
return false;
45+
return 0;
4346
}
47+
return parent::scanChildren($path, $recursive, $reuse, $folderId, $lock, $oldSize, $etagChanged);
48+
}
49+
50+
private function shouldScanPath(string $path): bool {
51+
$path = trim($path, '/');
4452
return $path === '' || str_starts_with($path, 'appdata_') || str_starts_with($path, '__groupfolders');
4553
}
4654
}

‎lib/private/Preview/BackgroundCleanupJob.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ public function __construct(
3535
public function run($argument): void {
3636
foreach ($this->getDeletedFiles() as $fileId) {
3737
$previewIds = [];
38-
foreach ($this->previewService->getAvailablePreviewForFile($fileId) as $preview) {
38+
foreach ($this->previewService->getAvailablePreviewsForFile($fileId) as $preview) {
3939
$this->previewService->deletePreview($preview);
4040
}
4141
}

0 commit comments

Comments
 (0)