feat(preview): Store original file mimetype in preview table

Allow to quickly query all the files from a specific mimetype like in
the ResetRenderedTexts command.

Signed-off-by: Carl Schwan <carl.schwan@nextcloud.com>

Author: CarlSchwan

core/BackgroundJobs/MovePreviewJob.php

@@ -24,7 +24,6 @@
 use OCP\Files\SimpleFS\ISimpleFolder;
 use OCP\IAppConfig;
 use OCP\IDBConnection;
-use OCP\IPreview;
 
 class MovePreviewJob extends TimedJob {
 	private IAppData $appData;
@@ -138,7 +137,7 @@ private function processPreviews(int|string $fileId, bool $simplePaths): void {
 		}
 
 		$qb = $this->connection->getQueryBuilder();
-		$qb->select('*')
+		$qb->select('storage', 'etag', 'mimetype')
 			->from('filecache')
 			->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId)))
 			->setMaxResults(1)
@@ -154,6 +153,7 @@ private function processPreviews(int|string $fileId, bool $simplePaths): void {
 				$file = $previewFile['file'];
 				$preview->setStorageId($result[0]['storage']);
 				$preview->setEtag($result[0]['etag']);
+				$preview->setSourceMimetype($result[0]['mimetype']);
 				try {
 					$preview = $this->previewMapper->insert($preview);
 				} catch (Exception $e) {

core/Command/Preview/ResetRenderedTexts.php

@@ -67,9 +67,7 @@ private function deleteAvatars(OutputInterface $output, bool $dryMode): void {
 
 			try {
 				$avatar->remove();
-			} catch (NotFoundException $e) {
-				// continue
-			} catch (NotPermittedException $e) {
+			} catch (NotFoundException|NotPermittedException) {
 				// continue
 			}
 		}
@@ -91,8 +89,8 @@ private function getAvatarsToDelete(): \Iterator {
 	private function deletePreviews(OutputInterface $output, bool $dryMode): void {
 		$previewsToDeleteCount = 0;
 
-		foreach ($this->getPreviewsToDelete() as ['path' => $filePath, 'preview' => $preview]) {
-			$output->writeln('Deleting previews for ' . $filePath, OutputInterface::VERBOSITY_VERBOSE);
+		foreach ($this->getPreviewsToDelete() as $preview) {
+			$output->writeln('Deleting preview ' . $preview->getName() . ' for fileId ' . $preview->getFileId(), OutputInterface::VERBOSITY_VERBOSE);
 
 			$previewsToDeleteCount++;
 
@@ -107,28 +105,13 @@ private function deletePreviews(OutputInterface $output, bool $dryMode): void {
 	}
 
 	/**
-	 * @return \Iterator<array{path: string, preview: Preview}>
+	 * @return \Generator<Preview>
 	 */
-	private function getPreviewsToDelete(): \Iterator {
-		$qb = $this->connection->getQueryBuilder();
-		$qb->select('fileid', 'path')
-			->from('filecache')
-			->where(
-				$qb->expr()->orX(
-					$qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/plain'))),
-					$qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/markdown'))),
-					$qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/x-markdown')))
-				)
-			);
-
-		$cursor = $qb->executeQuery();
-
-		while ($row = $cursor->fetch()) {
-			foreach ($this->previewService->getAvailablePreviewForFile($row['fileid']) as $preview) {
-				yield ['path' => $row['path'], 'preview' => $preview];
-			}
-		}
-
-		$cursor->closeCursor();
+	private function getPreviewsToDelete(): \Generator {
+		return $this->previewService->getPreviewsForMimeTypes([
+			$this->mimeTypeLoader->getId('text/plain'),
+			$this->mimeTypeLoader->getId('text/markdown'),
+			$this->mimeTypeLoader->getId('text/x-markdown'),
+		]);
 	}
 }

lib/private/Preview/Db/Preview.php

@@ -35,6 +35,8 @@
  * @method void setCropped(bool $cropped)
  * @method void setMimetype(int $mimetype) Set the mimetype of the preview.
  * @method int getMimetype() Get the mimetype of the preview.
+ * @method void setSourceMimetype(int $sourceMimetype) Set the mimetype of the source file.
+ * @method int getSourceMimetype() Get the mimetype of the source file.
  * @method int getMtime() Get the modification time of the preview.
  * @method void setMtime(int $mtime)
  * @method int getSize() Get the size of the preview.
@@ -61,6 +63,8 @@ class Preview extends Entity {
 	protected ?int $width = null;
 	protected ?int $height = null;
 	protected ?int $mimetype = null;
+
+	protected ?int $sourceMimetype = null;
 	protected ?int $mtime = null;
 	protected ?int $size = null;
 	protected ?bool $max = null;
@@ -77,6 +81,7 @@ public function __construct() {
 		$this->addType('width', Types::INTEGER);
 		$this->addType('height', Types::INTEGER);
 		$this->addType('mimetype', Types::INTEGER);
+		$this->addType('sourceMimetype', Types::INTEGER);
 		$this->addType('mtime', Types::INTEGER);
 		$this->addType('size', Types::INTEGER);
 		$this->addType('max', Types::BOOLEAN);

lib/private/Preview/Db/PreviewMapper.php

@@ -139,4 +139,19 @@ public function getPreviews(int $lastId, int $limit = 1000): \Generator {
 		return $this->yieldEntities($qb);
 
 	}
+
+	/**
+	 * @param int[] $mimeTypes
+	 * @return \Generator<Preview>
+	 */
+	public function getPreviewsForMimeTypes(array $mimeTypes): \Generator {
+		$qb = $this->db->getQueryBuilder();
+		$this->joinLocation($qb)
+			->where($qb->expr()->orX(
+				...array_map(function (int $mimeType) use ($qb) {
+					return $qb->expr()->eq('source_mimetype', $qb->createNamedParameter($mimeType, IQueryBuilder::PARAM_INT));
+				}, $mimeTypes)
+			));
+		return $this->yieldEntities($qb);
+	}
 }

lib/private/Preview/Generator.php

@@ -12,6 +12,7 @@
 use OC\Preview\Storage\StorageFactory;
 use OCP\EventDispatcher\IEventDispatcher;
 use OCP\Files\File;
+use OCP\Files\IMimeTypeLoader;
 use OCP\Files\InvalidPathException;
 use OCP\Files\NotFoundException;
 use OCP\Files\NotPermittedException;
@@ -38,6 +39,7 @@ public function __construct(
 		private LoggerInterface $logger,
 		private PreviewMapper $previewMapper,
 		private StorageFactory $storageFactory,
+		private IMimeTypeLoader $mimeTypeLoader,
 	) {
 	}
 
@@ -541,6 +543,7 @@ public function savePreview(File $file, int $width, int $height, bool $crop, boo
 		$previewEntry->setFileId($file->getId());
 		$previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId());
 		$previewEntry->setWidth($width);
+		$previewEntry->setSourceMimetype($this->mimeTypeLoader->getId($file->getMimeType()));
 		$previewEntry->setHeight($height);
 		$previewEntry->setVersion($version);
 		$previewEntry->setMax($max);

lib/private/Preview/PreviewService.php

@@ -71,7 +71,15 @@ public function getAvailableFileIds(): \Generator {
 	 * @return \Generator<Preview>
 	 */
 	public function getAvailablePreviewForFile(int $fileId): \Generator {
-		yield from $this->previewMapper->getAvailablePreviewForFile($fileId);
+		return $this->previewMapper->getAvailablePreviewForFile($fileId);
+	}
+
+	/**
+	 * @param int[] $mimeTypes
+	 * @return \Generator<Preview>
+	 */
+	public function getPreviewsForMimeTypes(array $mimeTypes): \Generator {
+		return $this->previewMapper->getPreviewsForMimeTypes($mimeTypes);
 	}
 
 	public function deleteAll(): void {

lib/private/Preview/Storage/LocalPreviewStorage.php

@@ -101,7 +101,7 @@ public function scan(): int {
 					$preview->setEncrypted(false);
 
 					$qb = $this->connection->getQueryBuilder();
-					$result = $qb->select('*')
+					$result = $qb->select('storage', 'etag', 'mimetype')
 						->from('filecache')
 						->where($qb->expr()->eq('fileid', $qb->createNamedParameter($preview->getFileId())))
 						->setMaxResults(1)
@@ -127,6 +127,7 @@ public function scan(): int {
 
 					$preview->setStorageId($result[0]['storage']);
 					$preview->setEtag($result[0]['etag']);
+					$preview->setSourceMimetype($result[0]['mimetype']);
 
 					// try to insert, if that fails the preview is already in the DB
 					$this->previewMapper->insert($preview);

lib/private/PreviewManager.php

@@ -16,6 +16,7 @@
 use OCP\AppFramework\QueryException;
 use OCP\EventDispatcher\IEventDispatcher;
 use OCP\Files\File;
+use OCP\Files\IMimeTypeLoader;
 use OCP\Files\IRootFolder;
 use OCP\Files\NotFoundException;
 use OCP\Files\SimpleFS\ISimpleFile;
@@ -139,6 +140,7 @@ private function getGenerator(): Generator {
 				$this->container->get(LoggerInterface::class),
 				$this->container->get(PreviewMapper::class),
 				$this->container->get(StorageFactory::class),
+				$this->container->get(IMimeTypeLoader::class),
 			);
 		}
 		return $this->generator;

tests/lib/Preview/GeneratorTest.php

@@ -14,6 +14,7 @@
 use OC\Preview\Storage\StorageFactory;
 use OCP\EventDispatcher\IEventDispatcher;
 use OCP\Files\File;
+use OCP\Files\IMimeTypeLoader;
 use OCP\Files\Mount\IMountPoint;
 use OCP\Files\NotFoundException;
 use OCP\IConfig;
@@ -34,6 +35,7 @@ class GeneratorTest extends TestCase {
 	private LoggerInterface&MockObject $logger;
 	private StorageFactory&MockObject $storageFactory;
 	private PreviewMapper&MockObject $previewMapper;
+	private IMimeTypeLoader&MockObject $mimeTypeLoader;
 
 	protected function setUp(): void {
 		parent::setUp();
@@ -45,6 +47,7 @@ protected function setUp(): void {
 		$this->logger = $this->createMock(LoggerInterface::class);
 		$this->previewMapper = $this->createMock(PreviewMapper::class);
 		$this->storageFactory = $this->createMock(StorageFactory::class);
+		$this->mimetypeLoader = $this->createMock(IMimeTypeLoader::class);
 
 		$this->generator = new Generator(
 			$this->config,
@@ -53,7 +56,8 @@ protected function setUp(): void {
 			$this->eventDispatcher,
 			$this->logger,
 			$this->previewMapper,
-			$this->storageFactory
+			$this->storageFactory,
+			$this->mimetypeLoader,
 		);
 	}
 

tests/lib/Preview/PreviewMapperTest.php

@@ -68,6 +68,7 @@ private function createPreviewForFileId(int $fileId, ?int $bucket = null) {
 		$preview->setCropped(true);
 		$preview->setMax(true);
 		$preview->setWidth(100);
+		$preview->setSourceMimetype(1);
 		$preview->setHeight(100);
 		$preview->setSize(100);
 		$preview->setMtime(time());