diff --git a/.env.example b/.env.example index da37b45f99..b1a17c6a89 100644 --- a/.env.example +++ b/.env.example @@ -34,6 +34,8 @@ MBIN_DEFAULT_THEME=default # Set the max image file size (in bytes) # This should be set to <= `upload_max_filesize` and `post_max_size` in the server's php.ini file MBIN_MAX_IMAGE_BYTES=6000000 +# Image compression quality, set to -1 to disable. A value between 0.1 and 0.95 should be used +MBIN_IMAGE_COMPRESSION_QUALITY=0.9 # Change the down vote behaviour. Possible values are: # 'enabled' => default mode downvotes are enabled # 'hidden' => downvotes are counted and users can downvote, but the number is hidden diff --git a/.env.example_docker b/.env.example_docker index 6b808e309b..c049dfa481 100644 --- a/.env.example_docker +++ b/.env.example_docker @@ -39,6 +39,8 @@ MBIN_DEFAULT_THEME=default # Set the max image file size (in bytes) # This should be set to <= `upload_max_filesize` and `post_max_size` in the server's php.ini file MBIN_MAX_IMAGE_BYTES=6000000 +# Image compression quality, set to -1 to disable. A value between 0.1 and 0.95 should be used +MBIN_IMAGE_COMPRESSION_QUALITY=0.9 # Change the down vote behaviour. Possible values are: # 'enabled' => default mode downvotes are enabled # 'hidden' => downvotes are counted and users can downvote, but the number is hidden diff --git a/composer.json b/composer.json index 64ec4678dc..1a9a4f20b9 100644 --- a/composer.json +++ b/composer.json @@ -26,9 +26,11 @@ "doctrine/orm": "^2.19.6", "embed/embed": "^4.4.12", "endroid/qr-code": "^6.0.3", + "firebase/php-jwt": "7.0.2 as 6.11.1", "friendsofsymfony/jsrouting-bundle": "^3.5.0", "furqansiddiqui/bip39-mnemonic-php": "^0.1.7", "gumlet/php-image-resize": "^2.0.4", + "imagine/imagine": "^1.5", "knplabs/knp-time-bundle": "^2.4.0", "knpuniversity/oauth2-client-bundle": "^2.18.1", "kornrunner/blurhash": "^1.2.2", @@ -114,8 +116,7 @@ "twig/intl-extra": "^3.10.0", "twig/twig": "^3.15.0", "webmozart/assert": "^1.11.0", - "wohali/oauth2-discord-new": "^1.2.1", - "firebase/php-jwt": "7.0.2 as 6.11.1" + "wohali/oauth2-discord-new": "^1.2.1" }, "require-dev": { "brianium/paratest": "^7.10.1", diff --git a/composer.lock b/composer.lock index 1b2ef80705..7daf7f8341 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "cf2ded08cd846a7c4c4c3e3b52c25909", + "content-hash": "18b005a08fc754499c4aa818a3190404", "packages": [ { "name": "aws/aws-crt-php", @@ -2943,16 +2943,16 @@ }, { "name": "imagine/imagine", - "version": "1.5.0", + "version": "1.5.2", "source": { "type": "git", "url": "https://github.com/php-imagine/Imagine.git", - "reference": "80ab21434890dee9ba54969d31c51ac8d4d551e0" + "reference": "f9ed796eefb77c2f0f2167e1d4e36bc2b5ed6b0c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-imagine/Imagine/zipball/80ab21434890dee9ba54969d31c51ac8d4d551e0", - "reference": "80ab21434890dee9ba54969d31c51ac8d4d551e0", + "url": "https://api.github.com/repos/php-imagine/Imagine/zipball/f9ed796eefb77c2f0f2167e1d4e36bc2b5ed6b0c", + "reference": "f9ed796eefb77c2f0f2167e1d4e36bc2b5ed6b0c", "shasum": "" }, "require": { @@ -2999,9 +2999,9 @@ ], "support": { "issues": "https://github.com/php-imagine/Imagine/issues", - "source": "https://github.com/php-imagine/Imagine/tree/1.5.0" + "source": "https://github.com/php-imagine/Imagine/tree/1.5.2" }, - "time": "2024-12-03T14:37:55+00:00" + "time": "2026-01-09T10:45:12+00:00" }, { "name": "knplabs/knp-time-bundle", diff --git a/config/services.yaml b/config/services.yaml index 9c32ed0320..fbca482ef3 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -117,6 +117,8 @@ parameters: mbin_max_image_bytes: '%env(int:default:mbin_max_image_bytes_default:MBIN_MAX_IMAGE_BYTES)%' mbin_max_image_bytes_default: 6000000 + mbin_image_compression_quality: '%env(float:default:mbin_image_compression_quality_default:MBIN_IMAGE_COMPRESSION_QUALITY)%' + mbin_image_compression_quality_default: -1 mbin_downvotes_mode_default: 'enabled' mbin_downvotes_mode: '%env(enum:\App\Utils\DownvotesMode:default:mbin_downvotes_mode_default:MBIN_DOWNVOTES_MODE)%' @@ -156,6 +158,7 @@ services: $monitoringTwigRendersPersistingEnabled: '%mbin_monitoring_twig_render_persisting_enabled%' $monitoringCurlRequestsEnabled: '%mbin_monitoring_curl_requests_enabled%' $monitoringCurlRequestPersistingEnabled: '%mbin_monitoring_curl_request_persisting_enabled%' + $imageCompressionQuality: '%mbin_image_compression_quality%' kbin.s3_client: class: Aws\S3\S3Client diff --git a/docs/02-admin/03-optional-features/09-image-compression.md b/docs/02-admin/03-optional-features/09-image-compression.md new file mode 100644 index 0000000000..5e121b7d25 --- /dev/null +++ b/docs/02-admin/03-optional-features/09-image-compression.md @@ -0,0 +1,34 @@ +# Image compression + +You can enable compression of images uploaded to mbin by users or downloaded from remote instances, +for increased compatibility, to save on size and for a better user experience. + +To enable image compression set `MBIN_IMAGE_COMPRESSION_QUALITY` in your `.env` file to a value between 0.1 and 0.95. +This setting is used as a starting point to compress the image. It is gradually lowered (in 0.05 steps) until the maximum size is no longer exceeded. + +> [!HINT] +> The maximum file size is determined by the `MBIN_MAX_IMAGE_BYTES` setting in your `.env` file + +> [!NOTE] +> Enabling this setting can cause a higher memory usage + +## Better compatibility + +If another instance shares a thread with an image attached that exceeds your maximum image size, it will not be downloaded, +but instead loaded directly from the other instance. This works most of the time, +but sometimes website settings will block it and thus your users will see an image that cannot be loaded. +This behavior also introduces web requests to other servers, which may unintentionally leak information to the remote instance. + +If instead your server compresses the image and saves it locally this will never happen. + +## Saving space + +When image compression is enabled you can reduce your maximum image size to, lets say 1MB. +Without the compression this might not be suitable, because too many images exceed that size, +and you don't want to risk compatibility problems, +but with it enabled the images will just be compressed, saving space. + +## A better user experience + +Normally there is a maximum image size your users must adhere to, but if image compression is enabled, +instead of showing your user an error that the image exceeds that size, the upload goes through and the image is compressed. diff --git a/docs/02-admin/03-optional-features/README.md b/docs/02-admin/03-optional-features/README.md index 14378ef63a..2e66346b42 100644 --- a/docs/02-admin/03-optional-features/README.md +++ b/docs/02-admin/03-optional-features/README.md @@ -12,3 +12,4 @@ Like setting-up: - [S3 storage](06-s3_storage.md) - Configure an object storage service (S3) compatible bucket for storing images. - [Anubis](07-anubis.md) - A service for weighing the incoming requests and may present them with a proof-of-work challenge. It is useful if your instance gets hit a lot of bot traffic that you're tired of filtering through - [Monitoring](08-monitoring.md) - Internal monitoring of requests and messengers +- [Image compression](09-image-compression.md) - compress images if they exceed your maximum image size diff --git a/docs/02-admin/04-running-mbin/05-cli.md b/docs/02-admin/04-running-mbin/05-cli.md index 52243c5a0c..330cc6f834 100644 --- a/docs/02-admin/04-running-mbin/05-cli.md +++ b/docs/02-admin/04-running-mbin/05-cli.md @@ -326,6 +326,44 @@ Arguments: ## Images +### Remove cached remote media + +This command allows you to remove the cached file of remote media, **without** deleting the reference. +You can run this command as a cron job to only keep cached media from the last 30 days for example. + +> [!TIP] +> If a thread or microblog is opened without a local cache of the attached image existing, the image will be downloaded again. +> Once an image is downloaded again, it will not get deleted for the number of days you set as a parameter. + +> [!NOTE] +> User avatars and covers and magazine icons and banners are not affected by this command, +> only images from threads, microblogs and comments. + +Usage: + +```bash +php bin/console mbin:images:remove-remote [--days|-d] [--batch-size] [--dry-run] +``` + +Options: +- `--days`|`-d`: the number of days of media you want to keep. Everything older than the amount of days will be deleted +- `--batch-size` (default `10000`): the number of images to retrieve per query from the DB. A higher number means less queries, but higher memory usage. +- `--dry-run`: if set, no images will be deleted + +### Refresh the meta data of stored images + +This command allows you to refresh the filesize of the stored media, as well as the status. +If an image is no longer present on storage this command adjusts it in the DB. + +Usage: + +```bash +php bin/console mbin:images:refresh-meta [--batch-size] [--dry-run] +``` + +Options: +- `--batch-size` (default `10000`): the number of images to retrieve per query from the DB. A higher number means less queries, but higher memory usage. +- `--dry-run`: if set, no metadata will be changed ### Remove old federated images diff --git a/migrations/Version20260303142852.php b/migrations/Version20260303142852.php new file mode 100644 index 0000000000..7fb4a125c9 --- /dev/null +++ b/migrations/Version20260303142852.php @@ -0,0 +1,75 @@ +addSql('ALTER TABLE image ADD is_compressed BOOLEAN DEFAULT false NOT NULL'); + $this->addSql('ALTER TABLE image ADD source_too_big BOOLEAN DEFAULT false NOT NULL'); + $this->addSql('ALTER TABLE image ADD downloaded_at TIMESTAMP(0) WITH TIME ZONE DEFAULT NULL'); + // init the column for all existing images, it gets overwritten in the big loop underneath + $this->addSql('ALTER TABLE image ADD created_at TIMESTAMP(0) WITH TIME ZONE NOT NULL DEFAULT current_timestamp'); + $this->addSql('ALTER TABLE image ALTER created_at DROP DEFAULT;'); + $this->addSql('ALTER TABLE image ADD original_size BIGINT DEFAULT 0 NOT NULL'); + $this->addSql('ALTER TABLE image ADD local_size BIGINT DEFAULT 0 NOT NULL'); + + // set the downloaded at value to something realistically + $this->addSql('DO +$do$ + declare tempRow record; +BEGIN + FOR tempRow IN + SELECT i.id, e.created_at as ec, ec.created_at as ecc, p.created_at as pc, pc.created_at as pcc, u.created_at as uc, u2.created_at as u2c, m.created_at as mc, m2.created_at as m2c FROM image i + LEFT JOIN entry e ON i.id = e.image_id + LEFT JOIN entry_comment ec ON i.id = ec.image_id + LEFT JOIN post p ON i.id = p.image_id + LEFT JOIN post_comment pc ON i.id = pc.image_id + LEFT JOIN "user" u ON i.id = u.avatar_id + LEFT JOIN "user" u2 ON i.id = u2.cover_id + LEFT JOIN magazine m ON i.id = m.icon_id + LEFT JOIN magazine m2 ON i.id = m2.banner_id + LOOP + IF tempRow.ec IS NOT NULL THEN + UPDATE image SET downloaded_at = tempRow.ec, created_at = tempRow.ec WHERE id = tempRow.id; + ELSIF tempRow.ecc IS NOT NULL THEN + UPDATE image SET downloaded_at = tempRow.ecc, created_at = tempRow.ecc WHERE id = tempRow.id; + ELSIF tempRow.pc IS NOT NULL THEN + UPDATE image SET downloaded_at = tempRow.pc, created_at = tempRow.pc WHERE id = tempRow.id; + ELSIF tempRow.pcc IS NOT NULL THEN + UPDATE image SET downloaded_at = tempRow.pcc, created_at = tempRow.pcc WHERE id = tempRow.id; + ELSIF tempRow.uc IS NOT NULL THEN + UPDATE image SET downloaded_at = tempRow.uc, created_at = tempRow.uc WHERE id = tempRow.id; + ELSIF tempRow.u2c IS NOT NULL THEN + UPDATE image SET downloaded_at = tempRow.u2c, created_at = tempRow.u2c WHERE id = tempRow.id; + ELSIF tempRow.mc IS NOT NULL THEN + UPDATE image SET downloaded_at = tempRow.mc, created_at = tempRow.mc WHERE id = tempRow.id; + ELSIF tempRow.m2c IS NOT NULL THEN + UPDATE image SET downloaded_at = tempRow.m2c, created_at = tempRow.m2c WHERE id = tempRow.id; + END IF; + END LOOP; +END +$do$;'); + } + + public function down(Schema $schema): void + { + $this->addSql('ALTER TABLE image DROP is_compressed'); + $this->addSql('ALTER TABLE image DROP source_too_big'); + $this->addSql('ALTER TABLE image DROP downloaded_at'); + $this->addSql('ALTER TABLE image DROP created_at'); + $this->addSql('ALTER TABLE image DROP original_size'); + $this->addSql('ALTER TABLE image DROP local_size'); + } +} diff --git a/src/Command/RefreshImageMetaDataCommand.php b/src/Command/RefreshImageMetaDataCommand.php new file mode 100644 index 0000000000..13a3f46e8c --- /dev/null +++ b/src/Command/RefreshImageMetaDataCommand.php @@ -0,0 +1,116 @@ +addOption('batch-size', null, InputOption::VALUE_REQUIRED, 'The number of images to handle at once, the higher the number the faster the command, but it also takes more memory', '10000'); + $this->addOption('dry-run', null, InputOption::VALUE_NONE, 'Do a trial without removing any media'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + GeneralUtil::useProgressbarFormatsWithMessage(); + + $dryRun = \boolval($input->getOption('dry-run')); + $batchSize = \intval($input->getOption('batch-size')); + $images = $this->imageRepository->findSavedImagesPaginated($batchSize); + $count = $images->count(); + $progressBar = $io->createProgressBar($count); + $progressBar->setMessage(''); + $progressBar->start(); + $totalCheckedFiles = 0; + $totalUpdateFiles = 0; + + for ($i = 0; $i < $images->getNbPages(); ++$i) { + $progressBar->setMessage(\sprintf('Fetching images %s - %s', ($i * $batchSize) + 1, ($i + 1) * $batchSize)); + $progressBar->display(); + foreach ($images->getCurrentPageResults() as $image) { + $progressBar->advance(); + ++$totalCheckedFiles; + + try { + if ($this->publicUploadsFilesystem->has($image->filePath)) { + ++$totalUpdateFiles; + $fileSize = $this->publicUploadsFilesystem->fileSize($image->filePath); + if (!$dryRun) { + $image->localSize = $fileSize; + $progressBar->setMessage(\sprintf('Refreshed meta data of "%s" (%s)', $image->filePath, $image->getId())); + $this->logger->debug('Refreshed meta data of "{path}" ({id})', ['path' => $image->filePath, 'id' => $image->getId()]); + } else { + $progressBar->setMessage(\sprintf('Would have refreshed meta data of "%s" (%s)', $image->filePath, $image->getId())); + } + $progressBar->display(); + } else { + $previousPath = $image->filePath; + // mark it as not present on the media storage + if (!$dryRun) { + $image->filePath = null; + $image->localSize = 0; + $image->downloadedAt = null; + $progressBar->setMessage(\sprintf('Marked "%s" (%s) as not present on the media storage', $previousPath, $image->getId())); + } else { + $progressBar->setMessage(\sprintf('Would have marked "%s" (%s) as not present on the media storage', $image->filePath, $image->getId())); + } + $progressBar->display(); + } + } catch (FilesystemException $e) { + $this->logger->error('There was an exception refreshing the meta data of "{path}" ({id}): {exClass} - {message}', [ + 'path' => $image->filePath, + 'id' => $image->getId(), + 'exClass' => \get_class($image), + 'message' => $e->getMessage(), + 'exception' => $e, + ]); + $progressBar->setMessage(\sprintf('Error checking meta data of "%s" (%s)', $image->filePath, $image->getId())); + $progressBar->display(); + } + } + if (!$dryRun) { + $this->entityManager->flush(); + } + if ($images->hasNextPage()) { + $images->setCurrentPage($images->getNextPage()); + } + } + $io->writeln(''); + if (!$dryRun) { + $io->success(\sprintf('Refreshed %s files', $totalUpdateFiles)); + } else { + $io->success(\sprintf('Would have refreshed %s files', $totalUpdateFiles)); + } + + return Command::SUCCESS; + } +} diff --git a/src/Command/RemoveRemoteMediaCommand.php b/src/Command/RemoveRemoteMediaCommand.php new file mode 100644 index 0000000000..1214de608f --- /dev/null +++ b/src/Command/RemoveRemoteMediaCommand.php @@ -0,0 +1,95 @@ +addOption('days', 'd', InputOption::VALUE_REQUIRED, 'Delete media that is older than x days, if you omit this parameter or set it to 0 it will remove all cached remote media'); + $this->addOption('batch-size', null, InputOption::VALUE_REQUIRED, 'The number of images to handle at once, the higher the number the faster the command, but it also takes more memory', '10000'); + $this->addOption('dry-run', null, InputOption::VALUE_NONE, 'Do a trial without removing any media'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + $days = \intval($input->getOption('days')); + if ($days < 0) { + $io->error('Days must be at least 0'); + + return Command::FAILURE; + } + + GeneralUtil::useProgressbarFormatsWithMessage(); + + $dryRun = \boolval($input->getOption('dry-run')); + $batchSize = \intval($input->getOption('batch-size')); + $images = $this->imageRepository->findOldRemoteMediaPaginated($days, $batchSize); + $count = $images->count(); + $progressBar = $io->createProgressBar($count); + $progressBar->setMessage(''); + $progressBar->start(); + $totalDeletedFiles = 0; + $totalDeletedSize = 0; + + for ($i = 0; $i < $images->getNbPages(); ++$i) { + $progressBar->setMessage(\sprintf('Fetching images %s - %s', ($i * $batchSize) + 1, ($i + 1) * $batchSize)); + $progressBar->display(); + foreach ($images->getCurrentPageResults() as $image) { + $progressBar->advance(); + ++$totalDeletedFiles; + $totalDeletedSize += $image->localSize; + + if (!$dryRun) { + if ($this->imageManager->removeCachedImage($image)) { + $progressBar->setMessage(\sprintf('Removed "%s" (%s)', $image->filePath, $image->getId())); + $progressBar->display(); + $this->logger->debug('Removed "{path}" ({id})', ['path' => $image->filePath, 'id' => $image->getId()]); + } + } else { + $progressBar->setMessage(\sprintf('Would have removed "%s" (%s)', $image->filePath, $image->getId())); + $this->logger->debug('Would have removed "{path}" ({id})', ['path' => $image->filePath, 'id' => $image->getId()]); + } + } + if ($images->hasNextPage()) { + $images->setCurrentPage($images->getNextPage()); + } + } + $io->writeln(''); + if (!$dryRun) { + $io->success(\sprintf('Removed %s files (~%sB)', $totalDeletedFiles, $this->formatter->abbreviateNumber($totalDeletedSize))); + } else { + $io->success(\sprintf('Would have removed %s files (~%sB)', $totalDeletedFiles, $this->formatter->abbreviateNumber($totalDeletedSize))); + } + + return Command::SUCCESS; + } +} diff --git a/src/Controller/Entry/EntrySingleController.php b/src/Controller/Entry/EntrySingleController.php index 78c085dd8d..3e07c632bc 100644 --- a/src/Controller/Entry/EntrySingleController.php +++ b/src/Controller/Entry/EntrySingleController.php @@ -15,9 +15,12 @@ use App\PageView\EntryCommentPageView; use App\Repository\Criteria; use App\Repository\EntryCommentRepository; +use App\Repository\ImageRepository; use App\Service\MentionManager; +use Doctrine\ORM\EntityManagerInterface; use Pagerfanta\PagerfantaInterface; use Psr\EventDispatcher\EventDispatcherInterface; +use Psr\Log\LoggerInterface; use Symfony\Bridge\Doctrine\Attribute\MapEntity; use Symfony\Bundle\SecurityBundle\Security; use Symfony\Component\HttpFoundation\JsonResponse; @@ -28,17 +31,24 @@ class EntrySingleController extends AbstractController { use PrivateContentTrait; + public function __construct( + private readonly Security $security, + private readonly ImageRepository $imageRepository, + private readonly EntryCommentRepository $commentRepository, + private readonly EventDispatcherInterface $dispatcher, + private readonly MentionManager $mentionManager, + private readonly LoggerInterface $logger, + private readonly EntityManagerInterface $entityManager, + ) { + } + public function __invoke( #[MapEntity(mapping: ['magazine_name' => 'name'])] Magazine $magazine, #[MapEntity(id: 'entry_id')] Entry $entry, ?string $sortBy, - EntryCommentRepository $repository, - EventDispatcherInterface $dispatcher, - MentionManager $mentionManager, Request $request, - Security $security, ): Response { if ($entry->magazine !== $magazine) { return $this->redirectToRoute( @@ -55,7 +65,14 @@ public function __invoke( $this->handlePrivateContent($entry); - $criteria = new EntryCommentPageView($this->getPageNb($request), $security); + $images = []; + if ($entry->image) { + $images[] = $entry->image; + } + $images = array_merge($images, $this->commentRepository->findImagesByEntry($entry)); + $this->imageRepository->redownloadImagesIfNecessary($images); + + $criteria = new EntryCommentPageView($this->getPageNb($request), $this->security); $criteria->showSortOption($criteria->resolveSort($sortBy)); $criteria->entry = $entry; @@ -67,13 +84,13 @@ public function __invoke( $criteria->onlyParents = false; } - $comments = $repository->findByCriteria($criteria); + $comments = $this->commentRepository->findByCriteria($criteria); $commentObjects = [...$comments->getCurrentPageResults()]; - $repository->hydrate(...$commentObjects); - $repository->hydrateChildren(...$commentObjects); + $this->commentRepository->hydrate(...$commentObjects); + $this->commentRepository->hydrateChildren(...$commentObjects); - $dispatcher->dispatch(new EntryHasBeenSeenEvent($entry)); + $this->dispatcher->dispatch(new EntryHasBeenSeenEvent($entry)); if ($request->isXmlHttpRequest()) { return $this->getJsonResponse($magazine, $entry, $comments); @@ -83,7 +100,7 @@ public function __invoke( $dto = new EntryCommentDto(); if ($user && $user->addMentionsEntries && $entry->user !== $user) { - $dto->body = $mentionManager->addHandle([$entry->user->username])[0]; + $dto->body = $this->mentionManager->addHandle([$entry->user->username])[0]; } return $this->render( diff --git a/src/Controller/Post/PostSingleController.php b/src/Controller/Post/PostSingleController.php index df45aa04df..d72bf37765 100644 --- a/src/Controller/Post/PostSingleController.php +++ b/src/Controller/Post/PostSingleController.php @@ -14,6 +14,7 @@ use App\Form\PostCommentType; use App\PageView\PostCommentPageView; use App\Repository\Criteria; +use App\Repository\ImageRepository; use App\Repository\PostCommentRepository; use App\Service\MentionManager; use Pagerfanta\PagerfantaInterface; @@ -28,17 +29,22 @@ class PostSingleController extends AbstractController { use PrivateContentTrait; + public function __construct( + private readonly PostCommentRepository $commentRepository, + private readonly EventDispatcherInterface $dispatcher, + private readonly MentionManager $mentionManager, + private readonly Security $security, + private readonly ImageRepository $imageRepository, + ) { + } + public function __invoke( #[MapEntity(mapping: ['magazine_name' => 'name'])] Magazine $magazine, #[MapEntity(id: 'post_id')] Post $post, ?string $sortBy, - PostCommentRepository $repository, - EventDispatcherInterface $dispatcher, - MentionManager $mentionManager, Request $request, - Security $security, ): Response { if ($post->magazine !== $magazine) { return $this->redirectToRoute( @@ -55,7 +61,14 @@ public function __invoke( $this->handlePrivateContent($post); - $criteria = new PostCommentPageView($this->getPageNb($request), $security); + $images = []; + if ($post->image) { + $images[] = $post->image; + } + $images = array_merge($images, $this->commentRepository->findImagesByPost($post)); + $this->imageRepository->redownloadImagesIfNecessary($images); + + $criteria = new PostCommentPageView($this->getPageNb($request), $this->security); $criteria->showSortOption($criteria->resolveSort($sortBy)); $criteria->content = Criteria::CONTENT_MICROBLOG; $criteria->post = $post; @@ -70,13 +83,13 @@ public function __invoke( $criteria->onlyParents = false; } - $comments = $repository->findByCriteria($criteria); + $comments = $this->commentRepository->findByCriteria($criteria); $commentObjects = [...$comments->getCurrentPageResults()]; - $repository->hydrate(...$commentObjects); - $repository->hydrateChildren(...$commentObjects); + $this->commentRepository->hydrate(...$commentObjects); + $this->commentRepository->hydrateChildren(...$commentObjects); - $dispatcher->dispatch(new PostHasBeenSeenEvent($post)); + $this->dispatcher->dispatch(new PostHasBeenSeenEvent($post)); if ($request->isXmlHttpRequest()) { return $this->getJsonResponse($magazine, $post, $comments); @@ -84,7 +97,7 @@ public function __invoke( $dto = new PostCommentDto(); if ($this->getUser() && $this->getUser()->addMentionsPosts && $post->user !== $this->getUser()) { - $dto->body = $mentionManager->addHandle([$post->user->username])[0]; + $dto->body = $this->mentionManager->addHandle([$post->user->username])[0]; } return $this->render( diff --git a/src/Entity/Image.php b/src/Entity/Image.php index 2afda77ccd..1bddf9d398 100644 --- a/src/Entity/Image.php +++ b/src/Entity/Image.php @@ -4,6 +4,7 @@ namespace App\Entity; +use App\Entity\Traits\CreatedAtTrait; use App\Repository\ImageRepository; use Doctrine\ORM\Mapping\Cache; use Doctrine\ORM\Mapping\Column; @@ -20,6 +21,9 @@ #[Cache(usage: 'NONSTRICT_READ_WRITE')] class Image { + use CreatedAtTrait { + CreatedAtTrait::__construct as createdAtTraitConstruct; + } #[Column(type: 'string', nullable: true)] /** * If this is NULL it is only a remote image, probably because the image was too big. @@ -39,6 +43,17 @@ class Image public ?string $altText = null; #[Column(type: 'text', nullable: true)] public ?string $sourceUrl = null; + #[Column(nullable: false, options: ['default' => false])] + public bool $isCompressed = false; + #[Column(nullable: false, options: ['default' => false])] + public bool $sourceTooBig = false; + #[Column(type: 'datetimetz_immutable', nullable: true, options: ['default' => null])] + public ?\DateTimeImmutable $downloadedAt; + #[Column(type: 'bigint', options: ['default' => 0])] + public int $localSize = 0; + #[Column(type: 'bigint', options: ['default' => 0])] + public int $originalSize = 0; + #[Id] #[GeneratedValue] #[Column(type: 'integer')] @@ -52,6 +67,7 @@ public function __construct( ?int $height, ?string $blurhash, ) { + $this->createdAtTraitConstruct(); $this->filePath = $filePath; $this->fileName = $fileName; $this->blurhash = $blurhash; diff --git a/src/Event/ImagePostProcessEvent.php b/src/Event/ImagePostProcessEvent.php index 67d341f0a6..5b5aee1566 100644 --- a/src/Event/ImagePostProcessEvent.php +++ b/src/Event/ImagePostProcessEvent.php @@ -11,6 +11,7 @@ class ImagePostProcessEvent extends Event { public function __construct( public string $source, + public string $targetFilePath, public ImageOrigin $origin, ) { } diff --git a/src/EventSubscriber/Image/ImageCompressSubscriber.php b/src/EventSubscriber/Image/ImageCompressSubscriber.php new file mode 100644 index 0000000000..664b0ac35a --- /dev/null +++ b/src/EventSubscriber/Image/ImageCompressSubscriber.php @@ -0,0 +1,38 @@ + 'compressImage', + ]; + } + + public function compressImage(ImagePostProcessEvent $event): void + { + $extension = pathinfo($event->targetFilePath, PATHINFO_EXTENSION); + if (!$this->imageManager->compressUntilSize($event->source, $extension, $this->settingsManager->getMaxImageBytes())) { + if (filesize($event->source) > $this->settingsManager->getMaxImageBytes()) { + $this->logger->warning('Was not able to compress image {i} to size {b}', ['i' => $event->source, 'b' => $this->settingsManager->getMaxImageBytes()]); + } + } + } +} diff --git a/src/Pagination/Transformation/ContentPopulationTransformer.php b/src/Pagination/Transformation/ContentPopulationTransformer.php index 7c3f5a30f4..b2c63ceadd 100644 --- a/src/Pagination/Transformation/ContentPopulationTransformer.php +++ b/src/Pagination/Transformation/ContentPopulationTransformer.php @@ -6,10 +6,12 @@ use App\Entity\Entry; use App\Entity\EntryComment; +use App\Entity\Image; use App\Entity\Magazine; use App\Entity\Post; use App\Entity\PostComment; use App\Entity\User; +use App\Utils\SqlHelpers; use Doctrine\ORM\EntityManagerInterface; class ContentPopulationTransformer implements ResultTransformer @@ -27,6 +29,7 @@ public function transform(iterable $input): iterable $postCommentRepository = $this->entityManager->getRepository(PostComment::class); $magazineRepository = $this->entityManager->getRepository(Magazine::class); $userRepository = $this->entityManager->getRepository(User::class); + $imageRepository = $this->entityManager->getRepository(Image::class); $positionsArray = $this->buildPositionArray($input); $entryIds = $this->getOverviewIds((array) $input, 'entry'); @@ -63,7 +66,12 @@ public function transform(iterable $input): iterable $users = $userRepository->findBy(['id' => $userIds]); } - return $this->applyPositions($positionsArray, $entries ?? [], $entryComments ?? [], $post ?? [], $postComment ?? [], $magazines ?? [], $users ?? []); + $imageIds = $this->getOverviewIds((array) $input, 'image'); + if (\count($imageIds) > 0) { + $images = SqlHelpers::findByAdjusted($imageRepository, 'id', $imageIds); + } + + return $this->applyPositions($positionsArray, $entries ?? [], $entryComments ?? [], $post ?? [], $postComment ?? [], $magazines ?? [], $users ?? [], $images ?? []); } private function getOverviewIds(array $result, string $type): array @@ -84,6 +92,7 @@ private function buildPositionArray(iterable $input): array $postCommentPositions = []; $userPositions = []; $magazinePositions = []; + $imagePositions = []; $i = 0; foreach ($input as $current) { switch ($current['type']) { @@ -105,6 +114,9 @@ private function buildPositionArray(iterable $input): array case 'user': $userPositions[$current['id']] = $i; break; + case 'image': + $imagePositions[$current['id']] = $i; + break; } ++$i; } @@ -116,6 +128,7 @@ private function buildPositionArray(iterable $input): array 'post_comment' => $postCommentPositions, 'magazine' => $magazinePositions, 'user' => $userPositions, + 'image' => $imagePositions, ]; } @@ -125,8 +138,10 @@ private function buildPositionArray(iterable $input): array * @param EntryComment[] $entryComments * @param Post[] $posts * @param PostComment[] $postComments + * @param User[] $users + * @param Image[] $images */ - private function applyPositions(array $positionsArray, array $entries, array $entryComments, array $posts, array $postComments, array $magazines, array $users): array + private function applyPositions(array $positionsArray, array $entries, array $entryComments, array $posts, array $postComments, array $magazines, array $users, array $images): array { $result = []; foreach ($entries as $entry) { @@ -147,6 +162,9 @@ private function applyPositions(array $positionsArray, array $entries, array $en foreach ($users as $user) { $result[$positionsArray['user'][$user->getId()]] = $user; } + foreach ($images as $image) { + $result[$positionsArray['image'][$image->getId()]] = $image; + } ksort($result, SORT_NUMERIC); return $result; diff --git a/src/Repository/EntryCommentRepository.php b/src/Repository/EntryCommentRepository.php index d31375e7e7..404d049a28 100644 --- a/src/Repository/EntryCommentRepository.php +++ b/src/Repository/EntryCommentRepository.php @@ -11,9 +11,11 @@ use App\Entity\Contracts\VisibilityInterface; use App\Entity\DomainBlock; use App\Entity\DomainSubscription; +use App\Entity\Entry; use App\Entity\EntryComment; use App\Entity\EntryCommentFavourite; use App\Entity\HashtagLink; +use App\Entity\Image; use App\Entity\MagazineBlock; use App\Entity\MagazineSubscription; use App\Entity\Moderator; @@ -267,6 +269,22 @@ private function filter(QueryBuilder $qb, Criteria $criteria): QueryBuilder return $qb; } + /** + * @return Image[] + */ + public function findImagesByEntry(Entry $entry): array + { + $results = $this->createQueryBuilder('c') + ->addSelect('i') + ->innerJoin('c.image', 'i') + ->andWhere('c.entry = :entry') + ->setParameter('entry', $entry) + ->getQuery() + ->getResult(); + + return array_map(fn (EntryComment $comment) => $comment->image, $results); + } + public function hydrateChildren(EntryComment ...$comments): void { $children = $this->createQueryBuilder('c') diff --git a/src/Repository/ImageRepository.php b/src/Repository/ImageRepository.php index 65925c6f6e..b154ea6979 100644 --- a/src/Repository/ImageRepository.php +++ b/src/Repository/ImageRepository.php @@ -7,9 +7,14 @@ use App\Entity\Image; use App\Event\ImagePostProcessEvent; use App\Exception\ImageDownloadTooLargeException; +use App\Pagination\NativeQueryAdapter; +use App\Pagination\Pagerfanta; +use App\Pagination\QueryAdapter; +use App\Pagination\Transformation\ContentPopulationTransformer; use App\Service\ImageManagerInterface; use App\Utils\ImageOrigin; use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository; +use Doctrine\DBAL\Exception; use Doctrine\Persistence\ManagerRegistry; use kornrunner\Blurhash\Blurhash; use Psr\EventDispatcher\EventDispatcherInterface; @@ -30,6 +35,7 @@ public function __construct( private readonly ImageManagerInterface $imageManager, private readonly EventDispatcherInterface $dispatcher, private readonly LoggerInterface $logger, + private readonly ContentPopulationTransformer $contentPopulationTransformer, ) { parent::__construct($registry, Image::class); } @@ -93,10 +99,17 @@ private function findOrCreateFromSource(string $source, ImageOrigin $origin): ?I $image->setDimensions($width, $height); } - $this->dispatcher->dispatch(new ImagePostProcessEvent($source, $origin)); + $previousFileSize = filesize($source); + $image->originalSize = $previousFileSize; + $this->dispatcher->dispatch(new ImagePostProcessEvent($source, $filePath, $origin)); + $afterProcessFileSize = filesize($source); + if ($afterProcessFileSize < $previousFileSize) { + $image->isCompressed = true; + } try { $this->imageManager->store($source, $filePath); + $image->localSize = $afterProcessFileSize; return $image; } catch (ImageDownloadTooLargeException $e) { @@ -106,6 +119,8 @@ private function findOrCreateFromSource(string $source, ImageOrigin $origin): ?I ['origin' => $origin, 'type' => \gettype($e)], ); $image->filePath = null; + $image->localSize = 0; + $image->sourceTooBig = true; return $image; } else { @@ -172,4 +187,128 @@ public function blurhash(string $filePath): ?string return null; } } + + /** + * @param int $limit use a high limit, as this query takes a few seconds and the limit does not affect that, so we are using as high a number as we can -> we're limited by memory + * + * @return Pagerfanta + * + * @throws Exception + */ + public function findOldRemoteMediaPaginated(int $olderThanDays, int $limit = 10000): Pagerfanta + { + $query = $this->createQueryBuilder('i') + ->andWhere('i.downloadedAt < :date') + ->andWhere('i.filePath IS NOT NULL') + ->andWhere('i.sourceUrl IS NOT NULL') + ->setParameter('date', new \DateTimeImmutable("now - $olderThanDays days")) + ->getQuery(); + // this complicated looking query makes sure to not include avatars, covers, icons or banners + $sql = 'SELECT id, MAX(last_active) as last_active, MAX(downloaded_at) as downloaded_at, \'image\' as type FROM ( + SELECT i.id, i.downloaded_at, e.last_active FROM image i + INNER JOIN entry e ON i.id = e.image_id + LEFT JOIN "user" u ON i.id = u.avatar_id + LEFT JOIN "user" u2 ON i.id = u2.cover_id + LEFT JOIN magazine m ON i.id = m.icon_id + LEFT JOIN magazine m2 ON i.id = m2.banner_id + WHERE u IS NULL AND u2 IS NULL AND m IS NULL AND m2 IS NULL AND i.file_path IS NOT NULL AND i.source_url IS NOT NULL + UNION ALL + SELECT i.id, i.downloaded_at, ec.last_active FROM image i + INNER JOIN entry_comment ec ON i.id = ec.image_id + LEFT JOIN "user" u ON i.id = u.avatar_id + LEFT JOIN "user" u2 ON i.id = u2.cover_id + LEFT JOIN magazine m ON i.id = m.icon_id + LEFT JOIN magazine m2 ON i.id = m2.banner_id + WHERE u IS NULL AND u2 IS NULL AND m IS NULL AND m2 IS NULL AND i.file_path IS NOT NULL AND i.source_url IS NOT NULL + UNION ALL + SELECT i.id, i.downloaded_at, p.last_active FROM image i + INNER JOIN post p ON i.id = p.image_id + LEFT JOIN "user" u ON i.id = u.avatar_id + LEFT JOIN "user" u2 ON i.id = u2.cover_id + LEFT JOIN magazine m ON i.id = m.icon_id + LEFT JOIN magazine m2 ON i.id = m2.banner_id + WHERE u IS NULL AND u2 IS NULL AND m IS NULL AND m2 IS NULL AND i.file_path IS NOT NULL AND i.source_url IS NOT NULL + UNION ALL + SELECT i.id, i.downloaded_at, pc.last_active FROM image i + INNER JOIN post_comment pc ON i.id = pc.image_id + LEFT JOIN "user" u ON i.id = u.avatar_id + LEFT JOIN "user" u2 ON i.id = u2.cover_id + LEFT JOIN magazine m ON i.id = m.icon_id + LEFT JOIN magazine m2 ON i.id = m2.banner_id + WHERE u IS NULL AND u2 IS NULL AND m IS NULL AND m2 IS NULL AND i.file_path IS NOT NULL AND i.source_url IS NOT NULL + ) images WHERE last_active < :date AND (downloaded_at < :date OR downloaded_at IS NULL) GROUP BY id'; + + $adapter = new NativeQueryAdapter($this->getEntityManager()->getConnection(), $sql, ['date' => new \DateTimeImmutable("now - $olderThanDays days")], transformer: $this->contentPopulationTransformer); + $fanta = new Pagerfanta($adapter); + $fanta->setCurrentPage(1); + $fanta->setMaxPerPage($limit); + + return $fanta; + } + + /** + * @return Pagerfanta + */ + public function findSavedImagesPaginated(int $pageSize): Pagerfanta + { + $query = $this->createQueryBuilder('i') + ->andWhere('i.filePath IS NOT NULL') + ->orderBy('i.filePath'); + + $adapter = new QueryAdapter($query); + $fanta = new Pagerfanta($adapter); + $fanta->setMaxPerPage($pageSize); + $fanta->setCurrentPage(1); + + return $fanta; + } + + public function redownloadImage(Image $image): void + { + if ($image->filePath || !$image->sourceUrl || $image->sourceTooBig) { + return; + } + + $tempFilePath = $this->imageManager->download($image->sourceUrl); + if (null === $tempFilePath) { + return; + } + + [$filePath, $fileName] = $this->imageManager->getFilePathAndName($tempFilePath); + + $previousFileSize = filesize($tempFilePath); + $image->originalSize = $previousFileSize; + $this->dispatcher->dispatch(new ImagePostProcessEvent($tempFilePath, $filePath, ImageOrigin::External)); + $afterProcessFileSize = filesize($tempFilePath); + if ($afterProcessFileSize < $previousFileSize) { + $image->isCompressed = true; + } + + try { + if ($this->imageManager->store($tempFilePath, $filePath)) { + $image->filePath = $filePath; + $image->localSize = $afterProcessFileSize; + $image->downloadedAt = new \DateTimeImmutable('now'); + } + } catch (ImageDownloadTooLargeException) { + $image->localSize = 0; + $image->sourceTooBig = true; + } catch (\Exception) { + } + } + + /** + * @param Image[] $images + */ + public function redownloadImagesIfNecessary(array $images): void + { + foreach ($images as $image) { + $this->logger->debug('Maybe redownloading images {i}', ['i' => implode(', ', array_map(fn (Image $image) => $image->getId(), $images))]); + if ($image && null === $image->filePath && !$image->sourceTooBig && $image->sourceUrl) { + // there is an image, but not locally, and it was not too big, and we have the source URL -> try redownloading it + $this->redownloadImage($image); + } + } + $this->getEntityManager()->flush(); + } } diff --git a/src/Repository/PostCommentRepository.php b/src/Repository/PostCommentRepository.php index 4ffe10e2a7..de5203d85c 100644 --- a/src/Repository/PostCommentRepository.php +++ b/src/Repository/PostCommentRepository.php @@ -10,6 +10,8 @@ use App\Entity\Contracts\VisibilityInterface; use App\Entity\HashtagLink; +use App\Entity\Image; +use App\Entity\Post; use App\Entity\PostComment; use App\Entity\UserBlock; use App\Entity\UserFollow; @@ -190,6 +192,22 @@ private function filter(QueryBuilder $qb, Criteria $criteria) $qb->addOrderBy('c.id', 'DESC'); } + /** + * @return Image[] + */ + public function findImagesByPost(Post $post): array + { + $results = $this->createQueryBuilder('c') + ->addSelect('i') + ->innerJoin('c.image', 'i') + ->andWhere('c.post = :post') + ->setParameter('post', $post) + ->getQuery() + ->getResult(); + + return array_map(fn (PostComment $comment) => $comment->image, $results); + } + public function hydrateChildren(PostComment ...$comments): void { $children = $this->createQueryBuilder('c') diff --git a/src/Service/ImageManager.php b/src/Service/ImageManager.php index 9ea74ac8a5..2e05141473 100644 --- a/src/Service/ImageManager.php +++ b/src/Service/ImageManager.php @@ -4,14 +4,19 @@ namespace App\Service; +use App\Entity\Image as MbinImage; use App\Exception\CorruptedFileException; use App\Exception\ImageDownloadTooLargeException; use App\Repository\ImageRepository; +use App\Twig\Runtime\FormattingExtensionRuntime; use App\Utils\GeneralUtil; +use Doctrine\ORM\EntityManagerInterface; +use Imagine\Gd\Imagine; use League\Flysystem\FileAttributes; use League\Flysystem\FilesystemException; use League\Flysystem\FilesystemOperator; use League\Flysystem\StorageAttributes; +use Liip\ImagineBundle\Imagine\Cache\CacheManager; use Psr\Log\LoggerInterface; use Symfony\Component\Messenger\Exception\UnrecoverableMessageHandlingException; use Symfony\Component\Mime\MimeTypesInterface; @@ -21,12 +26,12 @@ class ImageManager implements ImageManagerInterface { - public const IMAGE_MIMETYPES = [ + public const array IMAGE_MIMETYPES = [ 'image/jpeg', 'image/jpg', 'image/gif', 'image/png', 'image/jxl', 'image/heic', 'image/heif', 'image/webp', 'image/avif', ]; - public const IMAGE_MIMETYPE_STR = 'image/jpeg, image/jpg, image/gif, image/png, image/jxl, image/heic, image/heif, image/webp, image/avif'; + public const string IMAGE_MIMETYPE_STR = 'image/jpeg, image/jpg, image/gif, image/png, image/jxl, image/heic, image/heif, image/webp, image/avif'; public function __construct( private readonly string $storageUrl, @@ -36,6 +41,10 @@ public function __construct( private readonly ValidatorInterface $validator, private readonly LoggerInterface $logger, private readonly SettingsManager $settings, + private readonly FormattingExtensionRuntime $formattingExtensionRuntime, + private readonly float $imageCompressionQuality, + private readonly CacheManager $imagineCacheManager, + private readonly EntityManagerInterface $entityManager, ) { } @@ -79,6 +88,61 @@ public function store(string $source, string $filePath): bool } } + /** + * Tries to compress an image until its size is smaller than $maxBytes. This overwrites the existing image. + * + * @return bool whether the image was compressed + */ + public function compressUntilSize(string $filePath, string $extension, int $maxBytes): bool + { + if (-1 === $this->imageCompressionQuality || filesize($filePath) <= $maxBytes) { + // don't compress images if disabled or smaller than max bytes + return false; + } + $imagine = new Imagine(); + $image = $imagine->open($filePath); + $bytes = filesize($filePath); + $initialBytes = $bytes; + $tempPath = "{$filePath}_temp_compress.$extension"; + $compressed = false; + $quality = 0.9; + if (0.1 <= $this->imageCompressionQuality && 1 > $this->imageCompressionQuality) { + $quality = $this->imageCompressionQuality; + } + while ($bytes > $maxBytes && $quality > 0.1) { + $this->logger->debug('[ImageManager::compressUntilSize] Trying to compress "{path}" with {q}% quality', ['path' => $tempPath, 'q' => $quality * 100]); + $image->save($tempPath, [ + 'jpeg_quality' => $quality * 100, // jpeg max value is 100 + 'png_compression_level' => 9, // this is lossless compression, so always use the max + 'webp_quality' => $quality * 100, // webp quality max is 100 + ]); + $bytes = filesize($tempPath); + if ($initialBytes === $bytes) { + // there were no changes, so maybe it is in a format that cannot be compressed... + break; + } + $compressed = true; + $quality -= 0.05; + } + $copied = false; + if ($compressed) { + if (copy($tempPath, $filePath)) { + $copied = true; + $this->logger->debug('[ImageManager::compressUntilSize] successfully compressed "{path}" with {q}% quality: {bytesBefore} -> {bytesNow}', [ + 'path' => $filePath, + 'q' => ($quality + 0.05) * 100, // re-add the last step, because it is always subtracted in the end if successful + 'bytesBefore' => $this->formattingExtensionRuntime->abbreviateNumber($initialBytes).'B', + 'bytesNow' => $this->formattingExtensionRuntime->abbreviateNumber($bytes).'B', + ]); + } + } + if (file_exists($tempPath)) { + unlink($tempPath); + } + + return $copied; + } + private function validate(string $filePath): bool { $violations = $this->validator->validate( @@ -187,14 +251,15 @@ public function getFileName(string $file): string public function remove(string $path): void { $this->publicUploadsFilesystem->delete($path); + $this->imagineCacheManager->remove($path); } - public function getPath(\App\Entity\Image $image): string + public function getPath(MbinImage $image): string { return $this->publicUploadsFilesystem->read($image->filePath); } - public function getUrl(?\App\Entity\Image $image): ?string + public function getUrl(?MbinImage $image): ?string { if (!$image) { return null; @@ -207,7 +272,7 @@ public function getUrl(?\App\Entity\Image $image): ?string return $image->sourceUrl; } - public function getMimetype(\App\Entity\Image $image): string + public function getMimetype(MbinImage $image): string { try { return $this->publicUploadsFilesystem->mimeType($image->filePath); @@ -313,4 +378,31 @@ private function getInternalImagePathAndName(StorageAttributes $flySystemFile): return [$path, end($parts)]; } + + public function removeCachedImage(MbinImage $image): bool + { + if (!$image->filePath || !$image->sourceUrl) { + return false; + } + + try { + $this->publicUploadsFilesystem->delete($image->filePath); + $this->imagineCacheManager->remove($image->filePath); + $image->filePath = null; + $image->downloadedAt = null; + $this->entityManager->persist($image); + $this->entityManager->flush(); + + return true; + } catch (\Exception|FilesystemException $e) { + $this->logger->error('Unable to remove cached images for "{path}": {ex} - {m}', [ + 'path' => $image->filePath, + 'ex' => \get_class($e), + 'm' => $e->getMessage(), + 'exception' => $e, + ]); + + return false; + } + } } diff --git a/src/Service/ImageManagerInterface.php b/src/Service/ImageManagerInterface.php index b7c42d8bf3..e420c677f8 100644 --- a/src/Service/ImageManagerInterface.php +++ b/src/Service/ImageManagerInterface.php @@ -41,4 +41,6 @@ public function getMimetype(\App\Entity\Image $image): string; * @throws FilesystemException */ public function deleteOrphanedFiles(ImageRepository $repository, bool $dryRun, array $ignoredPaths): iterable; + + public function compressUntilSize(string $filePath, string $extension, int $maxBytes): bool; } diff --git a/src/Utils/ArrayUtils.php b/src/Utils/ArrayUtils.php index e58d3a49af..119027ddaa 100644 --- a/src/Utils/ArrayUtils.php +++ b/src/Utils/ArrayUtils.php @@ -23,4 +23,23 @@ public static function numCompareDescending(int $a, int $b): int return ($a < $b) ? 1 : -1; } + + /** + * @template-covariant T + * + * @param T[] $a + * + * @return T[][] + */ + public static function sliceArrayIntoEqualPieces(array $a, int $size): array + { + $arraySize = \sizeof($a); + $steps = $arraySize / $size; + $result = []; + for ($i = 0; $i < $steps; ++$i) { + $result[] = \array_slice($a, $i * $size, $size); + } + + return $result; + } } diff --git a/src/Utils/GeneralUtil.php b/src/Utils/GeneralUtil.php index 11c586f3a2..2186b0ebbb 100644 --- a/src/Utils/GeneralUtil.php +++ b/src/Utils/GeneralUtil.php @@ -4,6 +4,8 @@ namespace App\Utils; +use Symfony\Component\Console\Helper\ProgressBar; + class GeneralUtil { public static function shouldPathBeIgnored(array $ignoredPaths, string $path): bool @@ -18,4 +20,12 @@ public static function shouldPathBeIgnored(array $ignoredPaths, string $path): b return $isIgnored; } + + public static function useProgressbarFormatsWithMessage(): void + { + ProgressBar::setFormatDefinition(ProgressBar::FORMAT_NORMAL, ProgressBar::getFormatDefinition(ProgressBar::FORMAT_NORMAL).' - %message%'); + ProgressBar::setFormatDefinition(ProgressBar::FORMAT_VERBOSE, ProgressBar::getFormatDefinition(ProgressBar::FORMAT_VERBOSE).' - %message%'); + ProgressBar::setFormatDefinition(ProgressBar::FORMAT_VERY_VERBOSE, ProgressBar::getFormatDefinition(ProgressBar::FORMAT_VERY_VERBOSE).' - %message%'); + ProgressBar::setFormatDefinition(ProgressBar::FORMAT_DEBUG, ProgressBar::getFormatDefinition(ProgressBar::FORMAT_DEBUG).' - %message%'); + } } diff --git a/src/Utils/SqlHelpers.php b/src/Utils/SqlHelpers.php index 1ee569d8f8..67788c246b 100644 --- a/src/Utils/SqlHelpers.php +++ b/src/Utils/SqlHelpers.php @@ -7,6 +7,7 @@ use App\Entity\MagazineBlock; use App\Entity\User; use App\Entity\UserBlock; +use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository; use Doctrine\DBAL\Exception; use Doctrine\DBAL\ParameterType; use Doctrine\DBAL\Types\Types; @@ -371,4 +372,24 @@ public function fetchSingleColumnAsArray(string $sql, User $user): array return $result; } + + /** + * This method is useful for gathering more entities than the parameter limit allows for. + * + * @template-covariant T + * + * @param ServiceEntityRepository $repository + * + * @return T[] + */ + public static function findByAdjusted(ServiceEntityRepository $repository, string $columnName, array $values): array + { + $split = ArrayUtils::sliceArrayIntoEqualPieces($values, 65000); + $results = []; + foreach ($split as $part) { + $results[] = $repository->findBy([$columnName => $part]); + } + + return array_merge(...$results); + } } diff --git a/tests/Service/TestingImageManager.php b/tests/Service/TestingImageManager.php index aaf8c32b41..c7443fc620 100644 --- a/tests/Service/TestingImageManager.php +++ b/tests/Service/TestingImageManager.php @@ -9,7 +9,10 @@ use App\Service\ImageManager; use App\Service\ImageManagerInterface; use App\Service\SettingsManager; +use App\Twig\Runtime\FormattingExtensionRuntime; +use Doctrine\ORM\EntityManagerInterface; use League\Flysystem\FilesystemOperator; +use Liip\ImagineBundle\Imagine\Cache\CacheManager; use Psr\Log\LoggerInterface; use Symfony\Component\DependencyInjection\Attribute\When; use Symfony\Component\Mime\MimeTypesInterface; @@ -30,8 +33,12 @@ public function __construct( ValidatorInterface $validator, LoggerInterface $logger, SettingsManager $settings, + FormattingExtensionRuntime $formattingExtensionRuntime, + float $imageCompressionQuality, + CacheManager $imagineCacheManager, + EntityManagerInterface $entityManager, ) { - $this->innerImageManager = new ImageManager($storageUrl, $publicUploadsFilesystem, $httpClient, $mimeTypeGuesser, $validator, $logger, $settings); + $this->innerImageManager = new ImageManager($storageUrl, $publicUploadsFilesystem, $httpClient, $mimeTypeGuesser, $validator, $logger, $settings, $formattingExtensionRuntime, $imageCompressionQuality, $imagineCacheManager, $entityManager); } public function setKibbyPath(string $kibbyPath): void @@ -101,4 +108,9 @@ public function deleteOrphanedFiles(ImageRepository $repository, bool $dryRun, a yield $deletedPath; } } + + public function compressUntilSize(string $filePath, string $extension, int $maxBytes): bool + { + return $this->innerImageManager->compressUntilSize($filePath, $extension, $maxBytes); + } } diff --git a/tests/Unit/Utils/ArrayUtilTest.php b/tests/Unit/Utils/ArrayUtilTest.php new file mode 100644 index 0000000000..cac05d2725 --- /dev/null +++ b/tests/Unit/Utils/ArrayUtilTest.php @@ -0,0 +1,28 @@ +getService(ValidatorInterface::class), $this->getService(LoggerInterface::class), $this->getService(SettingsManager::class), + $this->getService(FormattingExtensionRuntime::class), + self::getContainer()->getParameter('mbin_image_compression_quality'), + $this->getService(CacheManager::class), + $this->getService(EntityManagerInterface::class), ); $this->imageManager->setKibbyPath($this->kibbyPath); self::getContainer()->set(ImageManagerInterface::class, $this->imageManager);