Improve image handling

.env.example

@@ -34,6 +34,8 @@ MBIN_DEFAULT_THEME=default
 # Set the max image file size (in bytes)
 # This should be set to <= `upload_max_filesize` and `post_max_size` in the server's php.ini file
 MBIN_MAX_IMAGE_BYTES=6000000
+# Image compression quality, set to -1 to disable. A value between 0.1 and 0.95 should be used
+MBIN_IMAGE_COMPRESSION_QUALITY=0.9
 # Change the down vote behaviour. Possible values are:
 # 'enabled' => default mode downvotes are enabled
 # 'hidden' => downvotes are counted and users can downvote, but the number is hidden

.env.example_docker

@@ -39,6 +39,8 @@ MBIN_DEFAULT_THEME=default
 # Set the max image file size (in bytes)
 # This should be set to <= `upload_max_filesize` and `post_max_size` in the server's php.ini file
 MBIN_MAX_IMAGE_BYTES=6000000
+# Image compression quality, set to -1 to disable. A value between 0.1 and 0.95 should be used
+MBIN_IMAGE_COMPRESSION_QUALITY=0.9
 # Change the down vote behaviour. Possible values are:
 # 'enabled' => default mode downvotes are enabled
 # 'hidden' => downvotes are counted and users can downvote, but the number is hidden

composer.json

@@ -26,9 +26,11 @@
         "doctrine/orm": "^2.19.6",
         "embed/embed": "^4.4.12",
         "endroid/qr-code": "^6.0.3",
+        "firebase/php-jwt": "7.0.2 as 6.11.1",
         "friendsofsymfony/jsrouting-bundle": "^3.5.0",
         "furqansiddiqui/bip39-mnemonic-php": "^0.1.7",
         "gumlet/php-image-resize": "^2.0.4",
+        "imagine/imagine": "^1.5",
         "knplabs/knp-time-bundle": "^2.4.0",
         "knpuniversity/oauth2-client-bundle": "^2.18.1",
         "kornrunner/blurhash": "^1.2.2",
@@ -114,8 +116,7 @@
         "twig/intl-extra": "^3.10.0",
         "twig/twig": "^3.15.0",
         "webmozart/assert": "^1.11.0",
-        "wohali/oauth2-discord-new": "^1.2.1",
-        "firebase/php-jwt": "7.0.2 as 6.11.1"
+        "wohali/oauth2-discord-new": "^1.2.1"
     },
     "require-dev": {
         "brianium/paratest": "^7.10.1",

config/services.yaml

@@ -117,6 +117,8 @@ parameters:
 
     mbin_max_image_bytes: '%env(int:default:mbin_max_image_bytes_default:MBIN_MAX_IMAGE_BYTES)%'
     mbin_max_image_bytes_default: 6000000
+    mbin_image_compression_quality: '%env(float:default:mbin_image_compression_quality_default:MBIN_IMAGE_COMPRESSION_QUALITY)%'
+    mbin_image_compression_quality_default: -1
 
     mbin_downvotes_mode_default: 'enabled'
     mbin_downvotes_mode: '%env(enum:\App\Utils\DownvotesMode:default:mbin_downvotes_mode_default:MBIN_DOWNVOTES_MODE)%'
@@ -156,6 +158,7 @@ services:
             $monitoringTwigRendersPersistingEnabled: '%mbin_monitoring_twig_render_persisting_enabled%'
             $monitoringCurlRequestsEnabled: '%mbin_monitoring_curl_requests_enabled%'
             $monitoringCurlRequestPersistingEnabled: '%mbin_monitoring_curl_request_persisting_enabled%'
+            $imageCompressionQuality: '%mbin_image_compression_quality%'
 
     kbin.s3_client:
         class: Aws\S3\S3Client

docs/02-admin/03-optional-features/09-image-compression.md

@@ -0,0 +1,34 @@
+# Image compression
+
+You can enable compression of images uploaded to mbin by users or downloaded from remote instances, 
+for increased compatibility, to save on size and for a better user experience.
+
+To enable image compression set `MBIN_IMAGE_COMPRESSION_QUALITY` in your `.env` file to a value between 0.1 and 0.95.
+This setting is used as a starting point to compress the image. It is gradually lowered (in 0.05 steps) until the maximum size is no longer exceeded.
+
+> [!HINT]
+> The maximum file size is determined by the `MBIN_MAX_IMAGE_BYTES` setting in your `.env` file
+
+> [!NOTE]
+> Enabling this setting can cause a higher memory usage
+
+## Better compatibility
+
+If another instance shares a thread with an image attached that exceeds your maximum image size, it will not be downloaded,
+but instead loaded directly from the other instance. This works most of the time, 
+but sometimes website settings will block it and thus your users will see an image that cannot be loaded.
+This behavior also introduces web requests to other servers, which may unintentionally leak information to the remote instance. 
+
+If instead your server compresses the image and saves it locally this will never happen.
+
+## Saving space
+
+When image compression is enabled you can reduce your maximum image size to, lets say 1MB. 
+Without the compression this might not be suitable, because too many images exceed that size,
+and you don't want to risk compatibility problems, 
+but with it enabled the images will just be compressed, saving space.
+
+## A better user experience
+
+Normally there is a maximum image size your users must adhere to, but if image compression is enabled,
+instead of showing your user an error that the image exceeds that size, the upload goes through and the image is compressed.

docs/02-admin/03-optional-features/README.md

@@ -12,3 +12,4 @@ Like setting-up:
 - [S3 storage](06-s3_storage.md) - Configure an object storage service (S3) compatible bucket for storing images.
 - [Anubis](07-anubis.md) - A service for weighing the incoming requests and may present them with a proof-of-work challenge. It is useful if your instance gets hit a lot of bot traffic that you're tired of filtering through
 - [Monitoring](08-monitoring.md) - Internal monitoring of requests and messengers
+- [Image compression](09-image-compression.md) - compress images if they exceed your maximum image size

docs/02-admin/04-running-mbin/05-cli.md

@@ -326,6 +326,44 @@ Arguments:
 
 ## Images
 
+### Remove cached remote media
+
+This command allows you to remove the cached file of remote media, **without** deleting the reference.
+You can run this command as a cron job to only keep cached media from the last 30 days for example.
+
+> [!TIP]
+> If a thread or microblog is opened without a local cache of the attached image existing, the image will be downloaded again.
+> Once an image is downloaded again, it will not get deleted for the number of days you set as a parameter.
+
+> [!NOTE]
+> User avatars and covers and magazine icons and banners are not affected by this command, 
+> only images from threads, microblogs and comments.
+
+Usage:
+
+```bash
+php bin/console mbin:images:remove-remote [--days|-d] [--batch-size] [--dry-run]
+```
+
+Options:
+- `--days`|`-d`: the number of days of media you want to keep. Everything older than the amount of days will be deleted
+- `--batch-size` (default `10000`): the number of images to retrieve per query from the DB. A higher number means less queries, but higher memory usage.
+- `--dry-run`: if set, no images will be deleted
+
+### Refresh the meta data of stored images
+
+This command allows you to refresh the filesize of the stored media, as well as the status.
+If an image is no longer present on storage this command adjusts it in the DB.
+
+Usage:
+
+```bash
+php bin/console mbin:images:refresh-meta [--batch-size] [--dry-run]
+```
+
+Options:
+- `--batch-size` (default `10000`): the number of images to retrieve per query from the DB. A higher number means less queries, but higher memory usage.
+- `--dry-run`: if set, no metadata will be changed
 
 ### Remove old federated images
 

migrations/Version20260303142852.php

@@ -0,0 +1,75 @@
+<?php
+
+declare(strict_types=1);
+
+namespace DoctrineMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\Migrations\AbstractMigration;
+
+final class Version20260303142852 extends AbstractMigration
+{
+    public function getDescription(): string
+    {
+        return 'Add local_size, original_size, is_compressed and source_too_big image columns';
+    }
+
+    public function up(Schema $schema): void
+    {
+        $this->addSql('ALTER TABLE image ADD is_compressed BOOLEAN DEFAULT false NOT NULL');
+        $this->addSql('ALTER TABLE image ADD source_too_big BOOLEAN DEFAULT false NOT NULL');
+        $this->addSql('ALTER TABLE image ADD downloaded_at TIMESTAMP(0) WITH TIME ZONE DEFAULT NULL');
+        // init the column for all existing images, it gets overwritten in the big loop underneath
+        $this->addSql('ALTER TABLE image ADD created_at TIMESTAMP(0) WITH TIME ZONE NOT NULL DEFAULT current_timestamp');
+        $this->addSql('ALTER TABLE image ALTER created_at DROP DEFAULT;');
+        $this->addSql('ALTER TABLE image ADD original_size BIGINT DEFAULT 0 NOT NULL');
+        $this->addSql('ALTER TABLE image ADD local_size BIGINT DEFAULT 0 NOT NULL');
+
+        // set the downloaded at value to something realistically
+        $this->addSql('DO
+$do$
+    declare tempRow record;
+BEGIN
+    FOR tempRow IN
+        SELECT i.id, e.created_at as ec, ec.created_at as ecc, p.created_at as pc, pc.created_at as pcc, u.created_at as uc, u2.created_at as u2c, m.created_at as mc, m2.created_at as m2c FROM image i
+            LEFT JOIN entry e ON i.id = e.image_id
+            LEFT JOIN entry_comment ec ON i.id = ec.image_id
+            LEFT JOIN post p ON i.id = p.image_id
+            LEFT JOIN post_comment pc ON i.id = pc.image_id
+            LEFT JOIN "user" u ON i.id = u.avatar_id
+            LEFT JOIN "user" u2 ON i.id = u2.cover_id
+            LEFT JOIN magazine m ON i.id = m.icon_id
+            LEFT JOIN magazine m2 ON i.id = m2.banner_id
+    LOOP
+        IF tempRow.ec IS NOT NULL THEN
+            UPDATE image SET downloaded_at = tempRow.ec, created_at = tempRow.ec WHERE id = tempRow.id;
+        ELSIF tempRow.ecc IS NOT NULL THEN
+            UPDATE image SET downloaded_at = tempRow.ecc, created_at = tempRow.ecc WHERE id = tempRow.id;
+        ELSIF tempRow.pc IS NOT NULL THEN
+            UPDATE image SET downloaded_at = tempRow.pc, created_at = tempRow.pc WHERE id = tempRow.id;
+        ELSIF tempRow.pcc IS NOT NULL THEN
+            UPDATE image SET downloaded_at = tempRow.pcc, created_at = tempRow.pcc WHERE id = tempRow.id;
+        ELSIF tempRow.uc IS NOT NULL THEN
+            UPDATE image SET downloaded_at = tempRow.uc, created_at = tempRow.uc WHERE id = tempRow.id;
+        ELSIF tempRow.u2c IS NOT NULL THEN
+            UPDATE image SET downloaded_at = tempRow.u2c, created_at = tempRow.u2c WHERE id = tempRow.id;
+        ELSIF tempRow.mc IS NOT NULL THEN
+            UPDATE image SET downloaded_at = tempRow.mc, created_at = tempRow.mc WHERE id = tempRow.id;
+        ELSIF tempRow.m2c IS NOT NULL THEN
+            UPDATE image SET downloaded_at = tempRow.m2c, created_at = tempRow.m2c WHERE id = tempRow.id;
+        END IF;
+    END LOOP;
+END
+$do$;');
+    }
+
+    public function down(Schema $schema): void
+    {
+        $this->addSql('ALTER TABLE image DROP is_compressed');
+        $this->addSql('ALTER TABLE image DROP source_too_big');
+        $this->addSql('ALTER TABLE image DROP downloaded_at');
+        $this->addSql('ALTER TABLE image DROP created_at');
+        $this->addSql('ALTER TABLE image DROP original_size');
+        $this->addSql('ALTER TABLE image DROP local_size');
+    }
+}

src/Command/RefreshImageMetaDataCommand.php

@@ -0,0 +1,116 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Command;
+
+use App\Repository\ImageRepository;
+use App\Utils\GeneralUtil;
+use Doctrine\ORM\EntityManagerInterface;
+use League\Flysystem\FilesystemException;
+use League\Flysystem\FilesystemOperator;
+use Psr\Log\LoggerInterface;
+use Symfony\Component\Console\Attribute\AsCommand;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Style\SymfonyStyle;
+
+#[AsCommand(
+    name: 'mbin:images:refresh-meta',
+    description: 'Refresh meta information about your media',
+)]
+class RefreshImageMetaDataCommand extends Command
+{
+    public function __construct(
+        private readonly ImageRepository $imageRepository,
+        private readonly FilesystemOperator $publicUploadsFilesystem,
+        private readonly LoggerInterface $logger,
+        private readonly EntityManagerInterface $entityManager,
+    ) {
+        parent::__construct();
+    }
+
+    protected function configure(): void
+    {
+        $this->addOption('batch-size', null, InputOption::VALUE_REQUIRED, 'The number of images to handle at once, the higher the number the faster the command, but it also takes more memory', '10000');
+        $this->addOption('dry-run', null, InputOption::VALUE_NONE, 'Do a trial without removing any media');
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output): int
+    {
+        $io = new SymfonyStyle($input, $output);
+        GeneralUtil::useProgressbarFormatsWithMessage();
+
+        $dryRun = \boolval($input->getOption('dry-run'));
+        $batchSize = \intval($input->getOption('batch-size'));
+        $images = $this->imageRepository->findSavedImagesPaginated($batchSize);
+        $count = $images->count();
+        $progressBar = $io->createProgressBar($count);
+        $progressBar->setMessage('');
+        $progressBar->start();
+        $totalCheckedFiles = 0;
+        $totalUpdateFiles = 0;
+
+        for ($i = 0; $i < $images->getNbPages(); ++$i) {
+            $progressBar->setMessage(\sprintf('Fetching images %s - %s', ($i * $batchSize) + 1, ($i + 1) * $batchSize));
+            $progressBar->display();
+            foreach ($images->getCurrentPageResults() as $image) {
+                $progressBar->advance();
+                ++$totalCheckedFiles;
+
+                try {
+                    if ($this->publicUploadsFilesystem->has($image->filePath)) {
+                        ++$totalUpdateFiles;
+                        $fileSize = $this->publicUploadsFilesystem->fileSize($image->filePath);
+                        if (!$dryRun) {
+                            $image->localSize = $fileSize;
+                            $progressBar->setMessage(\sprintf('Refreshed meta data of "%s" (%s)', $image->filePath, $image->getId()));
+                            $this->logger->debug('Refreshed meta data of "{path}" ({id})', ['path' => $image->filePath, 'id' => $image->getId()]);
+                        } else {
+                            $progressBar->setMessage(\sprintf('Would have refreshed meta data of "%s" (%s)', $image->filePath, $image->getId()));
+                        }
+                        $progressBar->display();
+                    } else {
+                        $previousPath = $image->filePath;
+                        // mark it as not present on the media storage
+                        if (!$dryRun) {
+                            $image->filePath = null;
+                            $image->localSize = 0;
+                            $image->downloadedAt = null;
+                            $progressBar->setMessage(\sprintf('Marked "%s" (%s) as not present on the media storage', $previousPath, $image->getId()));
+                        } else {
+                            $progressBar->setMessage(\sprintf('Would have marked "%s" (%s) as not present on the media storage', $image->filePath, $image->getId()));
+                        }
+                        $progressBar->display();
+                    }
+                } catch (FilesystemException $e) {
+                    $this->logger->error('There was an exception refreshing the meta data of "{path}" ({id}): {exClass} - {message}', [
+                        'path' => $image->filePath,
+                        'id' => $image->getId(),
+                        'exClass' => \get_class($image),
+                        'message' => $e->getMessage(),
+                        'exception' => $e,
+                    ]);
+                    $progressBar->setMessage(\sprintf('Error checking meta data of "%s" (%s)', $image->filePath, $image->getId()));
+                    $progressBar->display();
+                }
+            }
+            if (!$dryRun) {
+                $this->entityManager->flush();
+            }
+            if ($images->hasNextPage()) {
+                $images->setCurrentPage($images->getNextPage());
+            }
+        }
+        $io->writeln('');
+        if (!$dryRun) {
+            $io->success(\sprintf('Refreshed %s files', $totalUpdateFiles));
+        } else {
+            $io->success(\sprintf('Would have refreshed %s files', $totalUpdateFiles));
+        }
+
+        return Command::SUCCESS;
+    }
+}