Skip to content

Commit a7914f5

Browse files
BentiGorlichBentiGorlich
committed
Add command to refresh media metadata
1 parent 40e818b commit a7914f5

File tree

3 files changed

+149
-0
lines changed

3 files changed

+149
-0
lines changed

‎docs/02-admin/04-running-mbin/05-cli.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -350,6 +350,21 @@ Options:
350350
- `--batch-size` (default `10000`): the number of images to retrieve per query from the DB. A higher number means less queries, but higher memory usage.
351351
- `--dry-run`: if set, no images will be deleted
352352

353+
### Refresh the meta data of stored images
354+
355+
This command allows you to refresh the filesize of the stored media, as well as the status.
356+
If an image is no longer present on storage this command adjusts it in the DB.
357+
358+
Usage:
359+
360+
```bash
361+
php bin/console mbin:images:refresh-meta [--batch-size] [--dry-run]
362+
```
363+
364+
Options:
365+
- `--batch-size` (default `10000`): the number of images to retrieve per query from the DB. A higher number means less queries, but higher memory usage.
366+
- `--dry-run`: if set, no metadata will be changed
367+
353368
### Remove old federated images
354369

355370
This command allows you to remove old federated images, without removing the content.

‎src/Command/RefreshImageMetaDataCommand.php‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace App\Command;
6+
7+
use App\Repository\ImageRepository;
8+
use App\Utils\GeneralUtil;
9+
use Doctrine\ORM\EntityManagerInterface;
10+
use League\Flysystem\FilesystemException;
11+
use League\Flysystem\FilesystemOperator;
12+
use Psr\Log\LoggerInterface;
13+
use Symfony\Component\Console\Attribute\AsCommand;
14+
use Symfony\Component\Console\Command\Command;
15+
use Symfony\Component\Console\Input\InputInterface;
16+
use Symfony\Component\Console\Input\InputOption;
17+
use Symfony\Component\Console\Output\OutputInterface;
18+
use Symfony\Component\Console\Style\SymfonyStyle;
19+
20+
#[AsCommand(
21+
name: 'mbin:images:refresh-meta',
22+
description: 'Refresh meta information about your media',
23+
)]
24+
class RefreshImageMetaDataCommand extends Command
25+
{
26+
public function __construct(
27+
private readonly ImageRepository $imageRepository,
28+
private readonly FilesystemOperator $publicUploadsFilesystem,
29+
private readonly LoggerInterface $logger,
30+
private readonly EntityManagerInterface $entityManager,
31+
) {
32+
parent::__construct();
33+
}
34+
35+
protected function configure(): void
36+
{
37+
$this->addOption('batch-size', null, InputOption::VALUE_REQUIRED, 'The number of images to handle at once, the higher the number the faster the command, but it also takes more memory', '10000');
38+
$this->addOption('dry-run', null, InputOption::VALUE_NONE, 'Do a trial without removing any media');
39+
}
40+
41+
protected function execute(InputInterface $input, OutputInterface $output): int
42+
{
43+
$io = new SymfonyStyle($input, $output);
44+
GeneralUtil::useProgressbarFormatsWithMessage();
45+
46+
$dryRun = \boolval($input->getOption('dry-run'));
47+
$batchSize = \intval($input->getOption('batch-size'));
48+
$images = $this->imageRepository->findSavedImagesPaginated($batchSize);
49+
$count = $images->count();
50+
$progressBar = $io->createProgressBar($count);
51+
$progressBar->setMessage('');
52+
$progressBar->start();
53+
$totalCheckedFiles = 0;
54+
$totalUpdateFiles = 0;
55+
56+
for ($i = 0; $i < $images->getNbPages(); ++$i) {
57+
$progressBar->setMessage(\sprintf('Fetching images %s - %s', ($i * $batchSize) + 1, ($i + 1) * $batchSize));
58+
$progressBar->display();
59+
foreach ($images->getCurrentPageResults() as $image) {
60+
$progressBar->advance();
61+
++$totalCheckedFiles;
62+
63+
try {
64+
if ($this->publicUploadsFilesystem->has($image->filePath)) {
65+
++$totalUpdateFiles;
66+
$fileSize = $this->publicUploadsFilesystem->fileSize($image->filePath);
67+
if (!$dryRun) {
68+
$image->localSize = $fileSize;
69+
$progressBar->setMessage(\sprintf('Refreshed meta data of "%s" (%s)', $image->filePath, $image->getId()));
70+
$this->logger->debug('Refreshed meta data of "{path}" ({id})', ['path' => $image->filePath, 'id' => $image->getId()]);
71+
} else {
72+
$progressBar->setMessage(\sprintf('Would have refreshed meta data of "%s" (%s)', $image->filePath, $image->getId()));
73+
}
74+
$progressBar->display();
75+
} else {
76+
$previousPath = $image->filePath;
77+
// mark it as not present on the media storage
78+
if (!$dryRun) {
79+
$image->filePath = null;
80+
$image->localSize = 0;
81+
$image->downloadedAt = null;
82+
$progressBar->setMessage(\sprintf('Marked "%s" (%s) as not present on the media storage', $previousPath, $image->getId()));
83+
} else {
84+
$progressBar->setMessage(\sprintf('Would have marked "%s" (%s) as not present on the media storage', $image->filePath, $image->getId()));
85+
}
86+
$progressBar->display();
87+
}
88+
} catch (FilesystemException $e) {
89+
$this->logger->error('There was an exception refreshing the meta data of "{path}" ({id}): {exClass} - {message}', [
90+
'path' => $image->filePath,
91+
'id' => $image->getId(),
92+
'exClass' => \get_class($image),
93+
'message' => $e->getMessage(),
94+
'exception' => $e,
95+
]);
96+
$progressBar->setMessage(\sprintf('Error checking meta data of "%s" (%s)', $image->filePath, $image->getId()));
97+
$progressBar->display();
98+
}
99+
}
100+
if (!$dryRun) {
101+
$this->entityManager->flush();
102+
}
103+
if ($images->hasNextPage()) {
104+
$images->setCurrentPage($images->getNextPage());
105+
}
106+
}
107+
$io->writeln('');
108+
if (!$dryRun) {
109+
$io->success(\sprintf('Refreshed %s files', $totalUpdateFiles));
110+
} else {
111+
$io->success(\sprintf('Would have refreshed %s files', $totalUpdateFiles));
112+
}
113+
114+
return Command::SUCCESS;
115+
}
116+
}

‎src/Repository/ImageRepository.php‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
use App\Exception\ImageDownloadTooLargeException;
1010
use App\Pagination\NativeQueryAdapter;
1111
use App\Pagination\Pagerfanta;
12+
use App\Pagination\QueryAdapter;
1213
use App\Pagination\Transformation\ContentPopulationTransformer;
1314
use App\Service\ImageManagerInterface;
1415
use App\Utils\ImageOrigin;
@@ -245,6 +246,23 @@ public function findOldRemoteMediaPaginated(int $olderThanDays, int $limit = 100
245246
return $fanta;
246247
}
247248

249+
/**
250+
* @return Pagerfanta<Image>
251+
*/
252+
public function findSavedImagesPaginated(int $pageSize): Pagerfanta
253+
{
254+
$query = $this->createQueryBuilder('i')
255+
->andWhere('i.filePath IS NOT NULL')
256+
->orderBy('i.filePath');
257+
258+
$adapter = new QueryAdapter($query);
259+
$fanta = new Pagerfanta($adapter);
260+
$fanta->setMaxPerPage($pageSize);
261+
$fanta->setCurrentPage(1);
262+
263+
return $fanta;
264+
}
265+
248266
public function redownloadImage(Image $image): void
249267
{
250268
if ($image->filePath || !$image->sourceUrl || $image->sourceTooBig) {

0 commit comments

Comments
 (0)