Skip to content

Commit d7caf6a

Browse files
morjamorja
authored
Merge pull request #18 from morja/main
Added new options to allow forced updates and to keep old records. See issue #16 and #17.
2 parents 6f34ee4 + 59557fd commit d7caf6a

6 files changed

+170
-7
lines changed

‎Classes/Command/UnduplicateCommand.php‎

Lines changed: 51 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,16 @@ class UnduplicateCommand extends Command
6464
*/
6565
private $dryRun = false;
6666

67+
/**
68+
* @var String|bool
69+
*/
70+
private $force = false;
71+
72+
/**
73+
* @var bool
74+
*/
75+
private $keepOldest = false;
76+
6777
/**
6878
* @var SymfonyStyle
6979
*/
@@ -85,7 +95,8 @@ public function __construct($name = null, ConnectionPool $connectionPool = null)
8595
*/
8696
public function configure()
8797
{
88-
$this->setDescription('Finds duplicates in sys_file and unduplicates them');
98+
$this->setDescription('Finds duplicates in sys_file and unduplicates them.
99+
By default it will use the newest (highest uid) record as master and delete the older records.');
89100
$this->setHelp(
90101
'currently fix references in ' . LF .
91102
'- sys_file_reference::link ' . LF .
@@ -115,6 +126,20 @@ public function configure()
115126
'Only use this storage',
116127
-1
117128
)
129+
->addOption(
130+
'force',
131+
'f',
132+
InputOption::VALUE_OPTIONAL,
133+
'Force keep or overwrite of metadata of the master record in case of conflict. Possible values: keep, keep-nonempty.
134+
Default: overwrite. Keep-nonempty is keeping only nonempty metadata in master, but updating the empty.',
135+
false
136+
)
137+
->addOption(
138+
'keep-oldest',
139+
'o',
140+
InputOption::VALUE_NONE,
141+
'Use the oldest record as master instead of the newest',
142+
)
118143
->addOption(
119144
'meta-fields',
120145
'm',
@@ -146,6 +171,11 @@ protected function execute(InputInterface $input, OutputInterface $output): int
146171
$this->updateReferenceIndex($input);
147172

148173
$this->dryRun = $input->getOption('dry-run');
174+
$this->force = $input->getOption('force'); // force will be null if no value is passed -> overwrite
175+
if ($this->force === null) {
176+
$this->force = 'overwrite';
177+
}
178+
$this->keepOldest = $input->getOption('keep-oldest');
149179
$onlyThisIdentifier = $input->getOption('identifier');
150180
$onlyThisStorage = (int)$input->getOption('storage');
151181

@@ -258,7 +288,7 @@ private function findDuplicateFilesForIdentifier(string $identifier, int $storag
258288
'storage',
259289
$fileQueryBuilder->createNamedParameter($storage, Connection::PARAM_INT)
260290
)
261-
)->orderBy('uid', 'DESC');
291+
)->orderBy('uid', $this->keepOldest ? 'ASC' : 'DESC');
262292

263293
$whereClause = 'MD5(identifier) = MD5(' . $fileQueryBuilder->createNamedParameter($identifier, \PDO::PARAM_STR) . ')';
264294
$fileQueryBuilder->add('where', $whereClause);
@@ -370,22 +400,36 @@ public function getSysRefIndexData(int $oldFileUid): Result
370400
public function updateMetadataRecord(int $masterFileUid, array $referenceRow): bool
371401
{
372402

373-
$metadata = $this->isMetadataRecordPopulated($referenceRow['ref_uid']);
403+
$oldMetadata = $this->isMetadataRecordPopulated($referenceRow['ref_uid']);
374404
$masterFileMetadata = $this->isMetadataRecordPopulated($masterFileUid);
405+
$masterEmoty = !$masterFileMetadata;
375406

376-
if (!$metadata || $metadata === $masterFileMetadata) { // check if record is empty or if the values are the same as in master
407+
if (!$oldMetadata || $oldMetadata === $masterFileMetadata) { // check if record is empty or if the values are the same as in master
377408
$this->output->writeln('<info>Deleting old metadata record</info>');
378409

379410
if (!$this->dryRun) {
380411
$this->deleteReferencedRecord($referenceRow);
381412
$this->deleteReference($referenceRow);
382413
}
383414

384-
} elseif ($metadata && !$masterFileMetadata) { // check if master record has metadata, if not, copy the old ones
385-
$this->output->writeln('<info>Old metadata is not empty and master is empty, copying values to master</info>');
415+
} elseif ($oldMetadata && ($masterEmoty || $this->force !== false)) { // check if master record has metadata, if not, copy the old ones
416+
417+
if ($masterEmoty) {
418+
$this->output->writeln('<info>Old metadata is not empty and master is empty, copying values to master. Deleting old metadata record</info>');
419+
} else if ($this->force !== false) {
420+
if ($this->force !== 'keep') {
421+
$this->output->writeln('<info>Force overwriting metadata in master. Deleting old metadata record</info>');
422+
} else {
423+
$this->output->writeln('<info>Force keeping metadata in master. Deleting old metadata record. Action may be required.</info>');
424+
}
425+
}
386426

387427
if (!$this->dryRun) {
388-
$this->updateMasterFileMetadata($masterFileUid, $metadata);
428+
if ($this->force === false ||
429+
$this->force === 'overwrite' ||
430+
$masterEmoty && $this->force === 'keep-nonempty') {
431+
$this->updateMasterFileMetadata($masterFileUid, $oldMetadata);
432+
}
389433
$this->deleteReferencedRecord($referenceRow);
390434
$this->deleteReference($referenceRow);
391435
}

‎Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_force_keep_RESULT.csv‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
"sys_file",,,
2+
,"uid","identifier","storage"
3+
,2,"/test/abc.jpg",1
4+
,4,"/test/abcd.jpg",1
5+
,6,"/test/abcde.jpg",1
6+
,8,"/test/abcdef.jpg",1
7+
"sys_refindex",,,,,,,
8+
,"hash","tablename","recuid","field","softref_key","ref_table","ref_uid"
9+
,"507cf53279411321324cd2f424e0682e","sys_file_metadata","2","","","sys_file","2"
10+
,"507cf53279411321324cd2f424e0684e","sys_file_metadata","4","","","sys_file","4"
11+
,"507cf53279411321324cd2f424e0686e","sys_file_metadata","6","","","sys_file","6"
12+
,"507cf53279411321324cd2f424e0688e","sys_file_metadata","8","","","sys_file","8"
13+
"sys_file_metadata",,,
14+
,"uid","file","description"
15+
,2,2,"ABC"
16+
,4,4,"ABC"
17+
,6,6,"ABC"
18+
,8,8,""

‎Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_force_keep_nonempty_RESULT.csv‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
"sys_file",,,
2+
,"uid","identifier","storage"
3+
,2,"/test/abc.jpg",1
4+
,4,"/test/abcd.jpg",1
5+
,6,"/test/abcde.jpg",1
6+
,8,"/test/abcdef.jpg",1
7+
"sys_refindex",,,,,,,
8+
,"hash","tablename","recuid","field","softref_key","ref_table","ref_uid"
9+
,"507cf53279411321324cd2f424e0682e","sys_file_metadata","2","","","sys_file","2"
10+
,"507cf53279411321324cd2f424e0684e","sys_file_metadata","4","","","sys_file","4"
11+
,"507cf53279411321324cd2f424e0686e","sys_file_metadata","6","","","sys_file","6"
12+
,"507cf53279411321324cd2f424e0688e","sys_file_metadata","8","","","sys_file","8"
13+
"sys_file_metadata",,,
14+
,"uid","file","description"
15+
,2,2,"ABC"
16+
,4,4,"ABC"
17+
,6,6,"ABC"
18+
,8,8,"ABC"

‎Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_force_overwrite_RESULT.csv‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
"sys_file",,,
2+
,"uid","identifier","storage"
3+
,2,"/test/abc.jpg",1
4+
,4,"/test/abcd.jpg",1
5+
,6,"/test/abcde.jpg",1
6+
,8,"/test/abcdef.jpg",1
7+
"sys_refindex",,,,,,,
8+
,"hash","tablename","recuid","field","softref_key","ref_table","ref_uid"
9+
,"507cf53279411321324cd2f424e0682e","sys_file_metadata","2","","","sys_file","2"
10+
,"507cf53279411321324cd2f424e0684e","sys_file_metadata","4","","","sys_file","4"
11+
,"507cf53279411321324cd2f424e0686e","sys_file_metadata","6","","","sys_file","6"
12+
,"507cf53279411321324cd2f424e0688e","sys_file_metadata","8","","","sys_file","8"
13+
"sys_file_metadata",,,
14+
,"uid","file","description"
15+
,2,2,"ABC"
16+
,4,4,"ABCD"
17+
,6,6,"ABC"
18+
,8,8,"ABC"

‎Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_oldest_RESULT.csv‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
"sys_file",,,
2+
,"uid","identifier","storage"
3+
,1,"/test/abc.jpg",1
4+
,3,"/test/abcd.jpg",1
5+
,4,"/test/abcd.jpg",1
6+
,5,"/test/abcde.jpg",1
7+
,7,"/test/abcdef.jpg",1
8+
"sys_refindex",,,,,,,
9+
,"hash","tablename","recuid","field","softref_key","ref_table","ref_uid"
10+
,"507cf53279411321324cd2f424e0681e","sys_file_metadata","1","","","sys_file","1"
11+
,"507cf53279411321324cd2f424e0683e","sys_file_metadata","3","","","sys_file","3"
12+
,"507cf53279411321324cd2f424e0684e","sys_file_metadata","4","","","sys_file","4"
13+
,"507cf53279411321324cd2f424e0685e","sys_file_metadata","5","","","sys_file","5"
14+
,"507cf53279411321324cd2f424e0687e","sys_file_metadata","7","","","sys_file","7"
15+
"sys_file_metadata",,,
16+
,"uid","file","description"
17+
,1,1,"ABC"
18+
,3,3,"ABCD"
19+
,4,4,"ABC"
20+
,5,5,"ABC"
21+
,7,7,"ABC"

‎Tests/Functional/Command/UnduplicateCommandTest.php‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,50 @@ class UnduplicateCommandTest extends FunctionalTestCase
8989
self::assertEquals(0, $result['status']);
9090
}
9191

92+
#[Test] public function unduplicateCommandKeepOldestWithMetadata(): void
93+
{
94+
$this->importCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata.csv');
95+
96+
$result = $this->executeConsoleCommand(self::BASE_COMMAND . ' --keep-oldest');
97+
98+
// the references are updated, so that the newer sys_file entry (uid=2) is used
99+
$this->assertCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata_oldest_RESULT.csv');
100+
self::assertEquals(0, $result['status']);
101+
}
102+
103+
#[Test] public function unduplicateCommandForceOverwriteWithMetadata(): void
104+
{
105+
$this->importCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata.csv');
106+
107+
$result = $this->executeConsoleCommand(self::BASE_COMMAND . ' --force');
108+
109+
// the references are updated, so that the newer sys_file entry (uid=2) is used
110+
$this->assertCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata_force_overwrite_RESULT.csv');
111+
self::assertEquals(0, $result['status']);
112+
}
113+
114+
#[Test] public function unduplicateCommandForceKeepWithMetadata(): void
115+
{
116+
$this->importCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata.csv');
117+
118+
$result = $this->executeConsoleCommand(self::BASE_COMMAND . ' --force keep');
119+
120+
// the references are updated, so that the newer sys_file entry (uid=2) is used
121+
$this->assertCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata_force_keep_RESULT.csv');
122+
self::assertEquals(0, $result['status']);
123+
}
124+
125+
#[Test] public function unduplicateCommandForceKeepNonEmptyWithMetadata(): void
126+
{
127+
$this->importCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata.csv');
128+
129+
$result = $this->executeConsoleCommand(self::BASE_COMMAND . ' --force keep-nonempty');
130+
131+
// the references are updated, so that the newer sys_file entry (uid=2) is used
132+
$this->assertCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata_force_keep_nonempty_RESULT.csv');
133+
self::assertEquals(0, $result['status']);
134+
}
135+
92136
/**
93137
* Provide a processed file for the test run, so that it can be deleted
94138
* @var array<string, string>

0 commit comments

Comments
 (0)