diff --git a/appinfo/info.xml b/appinfo/info.xml index e3807cc72..c826d7757 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -101,6 +101,7 @@ The app does not send any sensitive data to cloud providers or similar services. OCA\Recognize\Migration\InstallDeps + OCA\Recognize\Migration\RemoveDuplicateFaceDetections OCA\Recognize\Migration\InstallDeps diff --git a/lib/Db/FaceDetectionMapper.php b/lib/Db/FaceDetectionMapper.php index 9f7cbb38a..5abc5b3dc 100644 --- a/lib/Db/FaceDetectionMapper.php +++ b/lib/Db/FaceDetectionMapper.php @@ -11,6 +11,7 @@ use OCP\AppFramework\Db\DoesNotExistException; use OCP\AppFramework\Db\Entity; use OCP\AppFramework\Db\QBMapper; +use OCP\DB\Exception; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\IConfig; use OCP\IDBConnection; @@ -42,6 +43,35 @@ public function find(int $id): FaceDetection { return $this->findEntity($qb); } + /** + * @throws \OCP\DB\Exception + */ + public function insert(Entity $entity): FaceDetection { + $qb = $this->db->getQueryBuilder(); + $qb->select(FaceDetection::$columns) + ->from('recognize_face_detections') + ->where($qb->expr()->eq('file_id', $qb->createPositionalParameter($entity->getFileId(), IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('user_id', $qb->createPositionalParameter($entity->getUserId(), IQueryBuilder::PARAM_STR))) + ->andWhere($qb->expr()->eq('x', $qb->createPositionalParameter($entity->getX(), IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('y', $qb->createPositionalParameter($entity->getY(), IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('height', $qb->createPositionalParameter($entity->getHeight(), IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('width', $qb->createPositionalParameter($entity->getWidth(), IQueryBuilder::PARAM_INT))); + $duplicates = $this->findEntities($qb); + + if (empty($duplicates)) { + return parent::insert($entity); + } + + return $duplicates[0]; + } + + /** + * @throws Exception + */ + public function insertWithoutDeduplication(Entity $entity): FaceDetection { + return parent::insert($entity); + } + /** * @throws \OCP\DB\Exception */ diff --git a/lib/Migration/RemoveDuplicateFaceDetections.php b/lib/Migration/RemoveDuplicateFaceDetections.php new file mode 100644 index 000000000..8bdbbe457 --- /dev/null +++ b/lib/Migration/RemoveDuplicateFaceDetections.php @@ -0,0 +1,61 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +namespace OCA\Recognize\Migration; + +use OCP\IDBConnection; +use OCP\Migration\IOutput; +use OCP\Migration\IRepairStep; +use Psr\Log\LoggerInterface; + +final class RemoveDuplicateFaceDetections implements IRepairStep { + + public function __construct( + private IDBConnection $db, + private LoggerInterface $logger, + ) { + } + + public function getName(): string { + return 'Remove duplicate face detections'; + } + + public function run(IOutput $output): void { + try { + $subQuery = $this->db->getQueryBuilder(); + $subQuery->select($subQuery->func()->min('id')) + ->from('recognize_face_detections') + ->groupBy('file_id', 'user_id', 'x', 'y', 'height', 'width'); + + $qb = $this->db->getQueryBuilder(); + $qb->delete('recognize_face_detections') + ->where($qb->expr()->notIn('id', $qb->createFunction('(' . $subQuery->getSQL() .')'))); + + $qb->executeStatement(); + } catch (\Throwable $e) { + $output->warning('Failed to automatically remove duplicate face detections for recognize.'); + $this->logger->error('Failed to automatically remove duplicate face detections', ['exception' => $e]); + } + } +} diff --git a/tests/RemoveDuplicateFaceDetectionsTest.php b/tests/RemoveDuplicateFaceDetectionsTest.php new file mode 100644 index 000000000..57d080152 --- /dev/null +++ b/tests/RemoveDuplicateFaceDetectionsTest.php @@ -0,0 +1,79 @@ +db = Server::get(IDBConnection::class); + $this->faceDetectionMapper = Server::get(FaceDetectionMapper::class); + + // Clear + $qb = $this->db->getQueryBuilder(); + $qb->delete('recognize_face_detections')->executeStatement(); + + // Generate 11 face detections per file (1000 files per user; 100 users) + // = 1.100.000 face detections out of which 900.000 are superfluous duplicates to be removed + // After the repair step there should be 200.000 left + for ($k = 0; $k < 100; $k++) { + for ($j = 0; $j < 1000; $j++) { + $user = 'user' . $k; + $x = rand(0, 100) / 100; + $y = rand(0, 100) / 100; + $height = rand(0, 100) / 100; + $width = rand(0, 100) / 100; + for ($i = 0; $i < 10; $i++) { + $face = new \OCA\Recognize\Db\FaceDetection(); + $face->setUserId($user); + $face->setX($x); + $face->setY($y); + $face->setHeight($height); + $face->setWidth($width); + $face->setFileId($j); + $face->setThreshold(0.5); + $face->setVector([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]); + $this->faceDetectionMapper->insertWithoutDeduplication($face); + } + $face2 = new \OCA\Recognize\Db\FaceDetection(); + $face2->setUserId($user); + $face2->setX(rand(0, 100) / 100); + $face2->setY(rand(0, 100) / 100); + $face2->setHeight(rand(0, 100) / 100); + $face2->setWidth(rand(0, 100) / 100); + $face2->setFileId($k * $j); + $face2->setThreshold(0.5); + $face2->setVector([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]); + $this->faceDetectionMapper->insertWithoutDeduplication($face2); + } + } + } + + public function testRepairStep() : void { + // Prepare + $repairStep = Server::get(RemoveDuplicateFaceDetections::class); + $output = $this->createMock(\OCP\Migration\IOutput::class); + + // Check + $qb = $this->db->getQueryBuilder(); + $count = $qb->select($qb->func()->count('*'))->from('recognize_face_detections')->executeQuery()->fetchOne(); + $this->assertEquals(1100000, (int)$count); + + // Run + $repairStep->run($output); + + // Assert + $qb = $this->db->getQueryBuilder(); + $count = $qb->select($qb->func()->count('*'))->from('recognize_face_detections')->executeQuery()->fetchOne(); + $this->assertEquals(200000, (int)$count); + } +}