Skip to content

Commit 5bd0276

Browse files
marcelklehrmarcelklehr
authored
Merge pull request #1429 from nextcloud/fix/async-event-processing
refactor(FileListener): Process file events asynchronously
2 parents 2080e17 + d710e38 commit 5bd0276

24 files changed

+1214
-279
lines changed

‎.github/workflows/cluster-faces-test.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,7 @@ jobs:
206206
- name: Run clustering
207207
if: steps.clustering-cache.outputs.cache-hit != 'true'
208208
run: |
209+
./occ upgrade # in case server master has new migrations in the meantime
209210
./occ recognize:cluster-faces -b 10000
210211
./occ recognize:cluster-faces -b 10000
211212
./occ recognize:cluster-faces -b 10000

‎.github/workflows/files-scan-test.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,9 +140,10 @@ jobs:
140140
GITHUB_REF: ${{ github.ref }}
141141
run: |
142142
./occ files:scan admin -vvv
143+
for run in {1..100}; do php cron.php -v; done
143144
rm data/admin/files/res/alpine.JPG
144145
./occ files:scan admin -vvv
145-
for run in {1..100}; do php cron.php; done
146+
for run in {1..100}; do php cron.php -v; done
146147
[ ! -z "$(./occ tag:list)" ]
147148
148149
- name: Print tags

‎.github/workflows/phpunit-mariadb.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,7 @@ jobs:
155155
mkdir data
156156
./occ maintenance:install --verbose --database=mysql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin
157157
./occ app:enable --force ${{ env.APP_NAME }}
158+
./occ app:enable files_external
158159
159160
- name: Check PHPUnit script is defined
160161
id: check_phpunit

‎.github/workflows/phpunit-mysql.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,7 @@ jobs:
153153
mkdir data
154154
./occ maintenance:install --verbose --database=mysql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin
155155
./occ app:enable --force ${{ env.APP_NAME }}
156+
./occ app:enable files_external
156157
157158
- name: Check PHPUnit script is defined
158159
id: check_phpunit

‎.github/workflows/phpunit-pgsql.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ jobs:
151151
mkdir data
152152
./occ maintenance:install --verbose --database=pgsql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin
153153
./occ app:enable --force ${{ env.APP_NAME }}
154+
./occ app:enable files_external
154155
155156
- name: Check PHPUnit script is defined
156157
id: check_phpunit

‎.github/workflows/phpunit-sqlite.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,7 @@ jobs:
140140
mkdir data
141141
./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin
142142
./occ app:enable --force ${{ env.APP_NAME }}
143+
./occ app:enable files_external
143144
144145
- name: Check PHPUnit script is defined
145146
id: check_phpunit

‎composer.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@
2222
"lint": "find . -name \\*.php -not -path './vendor/*' -print0 | xargs -0 -n1 php -l",
2323
"cs:check": "php-cs-fixer fix --dry-run --diff",
2424
"cs:fix": "php-cs-fixer fix",
25-
"psalm": "psalm.phar",
26-
"psalm:update-baseline": "psalm.phar --threads=1 --update-baseline",
27-
"psalm:update-baseline:force": "psalm.phar --threads=1 --update-baseline --set-baseline=psalm-baseline.xml",
25+
"psalm": "psalm",
26+
"psalm:update-baseline": "psalm --threads=1 --update-baseline",
27+
"psalm:update-baseline:force": "psalm --threads=1 --update-baseline --set-baseline=psalm-baseline.xml",
2828
"test:unit": "phpunit --config tests/phpunit.xml",
2929
"post-install-cmd": [
3030
"@composer bin all install --ansi",

‎lib/AppInfo/Application.php‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,6 @@
2323
use OCP\Files\Events\Node\NodeRenamedEvent;
2424
use OCP\Files\Events\NodeRemovedFromCache;
2525
use OCP\SabrePluginEvent;
26-
use OCP\Share\Events\ShareCreatedEvent;
27-
use OCP\Share\Events\ShareDeletedEvent;
2826

2927
final class Application extends App implements IBootstrap {
3028
public const APP_ID = 'recognize';
@@ -42,16 +40,10 @@ public function __construct() {
4240
$dispatcher->addServiceListener(BeforeNodeRenamedEvent::class, FileListener::class);
4341
$dispatcher->addServiceListener(CacheEntryInsertedEvent::class, FileListener::class);
4442
$dispatcher->addServiceListener(NodeRemovedFromCache::class, FileListener::class);
45-
// These events were added mid-way through NC 30, 31
46-
if (class_exists('OCP\Files\Config\Event\UserMountAddedEvent')) {
47-
$dispatcher->addServiceListener('OCP\Files\Config\Event\UserMountAddedEvent', FileListener::class);
48-
$dispatcher->addServiceListener('OCP\Files\Config\Event\UserMountRemovedEvent', FileListener::class);
49-
// it is not fired as of now, Added and Removed events are fired instead in that order
50-
// $context->addServiceListener('OCP\Files\Config\Event\UserMountUpdatedEvent', FileListener::class);
51-
} else {
52-
$dispatcher->addServiceListener(ShareCreatedEvent::class, FileListener::class);
53-
$dispatcher->addServiceListener(ShareDeletedEvent::class, FileListener::class);
54-
}
43+
$dispatcher->addServiceListener('OCP\Files\Config\Event\UserMountAddedEvent', FileListener::class);
44+
$dispatcher->addServiceListener('OCP\Files\Config\Event\UserMountRemovedEvent', FileListener::class);
45+
// it is not fired as of now, Added and Removed events are fired instead in that order
46+
// $context->addServiceListener('OCP\Files\Config\Event\UserMountUpdatedEvent', FileListener::class);
5547
}
5648

5749
public function register(IRegistrationContext $context): void {

‎lib/BackgroundJobs/ProcessFsActionsJob.php‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
<?php
2+
3+
/*
4+
* Copyright (c) 2021-2022 The Recognize contributors.
5+
* This file is licensed under the Affero General Public License version 3 or later. See the COPYING file.
6+
*/
7+
declare(strict_types=1);
8+
namespace OCA\Recognize\BackgroundJobs;
9+
10+
use OCA\Recognize\Db\FsAccessUpdate;
11+
use OCA\Recognize\Db\FsActionMapper;
12+
use OCA\Recognize\Db\FsCreation;
13+
use OCA\Recognize\Db\FsDeletion;
14+
use OCA\Recognize\Db\FsMove;
15+
use OCA\Recognize\Service\FsActionService;
16+
use OCP\AppFramework\Utility\ITimeFactory;
17+
use OCP\BackgroundJob\IJobList;
18+
use OCP\BackgroundJob\TimedJob;
19+
use OCP\DB\Exception;
20+
use Psr\Log\LoggerInterface;
21+
22+
final class ProcessFsActionsJob extends TimedJob {
23+
24+
public function __construct(
25+
ITimeFactory $timeFactory,
26+
private FsActionService $accessUpdateService,
27+
private IJobList $jobList,
28+
private FsActionMapper $accessUpdateMapper,
29+
private LoggerInterface $logger,
30+
) {
31+
parent::__construct($timeFactory);
32+
$this->setInterval(5 * 60);
33+
$this->setTimeSensitivity(self::TIME_SENSITIVE);
34+
}
35+
36+
/**
37+
* @param array{storage_id:int, type: class-string<FsAccessUpdate|FsCreation|FsDeletion|FsMove>} $argument
38+
* @return void
39+
*/
40+
protected function run($argument): void {
41+
$storageId = $argument['storage_id'] ?? null;
42+
$className = $argument['type'];
43+
44+
if (isset($storageId)) {
45+
$this->accessUpdateService->processActionsByClassAndStorageId($className, $storageId);
46+
try {
47+
$remainingCount = $this->accessUpdateMapper->countByStorageId($className, $storageId);
48+
} catch (Exception $e) {
49+
$this->logger->error('Failed to count fs actions: ' . $e->getMessage(), ['exception' => $e]);
50+
$remainingCount = 1;
51+
}
52+
} else {
53+
$this->accessUpdateService->processActionsByClass($className);
54+
try {
55+
$remainingCount = $this->accessUpdateMapper->count($className);
56+
} catch (Exception $e) {
57+
$this->logger->error('Failed to count fs actions: ' . $e->getMessage(), ['exception' => $e]);
58+
$remainingCount = 1;
59+
}
60+
}
61+
62+
63+
if ($remainingCount === 0) {
64+
// Remove job from queue
65+
$this->jobList->remove(self::class, $argument);
66+
}
67+
}
68+
}

‎lib/Clustering/DualTreeClique.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public function propagateSetChanges(array &$labelToSetId) {
6464

6565
$labels = $this->dataset->labels();
6666
$lastLabel = array_pop($labels);
67-
if (!isset($labelToSetId[$lastLabel])) {
67+
if (!isset($lastLabel) || !isset($labelToSetId[$lastLabel])) {
6868
return null;
6969
}
7070
$setId = $labelToSetId[$lastLabel];

0 commit comments

Comments
 (0)