Skip to content

Commit 69ec2ce

Browse files
authored
Merge pull request #56120 from nextcloud/feat/snowflake-file-sequence
2 parents a1d6af4 + 83dd9d3 commit 69ec2ce

File tree

11 files changed

+284
-41
lines changed

11 files changed

+284
-41
lines changed

lib/composer/composer/autoload_classmap.php

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2110,8 +2110,11 @@
21102110
'OC\\Share\\Constants' => $baseDir . '/lib/private/Share/Constants.php',
21112111
'OC\\Share\\Helper' => $baseDir . '/lib/private/Share/Helper.php',
21122112
'OC\\Share\\Share' => $baseDir . '/lib/private/Share/Share.php',
2113+
'OC\\Snowflake\\APCuSequence' => $baseDir . '/lib/private/Snowflake/APCuSequence.php',
21132114
'OC\\Snowflake\\Decoder' => $baseDir . '/lib/private/Snowflake/Decoder.php',
2115+
'OC\\Snowflake\\FileSequence' => $baseDir . '/lib/private/Snowflake/FileSequence.php',
21142116
'OC\\Snowflake\\Generator' => $baseDir . '/lib/private/Snowflake/Generator.php',
2117+
'OC\\Snowflake\\ISequence' => $baseDir . '/lib/private/Snowflake/ISequence.php',
21152118
'OC\\SpeechToText\\SpeechToTextManager' => $baseDir . '/lib/private/SpeechToText/SpeechToTextManager.php',
21162119
'OC\\SpeechToText\\TranscriptionJob' => $baseDir . '/lib/private/SpeechToText/TranscriptionJob.php',
21172120
'OC\\StreamImage' => $baseDir . '/lib/private/StreamImage.php',

lib/composer/composer/autoload_static.php

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,6 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
2121
array (
2222
'NCU\\' => 4,
2323
),
24-
'B' =>
25-
array (
26-
'Bamarni\\Composer\\Bin\\' => 21,
27-
),
2824
);
2925

3026
public static $prefixDirsPsr4 = array (
@@ -44,10 +40,6 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
4440
array (
4541
0 => __DIR__ . '/../../..' . '/lib/unstable',
4642
),
47-
'Bamarni\\Composer\\Bin\\' =>
48-
array (
49-
0 => __DIR__ . '/..' . '/bamarni/composer-bin-plugin/src',
50-
),
5143
);
5244

5345
public static $fallbackDirsPsr4 = array (
@@ -2159,8 +2151,11 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
21592151
'OC\\Share\\Constants' => __DIR__ . '/../../..' . '/lib/private/Share/Constants.php',
21602152
'OC\\Share\\Helper' => __DIR__ . '/../../..' . '/lib/private/Share/Helper.php',
21612153
'OC\\Share\\Share' => __DIR__ . '/../../..' . '/lib/private/Share/Share.php',
2154+
'OC\\Snowflake\\APCuSequence' => __DIR__ . '/../../..' . '/lib/private/Snowflake/APCuSequence.php',
21622155
'OC\\Snowflake\\Decoder' => __DIR__ . '/../../..' . '/lib/private/Snowflake/Decoder.php',
2156+
'OC\\Snowflake\\FileSequence' => __DIR__ . '/../../..' . '/lib/private/Snowflake/FileSequence.php',
21632157
'OC\\Snowflake\\Generator' => __DIR__ . '/../../..' . '/lib/private/Snowflake/Generator.php',
2158+
'OC\\Snowflake\\ISequence' => __DIR__ . '/../../..' . '/lib/private/Snowflake/ISequence.php',
21642159
'OC\\SpeechToText\\SpeechToTextManager' => __DIR__ . '/../../..' . '/lib/private/SpeechToText/SpeechToTextManager.php',
21652160
'OC\\SpeechToText\\TranscriptionJob' => __DIR__ . '/../../..' . '/lib/private/SpeechToText/TranscriptionJob.php',
21662161
'OC\\StreamImage' => __DIR__ . '/../../..' . '/lib/private/StreamImage.php',

lib/private/Server.php

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,8 +115,11 @@
115115
use OC\SetupCheck\SetupCheckManager;
116116
use OC\Share20\ProviderFactory;
117117
use OC\Share20\ShareHelper;
118+
use OC\Snowflake\APCuSequence;
118119
use OC\Snowflake\Decoder;
120+
use OC\Snowflake\FileSequence;
119121
use OC\Snowflake\Generator;
122+
use OC\Snowflake\ISequence;
120123
use OC\SpeechToText\SpeechToTextManager;
121124
use OC\SystemTag\ManagerFactory as SystemTagManagerFactory;
122125
use OC\Talk\Broker;
@@ -1262,6 +1265,16 @@ public function __construct($webRoot, \OC\Config $config) {
12621265
$this->registerAlias(ISignatureManager::class, SignatureManager::class);
12631266

12641267
$this->registerAlias(IGenerator::class, Generator::class);
1268+
$this->registerService(ISequence::class, function (ContainerInterface $c): ISequence {
1269+
if (PHP_SAPI !== 'cli') {
1270+
$sequence = $c->get(APCuSequence::class);
1271+
if ($sequence->isAvailable()) {
1272+
return $sequence;
1273+
}
1274+
}
1275+
1276+
return $c->get(FileSequence::class);
1277+
}, false);
12651278
$this->registerAlias(IDecoder::class, Decoder::class);
12661279

12671280
$this->connectDispatcher();
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-only
8+
*/
9+
10+
namespace OC\Snowflake;
11+
12+
use Override;
13+
14+
class APCuSequence implements ISequence {
15+
#[Override]
16+
public function isAvailable(): bool {
17+
return PHP_SAPI !== 'cli' && function_exists('apcu_enabled') && apcu_enabled();
18+
}
19+
20+
#[Override]
21+
public function nextId(int $serverId, int $seconds, int $milliseconds): int|false {
22+
if ((int)apcu_cache_info(true)['creation_time'] === $seconds) {
23+
// APCu cache was just started
24+
// It means a sequence was maybe deleted
25+
return false;
26+
}
27+
28+
$key = 'seq:' . $seconds . ':' . $milliseconds;
29+
$sequenceId = apcu_inc($key, success: $success, ttl: 1);
30+
if ($success === true) {
31+
return $sequenceId;
32+
}
33+
34+
throw new \Exception('Failed to generate SnowflakeId with APCu');
35+
}
36+
}
Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-only
8+
*/
9+
10+
namespace OC\Snowflake;
11+
12+
use OCP\ITempManager;
13+
use Override;
14+
15+
class FileSequence implements ISequence {
16+
/** Number of files to use */
17+
private const NB_FILES = 20;
18+
/** Lock filename format **/
19+
private const LOCK_FILE_FORMAT = 'seq-%03d.lock';
20+
/** Delete sequences after SEQUENCE_TTL seconds **/
21+
private const SEQUENCE_TTL = 30;
22+
23+
private string $workDir;
24+
25+
public function __construct(
26+
ITempManager $tempManager,
27+
) {
28+
$this->workDir = $tempManager->getTemporaryFolder('.snowflakes');
29+
}
30+
31+
#[Override]
32+
public function isAvailable(): bool {
33+
return true;
34+
}
35+
36+
#[Override]
37+
public function nextId(int $serverId, int $seconds, int $milliseconds): int {
38+
// Open lock file
39+
$filePath = $this->getFilePath($milliseconds % self::NB_FILES);
40+
$fp = fopen($filePath, 'c+');
41+
if ($fp === false) {
42+
throw new \Exception('Unable to open sequence ID file: ' . $filePath);
43+
}
44+
if (!flock($fp, LOCK_EX)) {
45+
throw new \Exception('Unable to acquire lock on sequence ID file: ' . $filePath);
46+
}
47+
48+
// Read content
49+
$content = (string)fgets($fp);
50+
$locks = $content === ''
51+
? []
52+
: json_decode($content, true, 3, JSON_THROW_ON_ERROR);
53+
54+
// Generate new ID
55+
if (isset($locks[$seconds])) {
56+
if (isset($locks[$seconds][$milliseconds])) {
57+
++$locks[$seconds][$milliseconds];
58+
} else {
59+
$locks[$seconds][$milliseconds] = 0;
60+
}
61+
} else {
62+
$locks[$seconds] = [
63+
$milliseconds => 0
64+
];
65+
}
66+
67+
// Clean old sequence IDs
68+
$cleanBefore = $seconds - self::SEQUENCE_TTL;
69+
$locks = array_filter($locks, static function ($key) use ($cleanBefore) {
70+
return $key >= $cleanBefore;
71+
}, ARRAY_FILTER_USE_KEY);
72+
73+
// Write data
74+
ftruncate($fp, 0);
75+
$content = json_encode($locks, JSON_THROW_ON_ERROR);
76+
rewind($fp);
77+
fwrite($fp, $content);
78+
fsync($fp);
79+
80+
// Release lock
81+
fclose($fp);
82+
83+
return $locks[$seconds][$milliseconds];
84+
}
85+
86+
private function getFilePath(int $fileId): string {
87+
return $this->workDir . sprintf(self::LOCK_FILE_FORMAT, $fileId);
88+
}
89+
}

lib/private/Snowflake/Generator.php

Lines changed: 3 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,18 @@
2323
final class Generator implements IGenerator {
2424
public function __construct(
2525
private readonly ITimeFactory $timeFactory,
26+
private readonly ISequence $sequenceGenerator,
2627
) {
2728
}
2829

2930
#[Override]
3031
public function nextId(): string {
31-
// Time related
32+
// Relative time
3233
[$seconds, $milliseconds] = $this->getCurrentTime();
3334

3435
$serverId = $this->getServerId() & 0x1FF; // Keep 9 bits
3536
$isCli = (int)$this->isCli(); // 1 bit
36-
$sequenceId = $this->getSequenceId($seconds, $milliseconds, $serverId); // 12 bits
37+
$sequenceId = $this->sequenceGenerator->nextId($seconds, $milliseconds, $serverId); // 12 bits
3738
if ($sequenceId > 0xFFF || $sequenceId === false) {
3839
// Throttle a bit, wait for next millisecond
3940
usleep(1000);
@@ -106,33 +107,4 @@ private function getServerId(): int {
106107
private function isCli(): bool {
107108
return PHP_SAPI === 'cli';
108109
}
109-
110-
/**
111-
* Generates sequence ID from APCu (general case) or random if APCu disabled or CLI
112-
*
113-
* @return int|false Sequence ID or false if APCu not ready
114-
* @throws \Exception if there is an error with APCu
115-
*/
116-
private function getSequenceId(int $seconds, int $milliseconds, int $serverId): int|false {
117-
$key = 'seq:' . $seconds . ':' . $milliseconds;
118-
119-
// Use APCu as fastest local cache, but not shared between processes in CLI
120-
if (!$this->isCli() && function_exists('apcu_enabled') && apcu_enabled()) {
121-
if ((int)apcu_cache_info(true)['creation_time'] === $seconds) {
122-
// APCu cache was just started
123-
// It means a sequence was maybe deleted
124-
return false;
125-
}
126-
127-
$sequenceId = apcu_inc($key, success: $success, ttl: 1);
128-
if ($success === true) {
129-
return $sequenceId;
130-
}
131-
132-
throw new \Exception('Failed to generate SnowflakeId with APCu');
133-
}
134-
135-
// Otherwise, just return a random number
136-
return random_int(0, 0xFFF - 1);
137-
}
138110
}
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-only
8+
*/
9+
10+
namespace OC\Snowflake;
11+
12+
/**
13+
* Generates sequence IDs
14+
*/
15+
interface ISequence {
16+
/**
17+
* Check if generator is available
18+
*/
19+
public function isAvailable(): bool;
20+
21+
/**
22+
* Returns next sequence ID for current time and server
23+
*/
24+
public function nextId(int $serverId, int $seconds, int $milliseconds): int|false;
25+
}

tests/lib/Snowflake/APCuTest.php

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace Test\Snowflake;
11+
12+
use OC\Snowflake\APCuSequence;
13+
14+
/**
15+
* @package Test
16+
*/
17+
class APCuTest extends ISequenceBase {
18+
private string $path;
19+
20+
public function setUp():void {
21+
$this->sequence = new APCuSequence();
22+
}
23+
}
Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace Test\Snowflake;
11+
12+
use OC\Snowflake\FileSequence;
13+
use OCP\ITempManager;
14+
15+
/**
16+
* @package Test
17+
*/
18+
class FileSequenceTest extends ISequenceBase {
19+
private string $path;
20+
21+
public function setUp():void {
22+
$tempManager = $this->createMock(ITempManager::class);
23+
$this->path = uniqid(sys_get_temp_dir() . '/php_test_seq_', true);
24+
mkdir($this->path);
25+
$tempManager->method('getTemporaryFolder')->willReturn($this->path);
26+
$this->sequence = new FileSequence($tempManager);
27+
}
28+
29+
public function tearDown():void {
30+
foreach (glob($this->path . '/*') as $file) {
31+
unlink($file);
32+
}
33+
rmdir($this->path);
34+
}
35+
}

tests/lib/Snowflake/GeneratorTest.php

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,22 +12,29 @@
1212
use OC\AppFramework\Utility\TimeFactory;
1313
use OC\Snowflake\Decoder;
1414
use OC\Snowflake\Generator;
15+
use OC\Snowflake\ISequence;
1516
use OCP\AppFramework\Utility\ITimeFactory;
1617
use OCP\Snowflake\IGenerator;
1718
use PHPUnit\Framework\Attributes\DataProvider;
19+
use PHPUnit\Framework\MockObject\MockObject;
1820
use Test\TestCase;
1921

2022
/**
2123
* @package Test
2224
*/
2325
class GeneratorTest extends TestCase {
2426
private Decoder $decoder;
27+
private ISequence&MockObject $sequence;
2528

2629
public function setUp():void {
2730
$this->decoder = new Decoder();
31+
$this->sequence = $this->createMock(ISequence::class);
32+
$this->sequence->method('isAvailable')->willReturn(true);
33+
$this->sequence->method('nextId')->willReturn(421);
2834
}
35+
2936
public function testGenerator(): void {
30-
$generator = new Generator(new TimeFactory());
37+
$generator = new Generator(new TimeFactory(), $this->sequence);
3138
$snowflakeId = $generator->nextId();
3239
$data = $this->decoder->decode($generator->nextId());
3340

@@ -53,7 +60,7 @@ public function testGeneratorWithFixedTime(string $date, int $expectedSeconds, i
5360
$timeFactory = $this->createMock(ITimeFactory::class);
5461
$timeFactory->method('now')->willReturn($dt);
5562

56-
$generator = new Generator($timeFactory);
63+
$generator = new Generator($timeFactory, $this->sequence);
5764
$data = $this->decoder->decode($generator->nextId());
5865

5966
$this->assertEquals($expectedSeconds, ($data['createdAt']->format('U') - IGenerator::TS_OFFSET));

0 commit comments

Comments
 (0)