Skip to content

Commit f898ffe

Browse files
AltahrimAltahrim
committed
feat(snowflakeids): add File Sequence Generator
Signed-off-by: Benjamin Gaussorgues <benjamin.gaussorgues@nextcloud.com>
1 parent 700f4db commit f898ffe

File tree

9 files changed

+269
-32
lines changed

9 files changed

+269
-32
lines changed

‎lib/private/Server.php‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,11 @@
114114
use OC\SetupCheck\SetupCheckManager;
115115
use OC\Share20\ProviderFactory;
116116
use OC\Share20\ShareHelper;
117+
use OC\Snowflake\APCuSequence;
117118
use OC\Snowflake\Decoder;
119+
use OC\Snowflake\FileSequence;
118120
use OC\Snowflake\Generator;
121+
use OC\Snowflake\ISequence;
119122
use OC\SpeechToText\SpeechToTextManager;
120123
use OC\SystemTag\ManagerFactory as SystemTagManagerFactory;
121124
use OC\Talk\Broker;
@@ -1250,6 +1253,16 @@ public function __construct($webRoot, \OC\Config $config) {
12501253
$this->registerAlias(ISignatureManager::class, SignatureManager::class);
12511254

12521255
$this->registerAlias(IGenerator::class, Generator::class);
1256+
$this->registerService(ISequence::class, function (ContainerInterface $c): ISequence {
1257+
if (PHP_SAPI !== 'cli') {
1258+
$sequence = $c->get(APCuSequence::class);
1259+
if ($sequence->isAvailable()) {
1260+
return $sequence;
1261+
}
1262+
}
1263+
1264+
return $c->get(FileSequence::class);
1265+
}, false);
12531266
$this->registerAlias(IDecoder::class, Decoder::class);
12541267

12551268
$this->connectDispatcher();

‎lib/private/Snowflake/APCuSequence.php‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-only
8+
*/
9+
10+
namespace OC\Snowflake;
11+
12+
use Override;
13+
14+
class APCuSequence implements ISequence {
15+
#[Override]
16+
public function isAvailable(): bool {
17+
return PHP_SAPI !== 'cli' && function_exists('apcu_enabled') && apcu_enabled();
18+
}
19+
20+
#[Override]
21+
public function nextId(int $serverId, int $seconds, int $milliseconds): int|false {
22+
if ((int)apcu_cache_info(true)['creation_time'] === $seconds) {
23+
// APCu cache was just started
24+
// It means a sequence was maybe deleted
25+
return false;
26+
}
27+
28+
$key = 'seq:' . $seconds . ':' . $milliseconds;
29+
$sequenceId = apcu_inc($key, success: $success, ttl: 1);
30+
if ($success === true) {
31+
return $sequenceId;
32+
}
33+
34+
throw new \Exception('Failed to generate SnowflakeId with APCu');
35+
}
36+
}

‎lib/private/Snowflake/FileSequence.php‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-only
8+
*/
9+
10+
namespace OC\Snowflake;
11+
12+
use OCP\ITempManager;
13+
14+
class FileSequence implements ISequence {
15+
/** Number of files to use */
16+
private const NB_FILES = 20;
17+
/** Lock filename format **/
18+
private const LOCK_FILE_FORMAT = 'seq-%03d.lock';
19+
/** Delete sequences after SEQUENCE_TTL seconds **/
20+
private const SEQUENCE_TTL = 30;
21+
22+
public function __construct(
23+
private readonly ITempManager $tempManager,
24+
) {
25+
}
26+
27+
public function isAvailable(): bool {
28+
return true;
29+
}
30+
31+
public function nextId(int $serverId, int $seconds, int $milliseconds): int {
32+
// Open lock file
33+
$filePath = $this->getFilePath($milliseconds % self::NB_FILES);
34+
$fp = fopen($filePath, 'cb+');
35+
if (!flock($fp, LOCK_EX)) {
36+
throw new \Exception('Unable to acquire lock on sequence id file: ' . $filePath);
37+
}
38+
39+
// Read content
40+
$content = (string)fgets($fp);
41+
$locks = $content === ''
42+
? []
43+
: json_decode($content, true, 3, JSON_THROW_ON_ERROR);
44+
45+
// Generate new ID
46+
$paddedMs = str_pad((string)$milliseconds, 3, '0');
47+
if (isset($locks[$seconds])) {
48+
if (isset($locks[$seconds][$paddedMs])) {
49+
++$locks[$seconds][$paddedMs];
50+
} else {
51+
$locks[$seconds][$paddedMs] = 0;
52+
}
53+
} else {
54+
$locks[$seconds] = [
55+
$paddedMs => 0
56+
];
57+
}
58+
59+
// Clean old sequence IDs
60+
$cleanBefore = $seconds - self::SEQUENCE_TTL;
61+
$locks = array_filter($locks, static function ($key) use ($cleanBefore) {
62+
return $key >= $cleanBefore;
63+
}, ARRAY_FILTER_USE_KEY);
64+
65+
// Write data
66+
ftruncate($fp, 0);
67+
$content = json_encode($locks, JSON_THROW_ON_ERROR);
68+
rewind($fp);
69+
fwrite($fp, $content);
70+
fsync($fp);
71+
72+
// Release lock
73+
fclose($fp);
74+
75+
return $locks[$seconds][$paddedMs];
76+
}
77+
78+
private function getFilePath(int $fileId): string {
79+
return $this->tempManager->getTemporaryFolder('.snowflakes') . sprintf(self::LOCK_FILE_FORMAT, $fileId);
80+
}
81+
}

‎lib/private/Snowflake/Generator.php‎

Lines changed: 2 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
final class Generator implements IGenerator {
2424
public function __construct(
2525
private readonly ITimeFactory $timeFactory,
26+
private readonly ISequence $sequenceGenerator,
2627
) {
2728
}
2829

@@ -33,7 +34,7 @@ public function nextId(): string {
3334

3435
$serverId = $this->getServerId() & 0x1FF; // Keep 9 bits
3536
$isCli = (int)$this->isCli(); // 1 bit
36-
$sequenceId = $this->getSequenceId($seconds, $milliseconds, $serverId); // 12 bits
37+
$sequenceId = $this->sequenceGenerator->nextId($seconds, $milliseconds, $serverId); // 12 bits
3738
if ($sequenceId > 0xFFF || $sequenceId === false) {
3839
// Throttle a bit, wait for next millisecond
3940
usleep(1000);
@@ -106,33 +107,4 @@ private function getServerId(): int {
106107
private function isCli(): bool {
107108
return PHP_SAPI === 'cli';
108109
}
109-
110-
/**
111-
* Generates sequence ID from APCu (general case) or random if APCu disabled or CLI
112-
*
113-
* @return int|false Sequence ID or false if APCu not ready
114-
* @throws \Exception if there is an error with APCu
115-
*/
116-
private function getSequenceId(int $seconds, int $milliseconds, int $serverId): int|false {
117-
$key = 'seq:' . $seconds . ':' . $milliseconds;
118-
119-
// Use APCu as fastest local cache, but not shared between processes in CLI
120-
if (!$this->isCli() && function_exists('apcu_enabled') && apcu_enabled()) {
121-
if ((int)apcu_cache_info(true)['creation_time'] === $seconds) {
122-
// APCu cache was just started
123-
// It means a sequence was maybe deleted
124-
return false;
125-
}
126-
127-
$sequenceId = apcu_inc($key, success: $success, ttl: 1);
128-
if ($success === true) {
129-
return $sequenceId;
130-
}
131-
132-
throw new \Exception('Failed to generate SnowflakeId with APCu');
133-
}
134-
135-
// Otherwise, just return a random number
136-
return random_int(0, 0xFFF - 1);
137-
}
138110
}

‎lib/private/Snowflake/ISequence.php‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-only
8+
*/
9+
10+
namespace OC\Snowflake;
11+
12+
/**
13+
* Generates sequence IDs
14+
*/
15+
interface ISequence {
16+
/**
17+
* Check if generator is available
18+
*/
19+
public function isAvailable(): bool;
20+
21+
/**
22+
* Returns next sequence ID for current time and server
23+
*/
24+
public function nextId(int $serverId, int $seconds, int $milliseconds): int|false;
25+
}

‎tests/lib/Snowflake/APCuTest.php‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace Test\Snowflake;
11+
12+
use OC\Snowflake\APCuSequence;
13+
14+
/**
15+
* @package Test
16+
*/
17+
class APCuTest extends ISequenceBase {
18+
private string $path;
19+
20+
public function setUp():void {
21+
$this->sequence = new APCuSequence();
22+
}
23+
}

‎tests/lib/Snowflake/FileSequenceTest.php‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace Test\Snowflake;
11+
12+
use OC\Snowflake\FileSequence;
13+
use OCP\ITempManager;
14+
15+
/**
16+
* @package Test
17+
*/
18+
class FileSequenceTest extends ISequenceBase {
19+
private string $path;
20+
21+
public function setUp():void {
22+
$tempManager = $this->createMock(ITempManager::class);
23+
$this->path = uniqid(sys_get_temp_dir() . '/php_test_seq_', true);
24+
mkdir($this->path);
25+
$tempManager->method('getTemporaryFolder')->willReturn($this->path);
26+
$this->sequence = new FileSequence($tempManager);
27+
}
28+
29+
public function tearDown():void {
30+
foreach (glob($this->path . '/*') as $file) {
31+
unlink($file);
32+
}
33+
rmdir($this->path);
34+
}
35+
}

‎tests/lib/Snowflake/GeneratorTest.php‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,22 +12,29 @@
1212
use OC\AppFramework\Utility\TimeFactory;
1313
use OC\Snowflake\Decoder;
1414
use OC\Snowflake\Generator;
15+
use OC\Snowflake\ISequence;
1516
use OCP\AppFramework\Utility\ITimeFactory;
1617
use OCP\Snowflake\IGenerator;
1718
use PHPUnit\Framework\Attributes\DataProvider;
19+
use PHPUnit\Framework\MockObject\MockObject;
1820
use Test\TestCase;
1921

2022
/**
2123
* @package Test
2224
*/
2325
class GeneratorTest extends TestCase {
2426
private Decoder $decoder;
27+
private ISequence&MockObject $sequence;
2528

2629
public function setUp():void {
2730
$this->decoder = new Decoder();
31+
$this->sequence = $this->createMock(ISequence::class);
32+
$this->sequence->method('isAvailable')->willReturn(true);
33+
$this->sequence->method('nextId')->willReturn(421);
2834
}
35+
2936
public function testGenerator(): void {
30-
$generator = new Generator(new TimeFactory());
37+
$generator = new Generator(new TimeFactory(), $this->sequence);
3138
$snowflakeId = $generator->nextId();
3239
$data = $this->decoder->decode($generator->nextId());
3340

@@ -53,7 +60,7 @@ public function testGeneratorWithFixedTime(string $date, int $expectedSeconds, i
5360
$timeFactory = $this->createMock(ITimeFactory::class);
5461
$timeFactory->method('now')->willReturn($dt);
5562

56-
$generator = new Generator($timeFactory);
63+
$generator = new Generator($timeFactory, $this->sequence);
5764
$data = $this->decoder->decode($generator->nextId());
5865

5966
$this->assertEquals($expectedSeconds, ($data['createdAt']->format('U') - IGenerator::TS_OFFSET));

‎tests/lib/Snowflake/ISequenceBase.php‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace Test\Snowflake;
11+
12+
use OC\Snowflake\ISequence;
13+
use Test\TestCase;
14+
15+
/**
16+
* @package Test
17+
*/
18+
abstract class ISequenceBase extends TestCase {
19+
protected ISequence $sequence;
20+
21+
public function testGenerator(): void {
22+
if (!$this->sequence->isAvailable()) {
23+
$this->markTestSkipped('Sequence ID generator ' . get_class($this->sequence) . 'is’nt available. Skip');
24+
}
25+
26+
$nb = 50;
27+
$ids = [];
28+
$server = 42;
29+
for ($i = 0; $i < $nb; ++$i) {
30+
$time = explode('.', (string)microtime(true));
31+
$seconds = (int)$time[0];
32+
$milliseconds = (int)substr($time[1] ?? '0', 0, 3);
33+
$id = $this->sequence->nextId($server, $seconds, $milliseconds);
34+
$ids[] = sprintf('%d_%03d_%d', $seconds, $milliseconds, $id);
35+
usleep(100);
36+
}
37+
38+
// Is it unique?
39+
$this->assertCount($nb, array_unique($ids));
40+
// Is it sequential?
41+
$sortedIds = $ids;
42+
sort($sortedIds);
43+
$this->assertSame($sortedIds, $ids);
44+
}
45+
}

0 commit comments

Comments
 (0)