Skip to content

Commit 827a5fd

Browse files
AltahrimAltahrim
committed
feat(snowflakeids): add File Sequence Generator
Signed-off-by: Benjamin Gaussorgues <[email protected]>
1 parent 700f4db commit 827a5fd

File tree

9 files changed

+274
-32
lines changed

9 files changed

+274
-32
lines changed

‎lib/private/Server.php‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,11 @@
114114
use OC\SetupCheck\SetupCheckManager;
115115
use OC\Share20\ProviderFactory;
116116
use OC\Share20\ShareHelper;
117+
use OC\Snowflake\APCuSequence;
117118
use OC\Snowflake\Decoder;
119+
use OC\Snowflake\FileSequence;
118120
use OC\Snowflake\Generator;
121+
use OC\Snowflake\ISequence;
119122
use OC\SpeechToText\SpeechToTextManager;
120123
use OC\SystemTag\ManagerFactory as SystemTagManagerFactory;
121124
use OC\Talk\Broker;
@@ -1250,6 +1253,16 @@ public function __construct($webRoot, \OC\Config $config) {
12501253
$this->registerAlias(ISignatureManager::class, SignatureManager::class);
12511254

12521255
$this->registerAlias(IGenerator::class, Generator::class);
1256+
$this->registerService(ISequence::class, function (ContainerInterface $c): ISequence {
1257+
if (PHP_SAPI !== 'cli') {
1258+
$sequence = $c->get(APCuSequence::class);
1259+
if ($sequence->isAvailable()) {
1260+
return $sequence;
1261+
}
1262+
}
1263+
1264+
return $c->get(FileSequence::class);
1265+
}, false);
12531266
$this->registerAlias(IDecoder::class, Decoder::class);
12541267

12551268
$this->connectDispatcher();

‎lib/private/Snowflake/APCuSequence.php‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-only
8+
*/
9+
10+
namespace OC\Snowflake;
11+
12+
use Override;
13+
14+
class APCuSequence implements ISequence {
15+
#[Override]
16+
public function isAvailable(): bool {
17+
return PHP_SAPI !== 'cli' && function_exists('apcu_enabled') && apcu_enabled();
18+
}
19+
20+
#[Override]
21+
public function nextId(int $serverId, int $seconds, int $milliseconds): int|false {
22+
if ((int)apcu_cache_info(true)['creation_time'] === $seconds) {
23+
// APCu cache was just started
24+
// It means a sequence was maybe deleted
25+
return false;
26+
}
27+
28+
$key = 'seq:' . $seconds . ':' . $milliseconds;
29+
$sequenceId = apcu_inc($key, success: $success, ttl: 1);
30+
if ($success === true) {
31+
return $sequenceId;
32+
}
33+
34+
throw new \Exception('Failed to generate SnowflakeId with APCu');
35+
}
36+
}

‎lib/private/Snowflake/FileSequence.php‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-only
8+
*/
9+
10+
namespace OC\Snowflake;
11+
12+
use OCP\ITempManager;
13+
use Override;
14+
15+
class FileSequence implements ISequence {
16+
/** Number of files to use */
17+
private const NB_FILES = 20;
18+
/** Lock filename format **/
19+
private const LOCK_FILE_FORMAT = 'seq-%03d.lock';
20+
/** Delete sequences after SEQUENCE_TTL seconds **/
21+
private const SEQUENCE_TTL = 30;
22+
23+
public function __construct(
24+
private readonly ITempManager $tempManager,
25+
) {
26+
}
27+
28+
#[Override]
29+
public function isAvailable(): bool {
30+
return true;
31+
}
32+
33+
#[Override]
34+
public function nextId(int $serverId, int $seconds, int $milliseconds): int {
35+
// Open lock file
36+
$filePath = $this->getFilePath($milliseconds % self::NB_FILES);
37+
$fp = fopen($filePath, 'c+');
38+
if ($fp === false) {
39+
throw new \Exception('Unable to open sequence ID file: ' . $filePath);
40+
}
41+
if (!flock($fp, LOCK_EX)) {
42+
throw new \Exception('Unable to acquire lock on sequence ID file: ' . $filePath);
43+
}
44+
45+
// Read content
46+
$content = (string)fgets($fp);
47+
$locks = $content === ''
48+
? []
49+
: json_decode($content, true, 3, JSON_THROW_ON_ERROR);
50+
51+
// Generate new ID
52+
if (isset($locks[$seconds])) {
53+
if (isset($locks[$seconds][$milliseconds])) {
54+
++$locks[$seconds][$milliseconds];
55+
} else {
56+
$locks[$seconds][$milliseconds] = 0;
57+
}
58+
} else {
59+
$locks[$seconds] = [
60+
$milliseconds => 0
61+
];
62+
}
63+
64+
// Clean old sequence IDs
65+
$cleanBefore = $seconds - self::SEQUENCE_TTL;
66+
$locks = array_filter($locks, static function ($key) use ($cleanBefore) {
67+
return $key >= $cleanBefore;
68+
}, ARRAY_FILTER_USE_KEY);
69+
70+
// Write data
71+
ftruncate($fp, 0);
72+
$content = json_encode($locks, JSON_THROW_ON_ERROR);
73+
rewind($fp);
74+
fwrite($fp, $content);
75+
fsync($fp);
76+
77+
// Release lock
78+
fclose($fp);
79+
80+
return $locks[$seconds][$milliseconds];
81+
}
82+
83+
private function getFilePath(int $fileId): string {
84+
return $this->tempManager->getTemporaryFolder('.snowflakes') . sprintf(self::LOCK_FILE_FORMAT, $fileId);
85+
}
86+
}

‎lib/private/Snowflake/Generator.php‎

Lines changed: 2 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
final class Generator implements IGenerator {
2424
public function __construct(
2525
private readonly ITimeFactory $timeFactory,
26+
private readonly ISequence $sequenceGenerator,
2627
) {
2728
}
2829

@@ -33,7 +34,7 @@ public function nextId(): string {
3334

3435
$serverId = $this->getServerId() & 0x1FF; // Keep 9 bits
3536
$isCli = (int)$this->isCli(); // 1 bit
36-
$sequenceId = $this->getSequenceId($seconds, $milliseconds, $serverId); // 12 bits
37+
$sequenceId = $this->sequenceGenerator->nextId($seconds, $milliseconds, $serverId); // 12 bits
3738
if ($sequenceId > 0xFFF || $sequenceId === false) {
3839
// Throttle a bit, wait for next millisecond
3940
usleep(1000);
@@ -106,33 +107,4 @@ private function getServerId(): int {
106107
private function isCli(): bool {
107108
return PHP_SAPI === 'cli';
108109
}
109-
110-
/**
111-
* Generates sequence ID from APCu (general case) or random if APCu disabled or CLI
112-
*
113-
* @return int|false Sequence ID or false if APCu not ready
114-
* @throws \Exception if there is an error with APCu
115-
*/
116-
private function getSequenceId(int $seconds, int $milliseconds, int $serverId): int|false {
117-
$key = 'seq:' . $seconds . ':' . $milliseconds;
118-
119-
// Use APCu as fastest local cache, but not shared between processes in CLI
120-
if (!$this->isCli() && function_exists('apcu_enabled') && apcu_enabled()) {
121-
if ((int)apcu_cache_info(true)['creation_time'] === $seconds) {
122-
// APCu cache was just started
123-
// It means a sequence was maybe deleted
124-
return false;
125-
}
126-
127-
$sequenceId = apcu_inc($key, success: $success, ttl: 1);
128-
if ($success === true) {
129-
return $sequenceId;
130-
}
131-
132-
throw new \Exception('Failed to generate SnowflakeId with APCu');
133-
}
134-
135-
// Otherwise, just return a random number
136-
return random_int(0, 0xFFF - 1);
137-
}
138110
}

‎lib/private/Snowflake/ISequence.php‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-only
8+
*/
9+
10+
namespace OC\Snowflake;
11+
12+
/**
13+
* Generates sequence IDs
14+
*/
15+
interface ISequence {
16+
/**
17+
* Check if generator is available
18+
*/
19+
public function isAvailable(): bool;
20+
21+
/**
22+
* Returns next sequence ID for current time and server
23+
*/
24+
public function nextId(int $serverId, int $seconds, int $milliseconds): int|false;
25+
}

‎tests/lib/Snowflake/APCuTest.php‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace Test\Snowflake;
11+
12+
use OC\Snowflake\APCuSequence;
13+
14+
/**
15+
* @package Test
16+
*/
17+
class APCuTest extends ISequenceBase {
18+
private string $path;
19+
20+
public function setUp():void {
21+
$this->sequence = new APCuSequence();
22+
}
23+
}

‎tests/lib/Snowflake/FileSequenceTest.php‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace Test\Snowflake;
11+
12+
use OC\Snowflake\FileSequence;
13+
use OCP\ITempManager;
14+
15+
/**
16+
* @package Test
17+
*/
18+
class FileSequenceTest extends ISequenceBase {
19+
private string $path;
20+
21+
public function setUp():void {
22+
$tempManager = $this->createMock(ITempManager::class);
23+
$this->path = uniqid(sys_get_temp_dir() . '/php_test_seq_', true);
24+
mkdir($this->path);
25+
$tempManager->method('getTemporaryFolder')->willReturn($this->path);
26+
$this->sequence = new FileSequence($tempManager);
27+
}
28+
29+
public function tearDown():void {
30+
foreach (glob($this->path . '/*') as $file) {
31+
unlink($file);
32+
}
33+
rmdir($this->path);
34+
}
35+
}

‎tests/lib/Snowflake/GeneratorTest.php‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,22 +12,29 @@
1212
use OC\AppFramework\Utility\TimeFactory;
1313
use OC\Snowflake\Decoder;
1414
use OC\Snowflake\Generator;
15+
use OC\Snowflake\ISequence;
1516
use OCP\AppFramework\Utility\ITimeFactory;
1617
use OCP\Snowflake\IGenerator;
1718
use PHPUnit\Framework\Attributes\DataProvider;
19+
use PHPUnit\Framework\MockObject\MockObject;
1820
use Test\TestCase;
1921

2022
/**
2123
* @package Test
2224
*/
2325
class GeneratorTest extends TestCase {
2426
private Decoder $decoder;
27+
private ISequence&MockObject $sequence;
2528

2629
public function setUp():void {
2730
$this->decoder = new Decoder();
31+
$this->sequence = $this->createMock(ISequence::class);
32+
$this->sequence->method('isAvailable')->willReturn(true);
33+
$this->sequence->method('nextId')->willReturn(421);
2834
}
35+
2936
public function testGenerator(): void {
30-
$generator = new Generator(new TimeFactory());
37+
$generator = new Generator(new TimeFactory(), $this->sequence);
3138
$snowflakeId = $generator->nextId();
3239
$data = $this->decoder->decode($generator->nextId());
3340

@@ -53,7 +60,7 @@ public function testGeneratorWithFixedTime(string $date, int $expectedSeconds, i
5360
$timeFactory = $this->createMock(ITimeFactory::class);
5461
$timeFactory->method('now')->willReturn($dt);
5562

56-
$generator = new Generator($timeFactory);
63+
$generator = new Generator($timeFactory, $this->sequence);
5764
$data = $this->decoder->decode($generator->nextId());
5865

5966
$this->assertEquals($expectedSeconds, ($data['createdAt']->format('U') - IGenerator::TS_OFFSET));

‎tests/lib/Snowflake/ISequenceBase.php‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace Test\Snowflake;
11+
12+
use OC\Snowflake\ISequence;
13+
use Test\TestCase;
14+
15+
/**
16+
* @package Test
17+
*/
18+
abstract class ISequenceBase extends TestCase {
19+
protected ISequence $sequence;
20+
21+
public function testGenerator(): void {
22+
if (!$this->sequence->isAvailable()) {
23+
$this->markTestSkipped('Sequence ID generator ' . get_class($this->sequence) . 'is’nt available. Skip');
24+
}
25+
26+
$nb = 1000;
27+
$ids = [];
28+
$server = 42;
29+
for ($i = 0; $i < $nb; ++$i) {
30+
$time = explode('.', (string)microtime(true));
31+
$seconds = (int)$time[0];
32+
$milliseconds = (int)substr($time[1] ?? '0', 0, 3);
33+
$id = $this->sequence->nextId($server, $seconds, $milliseconds);
34+
$ids[] = sprintf('%d_%-03d_%d', $seconds, $milliseconds, $id);
35+
usleep(100);
36+
}
37+
38+
// Is it unique?
39+
$this->assertCount($nb, array_unique($ids));
40+
// Is it sequential?
41+
$sortedIds = $ids;
42+
sort($sortedIds);
43+
$this->assertSame($sortedIds, $ids);
44+
}
45+
}

0 commit comments

Comments
 (0)