Skip to content

Commit 10f3a3b

Browse files
AltahrimAltahrim
committed
feat(database): introduce Snowflake IDs generator
Signed-off-by: Benjamin Gaussorgues <[email protected]>
1 parent 84be993 commit 10f3a3b

File tree

9 files changed

+405
-13
lines changed

9 files changed

+405
-13
lines changed

‎.github/workflows/phpunit-32bits.yml‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@ name: PHPUnit 32bits
55
on:
66
pull_request:
77
paths:
8-
- 'version.php'
9-
- '.github/workflows/phpunit-32bits.yml'
10-
- 'tests/phpunit-autotest.xml'
8+
- "version.php"
9+
- ".github/workflows/phpunit-32bits.yml"
10+
- "tests/phpunit-autotest.xml"
1111
workflow_dispatch:
1212
schedule:
1313
- cron: "15 1 * * 1-6"
@@ -30,7 +30,7 @@ jobs:
3030
strategy:
3131
fail-fast: false
3232
matrix:
33-
php-versions: ['8.2', '8.3', '8.4']
33+
php-versions: ["8.2", "8.3", "8.4"]
3434

3535
steps:
3636
- name: Checkout server
@@ -51,8 +51,7 @@ jobs:
5151
extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, imagick, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite, apcu, ldap
5252
coverage: none
5353
ini-file: development
54-
ini-values:
55-
apc.enabled=on, apc.enable_cli=on, disable_functions= # https://github.com/shivammathur/setup-php/discussions/573
54+
ini-values: apc.enabled=on, apc.enable_cli=on, disable_functions= # https://github.com/shivammathur/setup-php/discussions/573
5655
env:
5756
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
5857

@@ -68,4 +67,4 @@ jobs:
6867
php -f tests/enable_all.php
6968
7069
- name: PHPUnit
71-
run: composer run test -- --exclude-group PRIMARY-azure,PRIMARY-s3,PRIMARY-swift,Memcached,Redis,RoutingWeirdness
70+
run: composer run test -- --exclude-group PRIMARY-azure,PRIMARY-s3,PRIMARY-swift,Memcached,Redis,RoutingWeirdness tests/lib/SnowflakeIdTest.php

‎lib/composer/composer/autoload_classmap.php‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -628,6 +628,8 @@
628628
'OCP\\IRequestId' => $baseDir . '/lib/public/IRequestId.php',
629629
'OCP\\IServerContainer' => $baseDir . '/lib/public/IServerContainer.php',
630630
'OCP\\ISession' => $baseDir . '/lib/public/ISession.php',
631+
'OCP\\ISnowflakeId' => $baseDir . '/lib/public/ISnowflakeId.php',
632+
'OCP\\ISnowflakeIdGenerator' => $baseDir . '/lib/public/ISnowflakeIdGenerator.php',
631633
'OCP\\IStreamImage' => $baseDir . '/lib/public/IStreamImage.php',
632634
'OCP\\ITagManager' => $baseDir . '/lib/public/ITagManager.php',
633635
'OCP\\ITags' => $baseDir . '/lib/public/ITags.php',
@@ -1871,6 +1873,7 @@
18711873
'OC\\OCS\\CoreCapabilities' => $baseDir . '/lib/private/OCS/CoreCapabilities.php',
18721874
'OC\\OCS\\DiscoveryService' => $baseDir . '/lib/private/OCS/DiscoveryService.php',
18731875
'OC\\OCS\\Provider' => $baseDir . '/lib/private/OCS/Provider.php',
1876+
'OC\\OpenMetrics\\Exporters\\RunningJobs' => $baseDir . '/lib/private/OpenMetrics/Exporters/RunningJobs.php',
18741877
'OC\\PhoneNumberUtil' => $baseDir . '/lib/private/PhoneNumberUtil.php',
18751878
'OC\\PreviewManager' => $baseDir . '/lib/private/PreviewManager.php',
18761879
'OC\\PreviewNotAvailableException' => $baseDir . '/lib/private/PreviewNotAvailableException.php',
@@ -2103,6 +2106,8 @@
21032106
'OC\\Share\\Constants' => $baseDir . '/lib/private/Share/Constants.php',
21042107
'OC\\Share\\Helper' => $baseDir . '/lib/private/Share/Helper.php',
21052108
'OC\\Share\\Share' => $baseDir . '/lib/private/Share/Share.php',
2109+
'OC\\SnowflakeId' => $baseDir . '/lib/private/SnowflakeId.php',
2110+
'OC\\SnowflakeIdGenerator' => $baseDir . '/lib/private/SnowflakeIdGenerator.php',
21062111
'OC\\SpeechToText\\SpeechToTextManager' => $baseDir . '/lib/private/SpeechToText/SpeechToTextManager.php',
21072112
'OC\\SpeechToText\\TranscriptionJob' => $baseDir . '/lib/private/SpeechToText/TranscriptionJob.php',
21082113
'OC\\StreamImage' => $baseDir . '/lib/private/StreamImage.php',

‎lib/composer/composer/autoload_static.php‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,32 +11,32 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
1111
);
1212

1313
public static $prefixLengthsPsr4 = array (
14-
'O' =>
14+
'O' =>
1515
array (
1616
'OC\\Core\\' => 8,
1717
'OC\\' => 3,
1818
'OCP\\' => 4,
1919
),
20-
'N' =>
20+
'N' =>
2121
array (
2222
'NCU\\' => 4,
2323
),
2424
);
2525

2626
public static $prefixDirsPsr4 = array (
27-
'OC\\Core\\' =>
27+
'OC\\Core\\' =>
2828
array (
2929
0 => __DIR__ . '/../../..' . '/core',
3030
),
31-
'OC\\' =>
31+
'OC\\' =>
3232
array (
3333
0 => __DIR__ . '/../../..' . '/lib/private',
3434
),
35-
'OCP\\' =>
35+
'OCP\\' =>
3636
array (
3737
0 => __DIR__ . '/../../..' . '/lib/public',
3838
),
39-
'NCU\\' =>
39+
'NCU\\' =>
4040
array (
4141
0 => __DIR__ . '/../../..' . '/lib/unstable',
4242
),
@@ -669,6 +669,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
669669
'OCP\\IRequestId' => __DIR__ . '/../../..' . '/lib/public/IRequestId.php',
670670
'OCP\\IServerContainer' => __DIR__ . '/../../..' . '/lib/public/IServerContainer.php',
671671
'OCP\\ISession' => __DIR__ . '/../../..' . '/lib/public/ISession.php',
672+
'OCP\\ISnowflakeId' => __DIR__ . '/../../..' . '/lib/public/ISnowflakeId.php',
673+
'OCP\\ISnowflakeIdGenerator' => __DIR__ . '/../../..' . '/lib/public/ISnowflakeIdGenerator.php',
672674
'OCP\\IStreamImage' => __DIR__ . '/../../..' . '/lib/public/IStreamImage.php',
673675
'OCP\\ITagManager' => __DIR__ . '/../../..' . '/lib/public/ITagManager.php',
674676
'OCP\\ITags' => __DIR__ . '/../../..' . '/lib/public/ITags.php',
@@ -1912,6 +1914,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
19121914
'OC\\OCS\\CoreCapabilities' => __DIR__ . '/../../..' . '/lib/private/OCS/CoreCapabilities.php',
19131915
'OC\\OCS\\DiscoveryService' => __DIR__ . '/../../..' . '/lib/private/OCS/DiscoveryService.php',
19141916
'OC\\OCS\\Provider' => __DIR__ . '/../../..' . '/lib/private/OCS/Provider.php',
1917+
'OC\\OpenMetrics\\Exporters\\RunningJobs' => __DIR__ . '/../../..' . '/lib/private/OpenMetrics/Exporters/RunningJobs.php',
19151918
'OC\\PhoneNumberUtil' => __DIR__ . '/../../..' . '/lib/private/PhoneNumberUtil.php',
19161919
'OC\\PreviewManager' => __DIR__ . '/../../..' . '/lib/private/PreviewManager.php',
19171920
'OC\\PreviewNotAvailableException' => __DIR__ . '/../../..' . '/lib/private/PreviewNotAvailableException.php',
@@ -2144,6 +2147,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
21442147
'OC\\Share\\Constants' => __DIR__ . '/../../..' . '/lib/private/Share/Constants.php',
21452148
'OC\\Share\\Helper' => __DIR__ . '/../../..' . '/lib/private/Share/Helper.php',
21462149
'OC\\Share\\Share' => __DIR__ . '/../../..' . '/lib/private/Share/Share.php',
2150+
'OC\\SnowflakeId' => __DIR__ . '/../../..' . '/lib/private/SnowflakeId.php',
2151+
'OC\\SnowflakeIdGenerator' => __DIR__ . '/../../..' . '/lib/private/SnowflakeIdGenerator.php',
21472152
'OC\\SpeechToText\\SpeechToTextManager' => __DIR__ . '/../../..' . '/lib/private/SpeechToText/SpeechToTextManager.php',
21482153
'OC\\SpeechToText\\TranscriptionJob' => __DIR__ . '/../../..' . '/lib/private/SpeechToText/TranscriptionJob.php',
21492154
'OC\\StreamImage' => __DIR__ . '/../../..' . '/lib/private/StreamImage.php',

‎lib/private/SnowflakeId.php‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-only
8+
*/
9+
10+
namespace OC;
11+
12+
use OCP\ISnowflakeId;
13+
use Override;
14+
15+
/**
16+
* Nextcloud Snowflake ID
17+
*
18+
* Get information about Snowflake Id
19+
*
20+
* @since 33.0.0
21+
*/
22+
final class SnowflakeId implements ISnowflakeId {
23+
private int $seconds = 0;
24+
private int $milliseconds = 0;
25+
private bool $isCli = false;
26+
/** @var int<0, 511> */
27+
private int $serverId = 0;
28+
/** @var int<0, 4095> */
29+
private int $sequenceId = 0;
30+
31+
public function __construct(
32+
private readonly int|float $id,
33+
) {
34+
}
35+
36+
private function decode(): void {
37+
if ($this->seconds !== 0) {
38+
return;
39+
}
40+
41+
PHP_INT_SIZE === 9 // FIXME 8
42+
? $this->decode64bits()
43+
: $this->decode32bits();
44+
}
45+
46+
private function decode64bits(): void {
47+
$id = (int)$this->id;
48+
$firstHalf = $id >> 32;
49+
$secondHalf = $id & 0xFFFFFFFF;
50+
51+
// First half without first bit is seconds
52+
$this->seconds = $firstHalf & 0x7FFFFFFF;
53+
54+
// Decode second half
55+
$this->milliseconds = $secondHalf >> 22;
56+
$this->serverId = ($secondHalf >> 13) & 0x1FF;
57+
$this->isCli = (bool)(($secondHalf >> 12) & 0x1);
58+
$this->sequenceId = $secondHalf & 0xFFF;
59+
}
60+
61+
private function decode32bits(): void {
62+
printf("ID : %s\n", $this->id);
63+
printf("ID string: %s\n", (string)$this->id);
64+
printf("ID format: %s\n", number_format($this->id, 0, '', ''));
65+
printf("ID format: %s\n", number_format($this->id, 5, '.', ''));
66+
printf("ID base2 : %s\n", base_convert((string)$this->id, 10, 2));
67+
printf("ID base16: %s\n", base_convert((string)$this->id, 10, 16));
68+
69+
$id = str_pad(base_convert((string)$this->id, 10, 2), 64, '0', STR_PAD_LEFT);
70+
$firstQuarter = bindec(substr($id, 0, 16));
71+
$secondQuarter = bindec(substr($id, 16, 16));
72+
$thirdQuarter = bindec(substr($id, 32, 16));
73+
$fourthQuarter = bindec(substr($id, 48, 16));
74+
75+
echo PHP_EOL;
76+
printf("Debug : %04x %04x %04x %04x\n",
77+
$firstQuarter,
78+
$secondQuarter,
79+
$thirdQuarter,
80+
$fourthQuarter,
81+
);
82+
83+
$this->seconds = (($firstQuarter & 0x7FFF) << 16) | ($secondQuarter & 0xFFFF);
84+
85+
$this->milliseconds = ($thirdQuarter >> 6) & 0x3FF;
86+
87+
$this->serverId = (($thirdQuarter & 0x3F) << 3) | (($fourthQuarter >> 13) & 0x7);
88+
$this->isCli = (bool)(($fourthQuarter >> 12) & 0x1);
89+
$this->sequenceId = $fourthQuarter & 0xFFF;
90+
}
91+
92+
#[Override]
93+
public function isCli(): bool {
94+
return $this->isCli;
95+
}
96+
97+
#[Override]
98+
public function numeric(): int|float {
99+
return $this->id;
100+
}
101+
102+
#[Override]
103+
public function seconds(): int {
104+
$this->decode();
105+
return $this->seconds;
106+
}
107+
108+
#[Override]
109+
public function milliseconds(): int {
110+
$this->decode();
111+
return $this->milliseconds;
112+
}
113+
114+
#[Override]
115+
public function createdAt(): float {
116+
$this->decode();
117+
return $this->seconds + self::TS_OFFSET + ($this->milliseconds / 1000);
118+
}
119+
120+
#[Override]
121+
public function serverId(): int {
122+
$this->decode();
123+
return $this->serverId;
124+
}
125+
126+
#[Override]
127+
public function sequenceId(): int {
128+
$this->decode();
129+
return $this->sequenceId;
130+
}
131+
}

‎lib/private/SnowflakeIdGenerator.php‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-only
8+
*/
9+
10+
namespace OC;
11+
12+
/**
13+
* Nextcloud Snowflake ID generator
14+
*
15+
* Generates unique ID for database
16+
*
17+
* @since 33.0.0
18+
*/
19+
final class SnowflakeIdGenerator {
20+
public function __invoke(): int|float {
21+
// Time related
22+
$microtime = microtime(true);
23+
$seconds = ((int)$microtime) - SnowflakeId::TS_OFFSET;
24+
$milliseconds = ((int)($microtime * 1000)) % 1000;
25+
26+
$serverId = $this->getServerId() & 0x1FF; // Keep 9 bits
27+
$isCli = (int)$this->isCli(); // 1 bit
28+
$sequenceId = $this->getSequenceId($seconds, $milliseconds); // 12 bits
29+
if ($sequenceId > 0xFFF) {
30+
// Throttle a bit, wait for next millisecond
31+
usleep(1000);
32+
return $this();
33+
}
34+
35+
$firstHalf = $seconds & 0x7FFFFFFF;
36+
$secondHalf = (($milliseconds & 0x3FF) << 22) | ($serverId << 13) | ($isCli << 12) | $sequenceId;
37+
if (PHP_INT_SIZE === 8) {
38+
return $firstHalf << 32 | $secondHalf;
39+
}
40+
41+
// Fallback for 32 bits systems
42+
return hexdec(bin2hex(pack('LL', $firstHalf, $secondHalf)));
43+
}
44+
45+
private function getServerId(): int {
46+
return crc32(gethostname() ?: random_bytes(8));
47+
}
48+
49+
private function isCli() {
50+
return PHP_SAPI === 'cli';
51+
}
52+
53+
private function getSequenceId(int $seconds, int $milliseconds): int {
54+
if ($this->isCli()) {
55+
// APCu cache isn’t shared between CLI processes
56+
return random_int(0, 0xFFF - 1);
57+
}
58+
59+
if (function_exists('apcu_inc')) {
60+
$key = 'sequence:' . $seconds . ':' . $milliseconds;
61+
$sequenceId = apcu_inc($key, ttl: 1);
62+
if ($sequenceId === false) {
63+
throw new \Exception('Failed to generate SnowflakeId with APCu');
64+
}
65+
66+
return $sequenceId;
67+
}
68+
69+
// TODO Implement file fallback?
70+
throw new \Exception('Failed to get sequence Id');
71+
}
72+
}

0 commit comments

Comments
 (0)