Skip to content

Commit 5b83603

Browse files
prefiksprefiks
committed
New database export/import infrastructure
1 parent 03e535e commit 5b83603

File tree

6 files changed

+633
-32
lines changed

6 files changed

+633
-32
lines changed

‎include/ejabberd_db_serialize.hrl‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
%%%----------------------------------------------------------------------
2+
%%%
3+
%%% ejabberd, Copyright (C) 2002-2025 ProcessOne
4+
%%%
5+
%%% This program is free software; you can redistribute it and/or
6+
%%% modify it under the terms of the GNU General Public License as
7+
%%% published by the Free Software Foundation; either version 2 of the
8+
%%% License, or (at your option) any later version.
9+
%%%
10+
%%% This program is distributed in the hope that it will be useful,
11+
%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13+
%%% General Public License for more details.
14+
%%%
15+
%%% You should have received a copy of the GNU General Public License along
16+
%%% with this program; if not, write to the Free Software Foundation, Inc.,
17+
%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
18+
%%%
19+
%%%----------------------------------------------------------------------
20+
21+
-record(serialize_mam_v1, {
22+
serverhost :: binary(),
23+
username :: binary(),
24+
timestamp :: integer(),
25+
peer :: binary(),
26+
type :: chat | groupchat,
27+
nick :: binary(),
28+
origin_id :: binary(),
29+
packet :: binary()
30+
}).
31+
-record(serialize_mam_prefs_v1, {
32+
serverhost :: binary(),
33+
username :: binary(),
34+
default :: atom(),
35+
always :: term(),
36+
never :: term()
37+
}).
38+
39+
-record(serialize_roster_v1, {
40+
serverhost :: binary(),
41+
username :: binary(),
42+
version :: binary() | undefined,
43+
entries :: [{binary(),
44+
binary(),
45+
[binary()],
46+
both | from | to | none,
47+
subscribe | unsubscribe | both | in | out | none,
48+
binary()}]
49+
}).
50+
51+
-record(serialize_auth_v1, {
52+
serverhost :: binary(),
53+
username :: binary(),
54+
passwords :: [binary() | {atom(), binary(), binary(), binary(), integer()}]
55+
}).

‎src/ejabberd_auth_sql.erl‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,17 +29,20 @@
2929
-author('[email protected]').
3030

3131
-behaviour(ejabberd_auth).
32+
-behaviour(ejabberd_db_serialize).
3233

3334
-export([start/1, stop/1, set_password_multiple/3, try_register_multiple/3,
3435
get_users/2, count_users/2, get_password/2,
3536
remove_user/2, store_type/1, plain_password_required/1,
3637
export/1, which_users_exists/2, drop_password_type/2, set_password_instance/3]).
3738
-export([sql_schemas/0]).
39+
-export([serialize/3, deserialize_start/1, deserialize/2]).
3840

3941
-include_lib("xmpp/include/scram.hrl").
4042
-include("logger.hrl").
4143
-include("ejabberd_sql_pt.hrl").
4244
-include("ejabberd_auth.hrl").
45+
-include("ejabberd_db_serialize.hrl").
4346

4447
%%%----------------------------------------------------------------------
4548
%%% API
@@ -444,3 +447,102 @@ export(_Server) ->
444447
(_Host, _R) ->
445448
[]
446449
end}].
450+
451+
452+
serialize(LServer, BatchSize, Last) ->
453+
Offset = case Last of
454+
undefined -> 0;
455+
_ -> Last
456+
end,
457+
case ejabberd_sql:sql_query(
458+
LServer,
459+
?SQL("select @(username)s, @(type)d, @(password)s, @(serverkey)s, @(salt)s, @(iterationcount)d from users "
460+
"where %(LServer)H "
461+
"order by username, type "
462+
"limit %(BatchSize)d offset %(Offset)d")) of
463+
{selected, Rows} ->
464+
?DEBUG("GOT rows ~p", [Rows]),
465+
Data = lists:map(
466+
fun({Username, _, Password, <<>>, <<>>, 0}) ->
467+
#serialize_auth_v1{serverhost = LServer, username = Username, passwords = [Password]};
468+
({Username, 1, Password, _, _, _}) ->
469+
#serialize_auth_v1{serverhost = LServer, username = Username, passwords = [Password]};
470+
({Username, 0, Password, ServerKey, Salt, IterationCount}) ->
471+
{Hash, SK} = case Password of
472+
<<"sha256:", Rest/binary>> ->
473+
{sha256, Rest};
474+
<<"sha512:", Rest/binary>> ->
475+
{sha512, Rest};
476+
Other ->
477+
{sha, Other}
478+
end,
479+
#serialize_auth_v1{
480+
serverhost = LServer,
481+
username = Username,
482+
passwords = [{Hash, SK, ServerKey, Salt, IterationCount}]
483+
};
484+
({Username, Type, StoredKey, ServerKey, Salt, IterationCount}) ->
485+
#serialize_auth_v1{
486+
serverhost = LServer,
487+
username = Username,
488+
passwords = [{num_to_hash(Type),
489+
StoredKey,
490+
ServerKey,
491+
Salt,
492+
IterationCount}]
493+
}
494+
end,
495+
Rows),
496+
Data2 = case length(Rows) < BatchSize of
497+
true -> Data ++ [#serialize_auth_v1{}];
498+
_ -> Data
499+
end,
500+
{_, Data3, _, RC2} = lists:foldl(
501+
fun(Next, {undefined, Res, _, _}) ->
502+
{Next, Res, 1, 0};
503+
(#serialize_auth_v1{username = U1, passwords = [P1]},
504+
{#serialize_auth_v1{username = U2, passwords = P2} = Next, Res, NC, RC})
505+
when U1 == U2 ->
506+
{Next#serialize_auth_v1{passwords = [P1 | P2]}, Res, NC + 1, RC};
507+
(Current, {Next, Acc, NC, RC}) ->
508+
{Current, [Next | Acc], 1, RC + NC}
509+
end,
510+
{undefined, [], 0, 0},
511+
Data2),
512+
{ok, Data3, Offset + RC2};
513+
_ ->
514+
{error, io_lib:format("Error when retrieving passwords data from database", [])}
515+
end.
516+
517+
518+
deserialize_start(LServer) ->
519+
ejabberd_sql:sql_query(
520+
LServer,
521+
?SQL("delete from users where %(LServer)H")).
522+
523+
524+
deserialize(LServer, Batch) ->
525+
case ejabberd_sql:sql_transaction(LServer,
526+
fun() ->
527+
lists:foreach(
528+
fun(#serialize_auth_v1{username = Username, passwords = Passwords}) ->
529+
lists:foreach(
530+
fun({Hash, StoredKey, ServerKey, Salt, IterationCount}) ->
531+
set_password_scram_t(Username,
532+
LServer,
533+
Hash,
534+
StoredKey,
535+
ServerKey,
536+
Salt,
537+
IterationCount);
538+
(Password) ->
539+
set_password_t(Username, LServer, Password)
540+
end,
541+
Passwords)
542+
end,
543+
Batch)
544+
end) of
545+
{atomic, _} -> ok;
546+
_ ->
547+
{error, io_lib:format("Error when storing passwords in database", [])}
548+
end.

‎src/ejabberd_db_serialize.erl‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
%%%----------------------------------------------------------------------
2+
%%% File : ejabberd_db_serialize.erl
3+
%%% Author : Pawel Chmielowski <[email protected]>
4+
%%% Purpose : DB data de/serialization
5+
%%% Created : 20 Nov 2025 by Pawel Chmielowski <[email protected]>
6+
%%%
7+
%%%
8+
%%% ejabberd, Copyright (C) 2002-2025 ProcessOne
9+
%%%
10+
%%% This program is free software; you can redistribute it and/or
11+
%%% modify it under the terms of the GNU General Public License as
12+
%%% published by the Free Software Foundation; either version 2 of the
13+
%%% License, or (at your option) any later version.
14+
%%%
15+
%%% This program is distributed in the hope that it will be useful,
16+
%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
17+
%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18+
%%% General Public License for more details.
19+
%%%
20+
%%% You should have received a copy of the GNU General Public License along
21+
%%% with this program; if not, write to the Free Software Foundation, Inc.,
22+
%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
23+
%%%
24+
%%%----------------------------------------------------------------------
25+
-module(ejabberd_db_serialize).
26+
27+
%% API
28+
-export([serialize_module/3, deserialize_module/3]).
29+
30+
-callback serialize(binary(), non_neg_integer(), undefined | term()) ->
31+
{ok, [term()], term()} | {error, iolist()}.
32+
-callback deserialize_start(binary()) -> ok | {error, iolist()}.
33+
-callback deserialize(binary(), [term()]) -> ok | {error, iolist()}.
34+
35+
-define(BATCH_SIZE, 1000).
36+
37+
serialize_module(Host, ejabberd_auth, Dir) ->
38+
DbMod2 =
39+
lists:foldl(
40+
fun(Method, undefined) ->
41+
DbMod = ejabberd:module_name([<<"auth">>, Method]),
42+
case erlang:function_exported(DbMod, serialize, 3) of
43+
true -> DbMod;
44+
_ -> undefined
45+
end;
46+
(_, Mod) -> Mod
47+
end, undefined, ejabberd_option:auth_method(Host)),
48+
case DbMod2 of
49+
undefined ->
50+
{error, iolist_to_binary(io_lib:format("No auth module offer serialization",
51+
[]))};
52+
_ ->
53+
serialize_db_module(Host, ejabberd_auth, DbMod2, Dir)
54+
end;
55+
serialize_module(Host, Mod, Dir) ->
56+
case gen_mod:is_loaded(Host, Mod) of
57+
true ->
58+
serialize_db_module(Host, Mod, gen_mod:db_mod(Host, Mod), Dir);
59+
_ ->
60+
{error, iolist_to_binary(io_lib:format("Module ~s is not loaded on host ~s",
61+
[Mod, Host]))}
62+
end.
63+
64+
serialize_db_module(Host, Mod, DbMod, Dir) ->
65+
case erlang:function_exported(DbMod, serialize, 3) of
66+
true ->
67+
FN = <<(atom_to_binary(Mod, latin1))/binary, ".dbser">>,
68+
Path = filename:join([Dir, FN]),
69+
write_data(Path, fun(Next) -> DbMod:serialize(Host, ?BATCH_SIZE, Next) end);
70+
_ ->
71+
{error, iolist_to_binary(io_lib:format("Module ~s doesn't offer serialization",
72+
[Mod]))}
73+
end.
74+
75+
deserialize_module(Host, ejabberd_auth, Dir) ->
76+
DbMod2 =
77+
lists:foldl(
78+
fun(Method, undefined) ->
79+
DbMod = ejabberd:module_name([<<"auth">>, Method]),
80+
case erlang:function_exported(DbMod, deserialize, 2) andalso
81+
erlang:function_exported(DbMod, deserialize_start, 1) of
82+
true -> DbMod;
83+
_ -> undefined
84+
end;
85+
(_, Mod) -> Mod
86+
end, undefined, ejabberd_option:auth_method(Host)),
87+
case DbMod2 of
88+
undefined ->
89+
{error, iolist_to_binary(io_lib:format("No auth module offer serialization",
90+
[]))};
91+
_ ->
92+
deserialize_db_module(Host, ejabberd_auth, DbMod2, Dir)
93+
end;
94+
deserialize_module(Host, Mod, Dir) ->
95+
case gen_mod:is_loaded(Host, Mod) of
96+
true ->
97+
deserialize_db_module(Host, Mod, gen_mod:db_mod(Host, Mod), Dir);
98+
_ ->
99+
{error, iolist_to_binary(io_lib:format("Module ~s is not loaded on host ~s",
100+
[Mod, Host]))}
101+
end.
102+
103+
deserialize_db_module(Host, Mod, DbMod, Dir) ->
104+
case erlang:function_exported(DbMod, deserialize, 2) andalso
105+
erlang:function_exported(DbMod, deserialize_start, 1)
106+
of
107+
true ->
108+
FN = <<(atom_to_binary(Mod, latin1))/binary, ".dbser">>,
109+
Path = filename:join([Dir, FN]),
110+
DbMod:deserialize_start(Host),
111+
read_data(Path, fun(Chunk) -> DbMod:deserialize(Host, Chunk) end);
112+
_ ->
113+
{error, iolist_to_binary(io_lib:format("Module ~s doesn't offer serialization",
114+
[Mod]))}
115+
end.
116+
117+
write_loop(Path, Io, Producer, Key) ->
118+
case Producer(Key) of
119+
{ok, [], _} ->
120+
ok;
121+
{ok, Data, NextKey} ->
122+
Ser = term_to_binary(Data),
123+
SerLen = byte_size(Ser),
124+
maybe
125+
ok ?= file:write(Io, <<"dbdb", SerLen:32/unsigned-little-integer>>),
126+
ok ?= file:write(Io, Ser),
127+
write_loop(Path, Io, Producer, NextKey)
128+
else
129+
{error, Msg} ->
130+
{error, io_lib:format("Error when writing to file ~s: ~p",
131+
[Path, Msg])}
132+
end;
133+
Error -> Error
134+
end.
135+
136+
write_data(Path, Producer) ->
137+
case file:open(Path, [write, raw, binary]) of
138+
{ok, IO} ->
139+
case write_loop(Path, IO, Producer, undefined) of
140+
ok ->
141+
file:close(IO),
142+
ok;
143+
{error, Msg} ->
144+
file:close(IO),
145+
file:delete(Path),
146+
{error, iolist_to_binary(Msg)}
147+
end;
148+
{error, Msg} ->
149+
{error, iolist_to_binary(io_lib:format("Unable to open file ~s for write: ~p",
150+
[Path, Msg]))}
151+
end.
152+
153+
read_loop(Path, Io, Consumer) ->
154+
case file:read(Io, 8) of
155+
{ok, <<"dbdb", Len:32/unsigned-little-integer>>} ->
156+
case file:read(Io, Len) of
157+
{ok, Data} when byte_size(Data) == Len ->
158+
try
159+
Decoded = erlang:binary_to_term(iolist_to_binary(Data)),
160+
case Consumer(Decoded) of
161+
ok ->
162+
read_loop(Path, Io, Consumer);
163+
Err -> Err
164+
end
165+
catch
166+
_:_ ->
167+
{error, io_lib:format("Unable to decode data from file ~s", [Path])}
168+
end;
169+
{ok, _} ->
170+
{error, io_lib:format("File ~s is too short", [Path])};
171+
eof ->
172+
{error, io_lib:format("File ~s is too short", [Path])};
173+
{error, Msg} ->
174+
{error, io_lib:format("Error when reading file ~s: ~p", [Path, Msg])}
175+
end;
176+
{ok, _} ->
177+
{error, io_lib:format("File ~s has wrong header", [Path])};
178+
eof ->
179+
ok;
180+
{error, Msg} ->
181+
{error, io_lib:format("Error when reading file ~s: ~p", [Path, Msg])}
182+
end.
183+
184+
read_data(Path, Consumer) ->
185+
case file:open(Path, [raw, read, binary]) of
186+
{ok, IO} ->
187+
case read_loop(Path, IO, Consumer) of
188+
ok ->
189+
file:close(IO),
190+
ok;
191+
{error, Msg} ->
192+
file:close(IO),
193+
{error, iolist_to_binary(Msg)}
194+
end;
195+
{error, Msg} ->
196+
{error, iolist_to_binary(io_lib:format("Unable to open file ~s for write: ~p",
197+
[Path, Msg]))}
198+
end.

‎src/mod_mam.erl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1573,7 +1573,7 @@ filter_by_max(Msgs, Len) when is_integer(Len), Len >= 0 ->
15731573
filter_by_max(_Msgs, _Junk) ->
15741574
{[], true}.
15751575

1576-
-spec limit_max(rsm_set(), binary()) -> rsm_set() | undefined.
1576+
-spec limit_max(rsm_set() | undefined, binary()) -> rsm_set() | undefined.
15771577
limit_max(RSM, ?NS_MAM_TMP) ->
15781578
RSM; % XEP-0313 v0.2 doesn't require clients to support RSM.
15791579
limit_max(undefined, _NS) ->

0 commit comments

Comments
 (0)