Skip to content

Commit 1566907

Browse files
Peter-SearbyPeter-Searby
committed
Merge remote-tracking branch 'couchbase/phoenix' into morpheus
* couchbase/phoenix: MB-67857: Notify on fake_chronicle_kv key updates MB-67857: Simplify fake_chronicle_kv:update_snapshot/2 MB-67857: Use ?FUNCTION_NAME in mock_helpers MB-67857: Fix ns_rebalance_report_manager name in mock_helpers MB-67857[BP]: Support concurrent writers in fake_chronicle_kv MB-67857[BP]: Put fake_chronicle_kv seqno in snapshot MB-67857[BP]: Remove ns_pubsub mock MB-67857[BP]: Rename fake_chronicle_kv:new/unload to setup/teardown MB-67857[BP]: Refactor to avoid fake_ns_pubsub uses MB-67857[BP]: Pull failover test mocks into new module MB-67857[BP]: Simplify meck unloading MB-67857[BP]: Fix fake_chronicle_kv MB-66812: Add metakv API test Change-Id: Icdf47b7b9a1d1b118cf57e9498008c7b878eb165
2 parents 3140d3b + 9d18f4f commit 1566907

File tree

6 files changed

+272
-70
lines changed

6 files changed

+272
-70
lines changed

‎apps/ns_server/test/fake_chronicle_kv.erl‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -94,15 +94,7 @@ teardown() ->
9494
%% -------------------------
9595
-spec update_snapshot(atom(), term()) -> true.
9696
update_snapshot(Key, Value) ->
97-
OldSnapshot = get_ets_snapshot(),
98-
NewSnapshot = maps:put(Key, add_rev_to_value(Value), OldSnapshot),
99-
100-
case store_ets_snapshot(OldSnapshot, NewSnapshot) of
101-
ok ->
102-
ok;
103-
retry ->
104-
update_snapshot(Key, Value)
105-
end.
97+
update_snapshot(#{Key => Value}).
10698

10799
-spec update_snapshot(map()) -> true.
108100
update_snapshot(Map) when is_map(Map) ->
@@ -281,6 +273,21 @@ compare_and_swap(OldSnapshot, NewSnapshot) ->
281273
end),
282274
case ets:select_replace(?TABLE_NAME, MS) of
283275
1 ->
276+
Mgr = chronicle_kv:event_manager(kv),
277+
maps:foreach(
278+
fun (seqno, _) ->
279+
ok;
280+
(Key, NewValue) ->
281+
case maps:find(Key, OldSnapshot) of
282+
{ok, NewValue} ->
283+
ok;
284+
_ ->
285+
{Value, Rev} = NewValue,
286+
gen_event:notify(Mgr,
287+
{{key, Key}, Rev,
288+
{updated, Value}})
289+
end
290+
end, NewSnapshot),
284291
ok;
285292
0 ->
286293
retry

‎apps/ns_server/test/mock_helpers.erl‎

Lines changed: 57 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -116,54 +116,54 @@ auto_reprovision(PidMap) ->
116116
PidMap0 = setup_mocks([leader_registry],
117117
PidMap),
118118

119-
{ok, AutoReprovisionPid} = auto_reprovision:start_link(),
119+
{ok, AutoReprovisionPid} = ?FUNCTION_NAME:start_link(),
120120
PidMap0#{?FUNCTION_NAME => AutoReprovisionPid}.
121121

122-
rebalance_report_manager(PidMap) ->
122+
ns_rebalance_report_manager(PidMap) ->
123123
fake_ns_config:update_snapshot(rest_creds, null),
124124

125125
meck:new(cb_atomic_persistent_term, [passthrough]),
126126
meck:expect(cb_atomic_persistent_term, get_or_set_if_invalid,
127127
fun(_, _, F) ->
128128
F(undefined)
129129
end),
130-
{ok, RebalanceReportManagerPid} = ns_rebalance_report_manager:start_link(),
130+
{ok, RebalanceReportManagerPid} = ?FUNCTION_NAME:start_link(),
131131
PidMap#{?FUNCTION_NAME => RebalanceReportManagerPid}.
132132

133133
ns_orchestrator(PidMap) ->
134134
PidMap0 = setup_mocks([ns_janitor_server,
135135
ns_doctor,
136-
rebalance_report_manager,
136+
ns_rebalance_report_manager,
137137
testconditions],
138138
PidMap),
139139

140-
{ok, OrchestratorPid} = ns_orchestrator:start_link(),
140+
{ok, OrchestratorPid} = ?FUNCTION_NAME:start_link(),
141141
PidMap0#{?FUNCTION_NAME => OrchestratorPid}.
142142

143143
compat_mode_manager(PidMap) ->
144-
{ok, CompatModeManagerPid} = compat_mode_manager:start_link(),
144+
{ok, CompatModeManagerPid} = ?FUNCTION_NAME:start_link(),
145145
PidMap#{?FUNCTION_NAME => CompatModeManagerPid}.
146146

147147
auto_failover(PidMap) ->
148148
PidMap0 = setup_mocks([compat_mode_events], PidMap),
149149

150-
{ok, AutoFailoverPid} = auto_failover:start_link(),
150+
{ok, AutoFailoverPid} = ?FUNCTION_NAME:start_link(),
151151
PidMap0#{?FUNCTION_NAME => AutoFailoverPid}.
152152

153153
compat_mode_events(PidMap) ->
154154
{ok, CompatModeEventsPid} = gen_event:start_link({local,
155-
compat_mode_events}),
155+
?FUNCTION_NAME}),
156156
PidMap#{?FUNCTION_NAME => CompatModeEventsPid}.
157157

158158
ns_node_disco_events(PidMap) ->
159159
{ok, NSNodeDiscoEventsPid} = gen_event:start_link({local,
160-
ns_node_disco_events}),
160+
?FUNCTION_NAME}),
161161
PidMap#{?FUNCTION_NAME => NSNodeDiscoEventsPid}.
162162

163163
chronicle_master(PidMap) ->
164164
PidMap0 = setup_mocks([chronicle], PidMap),
165165

166-
{ok, ChronicleMasterPid} = chronicle_master:start_link(),
166+
{ok, ChronicleMasterPid} = ?FUNCTION_NAME:start_link(),
167167
PidMap0#{?FUNCTION_NAME => ChronicleMasterPid}.
168168

169169
%%%===================================================================
@@ -175,24 +175,24 @@ chronicle_master(PidMap) ->
175175
%%%===================================================================
176176

177177
chronicle(PidMap) ->
178-
meck:new(chronicle),
179-
meck:expect(chronicle, acquire_lock, fun() -> {ok, self()} end),
180-
meck:expect(chronicle, set_peer_roles, fun(_,_) -> ok end),
178+
meck:new(?FUNCTION_NAME),
179+
meck:expect(?FUNCTION_NAME, acquire_lock, fun() -> {ok, self()} end),
180+
meck:expect(?FUNCTION_NAME, set_peer_roles, fun(_,_) -> ok end),
181181

182182
PidMap#{?FUNCTION_NAME => mocked}.
183183

184184
janitor_agent(PidMap) ->
185185
%% Janitor_agent mecks required to perform a full failover (with map).
186-
meck:new(janitor_agent, []),
187-
meck:expect(janitor_agent, query_vbuckets,
186+
meck:new(?FUNCTION_NAME, [passthrough]),
187+
meck:expect(?FUNCTION_NAME, query_vbuckets,
188188
fun(_,_,_,_) ->
189189
%% We don't need to return anything useful for this
190190
%% failover, we are failing over all but one node so
191191
%% we don't have to choose between any.
192192
{dict:from_list([{1, []}]), []}
193193
end),
194194

195-
meck:expect(janitor_agent, fetch_vbucket_states,
195+
meck:expect(?FUNCTION_NAME, fetch_vbucket_states,
196196
fun(VBucket, _) ->
197197
%% We need to return some semi-valid vBucket stat map
198198
%% from this. We might use a couple of different maps
@@ -209,86 +209,86 @@ janitor_agent(PidMap) ->
209209
A ++ R
210210
end),
211211

212-
meck:expect(janitor_agent, find_vbucket_state,
212+
meck:expect(?FUNCTION_NAME, find_vbucket_state,
213213
fun(Node, States) ->
214214
meck:passthrough([Node, States])
215215
end),
216216

217-
meck:expect(janitor_agent, apply_new_bucket_config,
217+
meck:expect(?FUNCTION_NAME, apply_new_bucket_config,
218218
fun(_,_,_,_) ->
219219
%% Just sets stuff in memcached, uninteresting here
220220
ok
221221
end),
222222

223-
meck:expect(janitor_agent, mark_bucket_warmed,
223+
meck:expect(?FUNCTION_NAME, mark_bucket_warmed,
224224
fun(_,_) ->
225225
%% Just sets stuff in memcached, uninteresting here
226226
ok
227227
end),
228228

229-
meck:expect(janitor_agent, check_bucket_ready,
229+
meck:expect(?FUNCTION_NAME, check_bucket_ready,
230230
fun(_,_,_) ->
231231
ready
232232
end),
233-
meck:expect(janitor_agent, delete_vbucket_copies,
233+
meck:expect(?FUNCTION_NAME, delete_vbucket_copies,
234234
fun(_,_,_,_) ->
235235
ok
236236
end),
237-
meck:expect(janitor_agent, finish_rebalance,
237+
meck:expect(?FUNCTION_NAME, finish_rebalance,
238238
fun(_,_,_) ->
239239
ok
240240
end),
241-
meck:expect(janitor_agent, prepare_nodes_for_rebalance,
241+
meck:expect(?FUNCTION_NAME, prepare_nodes_for_rebalance,
242242
fun(_,_,_) ->
243243
ok
244244
end),
245245

246-
meck:expect(janitor_agent, inhibit_view_compaction,
246+
meck:expect(?FUNCTION_NAME, inhibit_view_compaction,
247247
fun (_, _, _) -> nack end),
248248

249-
meck:expect(janitor_agent, uninhibit_view_compaction,
249+
meck:expect(?FUNCTION_NAME, uninhibit_view_compaction,
250250
fun (_, _, _, _) -> ok end),
251251

252-
meck:expect(janitor_agent, get_mass_dcp_docs_estimate,
252+
meck:expect(?FUNCTION_NAME, get_mass_dcp_docs_estimate,
253253
fun (_, _, VBs) ->
254254
{ok, lists:duplicate(length(VBs), {0, 0, random_state})}
255255
end),
256256

257-
meck:expect(janitor_agent, bulk_set_vbucket_state,
257+
meck:expect(?FUNCTION_NAME, bulk_set_vbucket_state,
258258
fun (_, _, _, _) -> ok end),
259259

260-
meck:expect(janitor_agent, initiate_indexing,
260+
meck:expect(?FUNCTION_NAME, initiate_indexing,
261261
fun (_, _, _, _, _) -> ok end),
262262

263-
meck:expect(janitor_agent, wait_dcp_data_move,
263+
meck:expect(?FUNCTION_NAME, wait_dcp_data_move,
264264
fun (_, _, _, _, _) -> ok end),
265265

266-
meck:expect(janitor_agent, get_vbucket_high_seqno,
266+
meck:expect(?FUNCTION_NAME, get_vbucket_high_seqno,
267267
fun (_, _, _, _) -> 0 end),
268268

269-
meck:expect(janitor_agent, wait_seqno_persisted,
269+
meck:expect(?FUNCTION_NAME, wait_seqno_persisted,
270270
fun (_, _, _, _, _) -> ok end),
271271

272-
meck:expect(janitor_agent, set_vbucket_state,
272+
meck:expect(?FUNCTION_NAME, set_vbucket_state,
273273
fun (_, _, _, _, _, _, _, _) -> ok end),
274274

275-
meck:expect(janitor_agent, wait_index_updated,
275+
meck:expect(?FUNCTION_NAME, wait_index_updated,
276276
fun (_, _, _, _, _) -> ok end),
277277

278-
meck:expect(janitor_agent, dcp_takeover,
278+
meck:expect(?FUNCTION_NAME, dcp_takeover,
279279
fun (_, _, _, _, _) -> ok end),
280280

281-
meck:expect(janitor_agent, get_src_dst_vbucket_replications,
281+
meck:expect(?FUNCTION_NAME, get_src_dst_vbucket_replications,
282282
fun (_, _) -> {[], []} end),
283283

284284
PidMap#{?FUNCTION_NAME => mocked}.
285285

286286
%% auto failover for orchestrator
287287
ns_janitor_server(PidMap) ->
288-
meck:new(ns_janitor_server),
289-
meck:expect(ns_janitor_server, start_cleanup,
288+
meck:new(?FUNCTION_NAME),
289+
meck:expect(?FUNCTION_NAME, start_cleanup,
290290
fun(_) -> {ok, self()} end),
291-
meck:expect(ns_janitor_server, terminate_cleanup,
291+
meck:expect(?FUNCTION_NAME, terminate_cleanup,
292292
fun(_) ->
293293
CallerPid = self(),
294294
CallerPid ! {cleanup_done, foo, bar},
@@ -298,55 +298,55 @@ ns_janitor_server(PidMap) ->
298298

299299
%% from manual failover setup
300300
leader_activities(PidMap) ->
301-
meck:new(leader_activities),
302-
meck:expect(leader_activities, run_activity,
301+
meck:new(?FUNCTION_NAME),
302+
meck:expect(?FUNCTION_NAME, run_activity,
303303
fun(_Name, _Quorum, Body) ->
304304
Body()
305305
end),
306-
meck:expect(leader_activities, run_activity,
306+
meck:expect(?FUNCTION_NAME, run_activity,
307307
fun(_Name, _Quorum, Body, _Opts) ->
308308
Body()
309309
end),
310-
311-
meck:expect(leader_activities, activate_quorum_nodes,
310+
meck:expect(?FUNCTION_NAME, activate_quorum_nodes,
312311
fun(_) -> ok end),
313-
meck:expect(leader_activities, deactivate_quorum_nodes,
312+
meck:expect(?FUNCTION_NAME, deactivate_quorum_nodes,
314313
fun(_) -> ok end),
315314
PidMap#{?FUNCTION_NAME => mocked}.
316315

316+
317317
ns_doctor(PidMap) ->
318-
meck:new(ns_doctor),
319-
meck:expect(ns_doctor, get_nodes, fun() -> [] end),
318+
meck:new(?FUNCTION_NAME),
319+
meck:expect(?FUNCTION_NAME, get_nodes, fun() -> [] end),
320320

321321
meck:expect(
322-
ns_doctor, get_memory_data,
322+
?FUNCTION_NAME, get_memory_data,
323323
fun(Nodes) ->
324324
{ok, [{Node, memsup:get_system_memory_data()} || Node <- Nodes]}
325325
end),
326326

327327
PidMap#{?FUNCTION_NAME => mocked}.
328328

329329
rebalance_quirks(PidMap) ->
330-
meck:new(rebalance_quirks, [passthrough]),
331-
meck:expect(rebalance_quirks, get_quirks, fun(_,_) -> [] end),
330+
meck:new(?FUNCTION_NAME, [passthrough]),
331+
meck:expect(?FUNCTION_NAME, get_quirks, fun(_,_) -> [] end),
332332
PidMap#{?FUNCTION_NAME => mocked}.
333333

334334
testconditions(PidMap) ->
335-
meck:new(testconditions, [passthrough]),
336-
meck:expect(testconditions, get, fun(_) -> ok end),
335+
meck:new(?FUNCTION_NAME, [passthrough]),
336+
meck:expect(?FUNCTION_NAME, get, fun(_) -> ok end),
337337
PidMap#{?FUNCTION_NAME => mocked}.
338338

339339
rebalance_agent(PidMap) ->
340-
meck:new(rebalance_agent),
341-
meck:expect(rebalance_agent, prepare_rebalance, fun(_,_) -> ok end),
342-
meck:expect(rebalance_agent, deactivate_bucket_data, fun(_,_,_) -> ok end),
340+
meck:new(?FUNCTION_NAME),
341+
meck:expect(?FUNCTION_NAME, prepare_rebalance, fun(_,_) -> ok end),
342+
meck:expect(?FUNCTION_NAME, deactivate_bucket_data, fun(_,_,_) -> ok end),
343343

344-
meck:expect(rebalance_agent, unprepare_rebalance, fun(_,_) -> ok end),
344+
meck:expect(?FUNCTION_NAME, unprepare_rebalance, fun(_,_) -> ok end),
345345
PidMap#{?FUNCTION_NAME => mocked}.
346346

347347
ns_storage_conf(PidMap) ->
348-
meck:new(ns_storage_conf),
349-
meck:expect(ns_storage_conf, delete_unused_buckets_db_files,
348+
meck:new(?FUNCTION_NAME),
349+
meck:expect(?FUNCTION_NAME, delete_unused_buckets_db_files,
350350
fun() -> ok end),
351351
PidMap#{?FUNCTION_NAME => mocked}.
352352

‎cluster_tests/run.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,8 @@
7676
metakv2_tests, \
7777
app_telemetry_tests, \
7878
jwt_tests, \
79-
bucket_dirs_cleanup_tests
79+
bucket_dirs_cleanup_tests, \
80+
metakv_tests
8081

8182
tmp_cluster_dir = os.path.join(testlib.get_cluster_test_dir(),
8283
"test_cluster_data")

‎cluster_tests/testlib/testlib.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -552,6 +552,9 @@ def delete_fail(cluster_or_node, path, expected_code, **kwargs):
552552
return request('DELETE', cluster_or_node, path, expected_code=expected_code,
553553
**kwargs)
554554

555+
def delete_fail(cluster_or_node, path, expected_code, **kwargs):
556+
return request('DELETE', cluster_or_node, path, expected_code, **kwargs)
557+
555558

556559
def set_default_auth(cluster_or_node, **kwargs):
557560
if 'auth' not in kwargs and 'cert' not in kwargs:
@@ -681,12 +684,18 @@ def metakv_get_succ(cluster, key, **kwargs):
681684
def metakv_get(cluster, key, **kwargs):
682685
return get(cluster, f"/_metakv{key}", **kwargs)
683686

684-
def metakv_put_succ(cluster, key, value, **kwargs):
685-
return put_succ(cluster, f"/_metakv{key}", data={'value': value}, **kwargs)
687+
def metakv_put_succ(cluster, key, **kwargs):
688+
return put_succ(cluster, f"/_metakv{key}", **kwargs)
689+
690+
def metakv_put_fail(cluster, key, expected_code, **kwargs):
691+
return put_fail(cluster, f"/_metakv{key}", expected_code, **kwargs)
686692

687693
def metakv_delete_succ(cluster, key, **kwargs):
688694
return delete_succ(cluster, f"/_metakv{key}", **kwargs)
689695

696+
def metakv_delete_fail(cluster, key, expected_code, **kwargs):
697+
return delete_fail(cluster, f"/_metakv{key}", expected_code, **kwargs)
698+
690699
def diag_eval(cluster, code, **kwargs):
691700
return post_succ(cluster, '/diag/eval', data=code, **kwargs)
692701

0 commit comments

Comments
 (0)