Skip to content

Commit 347ba63

Browse files
JonBailey1993JonBailey1993
committed
common/io_exerciser: Add injecterror commands to ceph_test_rados_io_sequence interactive mode
Add injecterror commands that can be used in interactive mode to inject read and write errors as well as clear them Signed-off-by: Jon Bailey <[email protected]>
1 parent 23dd255 commit 347ba63

File tree

11 files changed

+1721
-175
lines changed

11 files changed

+1721
-175
lines changed

‎src/common/io_exerciser/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ add_library(object_io_exerciser STATIC
55
Model.cc
66
ObjectModel.cc
77
RadosIo.cc
8+
JsonStructures.cc
89
)
910

1011
target_link_libraries(object_io_exerciser

‎src/common/io_exerciser/IoOp.cc‎

Lines changed: 170 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
#include "fmt/format.h"
44

5+
#include "include/ceph_assert.h"
6+
57
using IoOp = ceph::io_exerciser::IoOp;
68
using OpType = ceph::io_exerciser::OpType;
79

@@ -15,6 +17,9 @@ using TripleReadOp = ceph::io_exerciser::TripleReadOp;
1517
using SingleWriteOp = ceph::io_exerciser::SingleWriteOp;
1618
using DoubleWriteOp = ceph::io_exerciser::DoubleWriteOp;
1719
using TripleWriteOp = ceph::io_exerciser::TripleWriteOp;
20+
using SingleFailedWriteOp = ceph::io_exerciser::SingleFailedWriteOp;
21+
using DoubleFailedWriteOp = ceph::io_exerciser::DoubleFailedWriteOp;
22+
using TripleFailedWriteOp = ceph::io_exerciser::TripleFailedWriteOp;
1823

1924
namespace
2025
{
@@ -164,17 +169,23 @@ std::string ceph::io_exerciser::ReadWriteOp<opType, numIOs>
164169
switch(opType)
165170
{
166171
case OpType::Read:
167-
[[fallthrough]];
172+
[[ fallthrough ]];
168173
case OpType::Read2:
169-
[[fallthrough]];
174+
[[ fallthrough ]];
170175
case OpType::Read3:
171176
return fmt::format("Read{} ({})", numIOs, offset_length_desc);
172177
case OpType::Write:
173-
[[fallthrough]];
178+
[[ fallthrough ]];
174179
case OpType::Write2:
175-
[[fallthrough]];
180+
[[ fallthrough ]];
176181
case OpType::Write3:
177182
return fmt::format("Write{} ({})", numIOs, offset_length_desc);
183+
case OpType::FailedWrite:
184+
[[ fallthrough ]];
185+
case OpType::FailedWrite2:
186+
[[ fallthrough ]];
187+
case OpType::FailedWrite3:
188+
return fmt::format("FailedWrite{} ({})", numIOs, offset_length_desc);
178189
default:
179190
ceph_abort_msg(fmt::format("Unsupported op type by ReadWriteOp ({})", opType));
180191
}
@@ -262,4 +273,159 @@ std::unique_ptr<TripleWriteOp> TripleWriteOp::generate(uint64_t offset1, uint64_
262273
return std::make_unique<TripleWriteOp>(offset1, length1,
263274
offset2, length2,
264275
offset3, length3);
276+
}
277+
278+
SingleFailedWriteOp::SingleFailedWriteOp(uint64_t offset, uint64_t length) :
279+
ReadWriteOp<OpType::FailedWrite, 1>({offset}, {length})
280+
{
281+
282+
}
283+
284+
std::unique_ptr<SingleFailedWriteOp> SingleFailedWriteOp::generate(uint64_t offset,
285+
uint64_t length)
286+
{
287+
return std::make_unique<SingleFailedWriteOp>(offset, length);
288+
}
289+
290+
DoubleFailedWriteOp::DoubleFailedWriteOp(uint64_t offset1, uint64_t length1,
291+
uint64_t offset2, uint64_t length2) :
292+
ReadWriteOp<OpType::FailedWrite2, 2>({offset1, offset2}, {length1, length2})
293+
{
294+
295+
}
296+
297+
std::unique_ptr<DoubleFailedWriteOp> DoubleFailedWriteOp::generate(uint64_t offset1,
298+
uint64_t length1,
299+
uint64_t offset2,
300+
uint64_t length2)
301+
{
302+
return std::make_unique<DoubleFailedWriteOp>(offset1, length1, offset2, length2);
303+
}
304+
305+
TripleFailedWriteOp::TripleFailedWriteOp(uint64_t offset1, uint64_t length1,
306+
uint64_t offset2, uint64_t length2,
307+
uint64_t offset3, uint64_t length3) :
308+
ReadWriteOp<OpType::FailedWrite3, 3>({offset1, offset2, offset3},
309+
{length1, length2, length3})
310+
{
311+
312+
}
313+
314+
std::unique_ptr<TripleFailedWriteOp> TripleFailedWriteOp::generate(uint64_t offset1,
315+
uint64_t length1,
316+
uint64_t offset2,
317+
uint64_t length2,
318+
uint64_t offset3,
319+
uint64_t length3)
320+
{
321+
return std::make_unique<TripleFailedWriteOp>(offset1, length1,
322+
offset2, length2,
323+
offset3, length3);
324+
}
325+
326+
template <ceph::io_exerciser::OpType opType>
327+
ceph::io_exerciser::InjectErrorOp<opType>
328+
::InjectErrorOp(int shard,
329+
const std::optional<uint64_t>& type,
330+
const std::optional<uint64_t>& when,
331+
const std::optional<uint64_t>& duration) :
332+
TestOp<opType>(),
333+
shard(shard),
334+
type(type),
335+
when(when),
336+
duration(duration)
337+
{
338+
339+
}
340+
341+
template <ceph::io_exerciser::OpType opType>
342+
std::string ceph::io_exerciser::InjectErrorOp<opType>::to_string(uint64_t blocksize) const
343+
{
344+
std::string_view inject_type = get_inject_type_string();
345+
return fmt::format("Inject {} error on shard {} of type {}"
346+
" after {} successful inject(s) lasting {} inject(s)",
347+
inject_type, shard, type.value_or(0),
348+
when.value_or(0), duration.value_or(1));
349+
}
350+
351+
ceph::io_exerciser::InjectReadErrorOp::InjectReadErrorOp(int shard,
352+
const std::optional<uint64_t>& type,
353+
const std::optional<uint64_t>& when,
354+
const std::optional<uint64_t>& duration) :
355+
InjectErrorOp<OpType::InjectReadError>(shard, type, when, duration)
356+
{
357+
358+
}
359+
360+
std::unique_ptr<ceph::io_exerciser::InjectReadErrorOp> ceph::io_exerciser
361+
::InjectReadErrorOp::generate(int shard,
362+
const std::optional<uint64_t>& type,
363+
const std::optional<uint64_t>& when,
364+
const std::optional<uint64_t>& duration)
365+
{
366+
return std::make_unique<InjectReadErrorOp>(shard, type, when, duration);
367+
}
368+
369+
ceph::io_exerciser::InjectWriteErrorOp::InjectWriteErrorOp(int shard,
370+
const std::optional<uint64_t>& type,
371+
const std::optional<uint64_t>& when,
372+
const std::optional<uint64_t>& duration) :
373+
InjectErrorOp<OpType::InjectWriteError>(shard, type, when, duration)
374+
{
375+
376+
}
377+
378+
std::unique_ptr<ceph::io_exerciser::InjectWriteErrorOp> ceph::io_exerciser
379+
::InjectWriteErrorOp::generate(int shard,
380+
const std::optional<uint64_t>& type,
381+
const std::optional<uint64_t>& when,
382+
const std::optional<uint64_t>& duration)
383+
{
384+
return std::make_unique<InjectWriteErrorOp>(shard, type, when, duration);
385+
}
386+
387+
388+
389+
template <ceph::io_exerciser::OpType opType>
390+
ceph::io_exerciser::ClearErrorInjectOp<opType>
391+
::ClearErrorInjectOp(int shard, const std::optional<uint64_t>& type) :
392+
TestOp<opType>(),
393+
shard(shard),
394+
type(type)
395+
{
396+
397+
}
398+
399+
template <ceph::io_exerciser::OpType opType>
400+
std::string ceph::io_exerciser::ClearErrorInjectOp<opType>::to_string(uint64_t blocksize) const
401+
{
402+
std::string_view inject_type = get_inject_type_string();
403+
return fmt::format("Clear {} injects on shard {} of type {}",
404+
inject_type, shard, type.value_or(0));
405+
}
406+
407+
ceph::io_exerciser::ClearReadErrorInjectOp::ClearReadErrorInjectOp(int shard,
408+
const std::optional<uint64_t>& type) :
409+
ClearErrorInjectOp<OpType::ClearReadErrorInject>(shard, type)
410+
{
411+
412+
}
413+
414+
std::unique_ptr<ceph::io_exerciser::ClearReadErrorInjectOp> ceph::io_exerciser
415+
::ClearReadErrorInjectOp::generate(int shard, const std::optional<uint64_t>& type)
416+
{
417+
return std::make_unique<ClearReadErrorInjectOp>(shard, type);
418+
}
419+
420+
ceph::io_exerciser::ClearWriteErrorInjectOp::ClearWriteErrorInjectOp(int shard,
421+
const std::optional<uint64_t>& type) :
422+
ClearErrorInjectOp<OpType::ClearWriteErrorInject>(shard, type)
423+
{
424+
425+
}
426+
427+
std::unique_ptr<ceph::io_exerciser::ClearWriteErrorInjectOp> ceph::io_exerciser
428+
::ClearWriteErrorInjectOp::generate(int shard, const std::optional<uint64_t>& type)
429+
{
430+
return std::make_unique<ClearWriteErrorInjectOp>(shard, type);
265431
}

‎src/common/io_exerciser/IoOp.h‎

Lines changed: 131 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
#pragma once
22

33
#include <array>
4+
#include <optional>
45
#include <string>
56
#include <memory>
6-
#include "include/ceph_assert.h"
77

88
#include "OpType.h"
99

@@ -145,5 +145,135 @@ namespace ceph {
145145
uint64_t offset3,
146146
uint64_t length3);
147147
};
148+
149+
class SingleFailedWriteOp : public ReadWriteOp<OpType::FailedWrite, 1>
150+
{
151+
public:
152+
SingleFailedWriteOp(uint64_t offset, uint64_t length);
153+
static std::unique_ptr<SingleFailedWriteOp> generate(uint64_t offset,
154+
uint64_t length);
155+
};
156+
157+
class DoubleFailedWriteOp : public ReadWriteOp<OpType::FailedWrite2, 2>
158+
{
159+
public:
160+
DoubleFailedWriteOp(uint64_t offset1, uint64_t length1,
161+
uint64_t offset2, uint64_t length2);
162+
static std::unique_ptr<DoubleFailedWriteOp> generate(uint64_t offset1,
163+
uint64_t length1,
164+
uint64_t offset2,
165+
uint64_t length2);
166+
};
167+
168+
class TripleFailedWriteOp : public ReadWriteOp<OpType::FailedWrite3, 3>
169+
{
170+
public:
171+
TripleFailedWriteOp(uint64_t offset1, uint64_t length1,
172+
uint64_t offset2, uint64_t length2,
173+
uint64_t offset3, uint64_t length3);
174+
static std::unique_ptr<TripleFailedWriteOp> generate(uint64_t offset1,
175+
uint64_t length1,
176+
uint64_t offset2,
177+
uint64_t length2,
178+
uint64_t offset3,
179+
uint64_t length3);
180+
};
181+
182+
template <ceph::io_exerciser::OpType opType>
183+
class InjectErrorOp : public TestOp<opType>
184+
{
185+
public:
186+
InjectErrorOp(int shard,
187+
const std::optional<uint64_t>& type,
188+
const std::optional<uint64_t>& when,
189+
const std::optional<uint64_t>& duration);
190+
191+
std::string to_string(uint64_t block_size) const override;
192+
193+
int shard;
194+
std::optional<uint64_t> type;
195+
std::optional<uint64_t> when;
196+
std::optional<uint64_t> duration;
197+
198+
protected:
199+
virtual inline constexpr std::string_view get_inject_type_string() const = 0;
200+
};
201+
202+
class InjectReadErrorOp : public InjectErrorOp<OpType::InjectReadError>
203+
{
204+
public:
205+
InjectReadErrorOp(int shard,
206+
const std::optional<uint64_t>& type,
207+
const std::optional<uint64_t>& when,
208+
const std::optional<uint64_t>& duration);
209+
210+
static std::unique_ptr<InjectReadErrorOp> generate(int shard,
211+
const std::optional<uint64_t>& type,
212+
const std::optional<uint64_t>& when,
213+
const std::optional<uint64_t>& duration);
214+
215+
protected:
216+
inline constexpr std::string_view get_inject_type_string() const override
217+
{ return "read"; }
218+
};
219+
220+
class InjectWriteErrorOp : public InjectErrorOp<OpType::InjectWriteError>
221+
{
222+
public:
223+
InjectWriteErrorOp(int shard,
224+
const std::optional<uint64_t>& type,
225+
const std::optional<uint64_t>& when,
226+
const std::optional<uint64_t>& duration);
227+
228+
static std::unique_ptr<InjectWriteErrorOp> generate(int shard,
229+
const std::optional<uint64_t>& type,
230+
const std::optional<uint64_t>& when,
231+
const std::optional<uint64_t>& duration);
232+
233+
protected:
234+
inline constexpr std::string_view get_inject_type_string() const override
235+
{ return "write"; }
236+
};
237+
238+
template <ceph::io_exerciser::OpType opType>
239+
class ClearErrorInjectOp : public TestOp<opType>
240+
{
241+
public:
242+
ClearErrorInjectOp(int shard, const std::optional<uint64_t>& type);
243+
244+
std::string to_string(uint64_t block_size) const override;
245+
246+
int shard;
247+
std::optional<uint64_t> type;
248+
249+
protected:
250+
virtual inline constexpr std::string_view get_inject_type_string() const = 0;
251+
};
252+
253+
class ClearReadErrorInjectOp : public ClearErrorInjectOp<OpType::ClearReadErrorInject>
254+
{
255+
public:
256+
ClearReadErrorInjectOp(int shard, const std::optional<uint64_t>& type);
257+
258+
static std::unique_ptr<ClearReadErrorInjectOp> generate(int shard,
259+
const std::optional<uint64_t>& type);
260+
261+
protected:
262+
inline constexpr std::string_view get_inject_type_string() const override
263+
{ return "read"; }
264+
};
265+
266+
class ClearWriteErrorInjectOp : public ClearErrorInjectOp<OpType::ClearWriteErrorInject>
267+
{
268+
public:
269+
ClearWriteErrorInjectOp(int shard, const std::optional<uint64_t>& type);
270+
271+
static std::unique_ptr<ClearWriteErrorInjectOp> generate(int shard,
272+
const std::optional<uint64_t>& type);
273+
274+
protected:
275+
inline constexpr std::string_view get_inject_type_string() const override
276+
{ return "write"; }
277+
};
148278
}
149279
}

0 commit comments

Comments
 (0)