Skip to content

Commit cd51f8a

Browse files
akashbc1989akashbc1989
authored
Merge pull request #266 from openebs/disk_detach
feat: adding disk utils for I/O error and I/O timeout
2 parents 330213b + f674364 commit cd51f8a

File tree

4 files changed

+575
-81
lines changed

4 files changed

+575
-81
lines changed

‎common/e2e_agent/client.go‎

Lines changed: 189 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ import (
77
"fmt"
88
"io"
99
"net/http"
10+
"strconv"
11+
"strings"
1012
"time"
1113

1214
"github.com/openebs/openebs-e2e/common"
@@ -122,6 +124,11 @@ type KernelModule struct {
122124
PersistentPath string `json:"persistentPath"`
123125
}
124126

127+
type DmDevice struct {
128+
Device string `json:"device"`
129+
Sectors uint64 `json:"sectors"`
130+
}
131+
125132
func sendRequest(reqType, url string, data interface{}) error {
126133
_, err := sendRequestGetResponse(reqType, url, data, true)
127134
return err
@@ -230,24 +237,68 @@ func DiskPartition(serverAddr string, cmd string) error {
230237
}
231238

232239
// CreateFaultyDevice creates a device which returns an error on write IOs
233-
func CreateFaultyDevice(serverAddr, device, table string) error {
240+
func CreateFaultyDevice(serverAddr, device, table string) (string, error) {
234241
url := "http://" + getAgentAddress(serverAddr) + "/createFaultyDevice"
242+
235243
data := Device{
236244
Device: device,
237245
Table: table,
238246
}
247+
239248
logf.Log.Info("Executing createFaultyDevice", "addr", serverAddr, "data", data)
240-
return sendRequest("POST", url, data)
249+
250+
// Send request and get wrapped response
251+
result, err := sendRequestGetResponse("POST", url, data, true)
252+
if err != nil {
253+
return result, fmt.Errorf("failed to send request: %v", err)
254+
}
255+
256+
// Unwrap response
257+
out, errCode, err := UnwrapResult(result)
258+
if err != nil {
259+
return out, fmt.Errorf("unwrap failed: %v", err)
260+
}
261+
262+
// Check agent error code
263+
if errCode != ErrNone {
264+
return out, fmt.Errorf(
265+
"createFaultyDevice failed: errCode=%d output=%s",
266+
errCode, out,
267+
)
268+
}
269+
270+
logf.Log.Info("createFaultyDevice succeeded", "output", out)
271+
272+
return out, nil
241273
}
242274

243275
// DeleteFaultyDevice deletes a device which returns an error on write IOs
244-
func DeleteFaultyDevice(serverAddr, device string) error {
276+
func DeleteFaultyDevice(serverAddr, device string) (string, error) {
245277
url := "http://" + getAgentAddress(serverAddr) + "/deleteFaultyDevice"
278+
246279
data := Device{
247280
Device: device,
248281
}
282+
249283
logf.Log.Info("Executing deleteFaultyDevice", "addr", serverAddr, "data", data)
250-
return sendRequest("POST", url, data)
284+
285+
result, err := sendRequestGetResponse("POST", url, data, true)
286+
if err != nil {
287+
return result, fmt.Errorf("failed to send request: %v", err)
288+
}
289+
290+
out, errCode, err := UnwrapResult(result)
291+
if err != nil {
292+
return out, fmt.Errorf("unwrap failed: %v", err)
293+
}
294+
295+
if errCode != ErrNone {
296+
return out, fmt.Errorf("deleteFaultyDevice failed: errCode=%d output=%s", errCode, out)
297+
}
298+
299+
logf.Log.Info("deleteFaultyDevice succeeded", "output", out)
300+
301+
return out, nil
251302
}
252303

253304
// ControlDevice sets the specified to the specified state
@@ -453,6 +504,7 @@ func FsUnfreezeDevice(serverAddr string, devicePath string) (string, error) {
453504
// ListDevice list device
454505
func ListDevice(serverAddr string) (string, error) {
455506
logf.Log.Info("Executing listdevice", "addr", serverAddr)
507+
logf.Log.Info("Executing getaddress", "addr", getAgentAddress(serverAddr))
456508
url := "http://" + getAgentAddress(serverAddr) + "/listdevice"
457509
return sendRequestGetResponse("POST", url, nil, false)
458510
}
@@ -1600,3 +1652,136 @@ func IsHugePagesPersistent(serverAddr string) (bool, error) {
16001652
logf.Log.Info("IsHugePagesPersistent succeeded", "output", out)
16011653
return out == HugePageCount, err
16021654
}
1655+
1656+
func GetDeviceSizeInSectors(nodeAddr, device string) (uint64, error) {
1657+
url := "http://" + getAgentAddress(nodeAddr) + "/dm/getDeviceSectors"
1658+
1659+
resp, err := sendRequestGetResponse(
1660+
"POST",
1661+
url,
1662+
DmDevice{Device: device},
1663+
false,
1664+
)
1665+
if err != nil {
1666+
return 0, err
1667+
}
1668+
1669+
sectors, err := strconv.ParseUint(strings.TrimSpace(resp), 10, 64)
1670+
if err != nil {
1671+
return 0, fmt.Errorf("failed to parse sectors %q: %w", resp, err)
1672+
}
1673+
1674+
return sectors, nil
1675+
}
1676+
1677+
func SetupTimeoutDevice(node, disk string) (string, error) {
1678+
sectors, err := GetDeviceSizeInSectors(node, disk)
1679+
if err != nil {
1680+
return "", err
1681+
}
1682+
1683+
url := "http://" + getAgentAddress(node) + "/dm/createPassThrough"
1684+
1685+
data := DmDevice{
1686+
Device: disk,
1687+
Sectors: sectors,
1688+
}
1689+
1690+
logf.Log.Info("Executing SetupTimeoutDevice", "addr", node, "data", data)
1691+
1692+
result, err := sendRequestGetResponse("POST", url, data, true)
1693+
if err != nil {
1694+
return result, fmt.Errorf("request failed: %v", err)
1695+
}
1696+
1697+
out, code, err := UnwrapResult(result)
1698+
if err != nil {
1699+
return out, fmt.Errorf("unwrap failed: %v", err)
1700+
}
1701+
1702+
if code != ErrNone {
1703+
return out, fmt.Errorf("setup timeout failed: errCode=%d output=%s", code, out)
1704+
}
1705+
1706+
logf.Log.Info("SetupTimeoutDevice succeeded", "output", out)
1707+
1708+
return out, nil
1709+
}
1710+
1711+
func InjectIOTimeout(nodeAddr, disk string) (string, error) {
1712+
url := "http://" + getAgentAddress(nodeAddr) + "/dm/suspend"
1713+
1714+
data := DmDevice{Device: disk}
1715+
1716+
logf.Log.Info("Executing InjectIOTimeout", "addr", nodeAddr, "data", data)
1717+
1718+
result, err := sendRequestGetResponse("POST", url, data, true)
1719+
if err != nil {
1720+
return result, fmt.Errorf("request failed: %v", err)
1721+
}
1722+
1723+
out, code, err := UnwrapResult(result)
1724+
if err != nil {
1725+
return out, fmt.Errorf("unwrap failed: %v", err)
1726+
}
1727+
1728+
if code != ErrNone {
1729+
return out, fmt.Errorf("inject timeout failed: errCode=%d output=%s", code, out)
1730+
}
1731+
1732+
logf.Log.Info("InjectIOTimeout succeeded", "output", out)
1733+
1734+
return out, nil
1735+
}
1736+
1737+
func RecoverIOTimeout(nodeAddr, disk string) (string, error) {
1738+
url := "http://" + getAgentAddress(nodeAddr) + "/dm/resume"
1739+
1740+
data := DmDevice{Device: disk}
1741+
1742+
logf.Log.Info("Executing RecoverIOTimeout", "addr", nodeAddr, "data", data)
1743+
1744+
result, err := sendRequestGetResponse("POST", url, data, true)
1745+
if err != nil {
1746+
return result, fmt.Errorf("request failed: %v", err)
1747+
}
1748+
1749+
out, code, err := UnwrapResult(result)
1750+
if err != nil {
1751+
return out, fmt.Errorf("unwrap failed: %v", err)
1752+
}
1753+
1754+
if code != ErrNone {
1755+
return out, fmt.Errorf("recover timeout failed: errCode=%d output=%s", code, out)
1756+
}
1757+
1758+
logf.Log.Info("RecoverIOTimeout succeeded", "output", out)
1759+
1760+
return out, nil
1761+
}
1762+
1763+
func CleanupTimeoutDevice(nodeAddr, disk string) (string, error) {
1764+
url := "http://" + getAgentAddress(nodeAddr) + "/dm/remove"
1765+
1766+
data := DmDevice{Device: disk}
1767+
1768+
logf.Log.Info("Executing CleanupTimeoutDevice", "addr", nodeAddr, "data", data)
1769+
1770+
result, err := sendRequestGetResponse("POST", url, data, true)
1771+
if err != nil {
1772+
return result, fmt.Errorf("request failed: %v", err)
1773+
}
1774+
1775+
out, code, err := UnwrapResult(result)
1776+
if err != nil {
1777+
return out, fmt.Errorf("unwrap failed: %v", err)
1778+
}
1779+
1780+
if code != ErrNone {
1781+
return out, fmt.Errorf("cleanup timeout failed: errCode=%d output=%s", code, out)
1782+
}
1783+
1784+
logf.Log.Info("CleanupTimeoutDevice succeeded", "output", out)
1785+
1786+
return out, nil
1787+
}

‎common/k8stest/util_disk_detach.go‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
package k8stest
2+
3+
import (
4+
"fmt"
5+
6+
e2eagent "github.com/openebs/openebs-e2e/common/e2e_agent"
7+
)
8+
9+
// InjectIOError injects IO error on the given pool device present on the given node.
10+
func InjectIOError(nodeAddr string, poolDevice string) (string, error) {
11+
table := fmt.Sprintf(
12+
"0 50000 linear %s 0\n"+
13+
"50000 5000000 error\n"+
14+
"5050000 8143000 linear %s 5050000",
15+
poolDevice,
16+
poolDevice,
17+
)
18+
19+
out, err := e2eagent.CreateFaultyDevice(nodeAddr, poolDevice, table)
20+
if err != nil {
21+
return out, fmt.Errorf(
22+
"failed to inject IO error on %s: %w (output=%s)",
23+
poolDevice,
24+
err,
25+
out,
26+
)
27+
}
28+
29+
return out, nil
30+
}
31+
32+
// RecoverIOError recovers from IO error on the given pool device present on the given node.
33+
func RecoverIOError(nodeAddr string, poolDevice string) (string, error) {
34+
out, err := e2eagent.DeleteFaultyDevice(nodeAddr, poolDevice)
35+
if err != nil {
36+
return "", fmt.Errorf(
37+
"failed to recover IO error on %s: %w (output=%s)",
38+
poolDevice,
39+
err,
40+
out,
41+
)
42+
}
43+
44+
return out, nil
45+
}
46+
47+
// SetupTimeoutDevice sets up a timeout device on the given pool device present on the given node.
48+
func SetupTimeoutDevice(nodeAddr, poolDevice string) (string, error) {
49+
resp, err := e2eagent.SetupTimeoutDevice(nodeAddr, poolDevice)
50+
if err != nil {
51+
return resp, fmt.Errorf(
52+
"setup timeout device failed on %s: %w",
53+
poolDevice, err,
54+
)
55+
}
56+
return resp, nil
57+
}
58+
59+
// InjectIOTimeout injects IO timeout on the given pool device present on the given node.
60+
func InjectIOTimeout(nodeAddr, poolDevice string) (string, error) {
61+
resp, err := e2eagent.InjectIOTimeout(nodeAddr, poolDevice)
62+
if err != nil {
63+
return resp, fmt.Errorf(
64+
"inject IO timeout failed on %s: %w",
65+
poolDevice, err,
66+
)
67+
}
68+
return resp, nil
69+
}
70+
71+
// RecoverIOTimeout recovers from IO timeout on the given pool device present on the given node.
72+
func RecoverIOTimeout(nodeAddr, poolDevice string) (string, error) {
73+
resp, err := e2eagent.RecoverIOTimeout(nodeAddr, poolDevice)
74+
if err != nil {
75+
return resp, fmt.Errorf(
76+
"recover IO timeout failed on %s: %w",
77+
poolDevice, err,
78+
)
79+
}
80+
return resp, nil
81+
}
82+
83+
// CleanupTimeoutDevice cleans up the timeout device on the given pool device present on the given node.
84+
func CleanupTimeoutDevice(nodeAddr, poolDevice string) (string, error) {
85+
resp, err := e2eagent.CleanupTimeoutDevice(nodeAddr, poolDevice)
86+
if err != nil {
87+
return resp, fmt.Errorf(
88+
"cleanup timeout device failed on %s: %w",
89+
poolDevice, err,
90+
)
91+
}
92+
return resp, nil
93+
}

‎tools/e2e-agent/build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
# as long as we do not make breaking changes.
1010
set -e
1111
IMAGE="openebs/e2e-agent"
12-
TAG="v3.1.3"
12+
TAG="v3.1.4"
1313
registry=""
1414
tag_as_latest=""
1515

0 commit comments

Comments
 (0)