Skip to content

Commit 3dedf4b

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#73525 from cofyc/fix71237-intree
Implement localDriver with various local volume types
2 parents 2d95638 + aa87e4b commit 3dedf4b

File tree

8 files changed

+696
-29
lines changed

8 files changed

+696
-29
lines changed

‎test/e2e/storage/drivers/in_tree.go‎

Lines changed: 202 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import (
3939
"fmt"
4040
"math/rand"
4141
"os/exec"
42+
"strconv"
4243
"strings"
4344
"time"
4445

@@ -120,7 +121,7 @@ func (n *nfsDriver) GetVolumeSource(readOnly bool, fsType string, testResource i
120121
}
121122
}
122123

123-
func (n *nfsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource {
124+
func (n *nfsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
124125
ntr, ok := testResource.(*nfsTestResource)
125126
Expect(ok).To(BeTrue(), "Failed to cast test resource to NFS Test Resource")
126127
return &v1.PersistentVolumeSource{
@@ -129,7 +130,7 @@ func (n *nfsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, test
129130
Path: "/",
130131
ReadOnly: readOnly,
131132
},
132-
}
133+
}, nil
133134
}
134135

135136
func (n *nfsDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass {
@@ -276,7 +277,7 @@ func (g *glusterFSDriver) GetVolumeSource(readOnly bool, fsType string, testReso
276277
}
277278
}
278279

279-
func (g *glusterFSDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource {
280+
func (g *glusterFSDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
280281
gtr, ok := testResource.(*glusterTestResource)
281282
Expect(ok).To(BeTrue(), "Failed to cast test resource to Gluster Test Resource")
282283

@@ -288,7 +289,7 @@ func (g *glusterFSDriver) GetPersistentVolumeSource(readOnly bool, fsType string
288289
Path: "test_vol",
289290
ReadOnly: readOnly,
290291
},
291-
}
292+
}, nil
292293
}
293294

294295
func (g *glusterFSDriver) CreateDriver() {
@@ -402,7 +403,7 @@ func (i *iSCSIDriver) GetVolumeSource(readOnly bool, fsType string, testResource
402403
return &volSource
403404
}
404405

405-
func (i *iSCSIDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource {
406+
func (i *iSCSIDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
406407
itr, ok := testResource.(*iSCSITestResource)
407408
Expect(ok).To(BeTrue(), "Failed to cast test resource to iSCSI Test Resource")
408409

@@ -417,7 +418,7 @@ func (i *iSCSIDriver) GetPersistentVolumeSource(readOnly bool, fsType string, te
417418
if fsType != "" {
418419
pvSource.ISCSI.FSType = fsType
419420
}
420-
return &pvSource
421+
return &pvSource, nil
421422
}
422423

423424
func (i *iSCSIDriver) CreateDriver() {
@@ -519,7 +520,7 @@ func (r *rbdDriver) GetVolumeSource(readOnly bool, fsType string, testResource i
519520
return &volSource
520521
}
521522

522-
func (r *rbdDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource {
523+
func (r *rbdDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
523524
f := r.driverInfo.Config.Framework
524525
ns := f.Namespace
525526

@@ -542,7 +543,7 @@ func (r *rbdDriver) GetPersistentVolumeSource(readOnly bool, fsType string, test
542543
if fsType != "" {
543544
pvSource.RBD.FSType = fsType
544545
}
545-
return &pvSource
546+
return &pvSource, nil
546547
}
547548

548549
func (r *rbdDriver) CreateDriver() {
@@ -637,7 +638,7 @@ func (c *cephFSDriver) GetVolumeSource(readOnly bool, fsType string, testResourc
637638
}
638639
}
639640

640-
func (c *cephFSDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource {
641+
func (c *cephFSDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
641642
f := c.driverInfo.Config.Framework
642643
ns := f.Namespace
643644

@@ -654,7 +655,7 @@ func (c *cephFSDriver) GetPersistentVolumeSource(readOnly bool, fsType string, t
654655
},
655656
ReadOnly: readOnly,
656657
},
657-
}
658+
}, nil
658659
}
659660

660661
func (c *cephFSDriver) CreateDriver() {
@@ -1026,7 +1027,7 @@ func (c *cinderDriver) GetVolumeSource(readOnly bool, fsType string, testResourc
10261027
return &volSource
10271028
}
10281029

1029-
func (c *cinderDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource {
1030+
func (c *cinderDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
10301031
ctr, ok := testResource.(*cinderTestResource)
10311032
Expect(ok).To(BeTrue(), "Failed to cast test resource to Cinder Test Resource")
10321033

@@ -1039,7 +1040,7 @@ func (c *cinderDriver) GetPersistentVolumeSource(readOnly bool, fsType string, t
10391040
if fsType != "" {
10401041
pvSource.Cinder.FSType = fsType
10411042
}
1042-
return &pvSource
1043+
return &pvSource, nil
10431044
}
10441045

10451046
func (c *cinderDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass {
@@ -1192,7 +1193,7 @@ func (g *gcePdDriver) GetVolumeSource(readOnly bool, fsType string, testResource
11921193
return &volSource
11931194
}
11941195

1195-
func (g *gcePdDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource {
1196+
func (g *gcePdDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
11961197
gtr, ok := testResource.(*gcePdTestResource)
11971198
Expect(ok).To(BeTrue(), "Failed to cast test resource to GCE PD Test Resource")
11981199
pvSource := v1.PersistentVolumeSource{
@@ -1204,7 +1205,7 @@ func (g *gcePdDriver) GetPersistentVolumeSource(readOnly bool, fsType string, te
12041205
if fsType != "" {
12051206
pvSource.GCEPersistentDisk.FSType = fsType
12061207
}
1207-
return &pvSource
1208+
return &pvSource, nil
12081209
}
12091210

12101211
func (g *gcePdDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass {
@@ -1315,14 +1316,14 @@ func (v *vSphereDriver) GetVolumeSource(readOnly bool, fsType string, testResour
13151316
return &volSource
13161317
}
13171318

1318-
func (v *vSphereDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource {
1319+
func (v *vSphereDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
13191320
vtr, ok := testResource.(*vSphereTestResource)
13201321
Expect(ok).To(BeTrue(), "Failed to cast test resource to vSphere Test Resource")
13211322

13221323
// vSphere driver doesn't seem to support readOnly volume
13231324
// TODO: check if it is correct
13241325
if readOnly {
1325-
return nil
1326+
return nil, nil
13261327
}
13271328
pvSource := v1.PersistentVolumeSource{
13281329
VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
@@ -1332,7 +1333,7 @@ func (v *vSphereDriver) GetPersistentVolumeSource(readOnly bool, fsType string,
13321333
if fsType != "" {
13331334
pvSource.VsphereVolume.FSType = fsType
13341335
}
1335-
return &pvSource
1336+
return &pvSource, nil
13361337
}
13371338

13381339
func (v *vSphereDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass {
@@ -1440,7 +1441,7 @@ func (a *azureDriver) GetVolumeSource(readOnly bool, fsType string, testResource
14401441
return &volSource
14411442
}
14421443

1443-
func (a *azureDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource {
1444+
func (a *azureDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
14441445
atr, ok := testResource.(*azureTestResource)
14451446
Expect(ok).To(BeTrue(), "Failed to cast test resource to Azure Test Resource")
14461447

@@ -1456,7 +1457,7 @@ func (a *azureDriver) GetPersistentVolumeSource(readOnly bool, fsType string, te
14561457
if fsType != "" {
14571458
pvSource.AzureDisk.FSType = &fsType
14581459
}
1459-
return &pvSource
1460+
return &pvSource, nil
14601461
}
14611462

14621463
func (a *azureDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass {
@@ -1558,7 +1559,7 @@ func (a *awsDriver) GetVolumeSource(readOnly bool, fsType string, testResource i
15581559
return &volSource
15591560
}
15601561
1561-
func (a *awsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource {
1562+
func (a *awsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
15621563
pvSource := v1.PersistentVolumeSource{
15631564
AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
15641565
VolumeID: a.volumeName,
@@ -1607,3 +1608,184 @@ func (a *awsDriver) DeleteVolume(volType testpatterns.TestVolType, testResource
16071608
framework.DeletePDWithRetry(a.volumeName)
16081609
}
16091610
*/
1611+
1612+
// local
1613+
type localDriver struct {
1614+
driverInfo testsuites.DriverInfo
1615+
node *v1.Node
1616+
hostExec utils.HostExec
1617+
// volumeType represents local volume type we are testing, e.g. tmpfs,
1618+
// directory, block device.
1619+
volumeType utils.LocalVolumeType
1620+
ltrMgr utils.LocalTestResourceManager
1621+
}
1622+
1623+
var (
1624+
// capabilities
1625+
defaultLocalVolumeCapabilities = map[testsuites.Capability]bool{
1626+
testsuites.CapPersistence: true,
1627+
testsuites.CapFsGroup: true,
1628+
testsuites.CapBlock: false,
1629+
testsuites.CapExec: true,
1630+
}
1631+
localVolumeCapabitilies = map[utils.LocalVolumeType]map[testsuites.Capability]bool{
1632+
utils.LocalVolumeBlock: {
1633+
testsuites.CapPersistence: true,
1634+
testsuites.CapFsGroup: true,
1635+
testsuites.CapBlock: true,
1636+
testsuites.CapExec: true,
1637+
},
1638+
}
1639+
// fstype
1640+
defaultLocalVolumeSupportedFsTypes = sets.NewString("")
1641+
localVolumeSupportedFsTypes = map[utils.LocalVolumeType]sets.String{
1642+
utils.LocalVolumeBlock: sets.NewString(
1643+
"", // Default fsType
1644+
"ext2",
1645+
"ext3",
1646+
"ext4",
1647+
"xfs",
1648+
),
1649+
}
1650+
// max file size
1651+
defaultLocalVolumeMaxFileSize = testpatterns.FileSizeSmall
1652+
localVolumeMaxFileSizes = map[utils.LocalVolumeType]int64{}
1653+
)
1654+
1655+
var _ testsuites.TestDriver = &localDriver{}
1656+
var _ testsuites.PreprovisionedVolumeTestDriver = &localDriver{}
1657+
var _ testsuites.PreprovisionedPVTestDriver = &localDriver{}
1658+
1659+
func InitLocalDriverWithVolumeType(volumeType utils.LocalVolumeType) func(config testsuites.TestConfig) testsuites.TestDriver {
1660+
maxFileSize := defaultLocalVolumeMaxFileSize
1661+
if maxFileSizeByVolType, ok := localVolumeMaxFileSizes[volumeType]; ok {
1662+
maxFileSize = maxFileSizeByVolType
1663+
}
1664+
supportedFsTypes := defaultLocalVolumeSupportedFsTypes
1665+
if supportedFsTypesByType, ok := localVolumeSupportedFsTypes[volumeType]; ok {
1666+
supportedFsTypes = supportedFsTypesByType
1667+
}
1668+
capabilities := defaultLocalVolumeCapabilities
1669+
if capabilitiesByType, ok := localVolumeCapabitilies[volumeType]; ok {
1670+
capabilities = capabilitiesByType
1671+
}
1672+
return func(config testsuites.TestConfig) testsuites.TestDriver {
1673+
hostExec := utils.NewHostExec(config.Framework)
1674+
// custom tag to distinguish from tests of other volume types
1675+
featureTag := fmt.Sprintf("[LocalVolumeType: %s]", volumeType)
1676+
// For GCE Local SSD volumes, we must run serially
1677+
if volumeType == utils.LocalVolumeGCELocalSSD {
1678+
featureTag += " [Serial]"
1679+
}
1680+
return &localDriver{
1681+
driverInfo: testsuites.DriverInfo{
1682+
Name: "local",
1683+
FeatureTag: featureTag,
1684+
MaxFileSize: maxFileSize,
1685+
SupportedFsType: supportedFsTypes,
1686+
Capabilities: capabilities,
1687+
Config: config,
1688+
},
1689+
hostExec: hostExec,
1690+
volumeType: volumeType,
1691+
ltrMgr: utils.NewLocalResourceManager("local-driver", hostExec, "/tmp"),
1692+
}
1693+
}
1694+
}
1695+
1696+
func (l *localDriver) GetDriverInfo() *testsuites.DriverInfo {
1697+
return &l.driverInfo
1698+
}
1699+
1700+
func (l *localDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
1701+
if l.volumeType == utils.LocalVolumeGCELocalSSD {
1702+
ssdInterface := "scsi"
1703+
filesystemType := "fs"
1704+
ssdCmd := fmt.Sprintf("ls -1 /mnt/disks/by-uuid/google-local-ssds-%s-%s/ | wc -l", ssdInterface, filesystemType)
1705+
res, err := l.hostExec.IssueCommandWithResult(ssdCmd, l.node)
1706+
Expect(err).NotTo(HaveOccurred())
1707+
num, err := strconv.Atoi(strings.TrimSpace(res))
1708+
Expect(err).NotTo(HaveOccurred())
1709+
if num < 1 {
1710+
framework.Skipf("Requires at least 1 %s %s localSSD ", ssdInterface, filesystemType)
1711+
}
1712+
}
1713+
}
1714+
1715+
func (l *localDriver) CreateDriver() {
1716+
// choose a randome node to test against
1717+
l.node = l.randomNode()
1718+
}
1719+
1720+
func (l *localDriver) CleanupDriver() {
1721+
l.hostExec.Cleanup()
1722+
}
1723+
1724+
func (l *localDriver) randomNode() *v1.Node {
1725+
f := l.driverInfo.Config.Framework
1726+
nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
1727+
node := nodes.Items[rand.Intn(len(nodes.Items))]
1728+
return &node
1729+
}
1730+
1731+
func (l *localDriver) CreateVolume(volType testpatterns.TestVolType) interface{} {
1732+
switch volType {
1733+
case testpatterns.PreprovisionedPV:
1734+
node := l.node
1735+
// assign this to schedule pod on this node
1736+
l.driverInfo.Config.ClientNodeName = node.Name
1737+
return l.ltrMgr.Create(node, l.volumeType, nil)
1738+
default:
1739+
framework.Failf("Unsupported volType: %v is specified", volType)
1740+
}
1741+
return nil
1742+
}
1743+
1744+
func (l *localDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) {
1745+
ltr, ok := testResource.(*utils.LocalTestResource)
1746+
Expect(ok).To(BeTrue(), "Failed to cast test resource to local Test Resource")
1747+
switch volType {
1748+
case testpatterns.PreprovisionedPV:
1749+
l.ltrMgr.Remove(ltr)
1750+
default:
1751+
framework.Failf("Unsupported volType: %v is specified", volType)
1752+
}
1753+
return
1754+
}
1755+
1756+
func (l *localDriver) nodeAffinityForNode(node *v1.Node) *v1.VolumeNodeAffinity {
1757+
nodeKey := "kubernetes.io/hostname"
1758+
if node.Labels == nil {
1759+
framework.Failf("Node does not have labels")
1760+
}
1761+
nodeValue, found := node.Labels[nodeKey]
1762+
if !found {
1763+
framework.Failf("Node does not have required label %q", nodeKey)
1764+
}
1765+
return &v1.VolumeNodeAffinity{
1766+
Required: &v1.NodeSelector{
1767+
NodeSelectorTerms: []v1.NodeSelectorTerm{
1768+
{
1769+
MatchExpressions: []v1.NodeSelectorRequirement{
1770+
{
1771+
Key: nodeKey,
1772+
Operator: v1.NodeSelectorOpIn,
1773+
Values: []string{nodeValue},
1774+
},
1775+
},
1776+
},
1777+
},
1778+
},
1779+
}
1780+
}
1781+
1782+
func (l *localDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
1783+
ltr, ok := testResource.(*utils.LocalTestResource)
1784+
Expect(ok).To(BeTrue(), "Failed to cast test resource to local Test Resource")
1785+
return &v1.PersistentVolumeSource{
1786+
Local: &v1.LocalVolumeSource{
1787+
Path: ltr.Path,
1788+
FSType: &fsType,
1789+
},
1790+
}, l.nodeAffinityForNode(ltr.Node)
1791+
}

‎test/e2e/storage/in_tree_volumes.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,14 @@ var testDrivers = []func(config testsuites.TestConfig) testsuites.TestDriver{
4040
drivers.InitVSphereDriver,
4141
drivers.InitAzureDriver,
4242
drivers.InitAwsDriver,
43+
drivers.InitLocalDriverWithVolumeType(utils.LocalVolumeDirectory),
44+
drivers.InitLocalDriverWithVolumeType(utils.LocalVolumeDirectoryLink),
45+
drivers.InitLocalDriverWithVolumeType(utils.LocalVolumeDirectoryBindMounted),
46+
drivers.InitLocalDriverWithVolumeType(utils.LocalVolumeDirectoryLinkBindMounted),
47+
drivers.InitLocalDriverWithVolumeType(utils.LocalVolumeTmpfs),
48+
drivers.InitLocalDriverWithVolumeType(utils.LocalVolumeBlock),
49+
drivers.InitLocalDriverWithVolumeType(utils.LocalVolumeBlockFS),
50+
drivers.InitLocalDriverWithVolumeType(utils.LocalVolumeGCELocalSSD),
4351
}
4452

4553
// List of testSuites to be executed in below loop

0 commit comments

Comments
 (0)