Skip to content

Commit 4258bcf

Browse files
helsaawyhelsaawy
committed
add multi-pod tmpfs support
Signed-off-by: Hamza El-Saawy <[email protected]>
1 parent b0a9d5f commit 4258bcf

File tree

3 files changed

+90
-36
lines changed

3 files changed

+90
-36
lines changed

‎internal/guest/runtime/hcsv2/uvm.go‎

Lines changed: 38 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -322,8 +322,7 @@ func setupSandboxMountsPath(id string) (err error) {
322322
return storage.MountRShared(mountPath)
323323
}
324324

325-
func setupSandboxTmpfsMountsPath(id string) error {
326-
var err error
325+
func setupSandboxTmpfsMountsPath(id string) (err error) {
327326
tmpfsDir := specGuest.SandboxTmpfsMountsDir(id)
328327
if err := os.MkdirAll(tmpfsDir, 0755); err != nil {
329328
return errors.Wrapf(err, "failed to create sandbox tmpfs mounts dir in sandbox %v", id)
@@ -473,16 +472,13 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM
473472

474473
if isVirtualPod {
475474
// For virtual pods, create virtual pod specific paths
476-
err = setupVirtualPodMountsPath(virtualPodID, id)
477-
if err != nil {
475+
if err = setupVirtualPodMountsPath(virtualPodID); err != nil {
478476
return nil, err
479477
}
480-
// Create hugepages path for virtual pod
481-
mountPath := specGuest.VirtualPodHugePagesMountsDir(virtualPodID)
482-
if err := os.MkdirAll(mountPath, 0755); err != nil {
483-
return nil, errors.Wrapf(err, "failed to create virtual pod hugepage mounts dir %v", virtualPodID)
478+
if err = setupVirtualPodTmpfsMountsPath(virtualPodID); err != nil {
479+
return nil, err
484480
}
485-
if err := storage.MountRShared(mountPath); err != nil {
481+
if err = setupVirtualPodHugePageMountsPath(virtualPodID); err != nil {
486482
return nil, err
487483
}
488484
} else {
@@ -1556,11 +1552,11 @@ func (h *Host) cleanupVirtualPod(virtualSandboxID string) {
15561552
}
15571553

15581554
// setupVirtualPodMountsPath creates mount directories for virtual pods
1559-
func setupVirtualPodMountsPath(virtualSandboxID, masterSandboxID string) (err error) {
1555+
func setupVirtualPodMountsPath(virtualSandboxID string) (err error) {
15601556
// Create virtual pod specific mount path using the new path generation functions
15611557
mountPath := specGuest.VirtualPodMountsDir(virtualSandboxID)
15621558
if err := os.MkdirAll(mountPath, 0755); err != nil {
1563-
return errors.Wrapf(err, "failed to create virtual pod sandboxMounts dir %v", virtualSandboxID)
1559+
return errors.Wrapf(err, "failed to create virtual pod mounts dir in sandbox %v", virtualSandboxID)
15641560
}
15651561
defer func() {
15661562
if err != nil {
@@ -1570,3 +1566,34 @@ func setupVirtualPodMountsPath(virtualSandboxID, masterSandboxID string) (err er
15701566

15711567
return storage.MountRShared(mountPath)
15721568
}
1569+
1570+
func setupVirtualPodTmpfsMountsPath(virtualSandboxID string) (err error) {
1571+
tmpfsDir := specGuest.VirtualPodTmpfsMountsDir(virtualSandboxID)
1572+
if err := os.MkdirAll(tmpfsDir, 0755); err != nil {
1573+
return errors.Wrapf(err, "failed to create virtual pod tmpfs mounts dir in sandbox %v", virtualSandboxID)
1574+
}
1575+
1576+
defer func() {
1577+
if err != nil {
1578+
_ = os.RemoveAll(tmpfsDir)
1579+
}
1580+
}()
1581+
1582+
// mount a tmpfs at the tmpfsDir
1583+
// this ensures that the tmpfsDir is a mount point and not just a directory
1584+
// we don't care if it is already mounted, so ignore EBUSY
1585+
if err := unix.Mount("tmpfs", tmpfsDir, "tmpfs", 0, ""); err != nil && !errors.Is(err, unix.EBUSY) {
1586+
return errors.Wrapf(err, "failed to mount tmpfs at %s", tmpfsDir)
1587+
}
1588+
1589+
return storage.MountRShared(tmpfsDir)
1590+
}
1591+
1592+
func setupVirtualPodHugePageMountsPath(virtualSandboxID string) error {
1593+
mountPath := specGuest.VirtualPodHugePagesMountsDir(virtualSandboxID)
1594+
if err := os.MkdirAll(mountPath, 0755); err != nil {
1595+
return errors.Wrapf(err, "failed to create virtual pod hugepage mounts dir %v", virtualSandboxID)
1596+
}
1597+
1598+
return storage.MountRShared(mountPath)
1599+
}

‎internal/guest/runtime/hcsv2/workload_container.go‎

Lines changed: 29 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -45,13 +45,15 @@ func updateSandboxMounts(sbid string, spec *oci.Spec) error {
4545
var sandboxSource string
4646
// if using `sandbox-tmp://` prefix, we mount a tmpfs in sandboxTmpfsMountsDir
4747
if strings.HasPrefix(m.Source, guestpath.SandboxTmpfsMountPrefix) {
48-
sandboxSource = specGuest.SandboxTmpfsMountSource(sbid, m.Source)
48+
// Use virtual pod aware mount source
49+
sandboxSource = specGuest.VirtualPodAwareSandboxTmpfsMountSource(sbid, virtualSandboxID, m.Source)
50+
expectedMountsDir := specGuest.VirtualPodAwareSandboxTmpfsMountsDir(sbid, virtualSandboxID)
51+
4952
// filepath.Join cleans the resulting path before returning, so it would resolve the relative path if one was given.
5053
// Hence, we need to ensure that the resolved path is still under the correct directory
51-
if !strings.HasPrefix(sandboxSource, specGuest.SandboxTmpfsMountsDir(sbid)) {
52-
return errors.Errorf("mount path %v for mount %v is not within sandboxTmpfsMountsDir", sandboxSource, m.Source)
54+
if !strings.HasPrefix(sandboxSource, expectedMountsDir) {
55+
return errors.Errorf("mount path %v for mount %v is not within sandbox's tmpfs mounts dir", sandboxSource, m.Source)
5356
}
54-
5557
} else {
5658
// Use virtual pod aware mount source
5759
sandboxSource = specGuest.VirtualPodAwareSandboxMountSource(sbid, virtualSandboxID, m.Source)
@@ -81,29 +83,31 @@ func updateHugePageMounts(sbid string, spec *oci.Spec) error {
8183
virtualSandboxID := spec.Annotations[annotations.VirtualPodID]
8284

8385
for i, m := range spec.Mounts {
84-
if strings.HasPrefix(m.Source, guestpath.HugePagesMountPrefix) {
85-
// Use virtual pod aware hugepages directory
86-
mountsDir := specGuest.VirtualPodAwareHugePagesMountsDir(sbid, virtualSandboxID)
87-
subPath := strings.TrimPrefix(m.Source, guestpath.HugePagesMountPrefix)
88-
pageSize := strings.Split(subPath, string(os.PathSeparator))[0]
89-
hugePageMountSource := filepath.Join(mountsDir, subPath)
90-
91-
// filepath.Join cleans the resulting path before returning so it would resolve the relative path if one was given.
92-
// Hence, we need to ensure that the resolved path is still under the correct directory
93-
if !strings.HasPrefix(hugePageMountSource, mountsDir) {
94-
return errors.Errorf("mount path %v for mount %v is not within hugepages's mounts dir", hugePageMountSource, m.Source)
95-
}
86+
if !strings.HasPrefix(m.Source, guestpath.HugePagesMountPrefix) {
87+
continue
88+
}
89+
90+
// Use virtual pod aware hugepages directory
91+
mountsDir := specGuest.VirtualPodAwareHugePagesMountsDir(sbid, virtualSandboxID)
92+
subPath := strings.TrimPrefix(m.Source, guestpath.HugePagesMountPrefix)
93+
pageSize := strings.Split(subPath, string(os.PathSeparator))[0]
94+
hugePageMountSource := filepath.Join(mountsDir, subPath)
95+
96+
// filepath.Join cleans the resulting path before returning so it would resolve the relative path if one was given.
97+
// Hence, we need to ensure that the resolved path is still under the correct directory
98+
if !strings.HasPrefix(hugePageMountSource, mountsDir) {
99+
return errors.Errorf("mount path %v for mount %v is not within hugepages's mounts dir", hugePageMountSource, m.Source)
100+
}
96101

97-
spec.Mounts[i].Source = hugePageMountSource
102+
spec.Mounts[i].Source = hugePageMountSource
98103

99-
_, err := os.Stat(hugePageMountSource)
100-
if os.IsNotExist(err) {
101-
if err := mkdirAllModePerm(hugePageMountSource); err != nil {
102-
return err
103-
}
104-
if err := unix.Mount("none", hugePageMountSource, "hugetlbfs", 0, "pagesize="+pageSize); err != nil {
105-
return errors.Errorf("mount operation failed for %v failed with error %v", hugePageMountSource, err)
106-
}
104+
_, err := os.Stat(hugePageMountSource)
105+
if os.IsNotExist(err) {
106+
if err := mkdirAllModePerm(hugePageMountSource); err != nil {
107+
return err
108+
}
109+
if err := unix.Mount("none", hugePageMountSource, "hugetlbfs", 0, "pagesize="+pageSize); err != nil {
110+
return errors.Errorf("mount operation failed for %v failed with error %v", hugePageMountSource, err)
107111
}
108112
}
109113
}

‎internal/guest/spec/spec.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,19 @@ func SandboxTmpfsMountsDir(sandboxID string) string {
122122
return filepath.Join(SandboxRootDir(sandboxID), "sandboxTmpfsMounts")
123123
}
124124

125+
// VirtualPodTmpfsMountsDir returns virtual pod tmpfs mounts directory inside UVM/host.
126+
func VirtualPodTmpfsMountsDir(virtualSandboxID string) string {
127+
return filepath.Join(VirtualPodRootDir(virtualSandboxID), "sandboxTmpfsMounts")
128+
}
129+
130+
// VirtualPodAwareSandboxMountsDir returns the appropriate tmpfs mounts directory
131+
func VirtualPodAwareSandboxTmpfsMountsDir(sandboxID, virtualSandboxID string) string {
132+
if virtualSandboxID != "" {
133+
return VirtualPodTmpfsMountsDir(virtualSandboxID)
134+
}
135+
return SandboxTmpfsMountsDir(sandboxID)
136+
}
137+
125138
// HugePagesMountsDir returns hugepages mounts directory inside UVM.
126139
func HugePagesMountsDir(sandboxID string) string {
127140
return filepath.Join(SandboxRootDir(sandboxID), "hugepages")
@@ -164,6 +177,16 @@ func SandboxTmpfsMountSource(sandboxID, path string) string {
164177
return filepath.Join(tmpfsMountDir, subPath)
165178
}
166179

180+
// VirtualPodAwareSandboxTmpfsMountSource returns tmpfs mount source path for virtual pod aware containers
181+
func VirtualPodAwareSandboxTmpfsMountSource(sandboxID, virtualSandboxID, path string) string {
182+
if virtualSandboxID != "" {
183+
mountsDir := VirtualPodTmpfsMountsDir(virtualSandboxID)
184+
subPath := strings.TrimPrefix(path, guestpath.SandboxTmpfsMountPrefix)
185+
return filepath.Join(mountsDir, subPath)
186+
}
187+
return SandboxTmpfsMountSource(sandboxID, path)
188+
}
189+
167190
// HugePagesMountSource returns hugepages mount path inside UVM
168191
func HugePagesMountSource(sandboxID, path string) string {
169192
mountsDir := HugePagesMountsDir(sandboxID)

0 commit comments

Comments
 (0)