Skip to content

Commit 72d8ecc

Browse files
helsaawyhelsaawy
committed
Add multi-pod tmpfs support; fix lint errors
Remove unused functions `get(Sandbox|Standalone)(Hostname|Hosts|Resolv)Path` and replace them with their `VirtualPodAware` counterparts to satisfy linter. (The original functions are already replaced wholesale). Expand multi-pod functionality to include tmpfs-backed sandbox mounts. Signed-off-by: Hamza El-Saawy <[email protected]>
1 parent b0a9d5f commit 72d8ecc

File tree

6 files changed

+120
-94
lines changed

6 files changed

+120
-94
lines changed

‎internal/guest/runtime/hcsv2/container.go‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -201,14 +201,17 @@ func (c *Container) Delete(ctx context.Context) error {
201201
}
202202

203203
// remove user mounts in sandbox container - use virtual pod aware paths
204-
mountsDir := specGuest.VirtualPodAwareSandboxMountsDir(c.id, virtualSandboxID)
205-
if err := storage.UnmountAllInPath(ctx, mountsDir, true); err != nil {
204+
if err := storage.UnmountAllInPath(ctx, specGuest.VirtualPodAwareSandboxMountsDir(c.id, virtualSandboxID), true); err != nil {
206205
entity.WithError(err).Error("failed to unmount sandbox mounts")
207206
}
208207

208+
// remove user mounts in tmpfs sandbox container - use virtual pod aware paths
209+
if err := storage.UnmountAllInPath(ctx, specGuest.VirtualPodAwareSandboxTmpfsMountsDir(c.id, virtualSandboxID), true); err != nil {
210+
entity.WithError(err).Error("failed to unmount tmpfs sandbox mounts")
211+
}
212+
209213
// remove hugepages mounts in sandbox container - use virtual pod aware paths
210-
hugePagesDir := specGuest.VirtualPodAwareHugePagesMountsDir(c.id, virtualSandboxID)
211-
if err := storage.UnmountAllInPath(ctx, hugePagesDir, true); err != nil {
214+
if err := storage.UnmountAllInPath(ctx, specGuest.VirtualPodAwareHugePagesMountsDir(c.id, virtualSandboxID), true); err != nil {
212215
entity.WithError(err).Error("failed to unmount hugepages mounts")
213216
}
214217
}

‎internal/guest/runtime/hcsv2/sandbox_container.go‎

Lines changed: 6 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -20,27 +20,15 @@ import (
2020
"github.com/Microsoft/hcsshim/pkg/annotations"
2121
)
2222

23-
func getSandboxHostnamePath(id string) string {
24-
return filepath.Join(specGuest.SandboxRootDir(id), "hostname")
25-
}
26-
27-
func getVirtualPodAwareSandboxHostnamePath(id, virtualSandboxID string) string {
23+
func getSandboxHostnamePath(id, virtualSandboxID string) string {
2824
return filepath.Join(specGuest.VirtualPodAwareSandboxRootDir(id, virtualSandboxID), "hostname")
2925
}
3026

31-
func getSandboxHostsPath(id string) string {
32-
return filepath.Join(specGuest.SandboxRootDir(id), "hosts")
33-
}
34-
35-
func getVirtualPodAwareSandboxHostsPath(id, virtualSandboxID string) string {
27+
func getSandboxHostsPath(id, virtualSandboxID string) string {
3628
return filepath.Join(specGuest.VirtualPodAwareSandboxRootDir(id, virtualSandboxID), "hosts")
3729
}
3830

39-
func getSandboxResolvPath(id string) string {
40-
return filepath.Join(specGuest.SandboxRootDir(id), "resolv.conf")
41-
}
42-
43-
func getVirtualPodAwareSandboxResolvPath(id, virtualSandboxID string) string {
31+
func getSandboxResolvPath(id, virtualSandboxID string) string {
4432
return filepath.Join(specGuest.VirtualPodAwareSandboxRootDir(id, virtualSandboxID), "resolv.conf")
4533
}
4634

@@ -74,19 +62,18 @@ func setupSandboxContainerSpec(ctx context.Context, id string, spec *oci.Spec) (
7462
}
7563
}
7664

77-
sandboxHostnamePath := getVirtualPodAwareSandboxHostnamePath(id, virtualSandboxID)
65+
sandboxHostnamePath := getSandboxHostnamePath(id, virtualSandboxID)
7866
if err := os.WriteFile(sandboxHostnamePath, []byte(hostname+"\n"), 0644); err != nil {
7967
return errors.Wrapf(err, "failed to write hostname to %q", sandboxHostnamePath)
8068
}
8169

8270
// Write the hosts
8371
sandboxHostsContent := network.GenerateEtcHostsContent(ctx, hostname)
84-
sandboxHostsPath := getVirtualPodAwareSandboxHostsPath(id, virtualSandboxID)
72+
sandboxHostsPath := getSandboxHostsPath(id, virtualSandboxID)
8573
if err := os.WriteFile(sandboxHostsPath, []byte(sandboxHostsContent), 0644); err != nil {
8674
return errors.Wrapf(err, "failed to write sandbox hosts to %q", sandboxHostsPath)
8775
}
8876

89-
log.G(ctx).Debug("quick setup network namespace, cflick")
9077
// Check if this is a virtual pod sandbox container by comparing container ID with virtual pod ID
9178
isVirtualPodSandbox := virtualSandboxID != "" && id == virtualSandboxID
9279
if strings.EqualFold(spec.Annotations[annotations.SkipPodNetworking], "true") || isVirtualPodSandbox {
@@ -97,7 +84,6 @@ func setupSandboxContainerSpec(ctx context.Context, id string, spec *oci.Spec) (
9784
}
9885
}
9986
// Write resolv.conf
100-
log.G(ctx).Debug("sandbox resolv.conf, cflick")
10187
ns, err := getNetworkNamespace(specGuest.GetNetworkNamespaceID(spec))
10288
if err != nil {
10389
if !strings.EqualFold(spec.Annotations[annotations.SkipPodNetworking], "true") {
@@ -119,7 +105,7 @@ func setupSandboxContainerSpec(ctx context.Context, id string, spec *oci.Spec) (
119105
if err != nil {
120106
return errors.Wrap(err, "failed to generate sandbox resolv.conf content")
121107
}
122-
sandboxResolvPath := getVirtualPodAwareSandboxResolvPath(id, virtualSandboxID)
108+
sandboxResolvPath := getSandboxResolvPath(id, virtualSandboxID)
123109
if err := os.WriteFile(sandboxResolvPath, []byte(resolvContent), 0644); err != nil {
124110
return errors.Wrap(err, "failed to write sandbox resolv.conf")
125111
}

‎internal/guest/runtime/hcsv2/standalone_container.go‎

Lines changed: 15 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -20,40 +20,24 @@ import (
2020
"github.com/Microsoft/hcsshim/pkg/annotations"
2121
)
2222

23-
func getStandaloneRootDir(id string) string {
24-
return filepath.Join(guestpath.LCOWRootPrefixInUVM, id)
25-
}
26-
27-
func getVirtualPodAwareStandaloneRootDir(id, virtualSandboxID string) string {
23+
func getStandaloneRootDir(id, virtualSandboxID string) string {
2824
if virtualSandboxID != "" {
2925
// Standalone container in virtual pod gets its own subdir
3026
return filepath.Join(guestpath.LCOWRootPrefixInUVM, "virtual-pods", virtualSandboxID, id)
3127
}
32-
return getStandaloneRootDir(id)
33-
}
34-
35-
func getStandaloneHostnamePath(id string) string {
36-
return filepath.Join(getStandaloneRootDir(id), "hostname")
37-
}
38-
39-
func getVirtualPodAwareStandaloneHostnamePath(id, virtualSandboxID string) string {
40-
return filepath.Join(getVirtualPodAwareStandaloneRootDir(id, virtualSandboxID), "hostname")
41-
}
42-
43-
func getStandaloneHostsPath(id string) string {
44-
return filepath.Join(getStandaloneRootDir(id), "hosts")
28+
return filepath.Join(guestpath.LCOWRootPrefixInUVM, id)
4529
}
4630

47-
func getVirtualPodAwareStandaloneHostsPath(id, virtualSandboxID string) string {
48-
return filepath.Join(getVirtualPodAwareStandaloneRootDir(id, virtualSandboxID), "hosts")
31+
func getStandaloneHostnamePath(id, virtualSandboxID string) string {
32+
return filepath.Join(getStandaloneRootDir(id, virtualSandboxID), "hostname")
4933
}
5034

51-
func getStandaloneResolvPath(id string) string {
52-
return filepath.Join(getStandaloneRootDir(id), "resolv.conf")
35+
func getStandaloneHostsPath(id, virtualSandboxID string) string {
36+
return filepath.Join(getStandaloneRootDir(id, virtualSandboxID), "hosts")
5337
}
5438

55-
func getVirtualPodAwareStandaloneResolvPath(id, virtualSandboxID string) string {
56-
return filepath.Join(getVirtualPodAwareStandaloneRootDir(id, virtualSandboxID), "resolv.conf")
39+
func getStandaloneResolvPath(id, virtualSandboxID string) string {
40+
return filepath.Join(getStandaloneRootDir(id, virtualSandboxID), "resolv.conf")
5741
}
5842

5943
func setupStandaloneContainerSpec(ctx context.Context, id string, spec *oci.Spec) (err error) {
@@ -66,7 +50,7 @@ func setupStandaloneContainerSpec(ctx context.Context, id string, spec *oci.Spec
6650
virtualSandboxID := spec.Annotations[annotations.VirtualPodID]
6751

6852
// Generate the standalone root dir - virtual pod aware
69-
rootDir := getVirtualPodAwareStandaloneRootDir(id, virtualSandboxID)
53+
rootDir := getStandaloneRootDir(id, virtualSandboxID)
7054
if err := os.MkdirAll(rootDir, 0755); err != nil {
7155
return errors.Wrapf(err, "failed to create container root directory %q", rootDir)
7256
}
@@ -87,15 +71,15 @@ func setupStandaloneContainerSpec(ctx context.Context, id string, spec *oci.Spec
8771

8872
// Write the hostname
8973
if !specGuest.MountPresent("/etc/hostname", spec.Mounts) {
90-
standaloneHostnamePath := getVirtualPodAwareStandaloneHostnamePath(id, virtualSandboxID)
74+
standaloneHostnamePath := getStandaloneHostnamePath(id, virtualSandboxID)
9175
if err := os.WriteFile(standaloneHostnamePath, []byte(hostname+"\n"), 0644); err != nil {
9276
return errors.Wrapf(err, "failed to write hostname to %q", standaloneHostnamePath)
9377
}
9478

9579
mt := oci.Mount{
9680
Destination: "/etc/hostname",
9781
Type: "bind",
98-
Source: getVirtualPodAwareStandaloneHostnamePath(id, virtualSandboxID),
82+
Source: getStandaloneHostnamePath(id, virtualSandboxID),
9983
Options: []string{"bind"},
10084
}
10185
if specGuest.IsRootReadonly(spec) {
@@ -107,15 +91,15 @@ func setupStandaloneContainerSpec(ctx context.Context, id string, spec *oci.Spec
10791
// Write the hosts
10892
if !specGuest.MountPresent("/etc/hosts", spec.Mounts) {
10993
standaloneHostsContent := network.GenerateEtcHostsContent(ctx, hostname)
110-
standaloneHostsPath := getVirtualPodAwareStandaloneHostsPath(id, virtualSandboxID)
94+
standaloneHostsPath := getStandaloneHostsPath(id, virtualSandboxID)
11195
if err := os.WriteFile(standaloneHostsPath, []byte(standaloneHostsContent), 0644); err != nil {
11296
return errors.Wrapf(err, "failed to write standalone hosts to %q", standaloneHostsPath)
11397
}
11498

11599
mt := oci.Mount{
116100
Destination: "/etc/hosts",
117101
Type: "bind",
118-
Source: getVirtualPodAwareStandaloneHostsPath(id, virtualSandboxID),
102+
Source: getStandaloneHostsPath(id, virtualSandboxID),
119103
Options: []string{"bind"},
120104
}
121105
if specGuest.IsRootReadonly(spec) {
@@ -140,15 +124,15 @@ func setupStandaloneContainerSpec(ctx context.Context, id string, spec *oci.Spec
140124
if err != nil {
141125
return errors.Wrap(err, "failed to generate standalone resolv.conf content")
142126
}
143-
standaloneResolvPath := getVirtualPodAwareStandaloneResolvPath(id, virtualSandboxID)
127+
standaloneResolvPath := getStandaloneResolvPath(id, virtualSandboxID)
144128
if err := os.WriteFile(standaloneResolvPath, []byte(resolvContent), 0644); err != nil {
145129
return errors.Wrap(err, "failed to write standalone resolv.conf")
146130
}
147131

148132
mt := oci.Mount{
149133
Destination: "/etc/resolv.conf",
150134
Type: "bind",
151-
Source: getVirtualPodAwareStandaloneResolvPath(id, virtualSandboxID),
135+
Source: getStandaloneResolvPath(id, virtualSandboxID),
152136
Options: []string{"bind"},
153137
}
154138
if specGuest.IsRootReadonly(spec) {

‎internal/guest/runtime/hcsv2/uvm.go‎

Lines changed: 40 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -322,8 +322,7 @@ func setupSandboxMountsPath(id string) (err error) {
322322
return storage.MountRShared(mountPath)
323323
}
324324

325-
func setupSandboxTmpfsMountsPath(id string) error {
326-
var err error
325+
func setupSandboxTmpfsMountsPath(id string) (err error) {
327326
tmpfsDir := specGuest.SandboxTmpfsMountsDir(id)
328327
if err := os.MkdirAll(tmpfsDir, 0755); err != nil {
329328
return errors.Wrapf(err, "failed to create sandbox tmpfs mounts dir in sandbox %v", id)
@@ -473,16 +472,13 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM
473472

474473
if isVirtualPod {
475474
// For virtual pods, create virtual pod specific paths
476-
err = setupVirtualPodMountsPath(virtualPodID, id)
477-
if err != nil {
475+
if err = setupVirtualPodMountsPath(virtualPodID); err != nil {
478476
return nil, err
479477
}
480-
// Create hugepages path for virtual pod
481-
mountPath := specGuest.VirtualPodHugePagesMountsDir(virtualPodID)
482-
if err := os.MkdirAll(mountPath, 0755); err != nil {
483-
return nil, errors.Wrapf(err, "failed to create virtual pod hugepage mounts dir %v", virtualPodID)
478+
if err = setupVirtualPodTmpfsMountsPath(virtualPodID); err != nil {
479+
return nil, err
484480
}
485-
if err := storage.MountRShared(mountPath); err != nil {
481+
if err = setupVirtualPodHugePageMountsPath(virtualPodID); err != nil {
486482
return nil, err
487483
}
488484
} else {
@@ -507,8 +503,7 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM
507503
if !ok || sid == "" {
508504
return nil, errors.Errorf("unsupported 'io.kubernetes.cri.sandbox-id': '%s'", sid)
509505
}
510-
err = setupWorkloadContainerSpec(ctx, sid, id, settings.OCISpecification, settings.OCIBundlePath)
511-
if err != nil {
506+
if err = setupWorkloadContainerSpec(ctx, sid, id, settings.OCISpecification, settings.OCIBundlePath); err != nil {
512507
return nil, err
513508
}
514509

@@ -672,9 +667,9 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM
672667
ns, err := getNetworkNamespace(namespaceID)
673668
// skip network activity for sandbox containers marked with skip uvm networking annotation
674669
if isCRI && err != nil && !strings.EqualFold(settings.OCISpecification.Annotations[annotations.SkipPodNetworking], "true") {
675-
// return nil, err
676670
return nil, err
677671
}
672+
// standalone is not required to have a networking namespace setup
678673
if ns != nil {
679674
if err := ns.AssignContainerPid(ctx, c.container.Pid()); err != nil {
680675
return nil, err
@@ -1556,11 +1551,11 @@ func (h *Host) cleanupVirtualPod(virtualSandboxID string) {
15561551
}
15571552

15581553
// setupVirtualPodMountsPath creates mount directories for virtual pods
1559-
func setupVirtualPodMountsPath(virtualSandboxID, masterSandboxID string) (err error) {
1554+
func setupVirtualPodMountsPath(virtualSandboxID string) (err error) {
15601555
// Create virtual pod specific mount path using the new path generation functions
15611556
mountPath := specGuest.VirtualPodMountsDir(virtualSandboxID)
15621557
if err := os.MkdirAll(mountPath, 0755); err != nil {
1563-
return errors.Wrapf(err, "failed to create virtual pod sandboxMounts dir %v", virtualSandboxID)
1558+
return errors.Wrapf(err, "failed to create virtual pod mounts dir in sandbox %v", virtualSandboxID)
15641559
}
15651560
defer func() {
15661561
if err != nil {
@@ -1570,3 +1565,34 @@ func setupVirtualPodMountsPath(virtualSandboxID, masterSandboxID string) (err er
15701565

15711566
return storage.MountRShared(mountPath)
15721567
}
1568+
1569+
func setupVirtualPodTmpfsMountsPath(virtualSandboxID string) (err error) {
1570+
tmpfsDir := specGuest.VirtualPodTmpfsMountsDir(virtualSandboxID)
1571+
if err := os.MkdirAll(tmpfsDir, 0755); err != nil {
1572+
return errors.Wrapf(err, "failed to create virtual pod tmpfs mounts dir in sandbox %v", virtualSandboxID)
1573+
}
1574+
1575+
defer func() {
1576+
if err != nil {
1577+
_ = os.RemoveAll(tmpfsDir)
1578+
}
1579+
}()
1580+
1581+
// mount a tmpfs at the tmpfsDir
1582+
// this ensures that the tmpfsDir is a mount point and not just a directory
1583+
// we don't care if it is already mounted, so ignore EBUSY
1584+
if err := unix.Mount("tmpfs", tmpfsDir, "tmpfs", 0, ""); err != nil && !errors.Is(err, unix.EBUSY) {
1585+
return errors.Wrapf(err, "failed to mount tmpfs at %s", tmpfsDir)
1586+
}
1587+
1588+
return storage.MountRShared(tmpfsDir)
1589+
}
1590+
1591+
func setupVirtualPodHugePageMountsPath(virtualSandboxID string) error {
1592+
mountPath := specGuest.VirtualPodHugePagesMountsDir(virtualSandboxID)
1593+
if err := os.MkdirAll(mountPath, 0755); err != nil {
1594+
return errors.Wrapf(err, "failed to create virtual pod hugepage mounts dir %v", virtualSandboxID)
1595+
}
1596+
1597+
return storage.MountRShared(mountPath)
1598+
}

‎internal/guest/runtime/hcsv2/workload_container.go‎

Lines changed: 29 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -45,13 +45,15 @@ func updateSandboxMounts(sbid string, spec *oci.Spec) error {
4545
var sandboxSource string
4646
// if using `sandbox-tmp://` prefix, we mount a tmpfs in sandboxTmpfsMountsDir
4747
if strings.HasPrefix(m.Source, guestpath.SandboxTmpfsMountPrefix) {
48-
sandboxSource = specGuest.SandboxTmpfsMountSource(sbid, m.Source)
48+
// Use virtual pod aware mount source
49+
sandboxSource = specGuest.VirtualPodAwareSandboxTmpfsMountSource(sbid, virtualSandboxID, m.Source)
50+
expectedMountsDir := specGuest.VirtualPodAwareSandboxTmpfsMountsDir(sbid, virtualSandboxID)
51+
4952
// filepath.Join cleans the resulting path before returning, so it would resolve the relative path if one was given.
5053
// Hence, we need to ensure that the resolved path is still under the correct directory
51-
if !strings.HasPrefix(sandboxSource, specGuest.SandboxTmpfsMountsDir(sbid)) {
52-
return errors.Errorf("mount path %v for mount %v is not within sandboxTmpfsMountsDir", sandboxSource, m.Source)
54+
if !strings.HasPrefix(sandboxSource, expectedMountsDir) {
55+
return errors.Errorf("mount path %v for mount %v is not within sandbox's tmpfs mounts dir", sandboxSource, m.Source)
5356
}
54-
5557
} else {
5658
// Use virtual pod aware mount source
5759
sandboxSource = specGuest.VirtualPodAwareSandboxMountSource(sbid, virtualSandboxID, m.Source)
@@ -81,29 +83,31 @@ func updateHugePageMounts(sbid string, spec *oci.Spec) error {
8183
virtualSandboxID := spec.Annotations[annotations.VirtualPodID]
8284

8385
for i, m := range spec.Mounts {
84-
if strings.HasPrefix(m.Source, guestpath.HugePagesMountPrefix) {
85-
// Use virtual pod aware hugepages directory
86-
mountsDir := specGuest.VirtualPodAwareHugePagesMountsDir(sbid, virtualSandboxID)
87-
subPath := strings.TrimPrefix(m.Source, guestpath.HugePagesMountPrefix)
88-
pageSize := strings.Split(subPath, string(os.PathSeparator))[0]
89-
hugePageMountSource := filepath.Join(mountsDir, subPath)
90-
91-
// filepath.Join cleans the resulting path before returning so it would resolve the relative path if one was given.
92-
// Hence, we need to ensure that the resolved path is still under the correct directory
93-
if !strings.HasPrefix(hugePageMountSource, mountsDir) {
94-
return errors.Errorf("mount path %v for mount %v is not within hugepages's mounts dir", hugePageMountSource, m.Source)
95-
}
86+
if !strings.HasPrefix(m.Source, guestpath.HugePagesMountPrefix) {
87+
continue
88+
}
89+
90+
// Use virtual pod aware hugepages directory
91+
mountsDir := specGuest.VirtualPodAwareHugePagesMountsDir(sbid, virtualSandboxID)
92+
subPath := strings.TrimPrefix(m.Source, guestpath.HugePagesMountPrefix)
93+
pageSize := strings.Split(subPath, string(os.PathSeparator))[0]
94+
hugePageMountSource := filepath.Join(mountsDir, subPath)
95+
96+
// filepath.Join cleans the resulting path before returning so it would resolve the relative path if one was given.
97+
// Hence, we need to ensure that the resolved path is still under the correct directory
98+
if !strings.HasPrefix(hugePageMountSource, mountsDir) {
99+
return errors.Errorf("mount path %v for mount %v is not within hugepages's mounts dir", hugePageMountSource, m.Source)
100+
}
96101

97-
spec.Mounts[i].Source = hugePageMountSource
102+
spec.Mounts[i].Source = hugePageMountSource
98103

99-
_, err := os.Stat(hugePageMountSource)
100-
if os.IsNotExist(err) {
101-
if err := mkdirAllModePerm(hugePageMountSource); err != nil {
102-
return err
103-
}
104-
if err := unix.Mount("none", hugePageMountSource, "hugetlbfs", 0, "pagesize="+pageSize); err != nil {
105-
return errors.Errorf("mount operation failed for %v failed with error %v", hugePageMountSource, err)
106-
}
104+
_, err := os.Stat(hugePageMountSource)
105+
if os.IsNotExist(err) {
106+
if err := mkdirAllModePerm(hugePageMountSource); err != nil {
107+
return err
108+
}
109+
if err := unix.Mount("none", hugePageMountSource, "hugetlbfs", 0, "pagesize="+pageSize); err != nil {
110+
return errors.Errorf("mount operation failed for %v failed with error %v", hugePageMountSource, err)
107111
}
108112
}
109113
}

0 commit comments

Comments
 (0)