Skip to content

Commit d3c914e

Browse files
char-1eechar-1ee
committed
Change logic flow of loadsnapshot
Signed-off-by: char-1ee <[email protected]>
1 parent e0bef9a commit d3c914e

File tree

4 files changed

+124
-75
lines changed

4 files changed

+124
-75
lines changed

‎configs/firecracker-containerd/firecracker-runtime.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,6 @@
44
"kernel_args": "console=ttyS0 noapic reboot=k panic=1 pci=off nomodules ro systemd.journald.forward_to_console systemd.unit=firecracker.target init=/sbin/overlay-init",
55
"root_drive": "/var/lib/firecracker-containerd/runtime/default-rootfs.img",
66
"cpu_template": "T2",
7-
"log_levels": ["info"]
7+
"log_fifo": "/tmp/fc-logs.fifo",
8+
"log_levels": ["debug"]
89
}

‎ctriface/iface.go‎

Lines changed: 78 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import (
3535
"time"
3636

3737
"github.com/vhive-serverless/vhive/snapshotting"
38+
"golang.org/x/sys/unix"
3839

3940
log "github.com/sirupsen/logrus"
4041

@@ -65,6 +66,13 @@ type StartVMResponse struct {
6566
GuestIP string
6667
}
6768

69+
type GuestRegionUffdMapping struct {
70+
BaseHostVirtAddr uint64 `json:"base_host_virt_addr"`
71+
Size uint64 `json:"size"`
72+
Offset uint64 `json:"offset"`
73+
PageSizeKiB uint64 `json:"page_size_kib"`
74+
}
75+
6876
const (
6977
testImageName = "ghcr.io/ease-lab/helloworld:var_workload"
7078
fileBackend = "File"
@@ -506,9 +514,6 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, originVmID string, vmID
506514
BackendPath: snap.GetMemFilePath(),
507515
}
508516

509-
var sendfdConn *net.UnixConn
510-
uffdListenerCh := make(chan struct{}, 1)
511-
512517
if o.GetUPFEnabled() {
513518
logger.Debug("TEST: UPF is enabled")
514519
conf.MemBackend.BackendType = uffdBackend
@@ -522,31 +527,9 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, originVmID string, vmID
522527
return nil, nil, err
523528
}
524529

525-
logger.Debug("TEST: start listening to uffd socket")
526-
if _, err := os.Stat(conf.MemBackend.BackendPath); err == nil {
527-
os.Remove(conf.MemBackend.BackendPath)
528-
}
529-
530-
go func() {
531-
listener, err := net.Listen("unix", conf.MemBackend.BackendPath)
532-
if err != nil {
533-
logger.Error("failed to listen to uffd socket")
534-
return
535-
}
536-
defer listener.Close()
537-
538-
logger.Debug("Listening ...")
539-
conn, err := listener.Accept()
540-
if err != nil {
541-
logger.Error("failed to accept connection to uffd socket")
542-
return
543-
}
530+
// ===========================
544531

545-
sendfdConn, _ = conn.(*net.UnixConn)
546-
close(uffdListenerCh)
547-
}()
548-
549-
time.Sleep(10 * time.Second) // TODO: sleep for 10 seconds to wait for the uffd socket to be ready
532+
// ===========================
550533
}
551534

552535
tStart = time.Now()
@@ -590,11 +573,76 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, originVmID string, vmID
590573
<-loadDone
591574

592575
if o.GetUPFEnabled() {
576+
logger.Debug("TEST: start listening to uffd socket")
577+
if _, err := os.Stat(conf.MemBackend.BackendPath); err == nil {
578+
os.Remove(conf.MemBackend.BackendPath)
579+
}
593580

594-
logger.Debug("TEST: Registering VM with snap with the memory manager")
581+
var sendfdConn *net.UnixConn
582+
uffdListenerCh := make(chan struct{}, 1)
583+
var userfaultFD *os.File
584+
var baseHostVirtAddr uint64
585+
586+
go func() {
587+
defer close(uffdListenerCh)
588+
listener, err := net.Listen("unix", conf.MemBackend.BackendPath) // TODO: find a better Listener API
589+
if err != nil {
590+
logger.Error("failed to listen to uffd socket")
591+
return
592+
}
593+
defer listener.Close()
594+
595+
logger.Debug("Listening ...")
596+
conn, err := listener.Accept() // blocking
597+
if err != nil {
598+
logger.Error("failed to accept connection to uffd socket")
599+
return
600+
}
601+
602+
sendfdConn, _ = conn.(*net.UnixConn)
603+
604+
// Parse mapping and fd
605+
buff := make([]byte, 256) // set a maximum buffer size
606+
oobBuff := make([]byte, unix.CmsgSpace(4))
607+
n, oobn, _, _, err := sendfdConn.ReadMsgUnix(buff, oobBuff)
608+
if err != nil {
609+
logger.Error("failed to reading from uffd socket")
610+
return
611+
}
612+
buff = buff[:n]
613+
614+
var fd int
615+
if oobn > 0 {
616+
scms, err := unix.ParseSocketControlMessage(oobBuff[:oobn])
617+
if err != nil {
618+
logger.Error("failed to parse socket control message")
619+
return
620+
}
621+
for _, scm := range scms {
622+
fds, err := unix.ParseUnixRights(&scm)
623+
if err != nil {
624+
logger.Error("failed to parse unix rights")
625+
return
626+
}
627+
if len(fds) > 0 {
628+
fd = fds[0] // Assuming only one fd is sent.
629+
break
630+
}
631+
}
632+
}
633+
userfaultFD = os.NewFile(uintptr(fd), "userfaultfd")
634+
635+
var mapping []GuestRegionUffdMapping
636+
if err := json.Unmarshal(buff, &mapping); err != nil {
637+
logger.Error("failed to unmarshal data")
638+
return
639+
}
640+
baseHostVirtAddr = mapping[0].BaseHostVirtAddr
641+
}()
595642

596643
<-uffdListenerCh
597644

645+
logger.Debug("TEST: Registering VM with snap with the memory manager")
598646
stateCfg := manager.SnapshotStateCfg{
599647
VMID: vmID,
600648
GuestMemPath: o.getMemoryFile(vmID),
@@ -609,8 +657,8 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, originVmID string, vmID
609657
logger.Error(err, "failed to register new VM with memory manager")
610658
}
611659

612-
logger.Debug("TEST: activate VM in mm")
613-
if activateErr = o.memoryManager.Activate(originVmID, sendfdConn); activateErr != nil {
660+
logger.Debug("TEST: activate VM in mm") // TODO: pass too many params in Activate
661+
if activateErr = o.memoryManager.Activate(originVmID, userfaultFD, baseHostVirtAddr); activateErr != nil {
614662
logger.Warn("Failed to activate VM in the memory manager", activateErr)
615663
}
616664
}

‎memory/manager/manager.go‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ import (
2626
"encoding/csv"
2727
"errors"
2828
"fmt"
29-
"net"
3029
"os"
3130
"strconv"
3231
"sync"
@@ -149,7 +148,7 @@ func (m *MemoryManager) DeregisterVM(vmID string) error {
149148
}
150149

151150
// Activate Creates an epoller to serve page faults for the VM
152-
func (m *MemoryManager) Activate(vmID string, conn *net.UnixConn) error {
151+
func (m *MemoryManager) Activate(vmID string, uffd *os.File, baseHostVirtAddr uint64) error {
153152
logger := log.WithFields(log.Fields{"vmID": vmID})
154153

155154
logger.Debug("Activating instance in the memory manager")
@@ -184,10 +183,13 @@ func (m *MemoryManager) Activate(vmID string, conn *net.UnixConn) error {
184183
return err
185184
}
186185

187-
if err := state.getUFFD(conn); err != nil {
188-
logger.Error("Failed to get uffd")
189-
return err
190-
}
186+
// if err := state.getUFFD(); err != nil {
187+
// logger.Error("Failed to get uffd")
188+
// return err
189+
// }
190+
191+
state.startAddress = baseHostVirtAddr
192+
state.userFaultFD = uffd
191193

192194
state.setupStateOnActivate()
193195

‎memory/manager/snapshot_state.go‎

Lines changed: 36 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,8 @@ import "C"
2929

3030
import (
3131
"encoding/binary"
32-
"encoding/json"
3332
"errors"
3433
"fmt"
35-
"net"
3634
"os"
3735
"path/filepath"
3836
"sort"
@@ -140,42 +138,42 @@ type GuestRegionUffdMapping struct {
140138
PageSizeKiB uint64 `json:"page_size_kib"`
141139
}
142140

143-
func (s *SnapshotState) getUFFD(sendfdConn *net.UnixConn) error {
144-
buff := make([]byte, 256) // set a maximum buffer size
145-
oobBuff := make([]byte, unix.CmsgSpace(4))
146-
147-
n, oobn, _, _, err := sendfdConn.ReadMsgUnix(buff, oobBuff)
148-
if err != nil {
149-
return fmt.Errorf("error reading message: %w", err)
150-
}
151-
buff = buff[:n]
152-
153-
var fd int
154-
if oobn > 0 {
155-
scms, err := unix.ParseSocketControlMessage(oobBuff[:oobn])
156-
if err != nil {
157-
return fmt.Errorf("error parsing socket control message: %w", err)
158-
}
159-
for _, scm := range scms {
160-
fds, err := unix.ParseUnixRights(&scm)
161-
if err != nil {
162-
return fmt.Errorf("error parsing unix rights: %w", err)
163-
}
164-
if len(fds) > 0 {
165-
fd = fds[0] // Assuming only one fd is sent.
166-
break
167-
}
168-
}
169-
}
170-
userfaultFD := os.NewFile(uintptr(fd), "userfaultfd")
171-
172-
var mapping []GuestRegionUffdMapping
173-
if err := json.Unmarshal(buff, &mapping); err != nil {
174-
return fmt.Errorf("error unmarshaling data: %w", err)
175-
}
176-
177-
s.startAddress = mapping[0].BaseHostVirtAddr
178-
s.userFaultFD = userfaultFD
141+
func (s *SnapshotState) getUFFD() error {
142+
// buff := make([]byte, 256) // set a maximum buffer size
143+
// oobBuff := make([]byte, unix.CmsgSpace(4))
144+
145+
// n, oobn, _, _, err := sendfdConn.ReadMsgUnix(buff, oobBuff)
146+
// if err != nil {
147+
// return fmt.Errorf("error reading message: %w", err)
148+
// }
149+
// buff = buff[:n]
150+
151+
// var fd int
152+
// if oobn > 0 {
153+
// scms, err := unix.ParseSocketControlMessage(oobBuff[:oobn])
154+
// if err != nil {
155+
// return fmt.Errorf("error parsing socket control message: %w", err)
156+
// }
157+
// for _, scm := range scms {
158+
// fds, err := unix.ParseUnixRights(&scm)
159+
// if err != nil {
160+
// return fmt.Errorf("error parsing unix rights: %w", err)
161+
// }
162+
// if len(fds) > 0 {
163+
// fd = fds[0] // Assuming only one fd is sent.
164+
// break
165+
// }
166+
// }
167+
// }
168+
// userfaultFD := os.NewFile(uintptr(fd), "userfaultfd")
169+
170+
// var mapping []GuestRegionUffdMapping
171+
// if err := json.Unmarshal(buff, &mapping); err != nil {
172+
// return fmt.Errorf("error unmarshaling data: %w", err)
173+
// }
174+
175+
// s.startAddress = baseHostVirtAddr
176+
// s.userFaultFD = userfaultFD
179177
return nil
180178
}
181179

0 commit comments

Comments
 (0)