Skip to content

Commit e78c5fd

Browse files
char-1eechar-1ee
committed
Test
1 parent e8092e4 commit e78c5fd

File tree

7 files changed

+111
-107
lines changed

7 files changed

+111
-107
lines changed

‎ctriface/iface.go‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ package ctriface
2525
import (
2626
"context"
2727
"encoding/json"
28+
"net"
2829
"os"
2930
"os/exec"
3031
"path/filepath"
@@ -505,6 +506,9 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, originVmID string, vmID
505506
BackendPath: snap.GetMemFilePath(),
506507
}
507508

509+
var sendfdConn *net.UnixConn
510+
uffdListenerCh := make(chan struct{}, 1)
511+
508512
if o.GetUPFEnabled() {
509513
logger.Debug("TEST: UPF is enabled")
510514
conf.MemBackend.BackendType = uffdBackend
@@ -517,6 +521,44 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, originVmID string, vmID
517521
if err := o.memoryManager.FetchState(originVmID); err != nil {
518522
return nil, nil, err
519523
}
524+
525+
logger.Debug("TEST: start listening to uffd socket")
526+
if _, err := os.Stat(conf.MemBackend.BackendPath); err == nil {
527+
os.Remove(conf.MemBackend.BackendPath)
528+
}
529+
530+
// connChan := make(chan *net.UnixConn, 1)
531+
errChan := make(chan error, 1)
532+
go func() {
533+
listener, err := net.Listen("unix", conf.MemBackend.BackendPath)
534+
if err != nil {
535+
errChan <- errors.Wrapf(err, "failed to listen to uffd socket")
536+
return
537+
// return nil, nil, errors.Wrapf(err, "failed to listen to uffd socket")
538+
}
539+
defer listener.Close()
540+
541+
logger.Debug("Listening ...")
542+
conn, err := listener.Accept()
543+
if err != nil {
544+
errChan <- errors.Wrapf(err, "failed to accept connection")
545+
return
546+
// return nil, nil, errors.Wrapf(err, "failed to accept connection")
547+
}
548+
549+
sendfdConn, _ = conn.(*net.UnixConn)
550+
close(uffdListenerCh)
551+
552+
// connChan <- sendfdConn
553+
}()
554+
555+
// select {
556+
// case sendfdConn = <-connChan:
557+
// logger.Debug("Connection accepted and type-asserted to *net.UnixConn")
558+
// case err := <-errChan:
559+
// logger.Errorf("Error occurred: %v\n", err)
560+
// }
561+
time.Sleep(10 * time.Second)
520562
}
521563

522564
tStart = time.Now()
@@ -556,12 +598,15 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, originVmID string, vmID
556598
}()
557599

558600
logger.Debug("TEST: CreatVM request sent")
601+
559602
<-loadDone
560603

561604
if o.GetUPFEnabled() {
562605

563606
logger.Debug("TEST: Registering VM with snap with the memory manager")
564607

608+
<-uffdListenerCh
609+
565610
stateCfg := manager.SnapshotStateCfg{
566611
VMID: vmID,
567612
GuestMemPath: o.getMemoryFile(vmID),
@@ -577,7 +622,7 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, originVmID string, vmID
577622
}
578623

579624
logger.Debug("TEST: activate VM in mm")
580-
if activateErr = o.memoryManager.Activate(vmID); activateErr != nil {
625+
if activateErr = o.memoryManager.Activate(vmID, sendfdConn); activateErr != nil {
581626
logger.Warn("Failed to activate VM in the memory manager", activateErr)
582627
}
583628
}

‎ctriface/iface_test.go‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,6 @@ import (
3333
log "github.com/sirupsen/logrus"
3434
"github.com/stretchr/testify/require"
3535
"github.com/vhive-serverless/vhive/snapshotting"
36-
37-
"github.com/vhive-serverless/vhive/lg"
3836
)
3937

4038
// TODO: Make it impossible to use lazy mode without UPF
@@ -82,38 +80,41 @@ func TestStartSnapStop(t *testing.T) {
8280

8381
vmID := "2"
8482

83+
log.Debug("STEP: StartVM")
8584
_, _, err := orch.StartVM(ctx, vmID, testImageName)
8685
require.NoError(t, err, "Failed to start VM")
8786

87+
log.Debug("STEP: PauseVM")
8888
err = orch.PauseVM(ctx, vmID)
8989
require.NoError(t, err, "Failed to pause VM")
9090

91+
log.Debug("STEP: NewSnapshot and CreateSnapshot")
9192
snap := snapshotting.NewSnapshot(vmID, "/fccd/snapshots", testImageName)
9293
err = orch.CreateSnapshot(ctx, vmID, snap)
9394
require.NoError(t, err, "Failed to create snapshot of VM")
9495

95-
// _, err = orch.ResumeVM(ctx, vmID)
96-
// require.NoError(t, err, "Failed to resume VM after created snapshot")
97-
96+
log.Debug("STEP: StopSingleVM")
9897
err = orch.StopSingleVM(ctx, vmID)
9998
require.NoError(t, err, "Failed to stop VM")
10099

101100
originVmID := vmID
102101
vmID = "3"
103102

103+
log.Debug("STEP: LoadSnapshot")
104104
_, _, err = orch.LoadSnapshot(ctx, originVmID, vmID, snap)
105105
require.NoError(t, err, "Failed to load snapshot of VM")
106106

107-
log.Debug("TEST: LoadSnapshot completed")
108-
lg.UniLogger.Println("This is a test")
107+
log.Debug("STEP: ResumeVM")
109108
_, err = orch.ResumeVM(ctx, vmID)
110109
require.NoError(t, err, "Failed to resume VM")
111110

112111
time.Sleep(30 * time.Second)
113112

113+
log.Debug("STEP: StopeSingleVM")
114114
err = orch.StopSingleVM(ctx, vmID)
115115
require.NoError(t, err, "Failed to stop VM")
116116

117+
log.Debug("STEP: Cleanup")
117118
_ = snap.Cleanup()
118119
orch.Cleanup()
119120
}

‎ctriface/orch.go‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -123,18 +123,21 @@ func NewOrchestrator(snapshotter, hostIface string, opts ...OrchestratorOption)
123123
}
124124

125125
if o.GetUPFEnabled() {
126-
file, err := os.Create(o.uffdSockAddr)
127-
if err != nil {
128-
log.Fatalf("Failed to create socket file: %v", err)
129-
}
130-
defer file.Close()
126+
// file, err := os.Create(o.uffdSockAddr)
127+
// if err != nil {
128+
// log.Fatalf("Failed to create socket file: %v", err)
129+
// }
130+
// defer file.Close()
131+
// lg.UniLogger.Println("TEST: created the uffd sock addr")
131132

132133
managerCfg := manager.MemoryManagerCfg{
133134
MetricsModeOn: o.isMetricsMode,
134135
UffdSockAddr: o.uffdSockAddr,
135136
}
136137
o.memoryManager = manager.NewMemoryManager(managerCfg)
137-
go o.memoryManager.ListenUffdSocket(o.uffdSockAddr)
138+
139+
// lg.UniLogger.Println("TEST: created a new memory manager. Start listen uffd socket")
140+
// go o.memoryManager.ListenUffdSocket(o.uffdSockAddr)
138141
}
139142

140143
log.Info("Creating containerd client")

‎lg/uni_logger.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
var UniLogger *log.Logger
1010

1111
func init() {
12-
file, err := os.OpenFile("output.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
12+
file, err := os.OpenFile("uni_output.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
1313
if err != nil {
1414
log.Fatalln("Failed to open log file:", err)
1515
}

‎memory/manager/manager.go‎

Lines changed: 12 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ func (m *MemoryManager) DeregisterVM(vmID string) error {
149149
}
150150

151151
// Activate Creates an epoller to serve page faults for the VM
152-
func (m *MemoryManager) Activate(vmID string) error {
152+
func (m *MemoryManager) Activate(vmID string, conn *net.UnixConn) error {
153153
logger := log.WithFields(log.Fields{"vmID": vmID})
154154

155155
logger.Debug("Activating instance in the memory manager")
@@ -164,14 +164,6 @@ func (m *MemoryManager) Activate(vmID string) error {
164164

165165
logger.Debug("TEST: Activate: fetch snapstate by vmID for UFFD")
166166

167-
// originID, ok := m.origins[vmID]
168-
169-
// if !ok {
170-
// logger.Debug("TEST: not loaded from snapshot")
171-
// }
172-
173-
// state, ok = m.instances[originID]
174-
175167
state, ok = m.instances[vmID]
176168

177169
if !ok {
@@ -187,29 +179,21 @@ func (m *MemoryManager) Activate(vmID string) error {
187179
return errors.New("VM already active")
188180
}
189181

190-
select {
191-
case <-m.startEpollingCh:
192-
if err := state.mapGuestMemory(); err != nil {
193-
logger.Error("Failed to map guest memory")
194-
return err
195-
}
196-
197-
if err := state.getUFFD(); err != nil {
198-
logger.Error("Failed to get uffd")
199-
return err
200-
}
182+
if err := state.mapGuestMemory(); err != nil {
183+
logger.Error("Failed to map guest memory")
184+
return err
185+
}
201186

202-
state.setupStateOnActivate()
187+
if err := state.getUFFD(conn); err != nil {
188+
logger.Error("Failed to get uffd")
189+
return err
190+
}
203191

204-
go state.pollUserPageFaults(readyCh)
192+
state.setupStateOnActivate()
205193

206-
<-readyCh
194+
go state.pollUserPageFaults(readyCh)
207195

208-
case <-time.After(100 * time.Second):
209-
return errors.New("Uffd connection to firecracker timeout")
210-
default:
211-
return errors.New("Failed to start epoller")
212-
}
196+
<-readyCh
213197

214198
return nil
215199
}
@@ -229,12 +213,6 @@ func (m *MemoryManager) FetchState(vmID string) error {
229213

230214
m.Lock()
231215

232-
// originID, ok := m.origins[vmID]
233-
// if !ok {
234-
// logger.Debug("TEST: not loaded from snapshot")
235-
// }
236-
// state, ok = m.instances[originID]
237-
238216
state, ok = m.instances[vmID]
239217
if !ok {
240218
m.Unlock()
@@ -409,39 +387,6 @@ func (m *MemoryManager) GetUPFLatencyStats(vmID string) ([]*metrics.Metric, erro
409387
return state.latencyMetrics, nil
410388
}
411389

412-
func (m *MemoryManager) ListenUffdSocket(uffdSockAddr string) error {
413-
log.Debug("Start listening to uffd socket")
414-
415-
m.startEpollingOnce.Do(func() {
416-
m.startEpollingCh = make(chan struct{})
417-
})
418-
419-
ln, err := net.Listen("unix", uffdSockAddr)
420-
if err != nil {
421-
log.Errorf("Failed to listen on uffd socket: %v", err)
422-
return errors.New("Failed to listen on uffd socket")
423-
}
424-
defer ln.Close()
425-
426-
for {
427-
conn, err := ln.Accept()
428-
if err != nil {
429-
log.Printf("Failed to accept connection on uffd socket: %v", err)
430-
continue
431-
}
432-
go func(conn net.Conn) {
433-
defer conn.Close()
434-
if err := ln.Close(); err != nil {
435-
log.Printf("Failed to close uffd socket listener: %v", err)
436-
}
437-
close(m.startEpollingCh)
438-
}(conn)
439-
break
440-
}
441-
442-
return nil
443-
}
444-
445390
// Deprecated
446391
// func (m *MemoryManager) GetUPFSockPath(vmID string, isSnapshotReady bool) (string, error) {
447392
// logger := log.WithFields(log.Fields{"vmID": vmID})

‎memory/manager/snapshot_state.go‎

Lines changed: 32 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ package manager
2828
import "C"
2929

3030
import (
31-
"context"
3231
"encoding/binary"
3332
"errors"
3433
"fmt"
@@ -134,37 +133,45 @@ func (s *SnapshotState) setupStateOnActivate() {
134133
}
135134
}
136135

137-
func (s *SnapshotState) getUFFD() error {
138-
var d net.Dialer
139-
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
140-
defer cancel()
136+
func (s *SnapshotState) getUFFD(sendfdConn *net.UnixConn) error {
137+
// var d net.Dialer
138+
// ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
139+
// defer cancel()
141140

142-
for {
143-
c, err := d.DialContext(ctx, "unix", s.InstanceSockAddr)
144-
if err != nil {
145-
if ctx.Err() != nil {
146-
log.Error("Failed to dial within the context timeout")
147-
return err
148-
}
149-
time.Sleep(1 * time.Millisecond)
150-
continue
151-
}
152-
log.Debugf("TEST: Dial uffd socket done: %s", s.InstanceSockAddr)
141+
// for {
142+
// c, err := d.DialContext(ctx, "unix", s.InstanceSockAddr)
143+
// if err != nil {
144+
// if ctx.Err() != nil {
145+
// log.Error("Failed to dial within the context timeout")
146+
// return err
147+
// }
148+
// time.Sleep(1 * time.Millisecond)
149+
// continue
150+
// }
153151

154-
defer c.Close()
152+
// defer c.Close()
155153

156-
sendfdConn := c.(*net.UnixConn)
154+
// sendfdConn := c.(*net.UnixConn)
157155

158-
fs, err := fd.Get(sendfdConn, 1, []string{"a file"})
159-
if err != nil {
160-
log.Error("Failed to receive the uffd")
161-
return err
162-
}
156+
// fs, err := fd.Get(sendfdConn, 1, []string{"a file"})
157+
// if err != nil {
158+
// log.Error("Failed to receive the uffd")
159+
// return err
160+
// }
163161

164-
s.userFaultFD = fs[0]
162+
// s.userFaultFD = fs[0]
165163

166-
return nil
164+
// return nil
165+
// }
166+
167+
fs, err := fd.Get(sendfdConn, 1, []string{"a file"})
168+
if err != nil {
169+
log.Error("Failed to receive the uffd")
170+
return err
167171
}
172+
173+
s.userFaultFD = fs[0]
174+
return nil
168175
}
169176

170177
func (s *SnapshotState) processMetrics() {

‎scripts/clean_fcctr.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,5 +80,8 @@ sudo rm /var/lib/cni/networks/fcnet*/19* || echo clean already
8080
echo Cleaning snapshots
8181
sudo rm -rf /fccd/snapshots/*
8282

83+
echo Cleaning UFFD socket
84+
sudo rm -f /tmp/uffd.sock
85+
8386
echo Creating a fresh devmapper
8487
source $DIR/create_devmapper.sh

0 commit comments

Comments
 (0)