Merge pull request opencontainers#1541 from adrianreber/lazy

checkpoint: support lazy migration

Author: Mrunal Patel

checkpoint.go

@@ -30,6 +30,8 @@ checkpointed.`,
 		cli.BoolFlag{Name: "tcp-established", Usage: "allow open tcp connections"},
 		cli.BoolFlag{Name: "ext-unix-sk", Usage: "allow external unix sockets"},
 		cli.BoolFlag{Name: "shell-job", Usage: "allow shell jobs"},
+		cli.BoolFlag{Name: "lazy-pages", Usage: "use userfaultfd to lazily restore memory pages"},
+		cli.StringFlag{Name: "status-fd", Value: "", Usage: "criu writes \\0 to this FD once lazy-pages is ready"},
 		cli.StringFlag{Name: "page-server", Value: "", Usage: "ADDRESS:PORT of the page server"},
 		cli.BoolFlag{Name: "file-locks", Usage: "handle file locks, for safety"},
 		cli.BoolFlag{Name: "pre-dump", Usage: "dump container's memory information only, leave the container running after this"},

libcontainer/container_linux.go

@@ -621,9 +621,24 @@ func (c *linuxContainer) checkCriuFeatures(criuOpts *CriuOpts, rpcOpts *criurpc.
 	logrus.Debugf("Feature check says: %s", criuFeatures)
 	missingFeatures := false
 
-	if *criuFeat.MemTrack && !*criuFeatures.MemTrack {
-		missingFeatures = true
-		logrus.Debugf("CRIU does not support MemTrack")
+	// The outer if checks if the fields actually exist
+	if (criuFeat.MemTrack != nil) &&
+		(criuFeatures.MemTrack != nil) {
+		// The inner if checks if they are set to true
+		if *criuFeat.MemTrack && !*criuFeatures.MemTrack {
+			missingFeatures = true
+			logrus.Debugf("CRIU does not support MemTrack")
+		}
+	}
+
+	// This needs to be repeated for every new feature check.
+	// Is there a way to put this in a function. Reflection?
+	if (criuFeat.LazyPages != nil) &&
+		(criuFeatures.LazyPages != nil) {
+		if *criuFeat.LazyPages && !*criuFeatures.LazyPages {
+			missingFeatures = true
+			logrus.Debugf("CRIU does not support LazyPages")
+		}
 	}
 
 	if missingFeatures {
@@ -779,6 +794,25 @@ func (c *linuxContainer) addMaskPaths(req *criurpc.CriuReq) error {
 		}
 		req.Opts.ExtMnt = append(req.Opts.ExtMnt, extMnt)
 	}
+	return nil
+}
+
+func waitForCriuLazyServer(r *os.File, status string) error {
+
+	data := make([]byte, 1)
+	_, err := r.Read(data)
+	if err != nil {
+		return err
+	}
+	fd, err := os.OpenFile(status, os.O_TRUNC|os.O_WRONLY, os.ModeAppend)
+	if err != nil {
+		return err
+	}
+	_, err = fd.Write(data)
+	if err != nil {
+		return err
+	}
+	fd.Close()
 
 	return nil
 }
@@ -846,6 +880,7 @@ func (c *linuxContainer) Checkpoint(criuOpts *CriuOpts) error {
 		EmptyNs:         proto.Uint32(criuOpts.EmptyNs),
 		OrphanPtsMaster: proto.Bool(true),
 		AutoDedup:       proto.Bool(criuOpts.AutoDedup),
+		LazyPages:       proto.Bool(criuOpts.LazyPages),
 	}
 
 	fcg := c.cgroupManager.GetPaths()["freezer"]
@@ -896,6 +931,24 @@ func (c *linuxContainer) Checkpoint(criuOpts *CriuOpts) error {
 		Opts: &rpcOpts,
 	}
 
+	if criuOpts.LazyPages {
+		// lazy migration requested; check if criu supports it
+		feat := criurpc.CriuFeatures{
+			LazyPages: proto.Bool(true),
+		}
+
+		if err := c.checkCriuFeatures(criuOpts, &rpcOpts, &feat); err != nil {
+			return err
+		}
+
+		statusRead, statusWrite, err := os.Pipe()
+		if err != nil {
+			return err
+		}
+		rpcOpts.StatusFd = proto.Int32(int32(statusWrite.Fd()))
+		go waitForCriuLazyServer(statusRead, criuOpts.StatusFd)
+	}
+
 	//no need to dump these information in pre-dump
 	if !criuOpts.PreDump {
 		for _, m := range c.config.Mounts {
@@ -1048,6 +1101,7 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error {
 			EmptyNs:         proto.Uint32(criuOpts.EmptyNs),
 			OrphanPtsMaster: proto.Bool(true),
 			AutoDedup:       proto.Bool(criuOpts.AutoDedup),
+			LazyPages:       proto.Bool(criuOpts.LazyPages),
 		},
 	}
 

libcontainer/criu_opts_linux.go

@@ -35,4 +35,6 @@ type CriuOpts struct {
 	ManageCgroupsMode       cgMode             // dump or restore cgroup mode
 	EmptyNs                 uint32             // don't c/r properties for namespace from this mask
 	AutoDedup               bool               // auto deduplication for incremental dumps
+	LazyPages               bool               // restore memory pages lazily using userfaultfd
+	StatusFd                string             // fd for feedback when lazy server is ready
 }

restore.go

@@ -86,6 +86,10 @@ using the runc checkpoint command.`,
 			Name:  "auto-dedup",
 			Usage: "enable auto deduplication of memory images",
 		},
+		cli.BoolFlag{
+			Name:  "lazy-pages",
+			Usage: "use userfaultfd to lazily restore memory pages",
+		},
 	},
 	Action: func(context *cli.Context) error {
 		if err := checkArgs(context, 1, exactArgs); err != nil {
@@ -128,5 +132,7 @@ func criuOptions(context *cli.Context) *libcontainer.CriuOpts {
 		FileLocks:               context.Bool("file-locks"),
 		PreDump:                 context.Bool("pre-dump"),
 		AutoDedup:               context.Bool("auto-dedup"),
+		LazyPages:               context.Bool("lazy-pages"),
+		StatusFd:                context.String("status-fd"),
 	}
 }

tests/integration/checkpoint.bats

@@ -122,3 +122,111 @@ function teardown() {
   [ "$status" -eq 0 ]
   [[ "${output}" == *"ponG Ping"* ]]
 }
+
+@test "checkpoint --lazy-pages and restore" {
+  # XXX: currently criu require root containers.
+  requires criu root
+
+  # check if lazy-pages is supported
+  run ${CRIU} check --feature lazy_pages
+  if [ "$status" -eq 1 ]; then
+    # this criu does not support lazy migration; skip the test
+    skip "this criu does not support lazy migration"
+  fi
+
+  sed -i 's;"terminal": true;"terminal": false;' config.json
+  sed -i 's;"readonly": true;"readonly": false;' config.json
+  sed -i 's/"sh"/"sh","-c","for i in `seq 10`; do read xxx || continue; echo ponG $xxx; done"/' config.json
+
+  # The following code creates pipes for stdin and stdout.
+  # CRIU can't handle fifo-s, so we need all these tricks.
+  fifo=`mktemp -u /tmp/runc-fifo-XXXXXX`
+  mkfifo $fifo
+
+  # For lazy migration we need to know when CRIU is ready to serve
+  # the memory pages via TCP.
+  lazy_pipe=`mktemp -u /tmp/lazy-pipe-XXXXXX`
+  mkfifo $lazy_pipe
+
+  # TCP port for lazy migration
+  port=27277
+
+  # stdout
+  cat $fifo | cat $fifo &
+  pid=$!
+  exec 50</proc/$pid/fd/0
+  exec 51>/proc/$pid/fd/0
+
+  # stdin
+  cat $fifo | cat $fifo &
+  pid=$!
+  exec 60</proc/$pid/fd/0
+  exec 61>/proc/$pid/fd/0
+
+  echo -n > $fifo
+  unlink $fifo
+
+  # run busybox
+  __runc run -d test_busybox <&60 >&51 2>&51
+  [ $? -eq 0 ]
+
+  testcontainer test_busybox running
+
+  # checkpoint the running container
+  mkdir image-dir
+  mkdir work-dir
+  # Double fork taken from helpers.bats
+  # We need to start 'runc checkpoint --lazy-pages' in the background,
+  # so we double fork in the shell.
+  (runc --criu "$CRIU" checkpoint --lazy-pages --page-server 0.0.0.0:${port} --status-fd ${lazy_pipe} --work-path ./work-dir --image-path ./image-dir test_busybox & ) &
+  # Sleeping here. This is ugly, but not sure how else to handle it.
+  # The return code of the in the background running runc is needed, if
+  # there is some basic error. If the lazy migration is ready can
+  # be handled by $lazy_pipe. Which probably will always be ready
+  # after sleeping two seconds.
+  sleep 2
+  # Check if inventory.img was written
+  [ -e image-dir/inventory.img ]
+  # If the inventory.img exists criu checkpointed some things, let's see
+  # if there were other errors in the log file.
+  run grep -B 5 Error ./work-dir/dump.log -q
+  [ "$status" -eq 1 ]
+
+  # This will block until CRIU is ready to serve memory pages
+  cat $lazy_pipe
+  [ "$status" -eq 1 ]
+
+  unlink $lazy_pipe
+
+  # Double fork taken from helpers.bats
+  # We need to start 'criu lazy-pages' in the background,
+  # so we double fork in the shell.
+  # Start CRIU in lazy-daemon mode
+  $(${CRIU} lazy-pages --page-server --address 127.0.0.1 --port ${port} -D image-dir &) &
+
+  # Restore lazily from checkpoint.
+  # The restored container needs a different name as the checkpointed
+  # container is not yet destroyed. It is only destroyed at that point
+  # in time when the last page is lazily transferred to the destination.
+  # Killing the CRIU on the checkpoint side will let the container
+  # continue to run if the migration failed at some point.
+  __runc --criu "$CRIU" restore -d --work-path ./image-dir --image-path ./image-dir --lazy-pages test_busybox_restore <&60 >&51 2>&51
+  ret=$?
+  [ $ret -eq 0 ]
+  run grep -B 5 Error ./work-dir/dump.log -q
+  [ "$status" -eq 1 ]
+
+  # busybox should be back up and running
+  testcontainer test_busybox_restore running
+
+  runc exec --cwd /bin test_busybox_restore echo ok
+  [ "$status" -eq 0 ]
+  [[ ${output} == "ok" ]]
+
+  echo Ping >&61
+  exec 61>&-
+  exec 51>&-
+  run cat <&50
+  [ "$status" -eq 0 ]
+  [[ "${output}" == *"ponG Ping"* ]]
+}