Freeze cgroup during live checkpoint

When checkpointing a container with --leave-running, libpod dumps the
container's memory via the OCI runtime (CRIU) first and only captures
the rootfs diff and named volumes afterwards. CRIU thaws the container
as soon as the memory dump finishes, so the processes inside the
container continue to run between the memory snapshot and the
file-system capture. As a result, the checkpoint can be inconsistent:
have CRIU images and a file system that reflect different points in time.

To fix this, we freeze the container's cgroup before invoking the OCI
runtime and thaw it again only after the checkpoint image/archive has
been written. The OCI runtime calls CRIU with the freezer cgroup and
restores it to its previous state once the dump completes, so a
container that was already frozen stays frozen across the dump and
the file system is captured at the same instant as the CRIU images.
This mirrors the approach other engines (e.g. CRI-O and containerd).

The default (stopping) checkpoint functionality is not affected by this
issue because CRIU leaves the tasks dead after the dump.

This patch also adds a regression test for the consistency of live
(--leave-running) checkpoints. The container runs a workload that
keeps an in-memory counter in sync with a value written to a file
on its root file system, maintaining the invariant that the on-disk
value never gets ahead of the in-memory counter.

Signed-off-by: Radostin Stoyanov <rstoyanov@fedoraproject.org>
(cherry picked from commit 2c7aeab70b)

docs/source/markdown/podman-container-checkpoint.1.md

@@ -126,6 +126,15 @@ Instead of providing the *container ID* or *name*, use the last created *contain
 Leave the *container* running after checkpointing instead of stopping it.\
 The default is **false**.
 
+To keep the checkpoint consistent, Podman freezes the *container's* cgroup
+before dumping its memory and only thaws it again after the root file-system
+changes and named volumes have been captured. This means the *container* is
+paused for the duration of the checkpoint, but its memory image and file system
+are guaranteed to reflect the same point in time. Containers started with
+**--cgroups=disabled** cannot be frozen and therefore do not get this
+guarantee. If freezing the *container* fails, Podman continues with checkpointing
+and warns that this consistency guarantee could not be provided.
+
 #### **--pre-checkpoint**, **-P**
 
 Dump the *container's* memory information only, leaving the *container* running. Later

libpod/container_internal_common.go

@@ -1275,6 +1275,55 @@ func (c *Container) checkpointRestoreSupported(version int) error {
 	return nil
 }
 
+// freezeForCheckpoint freezes the container's cgroup for the duration of a live
+// (options.KeepRunning) checkpoint so that the rootfs diff and named volumes are
+// captured at the same time as the CRIU images. It returns a thaw function that
+// the caller must defer.
+//
+// Freezing is best-effort: containers without cgroups cannot be frozen and a
+// freeze failure is non-fatal.
+func (c *Container) freezeForCheckpoint(options ContainerCheckpointOptions) func() {
+	noop := func() {}
+
+	if !options.KeepRunning || options.PreCheckPoint {
+		return noop
+	}
+
+	if c.config.NoCgroups {
+		logrus.Warnf("Container %s runs without cgroups, cannot freeze it during checkpoint: the file system of a --leave-running checkpoint may be inconsistent with CRIU images", c.ID())
+		return noop
+	}
+
+	// Use c.pause()/c.unpause() so the paused state is recorded in the
+	// database. If the checkpoint is then interrupted (e.g. by SIGKILL) Podman
+	// still knows the container is frozen and can recover it to a sane state.
+	if err := c.pause(); err != nil {
+		// Do not hard-fail a previously working checkpoint: warn that
+		// consistency cannot be guaranteed and continue.
+		logrus.Warnf("Freezing container %s during checkpoint failed, the file system of a --leave-running checkpoint may be inconsistent with CRIU images: %v", c.ID(), err)
+		return noop
+	}
+
+	return func() {
+		if err := c.unpause(); err != nil {
+			logrus.Errorf("Thawing container %s after checkpoint: %v", c.ID(), err)
+		}
+	}
+}
+
+// checkpoint dumps the container's state with the OCI runtime (CRIU) and, unless
+// options.KeepRunning is set, stops the container afterwards. The memory image is
+// written first; the root-fs diff and named volumes are captured later, in
+// exportCheckpoint/createCheckpointImage.
+//
+// For a live checkpoint (options.KeepRunning) CRIU thaws the tasks as soon as the
+// memory dump finishes, so without further action the process keeps running while
+// the file system is still being captured. The resulting checkpoint would then
+// have a memory image and a file system that reflect different points in time. To
+// keep them consistent, the container's cgroup is frozen before the runtime is
+// invoked and only thawed once the checkpoint image/archive has been written.
+// Containers running without cgroups cannot be frozen and keep the previous,
+// weaker guarantee. A freeze failure is non-fatal so existing setups keep working.
 func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointOptions) (*define.CRIUCheckpointRestoreStatistics, int64, error) {
 	if err := c.checkpointRestoreSupported(criu.MinCriuVersion); err != nil {
 		return nil, 0, err
@@ -1300,6 +1349,11 @@ func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointO
 	c.state.CheckpointLog = path.Join(c.bundlePath(), "dump.log")
 	c.state.CheckpointPath = c.CheckpointPath()
 
+	// Freeze a live checkpoint so its file system is captured at the same
+	// instant as the memory image; the deferred thaw runs once the checkpoint
+	// has been written.
+	defer c.freezeForCheckpoint(options)()
+
 	runtimeCheckpointDuration, err := c.ociRuntime.CheckpointContainer(c, options)
 	if err != nil {
 		return nil, 0, err

test/e2e/checkpoint_test.go

@@ -10,6 +10,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"slices"
+	"strconv"
 	"strings"
 	"time"
 
@@ -442,6 +443,72 @@ var _ = Describe("Podman checkpoint", func() {
 		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
 	})
 
+	It("podman checkpoint with --leave-running keeps the file system consistent with the memory image", func() {
+		// A live checkpoint must capture the memory image and the root
+		// file system at the same instant. The workload keeps an in-memory
+		// counter in sync with a value on the root file system; the on-disk
+		// value must never get ahead of the in-memory counter. If the file
+		// system is captured after CRIU resumed the container (the bug this
+		// guards against), restore either fails (diff tar caught a file
+		// mid-write) or observes an on-disk value ahead of memory.
+		script := `trap 'exit 0' TERM; f=/counter; n=0; echo "$n" > "$f"; ` +
+			`while true; do read d < "$f"; case "$d" in ""|*[!0-9]*) d=0;; esac; ` +
+			`if [ "$d" -gt "$n" ]; then echo "disk=$d mem=$n" >> /inconsistent; fi; ` +
+			`n=$((n+1)); echo "$n" > "$f"; sleep 0.1; done`
+
+		localRunString := getRunString([]string{ALPINE, "sh", "-c", script})
+		cid := podmanTest.PodmanExitCleanly(localRunString...).OutputToString()
+
+		// counter returns the workload's on-disk counter, or -1 if it cannot
+		// be read yet.
+		counter := func() int {
+			s := podmanTest.Podman([]string{"exec", cid, "cat", "/counter"})
+			s.WaitWithDefaultTimeout()
+			if s.ExitCode() != 0 {
+				return -1
+			}
+			n, err := strconv.Atoi(strings.TrimSpace(s.OutputToString()))
+			if err != nil {
+				return -1
+			}
+			return n
+		}
+
+		// Wait until the workload is up and has advanced its counter, instead
+		// of sleeping for a fixed amount of time.
+		Eventually(counter, "10s", "200ms").Should(BeNumerically(">", 0))
+
+		fileName := filepath.Join(podmanTest.TempDir, "consistency-"+cid+".tar")
+		podmanTest.PodmanExitCleanly("container", "checkpoint", "--leave-running", "--export", fileName, cid)
+
+		// The source container must be running and responsive (thawed) after
+		// a live checkpoint; a leaked freeze would make this exec hang.
+		podmanTest.PodmanExitCleanly("exec", cid, "true")
+
+		// Remove the original and restore from the checkpoint image. The
+		// restored process resumes from the captured memory image while its
+		// root file system comes from the captured diff. A torn diff (file
+		// captured mid-write) makes this restore fail.
+		podmanTest.PodmanExitCleanly("rm", "-t", "0", "-f", cid)
+
+		podmanTest.PodmanExitCleanly("container", "restore", "--import", fileName)
+		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
+
+		// Wait until the restored workload has run at least one more iteration
+		// (its counter advances) so the consistency check actually executes,
+		// rather than sleeping for a fixed amount of time.
+		var restored int
+		Eventually(func() bool {
+			restored = counter()
+			return restored >= 0
+		}, "10s", "200ms").Should(BeTrue())
+		Eventually(counter, "10s", "200ms").Should(BeNumerically(">", restored))
+
+		check := podmanTest.PodmanExitCleanly("exec", cid, "sh", "-c", "cat /inconsistent 2>/dev/null || true")
+		Expect(check.OutputToString()).To(BeEmpty(),
+			"restored container's file system is inconsistent with its memory image")
+	})
+
 	It("podman checkpoint and restore container with same IP", func() {
 		localRunString := getRunString([]string{"--name", "test_name", ALPINE, "top"})
 		session := podmanTest.Podman(localRunString)