Fix healthcheck failing silently with --transient-store

The systemd timer created for health checks did not pass global
podman flags to the subprocess, causing it to use default storage
settings instead of matching the parent process. This is most
visible with --transient-store, where the healthcheck looks up
the container in the default store instead of the volatile one.

Extract GlobalPodmanArgs() from CreateExitCommandArgs so both the
exit command and healthcheck timer share the same set of global
flags (--root, --runroot, --transient-store, --storage-driver, etc.).

Fixes: https://github.com/containers/podman/issues/28483

Signed-off-by: Jan Rodák <hony.com@seznam.cz>

libpod/healthcheck_linux.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/containers/podman/v6/pkg/errorhandling"
 	"github.com/containers/podman/v6/pkg/rootless"
+	"github.com/containers/podman/v6/pkg/specgenutil"
 	"github.com/containers/podman/v6/pkg/systemd"
 	"github.com/sirupsen/logrus"
 	systemdCommon "go.podman.io/common/pkg/systemd"
@@ -43,9 +44,7 @@ func (c *Container) createTimer(interval string, isStartup bool) error {
 	// StartLimitIntervalSec=0 so we don't hit the restart limit
 	cmd = append(cmd, "--unit", hcUnitName, fmt.Sprintf("--on-unit-inactive=%s", interval), "--timer-property=AccuracySec=1s", "--property=StartLimitIntervalSec=0", podman)
 
-	if logrus.IsLevelEnabled(logrus.DebugLevel) {
-		cmd = append(cmd, "--log-level=debug", "--syslog")
-	}
+	cmd = append(cmd, specgenutil.GlobalPodmanArgs(c.runtime.storageConfig, c.runtime.config, logrus.IsLevelEnabled(logrus.DebugLevel))...)
 
 	cmd = append(cmd, "healthcheck", "run", "--ignore-result", c.ID())
 

pkg/specgenutil/util.go

@@ -257,6 +257,48 @@ func parseAndValidatePort(port string) (uint16, error) {
 	return uint16(num), nil
 }
 
+// GlobalPodmanArgs returns the global podman CLI flags needed to ensure a
+// subprocess uses the same storage, runtime, and logging configuration as
+// the parent process. The returned slice does NOT include the podman binary
+// path or any subcommand-specific flags.
+func GlobalPodmanArgs(storageConfig storageTypes.StoreOptions, cfg *config.Config, syslog bool) []string {
+	args := []string{
+		"--root", storageConfig.GraphRoot,
+		"--runroot", storageConfig.RunRoot,
+		"--log-level", logrus.GetLevel().String(),
+		"--cgroup-manager", cfg.Engine.CgroupManager,
+		"--tmpdir", cfg.Engine.TmpDir,
+		"--network-config-dir", cfg.Network.NetworkConfigDir,
+		"--volumepath", cfg.Engine.VolumePath,
+		fmt.Sprintf("--transient-store=%t", storageConfig.TransientStore),
+	}
+	for _, dir := range cfg.Engine.HooksDir.Get() {
+		args = append(args, "--hooks-dir", dir)
+	}
+	if storageConfig.ImageStore != "" {
+		args = append(args, "--imagestore", storageConfig.ImageStore)
+	}
+	if cfg.Engine.OCIRuntime != "" {
+		args = append(args, "--runtime", cfg.Engine.OCIRuntime)
+	}
+	if storageConfig.GraphDriverName != "" {
+		args = append(args, "--storage-driver", storageConfig.GraphDriverName)
+	}
+	for _, opt := range storageConfig.GraphDriverOptions {
+		args = append(args, "--storage-opt", opt)
+	}
+	if cfg.Engine.EventsLogger != "" {
+		args = append(args, "--events-backend", cfg.Engine.EventsLogger)
+	}
+	if syslog {
+		args = append(args, "--syslog")
+	}
+	for _, module := range cfg.LoadedModules() {
+		args = append(args, "--module", module)
+	}
+	return args
+}
+
 func CreateExitCommandArgs(storageConfig storageTypes.StoreOptions, config *config.Config, syslog, rm, rmi, exec bool) ([]string, error) {
 	// We need a cleanup process for containers in the current model.
 	// But we can't assume that the caller is Podman - it could be another
@@ -269,49 +311,11 @@ func CreateExitCommandArgs(storageConfig storageTypes.StoreOptions, config *conf
 		return nil, err
 	}
 
-	command := []string{
-		podmanPath,
-		"--root", storageConfig.GraphRoot,
-		"--runroot", storageConfig.RunRoot,
-		"--log-level", logrus.GetLevel().String(),
-		"--cgroup-manager", config.Engine.CgroupManager,
-		"--tmpdir", config.Engine.TmpDir,
-		"--network-config-dir", config.Network.NetworkConfigDir,
-		"--volumepath", config.Engine.VolumePath,
-		fmt.Sprintf("--transient-store=%t", storageConfig.TransientStore),
-	}
-	for _, dir := range config.Engine.HooksDir.Get() {
-		command = append(command, []string{"--hooks-dir", dir}...)
-	}
-	if storageConfig.ImageStore != "" {
-		command = append(command, []string{"--imagestore", storageConfig.ImageStore}...)
-	}
-	if config.Engine.OCIRuntime != "" {
-		command = append(command, []string{"--runtime", config.Engine.OCIRuntime}...)
-	}
-	if storageConfig.GraphDriverName != "" {
-		command = append(command, []string{"--storage-driver", storageConfig.GraphDriverName}...)
-	}
-	for _, opt := range storageConfig.GraphDriverOptions {
-		command = append(command, []string{"--storage-opt", opt}...)
-	}
-	if config.Engine.EventsLogger != "" {
-		command = append(command, []string{"--events-backend", config.Engine.EventsLogger}...)
-	}
-
-	if syslog {
-		command = append(command, "--syslog")
-	}
-
-	// Make sure that loaded containers.conf modules are passed down to the
-	// callback as well.
-	for _, module := range config.LoadedModules() {
-		command = append(command, "--module", module)
-	}
+	command := append([]string{podmanPath}, GlobalPodmanArgs(storageConfig, config, syslog)...)
 
 	// --stopped-only is used to ensure we only cleanup stopped containers and do not race
 	// against other processes that did a cleanup() + init() again before we had the chance to run
-	command = append(command, []string{"container", "cleanup", "--stopped-only"}...)
+	command = append(command, "container", "cleanup", "--stopped-only")
 
 	if rm {
 		command = append(command, "--rm")

test/system/220-healthcheck.bats

@@ -101,7 +101,7 @@ Log[-1].Output   | \"Uh-oh on stdout!\\\nUh-oh on stderr!\\\n\"
     run -0 systemctl list-units
     cidmatch=$(grep "$cid" <<<"$output")
     echo "$cidmatch"
-    assert "$cidmatch" =~ " $cid-[0-9a-f]+\.timer  *.*/podman healthcheck run --ignore-result $cid" \
+    assert "$cidmatch" =~ " $cid-[0-9a-f]+\.timer  *.*/podman .* healthcheck run --ignore-result $cid" \
            "Healthcheck systemd unit exists"
 
     # Check that the right service option is applied so we don't hit the systemd restart limit.

test/system/221-healthcheck-transient.bats

@@ -0,0 +1,45 @@
+#!/usr/bin/env bats   -*- bats -*-
+#
+# tests for podman healthcheck with --transient-store
+#
+#
+
+load helpers
+load helpers.systemd
+
+
+function teardown() {
+    run_podman '?' --transient-store rm -t 0 -a -f
+    basic_teardown
+}
+
+# https://github.com/containers/podman/issues/28483
+@test "podman healthcheck --transient-store" {
+    skip_if_remote "transient-store is a local option"
+
+    ctr="c-h-$(safename)"
+
+    run_podman run -d --name $ctr --transient-store \
+           --health-cmd /home/podman/healthcheck    \
+           --health-interval 1s                     \
+           --health-retries 3                       \
+           $IMAGE /home/podman/pause
+    cid="$output"
+
+    # The systemd timer command line must include --transient-store=true so
+    # the healthcheck subprocess opens the correct (volatile) store.
+    run -0 systemctl list-units
+    cidmatch=$(grep "$cid" <<<"$output")
+    assert "$cidmatch" =~ "--transient-store=true .* healthcheck run --ignore-result $cid" \
+           "Healthcheck systemd unit includes --transient-store=true"
+
+    run_podman --transient-store wait --condition=healthy $ctr
+
+    run_podman --transient-store inspect $ctr \
+               --format "{{.State.Health.Status}} {{.State.Health.FailingStreak}}"
+    assert "$output" == "healthy 0" "health status and failing streak"
+
+    run_podman --transient-store rm -f -t0 $ctr
+}
+
+# vim: filetype=sh