From 9598b30ac2983ffaca61ac9fece9618b30eef349 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Rod=C3=A1k?= Date: Mon, 13 Apr 2026 18:33:29 +0200 Subject: [PATCH] Fix healthcheck failing silently with --transient-store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The systemd timer created for health checks did not pass global podman flags to the subprocess, causing it to use default storage settings instead of matching the parent process. This is most visible with --transient-store, where the healthcheck looks up the container in the default store instead of the volatile one. Extract GlobalPodmanArgs() from CreateExitCommandArgs so both the exit command and healthcheck timer share the same set of global flags (--root, --runroot, --transient-store, --storage-driver, etc.). Fixes: https://github.com/containers/podman/issues/28483 Signed-off-by: Jan Rodák --- libpod/healthcheck_linux.go | 5 +- pkg/specgenutil/util.go | 84 +++++++++++----------- test/system/220-healthcheck.bats | 2 +- test/system/221-healthcheck-transient.bats | 45 ++++++++++++ 4 files changed, 92 insertions(+), 44 deletions(-) create mode 100644 test/system/221-healthcheck-transient.bats diff --git a/libpod/healthcheck_linux.go b/libpod/healthcheck_linux.go index 49f4f329ee..965604dfeb 100644 --- a/libpod/healthcheck_linux.go +++ b/libpod/healthcheck_linux.go @@ -13,6 +13,7 @@ import ( "github.com/containers/podman/v6/pkg/errorhandling" "github.com/containers/podman/v6/pkg/rootless" + "github.com/containers/podman/v6/pkg/specgenutil" "github.com/containers/podman/v6/pkg/systemd" "github.com/sirupsen/logrus" systemdCommon "go.podman.io/common/pkg/systemd" @@ -43,9 +44,7 @@ func (c *Container) createTimer(interval string, isStartup bool) error { // StartLimitIntervalSec=0 so we don't hit the restart limit cmd = append(cmd, "--unit", hcUnitName, fmt.Sprintf("--on-unit-inactive=%s", interval), "--timer-property=AccuracySec=1s", "--property=StartLimitIntervalSec=0", podman) - if logrus.IsLevelEnabled(logrus.DebugLevel) { - cmd = append(cmd, "--log-level=debug", "--syslog") - } + cmd = append(cmd, specgenutil.GlobalPodmanArgs(c.runtime.storageConfig, c.runtime.config, logrus.IsLevelEnabled(logrus.DebugLevel))...) cmd = append(cmd, "healthcheck", "run", "--ignore-result", c.ID()) diff --git a/pkg/specgenutil/util.go b/pkg/specgenutil/util.go index 8fe0ae9ff8..225b42526e 100644 --- a/pkg/specgenutil/util.go +++ b/pkg/specgenutil/util.go @@ -257,6 +257,48 @@ func parseAndValidatePort(port string) (uint16, error) { return uint16(num), nil } +// GlobalPodmanArgs returns the global podman CLI flags needed to ensure a +// subprocess uses the same storage, runtime, and logging configuration as +// the parent process. The returned slice does NOT include the podman binary +// path or any subcommand-specific flags. +func GlobalPodmanArgs(storageConfig storageTypes.StoreOptions, cfg *config.Config, syslog bool) []string { + args := []string{ + "--root", storageConfig.GraphRoot, + "--runroot", storageConfig.RunRoot, + "--log-level", logrus.GetLevel().String(), + "--cgroup-manager", cfg.Engine.CgroupManager, + "--tmpdir", cfg.Engine.TmpDir, + "--network-config-dir", cfg.Network.NetworkConfigDir, + "--volumepath", cfg.Engine.VolumePath, + fmt.Sprintf("--transient-store=%t", storageConfig.TransientStore), + } + for _, dir := range cfg.Engine.HooksDir.Get() { + args = append(args, "--hooks-dir", dir) + } + if storageConfig.ImageStore != "" { + args = append(args, "--imagestore", storageConfig.ImageStore) + } + if cfg.Engine.OCIRuntime != "" { + args = append(args, "--runtime", cfg.Engine.OCIRuntime) + } + if storageConfig.GraphDriverName != "" { + args = append(args, "--storage-driver", storageConfig.GraphDriverName) + } + for _, opt := range storageConfig.GraphDriverOptions { + args = append(args, "--storage-opt", opt) + } + if cfg.Engine.EventsLogger != "" { + args = append(args, "--events-backend", cfg.Engine.EventsLogger) + } + if syslog { + args = append(args, "--syslog") + } + for _, module := range cfg.LoadedModules() { + args = append(args, "--module", module) + } + return args +} + func CreateExitCommandArgs(storageConfig storageTypes.StoreOptions, config *config.Config, syslog, rm, rmi, exec bool) ([]string, error) { // We need a cleanup process for containers in the current model. // But we can't assume that the caller is Podman - it could be another @@ -269,49 +311,11 @@ func CreateExitCommandArgs(storageConfig storageTypes.StoreOptions, config *conf return nil, err } - command := []string{ - podmanPath, - "--root", storageConfig.GraphRoot, - "--runroot", storageConfig.RunRoot, - "--log-level", logrus.GetLevel().String(), - "--cgroup-manager", config.Engine.CgroupManager, - "--tmpdir", config.Engine.TmpDir, - "--network-config-dir", config.Network.NetworkConfigDir, - "--volumepath", config.Engine.VolumePath, - fmt.Sprintf("--transient-store=%t", storageConfig.TransientStore), - } - for _, dir := range config.Engine.HooksDir.Get() { - command = append(command, []string{"--hooks-dir", dir}...) - } - if storageConfig.ImageStore != "" { - command = append(command, []string{"--imagestore", storageConfig.ImageStore}...) - } - if config.Engine.OCIRuntime != "" { - command = append(command, []string{"--runtime", config.Engine.OCIRuntime}...) - } - if storageConfig.GraphDriverName != "" { - command = append(command, []string{"--storage-driver", storageConfig.GraphDriverName}...) - } - for _, opt := range storageConfig.GraphDriverOptions { - command = append(command, []string{"--storage-opt", opt}...) - } - if config.Engine.EventsLogger != "" { - command = append(command, []string{"--events-backend", config.Engine.EventsLogger}...) - } - - if syslog { - command = append(command, "--syslog") - } - - // Make sure that loaded containers.conf modules are passed down to the - // callback as well. - for _, module := range config.LoadedModules() { - command = append(command, "--module", module) - } + command := append([]string{podmanPath}, GlobalPodmanArgs(storageConfig, config, syslog)...) // --stopped-only is used to ensure we only cleanup stopped containers and do not race // against other processes that did a cleanup() + init() again before we had the chance to run - command = append(command, []string{"container", "cleanup", "--stopped-only"}...) + command = append(command, "container", "cleanup", "--stopped-only") if rm { command = append(command, "--rm") diff --git a/test/system/220-healthcheck.bats b/test/system/220-healthcheck.bats index 20459a7cd6..6d1bb121b1 100644 --- a/test/system/220-healthcheck.bats +++ b/test/system/220-healthcheck.bats @@ -101,7 +101,7 @@ Log[-1].Output | \"Uh-oh on stdout!\\\nUh-oh on stderr!\\\n\" run -0 systemctl list-units cidmatch=$(grep "$cid" <<<"$output") echo "$cidmatch" - assert "$cidmatch" =~ " $cid-[0-9a-f]+\.timer *.*/podman healthcheck run --ignore-result $cid" \ + assert "$cidmatch" =~ " $cid-[0-9a-f]+\.timer *.*/podman .* healthcheck run --ignore-result $cid" \ "Healthcheck systemd unit exists" # Check that the right service option is applied so we don't hit the systemd restart limit. diff --git a/test/system/221-healthcheck-transient.bats b/test/system/221-healthcheck-transient.bats new file mode 100644 index 0000000000..02ee6e0ff8 --- /dev/null +++ b/test/system/221-healthcheck-transient.bats @@ -0,0 +1,45 @@ +#!/usr/bin/env bats -*- bats -*- +# +# tests for podman healthcheck with --transient-store +# +# + +load helpers +load helpers.systemd + + +function teardown() { + run_podman '?' --transient-store rm -t 0 -a -f + basic_teardown +} + +# https://github.com/containers/podman/issues/28483 +@test "podman healthcheck --transient-store" { + skip_if_remote "transient-store is a local option" + + ctr="c-h-$(safename)" + + run_podman run -d --name $ctr --transient-store \ + --health-cmd /home/podman/healthcheck \ + --health-interval 1s \ + --health-retries 3 \ + $IMAGE /home/podman/pause + cid="$output" + + # The systemd timer command line must include --transient-store=true so + # the healthcheck subprocess opens the correct (volatile) store. + run -0 systemctl list-units + cidmatch=$(grep "$cid" <<<"$output") + assert "$cidmatch" =~ "--transient-store=true .* healthcheck run --ignore-result $cid" \ + "Healthcheck systemd unit includes --transient-store=true" + + run_podman --transient-store wait --condition=healthy $ctr + + run_podman --transient-store inspect $ctr \ + --format "{{.State.Health.Status}} {{.State.Health.FailingStreak}}" + assert "$output" == "healthy 0" "health status and failing streak" + + run_podman --transient-store rm -f -t0 $ctr +} + +# vim: filetype=sh