mirror of
https://github.com/podman-container-tools/podman.git
synced 2026-09-03 14:17:50 +00:00
This commit addresses three intertwined bugs to fix an issue when using Gitlab runner on Podman. The three bug fixes are not split into separate commits as tests won't pass otherwise; avoidable noise when bisecting future issues. 1) Podman conflated states: even when asking to wait for the `exited` state, Podman returned as soon as a container transitioned to `stopped`. The issues surfaced in Gitlab tests to fail [1] as `conmon`'s buffers have not (yet) been emptied when attaching to a container right after a wait. The race window was extremely narrow, and I only managed to reproduce with the Gitlab runner [1] unit tests. 2) The clearer separation between `exited` and `stopped` revealed a race condition predating the changes. If a container is configured for autoremoval (e.g., via `run --rm`), the "run" process competes with the "cleanup" process running in the background. The window of the race condition was sufficiently large that the "cleanup" process has already removed the container and storage before the "run" process could read the exit code and hence waited indefinitely. Address the exit-code race condition by recording exit codes in the main libpod database. Exit codes can now be read from a database. When waiting for a container to exit, Podman first waits for the container to transition to `exited` and will then query the database for its exit code. Outdated exit codes are pruned during cleanup (i.e., non-performance critical) and when refreshing the database after a reboot. An exit code is considered outdated when it is older than 5 minutes. While the race condition predates this change, the waiting process has apparently always been fast enough in catching the exit code due to issue 1): `exited` and `stopped` were conflated. The waiting process hence caught the exit code after the container transitioned to `stopped` but before it `exited` and got removed. 3) With 1) and 2), Podman is now waiting for a container to properly transition to the `exited` state. Some tests did not pass after 1) and 2) which revealed the third bug: `conmon` was executed with its working directory pointing to the OCI runtime bundle of the container. The changed working directory broke resolving relative paths in the "cleanup" process. The "cleanup" process error'ed before actually cleaning up the container and waiting "main" process ran indefinitely - or until hitting a timeout. Fix the issue by executing `conmon` with the same working directory as Podman. Note that fixing 3) *may* address a number of issues we have seen in the past where for *some* reason cleanup processes did not fire. [1] https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27119#note_970712864 Signed-off-by: Valentin Rothberg <vrothberg@redhat.com> [MH: Minor reword of commit message] Signed-off-by: Matthew Heon <mheon@redhat.com>
248 lines
6.7 KiB
Go
248 lines
6.7 KiB
Go
package utils
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/containers/podman/v4/libpod/events"
|
|
api "github.com/containers/podman/v4/pkg/api/types"
|
|
"github.com/containers/podman/v4/pkg/domain/entities"
|
|
"github.com/containers/podman/v4/pkg/domain/infra/abi"
|
|
|
|
"github.com/containers/podman/v4/pkg/api/handlers"
|
|
"github.com/sirupsen/logrus"
|
|
|
|
"github.com/containers/podman/v4/libpod/define"
|
|
|
|
"github.com/containers/podman/v4/libpod"
|
|
"github.com/gorilla/schema"
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
type waitQueryDocker struct {
|
|
Condition string `schema:"condition"`
|
|
}
|
|
|
|
type waitQueryLibpod struct {
|
|
Interval string `schema:"interval"`
|
|
Condition []define.ContainerStatus `schema:"condition"`
|
|
}
|
|
|
|
func WaitContainerDocker(w http.ResponseWriter, r *http.Request) {
|
|
var err error
|
|
ctx := r.Context()
|
|
|
|
query := waitQueryDocker{}
|
|
|
|
decoder := ctx.Value(api.DecoderKey).(*schema.Decoder)
|
|
if err = decoder.Decode(&query, r.URL.Query()); err != nil {
|
|
Error(w, http.StatusBadRequest, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String()))
|
|
return
|
|
}
|
|
|
|
interval := time.Nanosecond
|
|
|
|
condition := "not-running"
|
|
if _, found := r.URL.Query()["condition"]; found {
|
|
condition = query.Condition
|
|
if !isValidDockerCondition(query.Condition) {
|
|
BadRequest(w, "condition", condition, errors.New("not a valid docker condition"))
|
|
return
|
|
}
|
|
}
|
|
|
|
name := GetName(r)
|
|
|
|
exists, err := containerExists(ctx, name)
|
|
if err != nil {
|
|
InternalServerError(w, err)
|
|
return
|
|
}
|
|
if !exists {
|
|
ContainerNotFound(w, name, define.ErrNoSuchCtr)
|
|
return
|
|
}
|
|
|
|
// In docker compatibility mode we have to send headers in advance,
|
|
// otherwise docker client would freeze.
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(200)
|
|
if flusher, ok := w.(http.Flusher); ok {
|
|
flusher.Flush()
|
|
}
|
|
|
|
exitCode, err := waitDockerCondition(ctx, name, interval, condition)
|
|
var errStruct *struct{ Message string }
|
|
if err != nil {
|
|
logrus.Errorf("While waiting on condition: %q", err)
|
|
errStruct = &struct {
|
|
Message string
|
|
}{
|
|
Message: err.Error(),
|
|
}
|
|
}
|
|
|
|
responseData := handlers.ContainerWaitOKBody{
|
|
StatusCode: int(exitCode),
|
|
Error: errStruct,
|
|
}
|
|
enc := json.NewEncoder(w)
|
|
enc.SetEscapeHTML(true)
|
|
err = enc.Encode(&responseData)
|
|
if err != nil {
|
|
logrus.Errorf("Unable to write json: %q", err)
|
|
}
|
|
}
|
|
|
|
func WaitContainerLibpod(w http.ResponseWriter, r *http.Request) {
|
|
var (
|
|
err error
|
|
interval = time.Millisecond * 250
|
|
conditions = []define.ContainerStatus{define.ContainerStateStopped, define.ContainerStateExited}
|
|
)
|
|
decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder)
|
|
query := waitQueryLibpod{}
|
|
if err := decoder.Decode(&query, r.URL.Query()); err != nil {
|
|
Error(w, http.StatusBadRequest, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String()))
|
|
return
|
|
}
|
|
|
|
if _, found := r.URL.Query()["interval"]; found {
|
|
interval, err = time.ParseDuration(query.Interval)
|
|
if err != nil {
|
|
InternalServerError(w, err)
|
|
return
|
|
}
|
|
}
|
|
|
|
if _, found := r.URL.Query()["condition"]; found {
|
|
if len(query.Condition) > 0 {
|
|
conditions = query.Condition
|
|
}
|
|
}
|
|
|
|
name := GetName(r)
|
|
|
|
waitFn := createContainerWaitFn(r.Context(), name, interval)
|
|
|
|
exitCode, err := waitFn(conditions...)
|
|
if err != nil {
|
|
if errors.Cause(err) == define.ErrNoSuchCtr {
|
|
ContainerNotFound(w, name, err)
|
|
return
|
|
}
|
|
InternalServerError(w, err)
|
|
return
|
|
}
|
|
WriteResponse(w, http.StatusOK, strconv.Itoa(int(exitCode)))
|
|
}
|
|
|
|
type containerWaitFn func(conditions ...define.ContainerStatus) (int32, error)
|
|
|
|
func createContainerWaitFn(ctx context.Context, containerName string, interval time.Duration) containerWaitFn {
|
|
runtime := ctx.Value(api.RuntimeKey).(*libpod.Runtime)
|
|
var containerEngine entities.ContainerEngine = &abi.ContainerEngine{Libpod: runtime}
|
|
|
|
return func(conditions ...define.ContainerStatus) (int32, error) {
|
|
opts := entities.WaitOptions{
|
|
Condition: conditions,
|
|
Interval: interval,
|
|
}
|
|
ctrWaitReport, err := containerEngine.ContainerWait(ctx, []string{containerName}, opts)
|
|
if err != nil {
|
|
return -1, err
|
|
}
|
|
if len(ctrWaitReport) != 1 {
|
|
return -1, fmt.Errorf("the ContainerWait() function returned unexpected count of reports: %d", len(ctrWaitReport))
|
|
}
|
|
return ctrWaitReport[0].ExitCode, ctrWaitReport[0].Error
|
|
}
|
|
}
|
|
|
|
func isValidDockerCondition(cond string) bool {
|
|
switch cond {
|
|
case "next-exit", "removed", "not-running", "":
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func waitDockerCondition(ctx context.Context, containerName string, interval time.Duration, dockerCondition string) (int32, error) {
|
|
containerWait := createContainerWaitFn(ctx, containerName, interval)
|
|
|
|
var err error
|
|
var code int32
|
|
switch dockerCondition {
|
|
case "next-exit":
|
|
code, err = waitNextExit(ctx, containerName)
|
|
case "removed":
|
|
code, err = waitRemoved(containerWait)
|
|
case "not-running", "":
|
|
code, err = waitNotRunning(containerWait)
|
|
default:
|
|
panic("not a valid docker condition")
|
|
}
|
|
return code, err
|
|
}
|
|
|
|
var notRunningStates = []define.ContainerStatus{
|
|
define.ContainerStateCreated,
|
|
define.ContainerStateRemoving,
|
|
define.ContainerStateExited,
|
|
define.ContainerStateConfigured,
|
|
}
|
|
|
|
func waitRemoved(ctrWait containerWaitFn) (int32, error) {
|
|
code, err := ctrWait(define.ContainerStateUnknown)
|
|
if err != nil && errors.Cause(err) == define.ErrNoSuchCtr {
|
|
return code, nil
|
|
}
|
|
return code, err
|
|
}
|
|
|
|
func waitNextExit(ctx context.Context, containerName string) (int32, error) {
|
|
runtime := ctx.Value(api.RuntimeKey).(*libpod.Runtime)
|
|
containerEngine := &abi.ContainerEngine{Libpod: runtime}
|
|
eventChannel := make(chan *events.Event)
|
|
errChannel := make(chan error)
|
|
opts := entities.EventsOptions{
|
|
EventChan: eventChannel,
|
|
Filter: []string{"event=died", fmt.Sprintf("container=%s", containerName)},
|
|
Stream: true,
|
|
}
|
|
|
|
// ctx is used to cancel event watching goroutine
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
go func() {
|
|
errChannel <- containerEngine.Events(ctx, opts)
|
|
}()
|
|
|
|
evt, ok := <-eventChannel
|
|
if ok {
|
|
return int32(evt.ContainerExitCode), nil
|
|
}
|
|
// if ok == false then containerEngine.Events() has exited
|
|
// it may happen if request was canceled (e.g. client closed connection prematurely) or
|
|
// the server is in process of shutting down
|
|
return -1, <-errChannel
|
|
}
|
|
|
|
func waitNotRunning(ctrWait containerWaitFn) (int32, error) {
|
|
return ctrWait(notRunningStates...)
|
|
}
|
|
|
|
func containerExists(ctx context.Context, name string) (bool, error) {
|
|
runtime := ctx.Value(api.RuntimeKey).(*libpod.Runtime)
|
|
var containerEngine entities.ContainerEngine = &abi.ContainerEngine{Libpod: runtime}
|
|
|
|
var ctrExistsOpts entities.ContainerExistsOptions
|
|
ctrExistRep, err := containerEngine.ContainerExists(ctx, name, ctrExistsOpts)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return ctrExistRep.Value, nil
|
|
}
|