mirror of
https://github.com/podman-container-tools/podman.git
synced 2026-09-15 12:07:53 +00:00
77bbf5fa79
Integration tests to verify podman-container-tools/container-libs#845 Signed-off-by: Jan Rodák <hony.com@seznam.cz>
887 lines
29 KiB
Go
887 lines
29 KiB
Go
//go:build linux || freebsd
|
|
|
|
package integration
|
|
|
|
import (
|
|
"archive/tar"
|
|
"bytes"
|
|
"compress/gzip"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"runtime"
|
|
"strings"
|
|
"sync/atomic"
|
|
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
"github.com/opencontainers/go-digest"
|
|
. "go.podman.io/podman/v6/test/utils"
|
|
"go.podman.io/storage/pkg/chunked/compressor"
|
|
)
|
|
|
|
// primaryAddr extracts the host:port from an httptest.Server.
|
|
func primaryAddr(s *httptest.Server) string {
|
|
return s.Listener.Addr().String()
|
|
}
|
|
|
|
// buildTarLayer creates a single tar archive containing a test data file for the given layer index.
|
|
func buildTarLayer(index int) []byte {
|
|
content := fmt.Appendf(nil, "layer-%d test data: the quick brown fox jumps over the lazy dog\n", index)
|
|
var raw bytes.Buffer
|
|
tw := tar.NewWriter(&raw)
|
|
Expect(tw.WriteHeader(&tar.Header{
|
|
Name: fmt.Sprintf("layer-%d/data.txt", index),
|
|
Mode: 0o644,
|
|
Size: int64(len(content)),
|
|
})).To(Succeed())
|
|
_, err := tw.Write(content)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(tw.Close()).To(Succeed())
|
|
return raw.Bytes()
|
|
}
|
|
|
|
// buildImageManifest creates the config blob, manifest JSON, and returns the content type.
|
|
func buildImageManifest(blobs map[string][]byte, diffIDs []string, layers []map[string]any, configMediaType, manifestMediaType string) ([]byte, string) {
|
|
configBytes, err := json.Marshal(map[string]any{
|
|
"architecture": runtime.GOARCH,
|
|
"os": "linux",
|
|
"rootfs": map[string]any{"type": "layers", "diff_ids": diffIDs},
|
|
})
|
|
Expect(err).ToNot(HaveOccurred())
|
|
configDigest := digest.FromBytes(configBytes)
|
|
blobs[configDigest.String()] = configBytes
|
|
|
|
manifestBytes, err := json.Marshal(map[string]any{
|
|
"schemaVersion": 2,
|
|
"mediaType": manifestMediaType,
|
|
"config": map[string]any{
|
|
"mediaType": configMediaType,
|
|
"size": len(configBytes),
|
|
"digest": configDigest.String(),
|
|
},
|
|
"layers": layers,
|
|
})
|
|
Expect(err).ToNot(HaveOccurred())
|
|
return manifestBytes, manifestMediaType
|
|
}
|
|
|
|
type testCompression int
|
|
|
|
const (
|
|
compressionGzip testCompression = iota
|
|
compressionZstdChunked
|
|
)
|
|
|
|
// buildTestImage constructs a valid container image with numLayers layers
|
|
// using the specified compression. Gzip produces a Docker schema2 manifest;
|
|
// zstd:chunked produces an OCI manifest with TOC annotations for partial pulls.
|
|
func buildTestImage(numLayers int, compression testCompression) (map[string][]byte, []byte, string, []digest.Digest) {
|
|
blobs := make(map[string][]byte)
|
|
var diffIDs []string
|
|
var layers []map[string]any
|
|
var layerDigests []digest.Digest
|
|
|
|
for i := range numLayers {
|
|
rawTar := buildTarLayer(i)
|
|
diffIDs = append(diffIDs, digest.FromBytes(rawTar).String())
|
|
|
|
var layerBytes []byte
|
|
layerDesc := map[string]any{}
|
|
|
|
switch compression {
|
|
case compressionZstdChunked:
|
|
var comp bytes.Buffer
|
|
annotations := make(map[string]string)
|
|
zw, err := compressor.ZstdCompressor(&comp, annotations, nil)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
_, err = zw.Write(rawTar)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(zw.Close()).To(Succeed())
|
|
layerBytes = append([]byte(nil), comp.Bytes()...)
|
|
layerDesc["mediaType"] = "application/vnd.oci.image.layer.v1.tar+zstd"
|
|
layerDesc["annotations"] = annotations
|
|
default:
|
|
var comp bytes.Buffer
|
|
gz := gzip.NewWriter(&comp)
|
|
_, err := gz.Write(rawTar)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(gz.Close()).To(Succeed())
|
|
layerBytes = append([]byte(nil), comp.Bytes()...)
|
|
layerDesc["mediaType"] = "application/vnd.docker.image.rootfs.diff.tar.gzip"
|
|
}
|
|
|
|
ld := digest.FromBytes(layerBytes)
|
|
blobs[ld.String()] = layerBytes
|
|
layerDigests = append(layerDigests, ld)
|
|
layerDesc["size"] = len(layerBytes)
|
|
layerDesc["digest"] = ld.String()
|
|
layers = append(layers, layerDesc)
|
|
}
|
|
|
|
var configMT, manifestMT string
|
|
switch compression {
|
|
case compressionZstdChunked:
|
|
configMT = "application/vnd.oci.image.config.v1+json"
|
|
manifestMT = "application/vnd.oci.image.manifest.v1+json"
|
|
default:
|
|
configMT = "application/vnd.docker.container.image.v1+json"
|
|
manifestMT = "application/vnd.docker.distribution.manifest.v2+json"
|
|
}
|
|
|
|
manifestBytes, ct := buildImageManifest(blobs, diffIDs, layers, configMT, manifestMT)
|
|
return blobs, manifestBytes, ct, layerDigests
|
|
}
|
|
|
|
// blobMiddleware wraps the default blob-serving handler. Middleware that wants
|
|
// to fail should write a response and not call next. Middleware that wants
|
|
// normal serving should call next.ServeHTTP(w, r).
|
|
type blobMiddleware func(next http.Handler) http.Handler
|
|
|
|
// registryHandler returns an http.Handler that implements a minimal Docker
|
|
// registry v2 API for the given repository name, serving the manifest and
|
|
// blobs. If mw is non-nil it wraps the default blob handler.
|
|
//
|
|
//nolint:unparam
|
|
func registryHandler(repo string, manifestBytes []byte, manifestCT string, blobs map[string][]byte, mw blobMiddleware) http.Handler {
|
|
manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", repo)
|
|
blobPrefix := fmt.Sprintf("/v2/%s/blobs/", repo)
|
|
|
|
defaultBlobHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
dgst := strings.TrimPrefix(r.URL.Path, blobPrefix)
|
|
data, ok := blobs[dgst]
|
|
if !ok {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(data)))
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write(data)
|
|
})
|
|
|
|
var blobHandler http.Handler = defaultBlobHandler
|
|
if mw != nil {
|
|
blobHandler = mw(defaultBlobHandler)
|
|
}
|
|
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch {
|
|
case r.URL.Path == "/v2/":
|
|
w.WriteHeader(http.StatusOK)
|
|
case r.URL.Path == manifestPath:
|
|
w.Header().Set("Content-Type", manifestCT)
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write(manifestBytes)
|
|
case strings.HasPrefix(r.URL.Path, blobPrefix):
|
|
blobHandler.ServeHTTP(w, r)
|
|
default:
|
|
http.NotFound(w, r)
|
|
}
|
|
})
|
|
}
|
|
|
|
// errorBlobMiddleware returns a blobMiddleware that always responds with the
|
|
// given status and optional JSON error body, ignoring the default handler.
|
|
func errorBlobMiddleware(status int, code, message string) blobMiddleware {
|
|
return func(_ http.Handler) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
if code != "" {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(status)
|
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
|
"errors": []map[string]string{
|
|
{"code": code, "message": message},
|
|
},
|
|
})
|
|
} else {
|
|
w.WriteHeader(status)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
var (
|
|
blobMW503 = errorBlobMiddleware(http.StatusServiceUnavailable, "", "")
|
|
blobMW500 = errorBlobMiddleware(http.StatusInternalServerError, "", "")
|
|
blobMWBlobUnknown = errorBlobMiddleware(http.StatusNotFound, "BLOB_UNKNOWN", "blob unknown to registry")
|
|
blobMW429 = errorBlobMiddleware(http.StatusTooManyRequests, "TOOMANYREQUESTS", "rate limit exceeded")
|
|
blobMW401 = errorBlobMiddleware(http.StatusUnauthorized, "UNAUTHORIZED", "authentication required")
|
|
blobMW403 = errorBlobMiddleware(http.StatusForbidden, "DENIED", "access denied")
|
|
)
|
|
|
|
// countingBlobMW returns a blobMiddleware that increments counter then
|
|
// delegates to the next handler (default blob serving).
|
|
func countingBlobMW(counter *atomic.Int32) blobMiddleware {
|
|
return func(next http.Handler) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
counter.Add(1)
|
|
next.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
}
|
|
|
|
// countingErrorBlobMW increments counter, then responds with an error
|
|
// (ignoring the default handler).
|
|
func countingErrorBlobMW(counter *atomic.Int32, errMW blobMiddleware) blobMiddleware {
|
|
return func(next http.Handler) http.Handler {
|
|
errHandler := errMW(next)
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
counter.Add(1)
|
|
errHandler.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
}
|
|
|
|
// rangeOnlyErrorBlobMW applies errMW only to range requests (GetBlobAt path).
|
|
// Non-range requests (GetBlob) pass through to the default handler unchanged.
|
|
func rangeOnlyErrorBlobMW(errMW blobMiddleware) blobMiddleware {
|
|
return func(next http.Handler) http.Handler {
|
|
errHandler := errMW(next)
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Header.Get("Range") != "" {
|
|
errHandler.ServeHTTP(w, r)
|
|
return
|
|
}
|
|
next.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
}
|
|
|
|
// pullAndVerifySuccess runs podman pull and asserts it succeeds, cleans up.
|
|
// globalArgs are placed before the "pull" subcommand (e.g. --pull-option=…).
|
|
//
|
|
//nolint:unparam
|
|
func pullAndVerifySuccess(imageRef string, globalArgs ...string) {
|
|
args := make([]string, 0, len(globalArgs)+4)
|
|
args = append(args, globalArgs...)
|
|
args = append(args, "pull", "-q", "--tls-verify=false", imageRef)
|
|
podmanTest.PodmanExitCleanly(args...)
|
|
podmanTest.PodmanExitCleanly("image", "exists", imageRef)
|
|
podmanTest.PodmanExitCleanly("rmi", imageRef)
|
|
}
|
|
|
|
func pullMirrorFallbackTests() {
|
|
Describe("blob-level mirror fallback", func() {
|
|
const repo = "library/mirrortest"
|
|
|
|
type singleMirrorCase struct {
|
|
description string
|
|
mw blobMiddleware
|
|
expectPullSuccess bool
|
|
expectedError string
|
|
}
|
|
|
|
singleMirrorCases := []singleMirrorCase{
|
|
{
|
|
description: "falls back to primary when mirror returns 503 on blobs",
|
|
mw: blobMW503,
|
|
expectPullSuccess: true,
|
|
},
|
|
{
|
|
description: "falls back to primary when mirror returns 500 on blobs",
|
|
mw: blobMW500,
|
|
expectPullSuccess: true,
|
|
},
|
|
{
|
|
description: "falls back to primary when mirror returns BLOB_UNKNOWN on blobs",
|
|
mw: blobMWBlobUnknown,
|
|
expectPullSuccess: true,
|
|
},
|
|
{
|
|
description: "falls back to primary when mirror returns 429 on blobs",
|
|
mw: blobMW429,
|
|
expectPullSuccess: true,
|
|
},
|
|
{
|
|
description: "does not fall back when mirror returns 401 on blobs",
|
|
mw: blobMW401,
|
|
expectPullSuccess: false,
|
|
expectedError: "unauthorized",
|
|
},
|
|
{
|
|
description: "does not fall back when mirror returns 403 on blobs",
|
|
mw: blobMW403,
|
|
expectPullSuccess: false,
|
|
expectedError: "denied",
|
|
},
|
|
}
|
|
|
|
for _, tc := range singleMirrorCases {
|
|
It(tc.description, func() {
|
|
SkipIfRemote("registries.conf is not used by the remote client")
|
|
|
|
blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionGzip)
|
|
|
|
primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil))
|
|
defer primary.Close()
|
|
|
|
var mirrorBlobHits atomic.Int32
|
|
mirror := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs,
|
|
countingErrorBlobMW(&mirrorBlobHits, tc.mw)))
|
|
defer mirror.Close()
|
|
|
|
conf := fmt.Sprintf(`[[registry]]
|
|
prefix = "mirrortest.local"
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
`, primaryAddr(primary), primaryAddr(mirror))
|
|
podmanTest.setRegistriesConfigEnv([]byte(conf))
|
|
defer resetRegistriesConfigEnv()
|
|
|
|
imageRef := "mirrortest.local/" + repo + ":latest"
|
|
session := podmanTest.Podman([]string{"pull", "-q", "--tls-verify=false", imageRef})
|
|
session.WaitWithDefaultTimeout()
|
|
|
|
if tc.expectPullSuccess {
|
|
Expect(session).Should(ExitCleanly())
|
|
Expect(mirrorBlobHits.Load()).To(BeNumerically(">", int32(0)),
|
|
"mirror should have been tried for blobs before fallback")
|
|
podmanTest.PodmanExitCleanly("image", "exists", imageRef)
|
|
podmanTest.PodmanExitCleanly("rmi", imageRef)
|
|
} else {
|
|
Expect(session).Should(ExitWithError(125, tc.expectedError))
|
|
}
|
|
})
|
|
}
|
|
|
|
It("skips multiple broken mirrors and falls back to primary", func() {
|
|
SkipIfRemote("registries.conf is not used by the remote client")
|
|
|
|
blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionGzip)
|
|
|
|
primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil))
|
|
defer primary.Close()
|
|
|
|
// Two broken mirrors: first returns 503, second returns BLOB_UNKNOWN.
|
|
var mirror1Hits, mirror2Hits atomic.Int32
|
|
mirror1 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs,
|
|
countingErrorBlobMW(&mirror1Hits, blobMW503)))
|
|
defer mirror1.Close()
|
|
mirror2 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs,
|
|
countingErrorBlobMW(&mirror2Hits, blobMWBlobUnknown)))
|
|
defer mirror2.Close()
|
|
|
|
conf := fmt.Sprintf(`[[registry]]
|
|
prefix = "mirrortest.local"
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
`, primaryAddr(primary), primaryAddr(mirror1), primaryAddr(mirror2))
|
|
podmanTest.setRegistriesConfigEnv([]byte(conf))
|
|
defer resetRegistriesConfigEnv()
|
|
|
|
imageRef := "mirrortest.local/" + repo + ":latest"
|
|
pullAndVerifySuccess(imageRef)
|
|
|
|
Expect(mirror1Hits.Load()).To(BeNumerically(">", int32(0)), "mirror1 should have been tried for blobs")
|
|
Expect(mirror2Hits.Load()).To(BeNumerically(">", int32(0)), "mirror2 should have been tried for blobs")
|
|
})
|
|
|
|
It("stops at first working mirror without reaching primary", func() {
|
|
SkipIfRemote("registries.conf is not used by the remote client")
|
|
|
|
blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionGzip)
|
|
|
|
var primaryBlobHits atomic.Int32
|
|
primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs,
|
|
countingBlobMW(&primaryBlobHits)))
|
|
defer primary.Close()
|
|
|
|
// mirror1: serves manifest, returns 503 on blobs.
|
|
mirror1 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMW503))
|
|
defer mirror1.Close()
|
|
|
|
// mirror2: serves everything. The fallback should stop here.
|
|
mirror2 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil))
|
|
defer mirror2.Close()
|
|
|
|
conf := fmt.Sprintf(`[[registry]]
|
|
prefix = "mirrortest.local"
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
`, primaryAddr(primary), primaryAddr(mirror1), primaryAddr(mirror2))
|
|
podmanTest.setRegistriesConfigEnv([]byte(conf))
|
|
defer resetRegistriesConfigEnv()
|
|
|
|
imageRef := "mirrortest.local/" + repo + ":latest"
|
|
pullAndVerifySuccess(imageRef)
|
|
|
|
Expect(primaryBlobHits.Load()).To(Equal(int32(0)), "primary should NOT be contacted when a working mirror is found first")
|
|
})
|
|
|
|
It("fails when all mirrors and primary return errors on blobs", func() {
|
|
SkipIfRemote("registries.conf is not used by the remote client")
|
|
|
|
blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionGzip)
|
|
|
|
// Primary also fails on blobs.
|
|
primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMW500))
|
|
defer primary.Close()
|
|
|
|
// Two mirrors, both broken.
|
|
mirror1 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMW503))
|
|
defer mirror1.Close()
|
|
mirror2 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMWBlobUnknown))
|
|
defer mirror2.Close()
|
|
|
|
conf := fmt.Sprintf(`[[registry]]
|
|
prefix = "mirrortest.local"
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
`, primaryAddr(primary), primaryAddr(mirror1), primaryAddr(mirror2))
|
|
podmanTest.setRegistriesConfigEnv([]byte(conf))
|
|
defer resetRegistriesConfigEnv()
|
|
|
|
imageRef := "mirrortest.local/" + repo + ":latest"
|
|
session := podmanTest.Podman([]string{"pull", "-q", "--tls-verify=false", imageRef})
|
|
session.WaitWithDefaultTimeout()
|
|
Expect(session).Should(ExitWithError(125, "fetching blob"))
|
|
})
|
|
|
|
It("stops fallback chain when mirror returns non-retriable 401", func() {
|
|
SkipIfRemote("registries.conf is not used by the remote client")
|
|
|
|
blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionGzip)
|
|
|
|
var primaryBlobHits atomic.Int32
|
|
primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs,
|
|
countingBlobMW(&primaryBlobHits)))
|
|
defer primary.Close()
|
|
|
|
// mirror1 (selected): serves manifest, returns 503 on blobs -> fallback.
|
|
mirror1 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMW503))
|
|
defer mirror1.Close()
|
|
|
|
// mirror2: returns 401 on blobs -> NOT retriable, should stop the chain.
|
|
mirror2 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMW401))
|
|
defer mirror2.Close()
|
|
|
|
conf := fmt.Sprintf(`[[registry]]
|
|
prefix = "mirrortest.local"
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
`, primaryAddr(primary), primaryAddr(mirror1), primaryAddr(mirror2))
|
|
podmanTest.setRegistriesConfigEnv([]byte(conf))
|
|
defer resetRegistriesConfigEnv()
|
|
|
|
imageRef := "mirrortest.local/" + repo + ":latest"
|
|
session := podmanTest.Podman([]string{"pull", "-q", "--tls-verify=false", imageRef})
|
|
session.WaitWithDefaultTimeout()
|
|
|
|
// 401 from mirror2 halts the chain. Primary is never reached.
|
|
Expect(session).Should(ExitWithError(125, "fetching blob"))
|
|
Expect(primaryBlobHits.Load()).To(Equal(int32(0)), "primary should not be reached after a non-retriable 401")
|
|
})
|
|
|
|
It("does not fall back past unreachable mirror (connection refused)", func() {
|
|
SkipIfRemote("registries.conf is not used by the remote client")
|
|
|
|
blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionGzip)
|
|
|
|
primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil))
|
|
defer primary.Close()
|
|
|
|
// mirror1 (selected): serves manifest, returns 503 on blobs.
|
|
mirror1 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMW503))
|
|
defer mirror1.Close()
|
|
|
|
// mirror2: start and immediately stop. Port is closed, server is unreachable.
|
|
mirror2 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil))
|
|
unreachableAddr := primaryAddr(mirror2)
|
|
mirror2.Close()
|
|
|
|
conf := fmt.Sprintf(`[[registry]]
|
|
prefix = "mirrortest.local"
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
`, primaryAddr(primary), primaryAddr(mirror1), unreachableAddr)
|
|
podmanTest.setRegistriesConfigEnv([]byte(conf))
|
|
defer resetRegistriesConfigEnv()
|
|
|
|
imageRef := "mirrortest.local/" + repo + ":latest"
|
|
// Connection refused is not a fallback-worthy error, so the
|
|
// chain stops at mirror2. Pull is expected to fail because
|
|
// primary is never tried.
|
|
session := podmanTest.Podman([]string{"pull", "-q", "--tls-verify=false", imageRef})
|
|
session.WaitWithDefaultTimeout()
|
|
Expect(session).Should(ExitWithError(125, "fetching blob"))
|
|
})
|
|
|
|
It("partial mirror: serves some layers, BLOB_UNKNOWN for the rest", func() {
|
|
SkipIfRemote("registries.conf is not used by the remote client")
|
|
|
|
blobs, manifestBytes, manifestCT, layerDigests := buildTestImage(3, compressionGzip)
|
|
Expect(layerDigests).To(HaveLen(3))
|
|
|
|
// Mirror has config + layer0, but NOT layer1 and layer2.
|
|
missing := map[string]bool{
|
|
layerDigests[1].String(): true,
|
|
layerDigests[2].String(): true,
|
|
}
|
|
blobPrefix := fmt.Sprintf("/v2/%s/blobs/", repo)
|
|
partialMW := func(next http.Handler) http.Handler {
|
|
errHandler := blobMWBlobUnknown(next)
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
dgst := strings.TrimPrefix(r.URL.Path, blobPrefix)
|
|
if missing[dgst] {
|
|
errHandler.ServeHTTP(w, r)
|
|
return
|
|
}
|
|
next.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
|
|
primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil))
|
|
defer primary.Close()
|
|
mirror := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, partialMW))
|
|
defer mirror.Close()
|
|
|
|
conf := fmt.Sprintf(`[[registry]]
|
|
prefix = "mirrortest.local"
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
`, primaryAddr(primary), primaryAddr(mirror))
|
|
podmanTest.setRegistriesConfigEnv([]byte(conf))
|
|
defer resetRegistriesConfigEnv()
|
|
|
|
imageRef := "mirrortest.local/" + repo + ":latest"
|
|
pullAndVerifySuccess(imageRef)
|
|
})
|
|
|
|
It("overloaded 503 mirror: retries on mirror before falling back", func() {
|
|
SkipIfRemote("registries.conf is not used by the remote client")
|
|
|
|
blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionGzip)
|
|
|
|
primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil))
|
|
defer primary.Close()
|
|
|
|
var mirrorBlobHits atomic.Int32
|
|
mirror := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs,
|
|
countingErrorBlobMW(&mirrorBlobHits, blobMW503)))
|
|
defer mirror.Close()
|
|
|
|
conf := fmt.Sprintf(`[[registry]]
|
|
prefix = "mirrortest.local"
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
`, primaryAddr(primary), primaryAddr(mirror))
|
|
podmanTest.setRegistriesConfigEnv([]byte(conf))
|
|
defer resetRegistriesConfigEnv()
|
|
|
|
imageRef := "mirrortest.local/" + repo + ":latest"
|
|
pullAndVerifySuccess(imageRef)
|
|
|
|
Expect(mirrorBlobHits.Load()).To(Equal(int32(2)), "mirror should be retried (initial + retry) for the first blob before fallback")
|
|
})
|
|
|
|
It("rate-limited mirror: serves first blobs then 429 on the rest", func() {
|
|
SkipIfRemote("registries.conf is not used by the remote client")
|
|
|
|
blobs, manifestBytes, manifestCT, _ := buildTestImage(4, compressionGzip)
|
|
|
|
primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil))
|
|
defer primary.Close()
|
|
|
|
// Mirror serves the first 2 blob requests successfully, then returns 429 for all subsequent ones.
|
|
var served atomic.Int32
|
|
rateLimitMW := func(next http.Handler) http.Handler {
|
|
errHandler := blobMW429(next)
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if served.Add(1) <= 2 {
|
|
next.ServeHTTP(w, r)
|
|
return
|
|
}
|
|
errHandler.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
|
|
mirror := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, rateLimitMW))
|
|
defer mirror.Close()
|
|
|
|
conf := fmt.Sprintf(`[[registry]]
|
|
prefix = "mirrortest.local"
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
`, primaryAddr(primary), primaryAddr(mirror))
|
|
podmanTest.setRegistriesConfigEnv([]byte(conf))
|
|
defer resetRegistriesConfigEnv()
|
|
|
|
imageRef := "mirrortest.local/" + repo + ":latest"
|
|
pullAndVerifySuccess(imageRef)
|
|
})
|
|
|
|
It("flaky mirror with multi-layer image: random 503 on half the requests", func() {
|
|
SkipIfRemote("registries.conf is not used by the remote client")
|
|
|
|
blobs, manifestBytes, manifestCT, _ := buildTestImage(5, compressionGzip)
|
|
|
|
primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil))
|
|
defer primary.Close()
|
|
|
|
// Mirror alternates between serving and 503ing based on a request counter.
|
|
var reqCount atomic.Int32
|
|
flakyMW := func(next http.Handler) http.Handler {
|
|
errHandler := blobMW503(next)
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if reqCount.Add(1)%2 == 0 {
|
|
errHandler.ServeHTTP(w, r)
|
|
return
|
|
}
|
|
next.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
|
|
mirror := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, flakyMW))
|
|
defer mirror.Close()
|
|
|
|
conf := fmt.Sprintf(`[[registry]]
|
|
prefix = "mirrortest.local"
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
`, primaryAddr(primary), primaryAddr(mirror))
|
|
podmanTest.setRegistriesConfigEnv([]byte(conf))
|
|
defer resetRegistriesConfigEnv()
|
|
|
|
imageRef := "mirrortest.local/" + repo + ":latest"
|
|
pullAndVerifySuccess(imageRef)
|
|
})
|
|
|
|
It("mixed failure modes across mirrors with different error types", func() {
|
|
SkipIfRemote("registries.conf is not used by the remote client")
|
|
|
|
blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionGzip)
|
|
|
|
primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil))
|
|
defer primary.Close()
|
|
|
|
// mirror1 (selected): 503 on blobs -> fallback-worthy.
|
|
mirror1 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMW503))
|
|
defer mirror1.Close()
|
|
// mirror2: BLOB_UNKNOWN on blobs -> fallback-worthy, continues chain.
|
|
mirror2 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMWBlobUnknown))
|
|
defer mirror2.Close()
|
|
// mirror3: 429 on blobs -> fallback-worthy, continues chain.
|
|
mirror3 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMW429))
|
|
defer mirror3.Close()
|
|
|
|
conf := fmt.Sprintf(`[[registry]]
|
|
prefix = "mirrortest.local"
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
`, primaryAddr(primary), primaryAddr(mirror1), primaryAddr(mirror2), primaryAddr(mirror3))
|
|
podmanTest.setRegistriesConfigEnv([]byte(conf))
|
|
defer resetRegistriesConfigEnv()
|
|
|
|
imageRef := "mirrortest.local/" + repo + ":latest"
|
|
pullAndVerifySuccess(imageRef)
|
|
})
|
|
|
|
It("zstd:chunked image pulls successfully from mirror via partial pull", func() {
|
|
SkipIfRemote("registries.conf is not used by the remote client")
|
|
|
|
blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionZstdChunked)
|
|
|
|
var primaryBlobHits atomic.Int32
|
|
primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs,
|
|
countingBlobMW(&primaryBlobHits)))
|
|
defer primary.Close()
|
|
|
|
var mirrorBlobHits atomic.Int32
|
|
mirror := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs,
|
|
countingBlobMW(&mirrorBlobHits)))
|
|
defer mirror.Close()
|
|
|
|
conf := fmt.Sprintf(`[[registry]]
|
|
prefix = "mirrortest.local"
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
`, primaryAddr(primary), primaryAddr(mirror))
|
|
podmanTest.setRegistriesConfigEnv([]byte(conf))
|
|
defer resetRegistriesConfigEnv()
|
|
|
|
imageRef := "mirrortest.local/" + repo + ":latest"
|
|
pullAndVerifySuccess(imageRef, "--pull-option=enable_partial_images=true")
|
|
|
|
Expect(mirrorBlobHits.Load()).To(BeNumerically(">", int32(0)),
|
|
"mirror should have served blob requests")
|
|
Expect(primaryBlobHits.Load()).To(Equal(int32(0)),
|
|
"primary should not be contacted when mirror works")
|
|
})
|
|
|
|
It("GetBlobAt falls back to primary when mirror returns 503 on range requests", func() {
|
|
SkipIfRemote("registries.conf is not used by the remote client")
|
|
|
|
blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionZstdChunked)
|
|
|
|
primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil))
|
|
defer primary.Close()
|
|
|
|
// Mirror: 503 only on range requests (GetBlobAt).
|
|
var mirrorRangeHits atomic.Int32
|
|
mirror := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs,
|
|
countingErrorBlobMW(&mirrorRangeHits, rangeOnlyErrorBlobMW(blobMW503))))
|
|
defer mirror.Close()
|
|
|
|
conf := fmt.Sprintf(`[[registry]]
|
|
prefix = "mirrortest.local"
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
`, primaryAddr(primary), primaryAddr(mirror))
|
|
podmanTest.setRegistriesConfigEnv([]byte(conf))
|
|
defer resetRegistriesConfigEnv()
|
|
|
|
imageRef := "mirrortest.local/" + repo + ":latest"
|
|
pullAndVerifySuccess(imageRef, "--pull-option=enable_partial_images=true")
|
|
})
|
|
|
|
It("GetBlobAt falls back to primary when mirror returns BLOB_UNKNOWN on range requests", func() {
|
|
SkipIfRemote("registries.conf is not used by the remote client")
|
|
|
|
blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionZstdChunked)
|
|
|
|
primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil))
|
|
defer primary.Close()
|
|
|
|
// Mirror: BLOB_UNKNOWN only on range requests.
|
|
mirror := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs,
|
|
rangeOnlyErrorBlobMW(blobMWBlobUnknown)))
|
|
defer mirror.Close()
|
|
|
|
conf := fmt.Sprintf(`[[registry]]
|
|
prefix = "mirrortest.local"
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
`, primaryAddr(primary), primaryAddr(mirror))
|
|
podmanTest.setRegistriesConfigEnv([]byte(conf))
|
|
defer resetRegistriesConfigEnv()
|
|
|
|
imageRef := "mirrortest.local/" + repo + ":latest"
|
|
pullAndVerifySuccess(imageRef, "--pull-option=enable_partial_images=true")
|
|
})
|
|
|
|
It("GetBlobAt falls back across multiple mirrors on range request failures", func() {
|
|
SkipIfRemote("registries.conf is not used by the remote client")
|
|
|
|
blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionZstdChunked)
|
|
|
|
var primaryBlobHits atomic.Int32
|
|
primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs,
|
|
countingBlobMW(&primaryBlobHits)))
|
|
defer primary.Close()
|
|
|
|
// mirror1: 503 on range requests only -> fallback-worthy.
|
|
mirror1 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs,
|
|
rangeOnlyErrorBlobMW(blobMW503)))
|
|
defer mirror1.Close()
|
|
|
|
// mirror2: BLOB_UNKNOWN on range requests only -> fallback-worthy.
|
|
mirror2 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs,
|
|
rangeOnlyErrorBlobMW(blobMWBlobUnknown)))
|
|
defer mirror2.Close()
|
|
|
|
conf := fmt.Sprintf(`[[registry]]
|
|
prefix = "mirrortest.local"
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
|
|
[[registry.mirror]]
|
|
location = %q
|
|
insecure = true
|
|
`, primaryAddr(primary), primaryAddr(mirror1), primaryAddr(mirror2))
|
|
podmanTest.setRegistriesConfigEnv([]byte(conf))
|
|
defer resetRegistriesConfigEnv()
|
|
|
|
imageRef := "mirrortest.local/" + repo + ":latest"
|
|
pullAndVerifySuccess(imageRef, "--pull-option=enable_partial_images=true")
|
|
})
|
|
})
|
|
}
|