From 77bbf5fa79daf7913f5dfff3adbb3ae724e4b1a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Rod=C3=A1k?= Date: Mon, 13 Jul 2026 14:28:56 +0200 Subject: [PATCH] Add blob-level mirror fallback integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integration tests to verify podman-container-tools/container-libs#845 Signed-off-by: Jan Rodák --- test/e2e/pull_mirror_fallback_test.go | 887 ++++++++++++++++++++++++++ test/e2e/pull_test.go | 2 + 2 files changed, 889 insertions(+) create mode 100644 test/e2e/pull_mirror_fallback_test.go diff --git a/test/e2e/pull_mirror_fallback_test.go b/test/e2e/pull_mirror_fallback_test.go new file mode 100644 index 0000000000..36d92ae28c --- /dev/null +++ b/test/e2e/pull_mirror_fallback_test.go @@ -0,0 +1,887 @@ +//go:build linux || freebsd + +package integration + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "runtime" + "strings" + "sync/atomic" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/opencontainers/go-digest" + . "go.podman.io/podman/v6/test/utils" + "go.podman.io/storage/pkg/chunked/compressor" +) + +// primaryAddr extracts the host:port from an httptest.Server. +func primaryAddr(s *httptest.Server) string { + return s.Listener.Addr().String() +} + +// buildTarLayer creates a single tar archive containing a test data file for the given layer index. +func buildTarLayer(index int) []byte { + content := fmt.Appendf(nil, "layer-%d test data: the quick brown fox jumps over the lazy dog\n", index) + var raw bytes.Buffer + tw := tar.NewWriter(&raw) + Expect(tw.WriteHeader(&tar.Header{ + Name: fmt.Sprintf("layer-%d/data.txt", index), + Mode: 0o644, + Size: int64(len(content)), + })).To(Succeed()) + _, err := tw.Write(content) + Expect(err).ToNot(HaveOccurred()) + Expect(tw.Close()).To(Succeed()) + return raw.Bytes() +} + +// buildImageManifest creates the config blob, manifest JSON, and returns the content type. +func buildImageManifest(blobs map[string][]byte, diffIDs []string, layers []map[string]any, configMediaType, manifestMediaType string) ([]byte, string) { + configBytes, err := json.Marshal(map[string]any{ + "architecture": runtime.GOARCH, + "os": "linux", + "rootfs": map[string]any{"type": "layers", "diff_ids": diffIDs}, + }) + Expect(err).ToNot(HaveOccurred()) + configDigest := digest.FromBytes(configBytes) + blobs[configDigest.String()] = configBytes + + manifestBytes, err := json.Marshal(map[string]any{ + "schemaVersion": 2, + "mediaType": manifestMediaType, + "config": map[string]any{ + "mediaType": configMediaType, + "size": len(configBytes), + "digest": configDigest.String(), + }, + "layers": layers, + }) + Expect(err).ToNot(HaveOccurred()) + return manifestBytes, manifestMediaType +} + +type testCompression int + +const ( + compressionGzip testCompression = iota + compressionZstdChunked +) + +// buildTestImage constructs a valid container image with numLayers layers +// using the specified compression. Gzip produces a Docker schema2 manifest; +// zstd:chunked produces an OCI manifest with TOC annotations for partial pulls. +func buildTestImage(numLayers int, compression testCompression) (map[string][]byte, []byte, string, []digest.Digest) { + blobs := make(map[string][]byte) + var diffIDs []string + var layers []map[string]any + var layerDigests []digest.Digest + + for i := range numLayers { + rawTar := buildTarLayer(i) + diffIDs = append(diffIDs, digest.FromBytes(rawTar).String()) + + var layerBytes []byte + layerDesc := map[string]any{} + + switch compression { + case compressionZstdChunked: + var comp bytes.Buffer + annotations := make(map[string]string) + zw, err := compressor.ZstdCompressor(&comp, annotations, nil) + Expect(err).ToNot(HaveOccurred()) + _, err = zw.Write(rawTar) + Expect(err).ToNot(HaveOccurred()) + Expect(zw.Close()).To(Succeed()) + layerBytes = append([]byte(nil), comp.Bytes()...) + layerDesc["mediaType"] = "application/vnd.oci.image.layer.v1.tar+zstd" + layerDesc["annotations"] = annotations + default: + var comp bytes.Buffer + gz := gzip.NewWriter(&comp) + _, err := gz.Write(rawTar) + Expect(err).ToNot(HaveOccurred()) + Expect(gz.Close()).To(Succeed()) + layerBytes = append([]byte(nil), comp.Bytes()...) + layerDesc["mediaType"] = "application/vnd.docker.image.rootfs.diff.tar.gzip" + } + + ld := digest.FromBytes(layerBytes) + blobs[ld.String()] = layerBytes + layerDigests = append(layerDigests, ld) + layerDesc["size"] = len(layerBytes) + layerDesc["digest"] = ld.String() + layers = append(layers, layerDesc) + } + + var configMT, manifestMT string + switch compression { + case compressionZstdChunked: + configMT = "application/vnd.oci.image.config.v1+json" + manifestMT = "application/vnd.oci.image.manifest.v1+json" + default: + configMT = "application/vnd.docker.container.image.v1+json" + manifestMT = "application/vnd.docker.distribution.manifest.v2+json" + } + + manifestBytes, ct := buildImageManifest(blobs, diffIDs, layers, configMT, manifestMT) + return blobs, manifestBytes, ct, layerDigests +} + +// blobMiddleware wraps the default blob-serving handler. Middleware that wants +// to fail should write a response and not call next. Middleware that wants +// normal serving should call next.ServeHTTP(w, r). +type blobMiddleware func(next http.Handler) http.Handler + +// registryHandler returns an http.Handler that implements a minimal Docker +// registry v2 API for the given repository name, serving the manifest and +// blobs. If mw is non-nil it wraps the default blob handler. +// +//nolint:unparam +func registryHandler(repo string, manifestBytes []byte, manifestCT string, blobs map[string][]byte, mw blobMiddleware) http.Handler { + manifestPath := fmt.Sprintf("/v2/%s/manifests/latest", repo) + blobPrefix := fmt.Sprintf("/v2/%s/blobs/", repo) + + defaultBlobHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + dgst := strings.TrimPrefix(r.URL.Path, blobPrefix) + data, ok := blobs[dgst] + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(data))) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(data) + }) + + var blobHandler http.Handler = defaultBlobHandler + if mw != nil { + blobHandler = mw(defaultBlobHandler) + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/v2/": + w.WriteHeader(http.StatusOK) + case r.URL.Path == manifestPath: + w.Header().Set("Content-Type", manifestCT) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(manifestBytes) + case strings.HasPrefix(r.URL.Path, blobPrefix): + blobHandler.ServeHTTP(w, r) + default: + http.NotFound(w, r) + } + }) +} + +// errorBlobMiddleware returns a blobMiddleware that always responds with the +// given status and optional JSON error body, ignoring the default handler. +func errorBlobMiddleware(status int, code, message string) blobMiddleware { + return func(_ http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if code != "" { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(map[string]any{ + "errors": []map[string]string{ + {"code": code, "message": message}, + }, + }) + } else { + w.WriteHeader(status) + } + }) + } +} + +var ( + blobMW503 = errorBlobMiddleware(http.StatusServiceUnavailable, "", "") + blobMW500 = errorBlobMiddleware(http.StatusInternalServerError, "", "") + blobMWBlobUnknown = errorBlobMiddleware(http.StatusNotFound, "BLOB_UNKNOWN", "blob unknown to registry") + blobMW429 = errorBlobMiddleware(http.StatusTooManyRequests, "TOOMANYREQUESTS", "rate limit exceeded") + blobMW401 = errorBlobMiddleware(http.StatusUnauthorized, "UNAUTHORIZED", "authentication required") + blobMW403 = errorBlobMiddleware(http.StatusForbidden, "DENIED", "access denied") +) + +// countingBlobMW returns a blobMiddleware that increments counter then +// delegates to the next handler (default blob serving). +func countingBlobMW(counter *atomic.Int32) blobMiddleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + counter.Add(1) + next.ServeHTTP(w, r) + }) + } +} + +// countingErrorBlobMW increments counter, then responds with an error +// (ignoring the default handler). +func countingErrorBlobMW(counter *atomic.Int32, errMW blobMiddleware) blobMiddleware { + return func(next http.Handler) http.Handler { + errHandler := errMW(next) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + counter.Add(1) + errHandler.ServeHTTP(w, r) + }) + } +} + +// rangeOnlyErrorBlobMW applies errMW only to range requests (GetBlobAt path). +// Non-range requests (GetBlob) pass through to the default handler unchanged. +func rangeOnlyErrorBlobMW(errMW blobMiddleware) blobMiddleware { + return func(next http.Handler) http.Handler { + errHandler := errMW(next) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Range") != "" { + errHandler.ServeHTTP(w, r) + return + } + next.ServeHTTP(w, r) + }) + } +} + +// pullAndVerifySuccess runs podman pull and asserts it succeeds, cleans up. +// globalArgs are placed before the "pull" subcommand (e.g. --pull-option=…). +// +//nolint:unparam +func pullAndVerifySuccess(imageRef string, globalArgs ...string) { + args := make([]string, 0, len(globalArgs)+4) + args = append(args, globalArgs...) + args = append(args, "pull", "-q", "--tls-verify=false", imageRef) + podmanTest.PodmanExitCleanly(args...) + podmanTest.PodmanExitCleanly("image", "exists", imageRef) + podmanTest.PodmanExitCleanly("rmi", imageRef) +} + +func pullMirrorFallbackTests() { + Describe("blob-level mirror fallback", func() { + const repo = "library/mirrortest" + + type singleMirrorCase struct { + description string + mw blobMiddleware + expectPullSuccess bool + expectedError string + } + + singleMirrorCases := []singleMirrorCase{ + { + description: "falls back to primary when mirror returns 503 on blobs", + mw: blobMW503, + expectPullSuccess: true, + }, + { + description: "falls back to primary when mirror returns 500 on blobs", + mw: blobMW500, + expectPullSuccess: true, + }, + { + description: "falls back to primary when mirror returns BLOB_UNKNOWN on blobs", + mw: blobMWBlobUnknown, + expectPullSuccess: true, + }, + { + description: "falls back to primary when mirror returns 429 on blobs", + mw: blobMW429, + expectPullSuccess: true, + }, + { + description: "does not fall back when mirror returns 401 on blobs", + mw: blobMW401, + expectPullSuccess: false, + expectedError: "unauthorized", + }, + { + description: "does not fall back when mirror returns 403 on blobs", + mw: blobMW403, + expectPullSuccess: false, + expectedError: "denied", + }, + } + + for _, tc := range singleMirrorCases { + It(tc.description, func() { + SkipIfRemote("registries.conf is not used by the remote client") + + blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionGzip) + + primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil)) + defer primary.Close() + + var mirrorBlobHits atomic.Int32 + mirror := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, + countingErrorBlobMW(&mirrorBlobHits, tc.mw))) + defer mirror.Close() + + conf := fmt.Sprintf(`[[registry]] +prefix = "mirrortest.local" +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true +`, primaryAddr(primary), primaryAddr(mirror)) + podmanTest.setRegistriesConfigEnv([]byte(conf)) + defer resetRegistriesConfigEnv() + + imageRef := "mirrortest.local/" + repo + ":latest" + session := podmanTest.Podman([]string{"pull", "-q", "--tls-verify=false", imageRef}) + session.WaitWithDefaultTimeout() + + if tc.expectPullSuccess { + Expect(session).Should(ExitCleanly()) + Expect(mirrorBlobHits.Load()).To(BeNumerically(">", int32(0)), + "mirror should have been tried for blobs before fallback") + podmanTest.PodmanExitCleanly("image", "exists", imageRef) + podmanTest.PodmanExitCleanly("rmi", imageRef) + } else { + Expect(session).Should(ExitWithError(125, tc.expectedError)) + } + }) + } + + It("skips multiple broken mirrors and falls back to primary", func() { + SkipIfRemote("registries.conf is not used by the remote client") + + blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionGzip) + + primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil)) + defer primary.Close() + + // Two broken mirrors: first returns 503, second returns BLOB_UNKNOWN. + var mirror1Hits, mirror2Hits atomic.Int32 + mirror1 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, + countingErrorBlobMW(&mirror1Hits, blobMW503))) + defer mirror1.Close() + mirror2 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, + countingErrorBlobMW(&mirror2Hits, blobMWBlobUnknown))) + defer mirror2.Close() + + conf := fmt.Sprintf(`[[registry]] +prefix = "mirrortest.local" +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true +`, primaryAddr(primary), primaryAddr(mirror1), primaryAddr(mirror2)) + podmanTest.setRegistriesConfigEnv([]byte(conf)) + defer resetRegistriesConfigEnv() + + imageRef := "mirrortest.local/" + repo + ":latest" + pullAndVerifySuccess(imageRef) + + Expect(mirror1Hits.Load()).To(BeNumerically(">", int32(0)), "mirror1 should have been tried for blobs") + Expect(mirror2Hits.Load()).To(BeNumerically(">", int32(0)), "mirror2 should have been tried for blobs") + }) + + It("stops at first working mirror without reaching primary", func() { + SkipIfRemote("registries.conf is not used by the remote client") + + blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionGzip) + + var primaryBlobHits atomic.Int32 + primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, + countingBlobMW(&primaryBlobHits))) + defer primary.Close() + + // mirror1: serves manifest, returns 503 on blobs. + mirror1 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMW503)) + defer mirror1.Close() + + // mirror2: serves everything. The fallback should stop here. + mirror2 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil)) + defer mirror2.Close() + + conf := fmt.Sprintf(`[[registry]] +prefix = "mirrortest.local" +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true +`, primaryAddr(primary), primaryAddr(mirror1), primaryAddr(mirror2)) + podmanTest.setRegistriesConfigEnv([]byte(conf)) + defer resetRegistriesConfigEnv() + + imageRef := "mirrortest.local/" + repo + ":latest" + pullAndVerifySuccess(imageRef) + + Expect(primaryBlobHits.Load()).To(Equal(int32(0)), "primary should NOT be contacted when a working mirror is found first") + }) + + It("fails when all mirrors and primary return errors on blobs", func() { + SkipIfRemote("registries.conf is not used by the remote client") + + blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionGzip) + + // Primary also fails on blobs. + primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMW500)) + defer primary.Close() + + // Two mirrors, both broken. + mirror1 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMW503)) + defer mirror1.Close() + mirror2 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMWBlobUnknown)) + defer mirror2.Close() + + conf := fmt.Sprintf(`[[registry]] +prefix = "mirrortest.local" +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true +`, primaryAddr(primary), primaryAddr(mirror1), primaryAddr(mirror2)) + podmanTest.setRegistriesConfigEnv([]byte(conf)) + defer resetRegistriesConfigEnv() + + imageRef := "mirrortest.local/" + repo + ":latest" + session := podmanTest.Podman([]string{"pull", "-q", "--tls-verify=false", imageRef}) + session.WaitWithDefaultTimeout() + Expect(session).Should(ExitWithError(125, "fetching blob")) + }) + + It("stops fallback chain when mirror returns non-retriable 401", func() { + SkipIfRemote("registries.conf is not used by the remote client") + + blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionGzip) + + var primaryBlobHits atomic.Int32 + primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, + countingBlobMW(&primaryBlobHits))) + defer primary.Close() + + // mirror1 (selected): serves manifest, returns 503 on blobs -> fallback. + mirror1 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMW503)) + defer mirror1.Close() + + // mirror2: returns 401 on blobs -> NOT retriable, should stop the chain. + mirror2 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMW401)) + defer mirror2.Close() + + conf := fmt.Sprintf(`[[registry]] +prefix = "mirrortest.local" +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true +`, primaryAddr(primary), primaryAddr(mirror1), primaryAddr(mirror2)) + podmanTest.setRegistriesConfigEnv([]byte(conf)) + defer resetRegistriesConfigEnv() + + imageRef := "mirrortest.local/" + repo + ":latest" + session := podmanTest.Podman([]string{"pull", "-q", "--tls-verify=false", imageRef}) + session.WaitWithDefaultTimeout() + + // 401 from mirror2 halts the chain. Primary is never reached. + Expect(session).Should(ExitWithError(125, "fetching blob")) + Expect(primaryBlobHits.Load()).To(Equal(int32(0)), "primary should not be reached after a non-retriable 401") + }) + + It("does not fall back past unreachable mirror (connection refused)", func() { + SkipIfRemote("registries.conf is not used by the remote client") + + blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionGzip) + + primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil)) + defer primary.Close() + + // mirror1 (selected): serves manifest, returns 503 on blobs. + mirror1 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMW503)) + defer mirror1.Close() + + // mirror2: start and immediately stop. Port is closed, server is unreachable. + mirror2 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil)) + unreachableAddr := primaryAddr(mirror2) + mirror2.Close() + + conf := fmt.Sprintf(`[[registry]] +prefix = "mirrortest.local" +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true +`, primaryAddr(primary), primaryAddr(mirror1), unreachableAddr) + podmanTest.setRegistriesConfigEnv([]byte(conf)) + defer resetRegistriesConfigEnv() + + imageRef := "mirrortest.local/" + repo + ":latest" + // Connection refused is not a fallback-worthy error, so the + // chain stops at mirror2. Pull is expected to fail because + // primary is never tried. + session := podmanTest.Podman([]string{"pull", "-q", "--tls-verify=false", imageRef}) + session.WaitWithDefaultTimeout() + Expect(session).Should(ExitWithError(125, "fetching blob")) + }) + + It("partial mirror: serves some layers, BLOB_UNKNOWN for the rest", func() { + SkipIfRemote("registries.conf is not used by the remote client") + + blobs, manifestBytes, manifestCT, layerDigests := buildTestImage(3, compressionGzip) + Expect(layerDigests).To(HaveLen(3)) + + // Mirror has config + layer0, but NOT layer1 and layer2. + missing := map[string]bool{ + layerDigests[1].String(): true, + layerDigests[2].String(): true, + } + blobPrefix := fmt.Sprintf("/v2/%s/blobs/", repo) + partialMW := func(next http.Handler) http.Handler { + errHandler := blobMWBlobUnknown(next) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + dgst := strings.TrimPrefix(r.URL.Path, blobPrefix) + if missing[dgst] { + errHandler.ServeHTTP(w, r) + return + } + next.ServeHTTP(w, r) + }) + } + + primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil)) + defer primary.Close() + mirror := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, partialMW)) + defer mirror.Close() + + conf := fmt.Sprintf(`[[registry]] +prefix = "mirrortest.local" +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true +`, primaryAddr(primary), primaryAddr(mirror)) + podmanTest.setRegistriesConfigEnv([]byte(conf)) + defer resetRegistriesConfigEnv() + + imageRef := "mirrortest.local/" + repo + ":latest" + pullAndVerifySuccess(imageRef) + }) + + It("overloaded 503 mirror: retries on mirror before falling back", func() { + SkipIfRemote("registries.conf is not used by the remote client") + + blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionGzip) + + primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil)) + defer primary.Close() + + var mirrorBlobHits atomic.Int32 + mirror := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, + countingErrorBlobMW(&mirrorBlobHits, blobMW503))) + defer mirror.Close() + + conf := fmt.Sprintf(`[[registry]] +prefix = "mirrortest.local" +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true +`, primaryAddr(primary), primaryAddr(mirror)) + podmanTest.setRegistriesConfigEnv([]byte(conf)) + defer resetRegistriesConfigEnv() + + imageRef := "mirrortest.local/" + repo + ":latest" + pullAndVerifySuccess(imageRef) + + Expect(mirrorBlobHits.Load()).To(Equal(int32(2)), "mirror should be retried (initial + retry) for the first blob before fallback") + }) + + It("rate-limited mirror: serves first blobs then 429 on the rest", func() { + SkipIfRemote("registries.conf is not used by the remote client") + + blobs, manifestBytes, manifestCT, _ := buildTestImage(4, compressionGzip) + + primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil)) + defer primary.Close() + + // Mirror serves the first 2 blob requests successfully, then returns 429 for all subsequent ones. + var served atomic.Int32 + rateLimitMW := func(next http.Handler) http.Handler { + errHandler := blobMW429(next) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if served.Add(1) <= 2 { + next.ServeHTTP(w, r) + return + } + errHandler.ServeHTTP(w, r) + }) + } + + mirror := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, rateLimitMW)) + defer mirror.Close() + + conf := fmt.Sprintf(`[[registry]] +prefix = "mirrortest.local" +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true +`, primaryAddr(primary), primaryAddr(mirror)) + podmanTest.setRegistriesConfigEnv([]byte(conf)) + defer resetRegistriesConfigEnv() + + imageRef := "mirrortest.local/" + repo + ":latest" + pullAndVerifySuccess(imageRef) + }) + + It("flaky mirror with multi-layer image: random 503 on half the requests", func() { + SkipIfRemote("registries.conf is not used by the remote client") + + blobs, manifestBytes, manifestCT, _ := buildTestImage(5, compressionGzip) + + primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil)) + defer primary.Close() + + // Mirror alternates between serving and 503ing based on a request counter. + var reqCount atomic.Int32 + flakyMW := func(next http.Handler) http.Handler { + errHandler := blobMW503(next) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if reqCount.Add(1)%2 == 0 { + errHandler.ServeHTTP(w, r) + return + } + next.ServeHTTP(w, r) + }) + } + + mirror := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, flakyMW)) + defer mirror.Close() + + conf := fmt.Sprintf(`[[registry]] +prefix = "mirrortest.local" +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true +`, primaryAddr(primary), primaryAddr(mirror)) + podmanTest.setRegistriesConfigEnv([]byte(conf)) + defer resetRegistriesConfigEnv() + + imageRef := "mirrortest.local/" + repo + ":latest" + pullAndVerifySuccess(imageRef) + }) + + It("mixed failure modes across mirrors with different error types", func() { + SkipIfRemote("registries.conf is not used by the remote client") + + blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionGzip) + + primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil)) + defer primary.Close() + + // mirror1 (selected): 503 on blobs -> fallback-worthy. + mirror1 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMW503)) + defer mirror1.Close() + // mirror2: BLOB_UNKNOWN on blobs -> fallback-worthy, continues chain. + mirror2 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMWBlobUnknown)) + defer mirror2.Close() + // mirror3: 429 on blobs -> fallback-worthy, continues chain. + mirror3 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, blobMW429)) + defer mirror3.Close() + + conf := fmt.Sprintf(`[[registry]] +prefix = "mirrortest.local" +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true +`, primaryAddr(primary), primaryAddr(mirror1), primaryAddr(mirror2), primaryAddr(mirror3)) + podmanTest.setRegistriesConfigEnv([]byte(conf)) + defer resetRegistriesConfigEnv() + + imageRef := "mirrortest.local/" + repo + ":latest" + pullAndVerifySuccess(imageRef) + }) + + It("zstd:chunked image pulls successfully from mirror via partial pull", func() { + SkipIfRemote("registries.conf is not used by the remote client") + + blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionZstdChunked) + + var primaryBlobHits atomic.Int32 + primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, + countingBlobMW(&primaryBlobHits))) + defer primary.Close() + + var mirrorBlobHits atomic.Int32 + mirror := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, + countingBlobMW(&mirrorBlobHits))) + defer mirror.Close() + + conf := fmt.Sprintf(`[[registry]] +prefix = "mirrortest.local" +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true +`, primaryAddr(primary), primaryAddr(mirror)) + podmanTest.setRegistriesConfigEnv([]byte(conf)) + defer resetRegistriesConfigEnv() + + imageRef := "mirrortest.local/" + repo + ":latest" + pullAndVerifySuccess(imageRef, "--pull-option=enable_partial_images=true") + + Expect(mirrorBlobHits.Load()).To(BeNumerically(">", int32(0)), + "mirror should have served blob requests") + Expect(primaryBlobHits.Load()).To(Equal(int32(0)), + "primary should not be contacted when mirror works") + }) + + It("GetBlobAt falls back to primary when mirror returns 503 on range requests", func() { + SkipIfRemote("registries.conf is not used by the remote client") + + blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionZstdChunked) + + primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil)) + defer primary.Close() + + // Mirror: 503 only on range requests (GetBlobAt). + var mirrorRangeHits atomic.Int32 + mirror := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, + countingErrorBlobMW(&mirrorRangeHits, rangeOnlyErrorBlobMW(blobMW503)))) + defer mirror.Close() + + conf := fmt.Sprintf(`[[registry]] +prefix = "mirrortest.local" +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true +`, primaryAddr(primary), primaryAddr(mirror)) + podmanTest.setRegistriesConfigEnv([]byte(conf)) + defer resetRegistriesConfigEnv() + + imageRef := "mirrortest.local/" + repo + ":latest" + pullAndVerifySuccess(imageRef, "--pull-option=enable_partial_images=true") + }) + + It("GetBlobAt falls back to primary when mirror returns BLOB_UNKNOWN on range requests", func() { + SkipIfRemote("registries.conf is not used by the remote client") + + blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionZstdChunked) + + primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, nil)) + defer primary.Close() + + // Mirror: BLOB_UNKNOWN only on range requests. + mirror := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, + rangeOnlyErrorBlobMW(blobMWBlobUnknown))) + defer mirror.Close() + + conf := fmt.Sprintf(`[[registry]] +prefix = "mirrortest.local" +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true +`, primaryAddr(primary), primaryAddr(mirror)) + podmanTest.setRegistriesConfigEnv([]byte(conf)) + defer resetRegistriesConfigEnv() + + imageRef := "mirrortest.local/" + repo + ":latest" + pullAndVerifySuccess(imageRef, "--pull-option=enable_partial_images=true") + }) + + It("GetBlobAt falls back across multiple mirrors on range request failures", func() { + SkipIfRemote("registries.conf is not used by the remote client") + + blobs, manifestBytes, manifestCT, _ := buildTestImage(1, compressionZstdChunked) + + var primaryBlobHits atomic.Int32 + primary := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, + countingBlobMW(&primaryBlobHits))) + defer primary.Close() + + // mirror1: 503 on range requests only -> fallback-worthy. + mirror1 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, + rangeOnlyErrorBlobMW(blobMW503))) + defer mirror1.Close() + + // mirror2: BLOB_UNKNOWN on range requests only -> fallback-worthy. + mirror2 := httptest.NewServer(registryHandler(repo, manifestBytes, manifestCT, blobs, + rangeOnlyErrorBlobMW(blobMWBlobUnknown))) + defer mirror2.Close() + + conf := fmt.Sprintf(`[[registry]] +prefix = "mirrortest.local" +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true + +[[registry.mirror]] +location = %q +insecure = true +`, primaryAddr(primary), primaryAddr(mirror1), primaryAddr(mirror2)) + podmanTest.setRegistriesConfigEnv([]byte(conf)) + defer resetRegistriesConfigEnv() + + imageRef := "mirrortest.local/" + repo + ":latest" + pullAndVerifySuccess(imageRef, "--pull-option=enable_partial_images=true") + }) + }) +} diff --git a/test/e2e/pull_test.go b/test/e2e/pull_test.go index 1586bbdc68..4b0c324415 100644 --- a/test/e2e/pull_test.go +++ b/test/e2e/pull_test.go @@ -325,6 +325,8 @@ var _ = Describe("Podman pull", func() { pullChunkedTests() + pullMirrorFallbackTests() + It("podman pull from docker-archive", func() { SkipIfNotAMD64() // https://github.com/containers/podman/issues/28273 SkipIfRemote("podman-remote does not support pulling from docker-archive")