Merge pull request #2043 from aaronlehmann/retry-on-5xx

Add retry on image push 5xx errors

util/contentutil/copy.go

@@ -65,7 +65,7 @@ func CopyChain(ctx context.Context, ingester content.Ingester, provider content.
 	handlers := []images.Handler{
 		images.ChildrenHandler(provider),
 		filterHandler,
-		retryhandler.New(remotes.FetchHandler(ingester, &localFetcher{provider}), nil),
+		retryhandler.New(remotes.FetchHandler(ingester, &localFetcher{provider}), func(_ []byte) {}),
 	}
 
 	if err := images.Dispatch(ctx, images.Handlers(handlers...), nil, desc); err != nil {

util/imageutil/config.go

@@ -101,7 +101,7 @@ func Config(ctx context.Context, str string, resolver remotes.Resolver, cache Co
 	children := childrenConfigHandler(cache, platform)
 
 	handlers := []images.Handler{
-		retryhandler.New(remotes.FetchHandler(cache, fetcher), nil),
+		retryhandler.New(remotes.FetchHandler(cache, fetcher), func(_ []byte) {}),
 		children,
 	}
 	if err := images.Dispatch(ctx, images.Handlers(handlers...), nil, desc); err != nil {

util/progress/logs/logs.go

@@ -133,6 +133,7 @@ func (sw *streamWriter) Close() error {
 func LoggerFromContext(ctx context.Context) func([]byte) {
 	return func(dt []byte) {
 		pw, _, _ := progress.FromContext(ctx)
+		defer pw.Close()
 		pw.Write(identity.NewID(), client.VertexLog{
 			Stream: stderr,
 			Data:   []byte(dt),

util/resolver/retryhandler/retry.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/containerd/containerd/images"
+	remoteserrors "github.com/containerd/containerd/remotes/errors"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
@@ -48,6 +49,14 @@ func New(f images.HandlerFunc, logger func([]byte)) images.HandlerFunc {
 }
 
 func retryError(err error) bool {
+	// Retry on 5xx errors
+	var errUnexpectedStatus remoteserrors.ErrUnexpectedStatus
+	if errors.As(err, &errUnexpectedStatus) &&
+		errUnexpectedStatus.StatusCode >= 500 &&
+		errUnexpectedStatus.StatusCode <= 599 {
+		return true
+	}
+
 	if errors.Is(err, io.EOF) || errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.EPIPE) {
 		return true
 	}