diff --git a/util/resolver/authorizer.go b/util/resolver/authorizer.go
index e58038c4..32a3b23c 100644
--- a/util/resolver/authorizer.go
+++ b/util/resolver/authorizer.go
@@ -400,6 +400,11 @@ func (ah *authHandler) fetchToken(ctx context.Context, sm *session.Manager, g se
 }
 
 func invalidAuthorization(c auth.Challenge, responses []*http.Response) error {
+	lastResponse := responses[len(responses)-1]
+	if lastResponse.StatusCode == http.StatusUnauthorized {
+		return errors.Wrapf(docker.ErrInvalidAuthorization, "authorization status: %v", lastResponse.StatusCode)
+	}
+
 	errStr := c.Parameters["error"]
 	if errStr == "" {
 		return nil