diff --git a/cache/refs.go b/cache/refs.go
index 2b25235e..2a8aeb30 100644
--- a/cache/refs.go
+++ b/cache/refs.go
@@ -399,8 +399,10 @@ func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr erro
 		ctx = winlayers.UseWindowsLayerMode(ctx)
 	}
 
-	if _, err := sr.prepareRemoteSnapshots(ctx, sr.descHandlers); err != nil {
-		return err
+	if sr.cm.Snapshotter.Name() == "stargz" {
+		if _, err := sr.prepareRemoteSnapshots(ctx, sr.descHandlers); err != nil {
+			return err
+		}
 	}
 
 	return sr.extract(ctx, sr.descHandlers, s)