Bump github.com/containerd/stargz-snapshotter to v0.3.0

Signed-off-by: ktock <ktokunaga.mail@gmail.com>

Dockerfile

@@ -13,7 +13,7 @@ ARG CNI_VERSION=v0.8.7
 ARG SHADOW_VERSION=4.8.1
 ARG FUSEOVERLAYFS_VERSION=v1.3.0
 # master (Dec 17, 2020)
-ARG STARGZ_SNAPSHOTTER_VERSION=2b97b583765b26f7284fe13f39f4ef925ecf87c3
+ARG STARGZ_SNAPSHOTTER_VERSION=v0.3.0
 
 ARG ALPINE_VERSION=3.12
 

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7
 	github.com/containerd/go-cni v1.0.1
 	github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0
-	github.com/containerd/stargz-snapshotter v0.2.1-0.20201217071531-2b97b583765b
+	github.com/containerd/stargz-snapshotter v0.3.0
 	github.com/containerd/typeurl v1.0.1
 	github.com/coreos/go-systemd/v22 v22.1.0
 	github.com/docker/cli v20.10.0-beta1.0.20201029214301-1d20b15adc38+incompatible

go.sum

@@ -222,8 +222,8 @@ github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0 h1:e+50zk22gvHL
 github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
 github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0=
 github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c=
-github.com/containerd/stargz-snapshotter v0.2.1-0.20201217071531-2b97b583765b h1:mcOHCuHEZLGFTCdS6LYjsbm4iePr2j7towTjtb8as0I=
+github.com/containerd/stargz-snapshotter v0.3.0 h1:80K+e6RHxCZhUGUMWGeMPb9wxLHrbwYN8nOTrDfrEHA=
-github.com/containerd/stargz-snapshotter v0.2.1-0.20201217071531-2b97b583765b/go.mod h1:sOeBRUp2i02FfLARbAMF7AUS8FrzLHL8txn4R94JeEI=
+github.com/containerd/stargz-snapshotter v0.3.0/go.mod h1:sOeBRUp2i02FfLARbAMF7AUS8FrzLHL8txn4R94JeEI=
 github.com/containerd/stargz-snapshotter/estargz v0.0.0-20201217071531-2b97b583765b h1:tnP4txDzNKsBOISNYG/f48Mt477CBeh9sS5rlu8MvSY=
 github.com/containerd/stargz-snapshotter/estargz v0.0.0-20201217071531-2b97b583765b/go.mod h1:E9uVkkBKf0EaC39j2JVW9EzdNhYvpz6eQIjILHebruk=
 github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=

vendor/github.com/containerd/stargz-snapshotter/fs/config/config.go

@@ -44,6 +44,7 @@ type Config struct {
 	Debug               bool   `toml:"debug"`
 	AllowNoVerification bool   `toml:"allow_no_verification"`
 	DisableVerification bool   `toml:"disable_verification"`
+	MaxConcurrency      int64  `toml:"max_concurrency"`
 
 	// BlobConfig is config for layer blob management.
 	BlobConfig `toml:"blob"`
@@ -53,9 +54,10 @@ type Config struct {
 }
 
 type BlobConfig struct {
-	ValidInterval int64 `toml:"valid_interval"`
+	ValidInterval   int64 `toml:"valid_interval"`
-	CheckAlways   bool  `toml:"check_always"`
+	CheckAlways     bool  `toml:"check_always"`
-	ChunkSize     int64 `toml:"chunk_size"`
+	ChunkSize       int64 `toml:"chunk_size"`
+	FetchTimeoutSec int64 `toml:"fetching_timeout_sec"`
 }
 
 type DirectoryCacheConfig struct {

vendor/github.com/containerd/stargz-snapshotter/fs/fs.go

@@ -45,6 +45,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -83,6 +84,7 @@ const (
 	stateDirName              = ".stargz-snapshotter"
 	defaultResolveResultEntry = 100
 	defaultPrefetchTimeoutSec = 10
+	defaultMaxConcurrency     = 2
 	statFileMode              = syscall.S_IFREG | 0400 // -r--------
 	stateDirMode              = syscall.S_IFDIR | 0500 // dr-x------
 )
@@ -144,6 +146,10 @@ func NewFilesystem(root string, cfg config.Config, opts ...Option) (_ snbase.Fil
 	if prefetchTimeout == 0 {
 		prefetchTimeout = defaultPrefetchTimeoutSec * time.Second
 	}
+	maxConcurrency := cfg.MaxConcurrency
+	if maxConcurrency == 0 {
+		maxConcurrency = defaultMaxConcurrency
+	}
 	getSources := fsOpts.getSources
 	if getSources == nil {
 		getSources = source.FromDefaultLabels(
@@ -161,7 +167,7 @@ func NewFilesystem(root string, cfg config.Config, opts ...Option) (_ snbase.Fil
 		layer:                 make(map[string]*layer),
 		resolveResult:         lru.New(resolveResultEntry),
 		blobResult:            lru.New(resolveResultEntry),
-		backgroundTaskManager: task.NewBackgroundTaskManager(2, 5*time.Second),
+		backgroundTaskManager: task.NewBackgroundTaskManager(maxConcurrency, 5*time.Second),
 		allowNoVerification:   cfg.AllowNoVerification,
 		disableVerification:   cfg.DisableVerification,
 	}, nil
@@ -702,6 +708,11 @@ func (n *node) Readdir(ctx context.Context) (fusefs.DirStream, syscall.Errno) {
 		}
 	}
 
+	// Avoid undeterministic order of entries on each call
+	sort.Slice(ents, func(i, j int) bool {
+		return ents[i].Name < ents[j].Name
+	})
+
 	return fusefs.NewListDirStream(ents), 0
 }
 

vendor/github.com/containerd/stargz-snapshotter/fs/reader/reader.go

@@ -40,6 +40,8 @@ import (
 	"golang.org/x/sync/semaphore"
 )
 
+const maxWalkDepth = 10000
+
 type Reader interface {
 	OpenFile(name string) (io.ReaderAt, error)
 	Lookup(name string) (*estargz.TOCEntry, bool)
@@ -166,13 +168,17 @@ func (gr *reader) Cache(opts ...CacheOption) (err error) {
 
 	eg, egCtx := errgroup.WithContext(context.Background())
 	eg.Go(func() error {
-		return gr.cacheWithReader(egCtx, eg, semaphore.NewWeighted(int64(runtime.GOMAXPROCS(0))),
+		return gr.cacheWithReader(egCtx,
+			0, eg, semaphore.NewWeighted(int64(runtime.GOMAXPROCS(0))),
 			root, r, filter, cacheOpts.cacheOpts...)
 	})
 	return eg.Wait()
 }
 
-func (gr *reader) cacheWithReader(ctx context.Context, eg *errgroup.Group, sem *semaphore.Weighted, dir *estargz.TOCEntry, r *estargz.Reader, filter func(*estargz.TOCEntry) bool, opts ...cache.Option) (rErr error) {
+func (gr *reader) cacheWithReader(ctx context.Context, currentDepth int, eg *errgroup.Group, sem *semaphore.Weighted, dir *estargz.TOCEntry, r *estargz.Reader, filter func(*estargz.TOCEntry) bool, opts ...cache.Option) (rErr error) {
+	if currentDepth > maxWalkDepth {
+		return fmt.Errorf("TOCEntry tree is too deep (depth:%d)", currentDepth)
+	}
 	dir.ForeachChild(func(_ string, e *estargz.TOCEntry) bool {
 		if e.Type == "dir" {
 			// Walk through all files on this stargz file.
@@ -189,7 +195,7 @@ func (gr *reader) cacheWithReader(ctx context.Context, eg *errgroup.Group, sem *
 					e.Name, dir.Name)
 				return false
 			}
-			if err := gr.cacheWithReader(ctx, eg, sem, e, r, filter, opts...); err != nil {
+			if err := gr.cacheWithReader(ctx, currentDepth+1, eg, sem, e, r, filter, opts...); err != nil {
 				rErr = err
 				return false
 			}

vendor/github.com/containerd/stargz-snapshotter/fs/remote/blob.go

@@ -60,6 +60,7 @@ type blob struct {
 	lastCheck     time.Time
 	lastCheckMu   sync.Mutex
 	checkInterval time.Duration
+	fetchTimeout  time.Duration
 
 	fetchedRegionSet   regionSet
 	fetchedRegionSetMu sync.Mutex
@@ -267,7 +268,7 @@ func (b *blob) fetchRange(allData map[region]io.Writer, opts *options) error {
 		req = append(req, reg)
 		fetched[reg] = false
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), b.fetchTimeout)
 	defer cancel()
 	mr, err := fr.fetch(ctx, req, true, opts)
 	if err != nil {

vendor/github.com/containerd/stargz-snapshotter/fs/remote/resolver.go

@@ -51,6 +51,7 @@ import (
 const (
 	defaultChunkSize        = 50000
 	defaultValidIntervalSec = 60
+	defaultFetchTimeoutSec  = 300
 )
 
 func NewResolver(cache cache.BlobCache, cfg config.BlobConfig) *Resolver {
@@ -63,6 +64,9 @@ func NewResolver(cache cache.BlobCache, cfg config.BlobConfig) *Resolver {
 	if cfg.CheckAlways {
 		cfg.ValidInterval = 0
 	}
+	if cfg.FetchTimeoutSec == 0 {
+		cfg.FetchTimeoutSec = defaultFetchTimeoutSec
+	}
 
 	return &Resolver{
 		bufPool: sync.Pool{
@@ -94,6 +98,7 @@ func (r *Resolver) Resolve(ctx context.Context, hosts docker.RegistryHosts, refs
 		lastCheck:     time.Now(),
 		checkInterval: time.Duration(r.blobConfig.ValidInterval) * time.Second,
 		resolver:      r,
+		fetchTimeout:  time.Duration(r.blobConfig.FetchTimeoutSec) * time.Second,
 	}, nil
 }
 

vendor/modules.txt

@@ -120,7 +120,7 @@ github.com/containerd/fifo
 github.com/containerd/go-cni
 # github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0
 github.com/containerd/go-runc
-# github.com/containerd/stargz-snapshotter v0.2.1-0.20201217071531-2b97b583765b
+# github.com/containerd/stargz-snapshotter v0.3.0
 github.com/containerd/stargz-snapshotter/cache
 github.com/containerd/stargz-snapshotter/fs
 github.com/containerd/stargz-snapshotter/fs/config