From db794baea73befbe240e6b3209de11ed7ef9b359 Mon Sep 17 00:00:00 2001 From: ktock Date: Thu, 21 Jan 2021 18:27:01 +0900 Subject: [PATCH] Bump github.com/containerd/stargz-snapshotter to v0.3.0 Signed-off-by: ktock --- Dockerfile | 2 +- go.mod | 2 +- go.sum | 4 ++-- .../stargz-snapshotter/fs/config/config.go | 8 +++++--- .../containerd/stargz-snapshotter/fs/fs.go | 13 ++++++++++++- .../stargz-snapshotter/fs/reader/reader.go | 12 +++++++++--- .../containerd/stargz-snapshotter/fs/remote/blob.go | 3 ++- .../stargz-snapshotter/fs/remote/resolver.go | 5 +++++ vendor/modules.txt | 2 +- 9 files changed, 38 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 64b6c1a2..1240318b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ ARG CNI_VERSION=v0.8.7 ARG SHADOW_VERSION=4.8.1 ARG FUSEOVERLAYFS_VERSION=v1.3.0 # master (Dec 17, 2020) -ARG STARGZ_SNAPSHOTTER_VERSION=2b97b583765b26f7284fe13f39f4ef925ecf87c3 +ARG STARGZ_SNAPSHOTTER_VERSION=v0.3.0 ARG ALPINE_VERSION=3.12 diff --git a/go.mod b/go.mod index 8b77392b..6fdfba67 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7 github.com/containerd/go-cni v1.0.1 github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0 - github.com/containerd/stargz-snapshotter v0.2.1-0.20201217071531-2b97b583765b + github.com/containerd/stargz-snapshotter v0.3.0 github.com/containerd/typeurl v1.0.1 github.com/coreos/go-systemd/v22 v22.1.0 github.com/docker/cli v20.10.0-beta1.0.20201029214301-1d20b15adc38+incompatible diff --git a/go.sum b/go.sum index 90e5929a..5afcbd6c 100644 --- a/go.sum +++ b/go.sum @@ -222,8 +222,8 @@ github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0 h1:e+50zk22gvHL github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= -github.com/containerd/stargz-snapshotter v0.2.1-0.20201217071531-2b97b583765b h1:mcOHCuHEZLGFTCdS6LYjsbm4iePr2j7towTjtb8as0I= -github.com/containerd/stargz-snapshotter v0.2.1-0.20201217071531-2b97b583765b/go.mod h1:sOeBRUp2i02FfLARbAMF7AUS8FrzLHL8txn4R94JeEI= +github.com/containerd/stargz-snapshotter v0.3.0 h1:80K+e6RHxCZhUGUMWGeMPb9wxLHrbwYN8nOTrDfrEHA= +github.com/containerd/stargz-snapshotter v0.3.0/go.mod h1:sOeBRUp2i02FfLARbAMF7AUS8FrzLHL8txn4R94JeEI= github.com/containerd/stargz-snapshotter/estargz v0.0.0-20201217071531-2b97b583765b h1:tnP4txDzNKsBOISNYG/f48Mt477CBeh9sS5rlu8MvSY= github.com/containerd/stargz-snapshotter/estargz v0.0.0-20201217071531-2b97b583765b/go.mod h1:E9uVkkBKf0EaC39j2JVW9EzdNhYvpz6eQIjILHebruk= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= diff --git a/vendor/github.com/containerd/stargz-snapshotter/fs/config/config.go b/vendor/github.com/containerd/stargz-snapshotter/fs/config/config.go index 93e937ce..a0ec0e38 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/fs/config/config.go +++ b/vendor/github.com/containerd/stargz-snapshotter/fs/config/config.go @@ -44,6 +44,7 @@ type Config struct { Debug bool `toml:"debug"` AllowNoVerification bool `toml:"allow_no_verification"` DisableVerification bool `toml:"disable_verification"` + MaxConcurrency int64 `toml:"max_concurrency"` // BlobConfig is config for layer blob management. BlobConfig `toml:"blob"` @@ -53,9 +54,10 @@ type Config struct { } type BlobConfig struct { - ValidInterval int64 `toml:"valid_interval"` - CheckAlways bool `toml:"check_always"` - ChunkSize int64 `toml:"chunk_size"` + ValidInterval int64 `toml:"valid_interval"` + CheckAlways bool `toml:"check_always"` + ChunkSize int64 `toml:"chunk_size"` + FetchTimeoutSec int64 `toml:"fetching_timeout_sec"` } type DirectoryCacheConfig struct { diff --git a/vendor/github.com/containerd/stargz-snapshotter/fs/fs.go b/vendor/github.com/containerd/stargz-snapshotter/fs/fs.go index bbeda21a..dc789006 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/fs/fs.go +++ b/vendor/github.com/containerd/stargz-snapshotter/fs/fs.go @@ -45,6 +45,7 @@ import ( "io" "os" "path/filepath" + "sort" "strconv" "strings" "sync" @@ -83,6 +84,7 @@ const ( stateDirName = ".stargz-snapshotter" defaultResolveResultEntry = 100 defaultPrefetchTimeoutSec = 10 + defaultMaxConcurrency = 2 statFileMode = syscall.S_IFREG | 0400 // -r-------- stateDirMode = syscall.S_IFDIR | 0500 // dr-x------ ) @@ -144,6 +146,10 @@ func NewFilesystem(root string, cfg config.Config, opts ...Option) (_ snbase.Fil if prefetchTimeout == 0 { prefetchTimeout = defaultPrefetchTimeoutSec * time.Second } + maxConcurrency := cfg.MaxConcurrency + if maxConcurrency == 0 { + maxConcurrency = defaultMaxConcurrency + } getSources := fsOpts.getSources if getSources == nil { getSources = source.FromDefaultLabels( @@ -161,7 +167,7 @@ func NewFilesystem(root string, cfg config.Config, opts ...Option) (_ snbase.Fil layer: make(map[string]*layer), resolveResult: lru.New(resolveResultEntry), blobResult: lru.New(resolveResultEntry), - backgroundTaskManager: task.NewBackgroundTaskManager(2, 5*time.Second), + backgroundTaskManager: task.NewBackgroundTaskManager(maxConcurrency, 5*time.Second), allowNoVerification: cfg.AllowNoVerification, disableVerification: cfg.DisableVerification, }, nil @@ -702,6 +708,11 @@ func (n *node) Readdir(ctx context.Context) (fusefs.DirStream, syscall.Errno) { } } + // Avoid undeterministic order of entries on each call + sort.Slice(ents, func(i, j int) bool { + return ents[i].Name < ents[j].Name + }) + return fusefs.NewListDirStream(ents), 0 } diff --git a/vendor/github.com/containerd/stargz-snapshotter/fs/reader/reader.go b/vendor/github.com/containerd/stargz-snapshotter/fs/reader/reader.go index d8438b3e..dcd6a10c 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/fs/reader/reader.go +++ b/vendor/github.com/containerd/stargz-snapshotter/fs/reader/reader.go @@ -40,6 +40,8 @@ import ( "golang.org/x/sync/semaphore" ) +const maxWalkDepth = 10000 + type Reader interface { OpenFile(name string) (io.ReaderAt, error) Lookup(name string) (*estargz.TOCEntry, bool) @@ -166,13 +168,17 @@ func (gr *reader) Cache(opts ...CacheOption) (err error) { eg, egCtx := errgroup.WithContext(context.Background()) eg.Go(func() error { - return gr.cacheWithReader(egCtx, eg, semaphore.NewWeighted(int64(runtime.GOMAXPROCS(0))), + return gr.cacheWithReader(egCtx, + 0, eg, semaphore.NewWeighted(int64(runtime.GOMAXPROCS(0))), root, r, filter, cacheOpts.cacheOpts...) }) return eg.Wait() } -func (gr *reader) cacheWithReader(ctx context.Context, eg *errgroup.Group, sem *semaphore.Weighted, dir *estargz.TOCEntry, r *estargz.Reader, filter func(*estargz.TOCEntry) bool, opts ...cache.Option) (rErr error) { +func (gr *reader) cacheWithReader(ctx context.Context, currentDepth int, eg *errgroup.Group, sem *semaphore.Weighted, dir *estargz.TOCEntry, r *estargz.Reader, filter func(*estargz.TOCEntry) bool, opts ...cache.Option) (rErr error) { + if currentDepth > maxWalkDepth { + return fmt.Errorf("TOCEntry tree is too deep (depth:%d)", currentDepth) + } dir.ForeachChild(func(_ string, e *estargz.TOCEntry) bool { if e.Type == "dir" { // Walk through all files on this stargz file. @@ -189,7 +195,7 @@ func (gr *reader) cacheWithReader(ctx context.Context, eg *errgroup.Group, sem * e.Name, dir.Name) return false } - if err := gr.cacheWithReader(ctx, eg, sem, e, r, filter, opts...); err != nil { + if err := gr.cacheWithReader(ctx, currentDepth+1, eg, sem, e, r, filter, opts...); err != nil { rErr = err return false } diff --git a/vendor/github.com/containerd/stargz-snapshotter/fs/remote/blob.go b/vendor/github.com/containerd/stargz-snapshotter/fs/remote/blob.go index 89d676b3..eced44f0 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/fs/remote/blob.go +++ b/vendor/github.com/containerd/stargz-snapshotter/fs/remote/blob.go @@ -60,6 +60,7 @@ type blob struct { lastCheck time.Time lastCheckMu sync.Mutex checkInterval time.Duration + fetchTimeout time.Duration fetchedRegionSet regionSet fetchedRegionSetMu sync.Mutex @@ -267,7 +268,7 @@ func (b *blob) fetchRange(allData map[region]io.Writer, opts *options) error { req = append(req, reg) fetched[reg] = false } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), b.fetchTimeout) defer cancel() mr, err := fr.fetch(ctx, req, true, opts) if err != nil { diff --git a/vendor/github.com/containerd/stargz-snapshotter/fs/remote/resolver.go b/vendor/github.com/containerd/stargz-snapshotter/fs/remote/resolver.go index 9b086465..f4548f68 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/fs/remote/resolver.go +++ b/vendor/github.com/containerd/stargz-snapshotter/fs/remote/resolver.go @@ -51,6 +51,7 @@ import ( const ( defaultChunkSize = 50000 defaultValidIntervalSec = 60 + defaultFetchTimeoutSec = 300 ) func NewResolver(cache cache.BlobCache, cfg config.BlobConfig) *Resolver { @@ -63,6 +64,9 @@ func NewResolver(cache cache.BlobCache, cfg config.BlobConfig) *Resolver { if cfg.CheckAlways { cfg.ValidInterval = 0 } + if cfg.FetchTimeoutSec == 0 { + cfg.FetchTimeoutSec = defaultFetchTimeoutSec + } return &Resolver{ bufPool: sync.Pool{ @@ -94,6 +98,7 @@ func (r *Resolver) Resolve(ctx context.Context, hosts docker.RegistryHosts, refs lastCheck: time.Now(), checkInterval: time.Duration(r.blobConfig.ValidInterval) * time.Second, resolver: r, + fetchTimeout: time.Duration(r.blobConfig.FetchTimeoutSec) * time.Second, }, nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index 47a2a474..036f0590 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -120,7 +120,7 @@ github.com/containerd/fifo github.com/containerd/go-cni # github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0 github.com/containerd/go-runc -# github.com/containerd/stargz-snapshotter v0.2.1-0.20201217071531-2b97b583765b +# github.com/containerd/stargz-snapshotter v0.3.0 github.com/containerd/stargz-snapshotter/cache github.com/containerd/stargz-snapshotter/fs github.com/containerd/stargz-snapshotter/fs/config