Merge pull request #2452 from crazy-max/update-hashstructure

update to github.com/mitchellh/hashstructure v2.0.2

executor/oci/spec.go

@@ -13,7 +13,7 @@ import (
 	"github.com/containerd/containerd/oci"
 	"github.com/containerd/continuity/fs"
 	"github.com/docker/docker/pkg/idtools"
-	"github.com/mitchellh/hashstructure"
+	"github.com/mitchellh/hashstructure/v2"
 	"github.com/moby/buildkit/executor"
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/util/network"
@@ -200,7 +200,7 @@ func (s *submounts) subMount(m mount.Mount, subPath string) (mount.Mount, error)
 	if s.m == nil {
 		s.m = map[uint64]mountRef{}
 	}
-	h, err := hashstructure.Hash(m, nil)
+	h, err := hashstructure.Hash(m, hashstructure.FormatV2, nil)
 	if err != nil {
 		return mount.Mount{}, nil
 	}

go.mod

@@ -34,7 +34,7 @@ require (
 	github.com/hashicorp/golang-lru v0.5.3
 	github.com/ishidawataru/sctp v0.0.0-20210226210310-f2269e66cdee // indirect
 	github.com/klauspost/compress v1.13.5
-	github.com/mitchellh/hashstructure v1.0.0
+	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/moby/locker v1.0.1
 	github.com/moby/sys/mount v0.3.0 // indirect
 	github.com/moby/sys/signal v0.6.0

go.sum

@@ -864,6 +864,8 @@ github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eI
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
 github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y=
 github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ=
+github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
+github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=

solver/llbsolver/bridge.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/containerd/containerd/platforms"
-	"github.com/mitchellh/hashstructure"
+	"github.com/mitchellh/hashstructure/v2"
 	"github.com/moby/buildkit/cache/remotecache"
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/frontend"
@@ -345,7 +345,7 @@ func cmKey(im gw.CacheOptionsEntry) (string, error) {
 	if im.Type == "registry" && im.Attrs["ref"] != "" {
 		return im.Attrs["ref"], nil
 	}
-	i, err := hashstructure.Hash(im, nil)
+	i, err := hashstructure.Hash(im, hashstructure.FormatV2, nil)
 	if err != nil {
 		return "", err
 	}

vendor/github.com/mitchellh/hashstructure/v2/README.md

@@ -17,20 +17,31 @@ sending data across the network, caching values locally (de-dup), and so on.
     doesn't affect the hash code but the field itself is still taken into
     account to create the hash value.
 
-  * Optionally specify a custom hash function to optimize for speed, collision
+  * Optionally, specify a custom hash function to optimize for speed, collision
     avoidance for your data set, etc.
 
-  * Optionally hash the output of `.String()` on structs that implement fmt.Stringer,
+  * Optionally, hash the output of `.String()` on structs that implement fmt.Stringer,
     allowing effective hashing of time.Time
 
+  * Optionally, override the hashing process by implementing `Hashable`.
+
 ## Installation
 
 Standard `go get`:
 
 ```
-$ go get github.com/mitchellh/hashstructure
+$ go get github.com/mitchellh/hashstructure/v2
 ```
 
+**Note on v2:** It is highly recommended you use the "v2" release since this
+fixes some significant hash collisions issues from v1. In practice, we used
+v1 for many years in real projects at HashiCorp and never had issues, but it
+is highly dependent on the shape of the data you're hashing and how you use
+those hashes.
+
+When using v2+, you can still generate weaker v1 hashes by using the
+`FormatV1` format when calling `Hash`.
+
 ## Usage & Example
 
 For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/hashstructure).
@@ -54,7 +65,7 @@ v := ComplexStruct{
     },
 }
 
-hash, err := hashstructure.Hash(v, nil)
+hash, err := hashstructure.Hash(v, hashstructure.FormatV2, nil)
 if err != nil {
     panic(err)
 }

vendor/github.com/mitchellh/hashstructure/v2/errors.go

@@ -0,0 +1,22 @@
+package hashstructure
+
+import (
+	"fmt"
+)
+
+// ErrNotStringer is returned when there's an error with hash:"string"
+type ErrNotStringer struct {
+	Field string
+}
+
+// Error implements error for ErrNotStringer
+func (ens *ErrNotStringer) Error() string {
+	return fmt.Sprintf("hashstructure: %s has hash:\"string\" set, but does not implement fmt.Stringer", ens.Field)
+}
+
+// ErrFormat is returned when an invalid format is given to the Hash function.
+type ErrFormat struct{}
+
+func (*ErrFormat) Error() string {
+	return "format must be one of the defined Format values in the hashstructure library"
+}

vendor/github.com/mitchellh/hashstructure/v2/hashstructure.go

@@ -6,18 +6,9 @@ import (
 	"hash"
 	"hash/fnv"
 	"reflect"
+	"time"
 )
 
-// ErrNotStringer is returned when there's an error with hash:"string"
-type ErrNotStringer struct {
-	Field string
-}
-
-// Error implements error for ErrNotStringer
-func (ens *ErrNotStringer) Error() string {
-	return fmt.Sprintf("hashstructure: %s has hash:\"string\" set, but does not implement fmt.Stringer", ens.Field)
-}
-
 // HashOptions are options that are available for hashing.
 type HashOptions struct {
 	// Hasher is the hash function to use. If this isn't set, it will
@@ -31,8 +22,42 @@ type HashOptions struct {
 	// ZeroNil is flag determining if nil pointer should be treated equal
 	// to a zero value of pointed type. By default this is false.
 	ZeroNil bool
+
+	// IgnoreZeroValue is determining if zero value fields should be
+	// ignored for hash calculation.
+	IgnoreZeroValue bool
+
+	// SlicesAsSets assumes that a `set` tag is always present for slices.
+	// Default is false (in which case the tag is used instead)
+	SlicesAsSets bool
+
+	// UseStringer will attempt to use fmt.Stringer always. If the struct
+	// doesn't implement fmt.Stringer, it'll fall back to trying usual tricks.
+	// If this is true, and the "string" tag is also set, the tag takes
+	// precedence (meaning that if the type doesn't implement fmt.Stringer, we
+	// panic)
+	UseStringer bool
 }
 
+// Format specifies the hashing process used. Different formats typically
+// generate different hashes for the same value and have different properties.
+type Format uint
+
+const (
+	// To disallow the zero value
+	formatInvalid Format = iota
+
+	// FormatV1 is the format used in v1.x of this library. This has the
+	// downsides noted in issue #18 but allows simultaneous v1/v2 usage.
+	FormatV1
+
+	// FormatV2 is the current recommended format and fixes the issues
+	// noted in FormatV1.
+	FormatV2
+
+	formatMax // so we can easily find the end
+)
+
 // Hash returns the hash value of an arbitrary value.
 //
 // If opts is nil, then default options will be used. See HashOptions
@@ -40,6 +65,11 @@ type HashOptions struct {
 // concurrently. None of the values within a *HashOptions struct are
 // safe to read/write while hashing is being done.
 //
+// The "format" is required and must be one of the format values defined
+// by this library. You should probably just use "FormatV2". This allows
+// generated hashes uses alternate logic to maintain compatibility with
+// older versions.
+//
 // Notes on the value:
 //
 //   * Unexported fields on structs are ignored and do not affect the
@@ -65,7 +95,12 @@ type HashOptions struct {
 //   * "string" - The field will be hashed as a string, only works when the
 //                field implements fmt.Stringer
 //
-func Hash(v interface{}, opts *HashOptions) (uint64, error) {
+func Hash(v interface{}, format Format, opts *HashOptions) (uint64, error) {
+	// Validate our format
+	if format <= formatInvalid || format >= formatMax {
+		return 0, &ErrFormat{}
+	}
+
 	// Create default options
 	if opts == nil {
 		opts = &HashOptions{}
@@ -82,17 +117,25 @@ func Hash(v interface{}, opts *HashOptions) (uint64, error) {
 
 	// Create our walker and walk the structure
 	w := &walker{
+		format:          format,
 		h:               opts.Hasher,
 		tag:             opts.TagName,
 		zeronil:         opts.ZeroNil,
+		ignorezerovalue: opts.IgnoreZeroValue,
+		sets:            opts.SlicesAsSets,
+		stringer:        opts.UseStringer,
 	}
 	return w.visit(reflect.ValueOf(v), nil)
 }
 
 type walker struct {
+	format          Format
 	h               hash.Hash64
 	tag             string
 	zeronil         bool
+	ignorezerovalue bool
+	sets            bool
+	stringer        bool
 }
 
 type visitOpts struct {
@@ -104,6 +147,8 @@ type visitOpts struct {
 	StructField string
 }
 
+var timeType = reflect.TypeOf(time.Time{})
+
 func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
 	t := reflect.TypeOf(0)
 
@@ -159,6 +204,18 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
 		return w.h.Sum64(), err
 	}
 
+	switch v.Type() {
+	case timeType:
+		w.h.Reset()
+		b, err := v.Interface().(time.Time).MarshalBinary()
+		if err != nil {
+			return 0, err
+		}
+
+		err = binary.Write(w.h, binary.LittleEndian, b)
+		return w.h.Sum64(), err
+	}
+
 	switch k {
 	case reflect.Array:
 		var h uint64
@@ -211,6 +268,11 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
 			h = hashUpdateUnordered(h, fieldHash)
 		}
 
+		if w.format != FormatV1 {
+			// Important: read the docs for hashFinishUnordered
+			h = hashFinishUnordered(w.h, h)
+		}
+
 		return h, nil
 
 	case reflect.Struct:
@@ -220,6 +282,24 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
 			include = impl
 		}
 
+		if impl, ok := parent.(Hashable); ok {
+			return impl.Hash()
+		}
+
+		// If we can address this value, check if the pointer value
+		// implements our interfaces and use that if so.
+		if v.CanAddr() {
+			vptr := v.Addr()
+			parentptr := vptr.Interface()
+			if impl, ok := parentptr.(Includable); ok {
+				include = impl
+			}
+
+			if impl, ok := parentptr.(Hashable); ok {
+				return impl.Hash()
+			}
+		}
+
 		t := v.Type()
 		h, err := w.visit(reflect.ValueOf(t.Name()), nil)
 		if err != nil {
@@ -242,11 +322,19 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
 					continue
 				}
 
+				if w.ignorezerovalue {
+					if innerV.IsZero() {
+						continue
+					}
+				}
+
 				// if string is set, use the string value
-				if tag == "string" {
+				if tag == "string" || w.stringer {
 					if impl, ok := innerV.Interface().(fmt.Stringer); ok {
 						innerV = reflect.ValueOf(impl.String())
-					} else {
+					} else if tag == "string" {
+						// We only show this error if the tag explicitly
+						// requests a stringer.
 						return 0, &ErrNotStringer{
 							Field: v.Type().Field(i).Name,
 						}
@@ -286,6 +374,11 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
 				fieldHash := hashUpdateOrdered(w.h, kh, vh)
 				h = hashUpdateUnordered(h, fieldHash)
 			}
+
+			if w.format != FormatV1 {
+				// Important: read the docs for hashFinishUnordered
+				h = hashFinishUnordered(w.h, h)
+			}
 		}
 
 		return h, nil
@@ -306,13 +399,18 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
 				return 0, err
 			}
 
-			if set {
+			if set || w.sets {
 				h = hashUpdateUnordered(h, current)
 			} else {
 				h = hashUpdateOrdered(w.h, h, current)
 			}
 		}
 
+		if set && w.format != FormatV1 {
+			// Important: read the docs for hashFinishUnordered
+			h = hashFinishUnordered(w.h, h)
+		}
+
 		return h, nil
 
 	case reflect.String:
@@ -349,6 +447,32 @@ func hashUpdateUnordered(a, b uint64) uint64 {
 	return a ^ b
 }
 
+// After mixing a group of unique hashes with hashUpdateUnordered, it's always
+// necessary to call hashFinishUnordered. Why? Because hashUpdateUnordered
+// is a simple XOR, and calling hashUpdateUnordered on hashes produced by
+// hashUpdateUnordered can effectively cancel out a previous change to the hash
+// result if the same hash value appears later on. For example, consider:
+//
+//   hashUpdateUnordered(hashUpdateUnordered("A", "B"), hashUpdateUnordered("A", "C")) =
+//   H("A") ^ H("B")) ^ (H("A") ^ H("C")) =
+//   (H("A") ^ H("A")) ^ (H("B") ^ H(C)) =
+//   H(B) ^ H(C) =
+//   hashUpdateUnordered(hashUpdateUnordered("Z", "B"), hashUpdateUnordered("Z", "C"))
+//
+// hashFinishUnordered "hardens" the result, so that encountering partially
+// overlapping input data later on in a different context won't cancel out.
+func hashFinishUnordered(h hash.Hash64, a uint64) uint64 {
+	h.Reset()
+
+	// We just panic if the writes fail
+	e1 := binary.Write(h, binary.LittleEndian, a)
+	if e1 != nil {
+		panic(e1)
+	}
+
+	return h.Sum64()
+}
+
 // visitFlag is used as a bitmask for affecting visit behavior
 type visitFlag uint
 

vendor/github.com/mitchellh/hashstructure/v2/include.go

@@ -13,3 +13,10 @@ type Includable interface {
 type IncludableMap interface {
 	HashIncludeMap(field string, k, v interface{}) (bool, error)
 }
+
+// Hashable is an interface that can optionally be implemented by a struct
+// to override the hash value. This value will override the hash value for
+// the entire struct. Entries in the struct will not be hashed.
+type Hashable interface {
+	Hash() (uint64, error)
+}

vendor/modules.txt

@@ -365,9 +365,9 @@ github.com/klauspost/compress/zstd/internal/xxhash
 # github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369
 ## explicit; go 1.9
 github.com/matttproud/golang_protobuf_extensions/pbutil
-# github.com/mitchellh/hashstructure v1.0.0
-## explicit
-github.com/mitchellh/hashstructure
+# github.com/mitchellh/hashstructure/v2 v2.0.2
+## explicit; go 1.14
+github.com/mitchellh/hashstructure/v2
 # github.com/moby/locker v1.0.1
 ## explicit; go 1.13
 github.com/moby/locker