update to github.com/mitchellh/hashstructure v2.0.2

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
master
CrazyMax 2021-11-09 09:56:04 +01:00
parent 0ad26d49f2
commit d81e79540b
No known key found for this signature in database
GPG Key ID: 3248E46B6BB8C7F7
10 changed files with 199 additions and 33 deletions

View File

@ -13,7 +13,7 @@ import (
"github.com/containerd/containerd/oci"
"github.com/containerd/continuity/fs"
"github.com/docker/docker/pkg/idtools"
"github.com/mitchellh/hashstructure"
"github.com/mitchellh/hashstructure/v2"
"github.com/moby/buildkit/executor"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/util/network"
@ -200,7 +200,7 @@ func (s *submounts) subMount(m mount.Mount, subPath string) (mount.Mount, error)
if s.m == nil {
s.m = map[uint64]mountRef{}
}
h, err := hashstructure.Hash(m, nil)
h, err := hashstructure.Hash(m, hashstructure.FormatV2, nil)
if err != nil {
return mount.Mount{}, nil
}

2
go.mod
View File

@ -34,7 +34,7 @@ require (
github.com/hashicorp/golang-lru v0.5.3
github.com/ishidawataru/sctp v0.0.0-20210226210310-f2269e66cdee // indirect
github.com/klauspost/compress v1.13.5
github.com/mitchellh/hashstructure v1.0.0
github.com/mitchellh/hashstructure/v2 v2.0.2
github.com/moby/locker v1.0.1
github.com/moby/sys/mount v0.3.0 // indirect
github.com/moby/sys/signal v0.6.0

2
go.sum
View File

@ -864,6 +864,8 @@ github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eI
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y=
github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ=
github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=

View File

@ -8,7 +8,7 @@ import (
"time"
"github.com/containerd/containerd/platforms"
"github.com/mitchellh/hashstructure"
"github.com/mitchellh/hashstructure/v2"
"github.com/moby/buildkit/cache/remotecache"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/frontend"
@ -345,7 +345,7 @@ func cmKey(im gw.CacheOptionsEntry) (string, error) {
if im.Type == "registry" && im.Attrs["ref"] != "" {
return im.Attrs["ref"], nil
}
i, err := hashstructure.Hash(im, nil)
i, err := hashstructure.Hash(im, hashstructure.FormatV2, nil)
if err != nil {
return "", err
}

View File

@ -17,20 +17,31 @@ sending data across the network, caching values locally (de-dup), and so on.
doesn't affect the hash code but the field itself is still taken into
account to create the hash value.
* Optionally specify a custom hash function to optimize for speed, collision
* Optionally, specify a custom hash function to optimize for speed, collision
avoidance for your data set, etc.
* Optionally hash the output of `.String()` on structs that implement fmt.Stringer,
* Optionally, hash the output of `.String()` on structs that implement fmt.Stringer,
allowing effective hashing of time.Time
* Optionally, override the hashing process by implementing `Hashable`.
## Installation
Standard `go get`:
```
$ go get github.com/mitchellh/hashstructure
$ go get github.com/mitchellh/hashstructure/v2
```
**Note on v2:** It is highly recommended you use the "v2" release since this
fixes some significant hash collisions issues from v1. In practice, we used
v1 for many years in real projects at HashiCorp and never had issues, but it
is highly dependent on the shape of the data you're hashing and how you use
those hashes.
When using v2+, you can still generate weaker v1 hashes by using the
`FormatV1` format when calling `Hash`.
## Usage & Example
For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/hashstructure).
@ -54,7 +65,7 @@ v := ComplexStruct{
},
}
hash, err := hashstructure.Hash(v, nil)
hash, err := hashstructure.Hash(v, hashstructure.FormatV2, nil)
if err != nil {
panic(err)
}

22
vendor/github.com/mitchellh/hashstructure/v2/errors.go generated vendored Normal file
View File

@ -0,0 +1,22 @@
package hashstructure
import (
"fmt"
)
// ErrNotStringer is returned when there's an error with hash:"string"
type ErrNotStringer struct {
Field string
}
// Error implements error for ErrNotStringer
func (ens *ErrNotStringer) Error() string {
return fmt.Sprintf("hashstructure: %s has hash:\"string\" set, but does not implement fmt.Stringer", ens.Field)
}
// ErrFormat is returned when an invalid format is given to the Hash function.
type ErrFormat struct{}
func (*ErrFormat) Error() string {
return "format must be one of the defined Format values in the hashstructure library"
}

View File

@ -6,18 +6,9 @@ import (
"hash"
"hash/fnv"
"reflect"
"time"
)
// ErrNotStringer is returned when there's an error with hash:"string"
type ErrNotStringer struct {
Field string
}
// Error implements error for ErrNotStringer
func (ens *ErrNotStringer) Error() string {
return fmt.Sprintf("hashstructure: %s has hash:\"string\" set, but does not implement fmt.Stringer", ens.Field)
}
// HashOptions are options that are available for hashing.
type HashOptions struct {
// Hasher is the hash function to use. If this isn't set, it will
@ -31,8 +22,42 @@ type HashOptions struct {
// ZeroNil is flag determining if nil pointer should be treated equal
// to a zero value of pointed type. By default this is false.
ZeroNil bool
// IgnoreZeroValue is determining if zero value fields should be
// ignored for hash calculation.
IgnoreZeroValue bool
// SlicesAsSets assumes that a `set` tag is always present for slices.
// Default is false (in which case the tag is used instead)
SlicesAsSets bool
// UseStringer will attempt to use fmt.Stringer always. If the struct
// doesn't implement fmt.Stringer, it'll fall back to trying usual tricks.
// If this is true, and the "string" tag is also set, the tag takes
// precedence (meaning that if the type doesn't implement fmt.Stringer, we
// panic)
UseStringer bool
}
// Format specifies the hashing process used. Different formats typically
// generate different hashes for the same value and have different properties.
type Format uint
const (
// To disallow the zero value
formatInvalid Format = iota
// FormatV1 is the format used in v1.x of this library. This has the
// downsides noted in issue #18 but allows simultaneous v1/v2 usage.
FormatV1
// FormatV2 is the current recommended format and fixes the issues
// noted in FormatV1.
FormatV2
formatMax // so we can easily find the end
)
// Hash returns the hash value of an arbitrary value.
//
// If opts is nil, then default options will be used. See HashOptions
@ -40,6 +65,11 @@ type HashOptions struct {
// concurrently. None of the values within a *HashOptions struct are
// safe to read/write while hashing is being done.
//
// The "format" is required and must be one of the format values defined
// by this library. You should probably just use "FormatV2". This allows
// generated hashes uses alternate logic to maintain compatibility with
// older versions.
//
// Notes on the value:
//
// * Unexported fields on structs are ignored and do not affect the
@ -65,7 +95,12 @@ type HashOptions struct {
// * "string" - The field will be hashed as a string, only works when the
// field implements fmt.Stringer
//
func Hash(v interface{}, opts *HashOptions) (uint64, error) {
func Hash(v interface{}, format Format, opts *HashOptions) (uint64, error) {
// Validate our format
if format <= formatInvalid || format >= formatMax {
return 0, &ErrFormat{}
}
// Create default options
if opts == nil {
opts = &HashOptions{}
@ -82,17 +117,25 @@ func Hash(v interface{}, opts *HashOptions) (uint64, error) {
// Create our walker and walk the structure
w := &walker{
h: opts.Hasher,
tag: opts.TagName,
zeronil: opts.ZeroNil,
format: format,
h: opts.Hasher,
tag: opts.TagName,
zeronil: opts.ZeroNil,
ignorezerovalue: opts.IgnoreZeroValue,
sets: opts.SlicesAsSets,
stringer: opts.UseStringer,
}
return w.visit(reflect.ValueOf(v), nil)
}
type walker struct {
h hash.Hash64
tag string
zeronil bool
format Format
h hash.Hash64
tag string
zeronil bool
ignorezerovalue bool
sets bool
stringer bool
}
type visitOpts struct {
@ -104,6 +147,8 @@ type visitOpts struct {
StructField string
}
var timeType = reflect.TypeOf(time.Time{})
func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
t := reflect.TypeOf(0)
@ -159,6 +204,18 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
return w.h.Sum64(), err
}
switch v.Type() {
case timeType:
w.h.Reset()
b, err := v.Interface().(time.Time).MarshalBinary()
if err != nil {
return 0, err
}
err = binary.Write(w.h, binary.LittleEndian, b)
return w.h.Sum64(), err
}
switch k {
case reflect.Array:
var h uint64
@ -211,6 +268,11 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
h = hashUpdateUnordered(h, fieldHash)
}
if w.format != FormatV1 {
// Important: read the docs for hashFinishUnordered
h = hashFinishUnordered(w.h, h)
}
return h, nil
case reflect.Struct:
@ -220,6 +282,24 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
include = impl
}
if impl, ok := parent.(Hashable); ok {
return impl.Hash()
}
// If we can address this value, check if the pointer value
// implements our interfaces and use that if so.
if v.CanAddr() {
vptr := v.Addr()
parentptr := vptr.Interface()
if impl, ok := parentptr.(Includable); ok {
include = impl
}
if impl, ok := parentptr.(Hashable); ok {
return impl.Hash()
}
}
t := v.Type()
h, err := w.visit(reflect.ValueOf(t.Name()), nil)
if err != nil {
@ -242,11 +322,19 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
continue
}
if w.ignorezerovalue {
if innerV.IsZero() {
continue
}
}
// if string is set, use the string value
if tag == "string" {
if tag == "string" || w.stringer {
if impl, ok := innerV.Interface().(fmt.Stringer); ok {
innerV = reflect.ValueOf(impl.String())
} else {
} else if tag == "string" {
// We only show this error if the tag explicitly
// requests a stringer.
return 0, &ErrNotStringer{
Field: v.Type().Field(i).Name,
}
@ -286,6 +374,11 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
fieldHash := hashUpdateOrdered(w.h, kh, vh)
h = hashUpdateUnordered(h, fieldHash)
}
if w.format != FormatV1 {
// Important: read the docs for hashFinishUnordered
h = hashFinishUnordered(w.h, h)
}
}
return h, nil
@ -306,13 +399,18 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
return 0, err
}
if set {
if set || w.sets {
h = hashUpdateUnordered(h, current)
} else {
h = hashUpdateOrdered(w.h, h, current)
}
}
if set && w.format != FormatV1 {
// Important: read the docs for hashFinishUnordered
h = hashFinishUnordered(w.h, h)
}
return h, nil
case reflect.String:
@ -349,6 +447,32 @@ func hashUpdateUnordered(a, b uint64) uint64 {
return a ^ b
}
// After mixing a group of unique hashes with hashUpdateUnordered, it's always
// necessary to call hashFinishUnordered. Why? Because hashUpdateUnordered
// is a simple XOR, and calling hashUpdateUnordered on hashes produced by
// hashUpdateUnordered can effectively cancel out a previous change to the hash
// result if the same hash value appears later on. For example, consider:
//
// hashUpdateUnordered(hashUpdateUnordered("A", "B"), hashUpdateUnordered("A", "C")) =
// H("A") ^ H("B")) ^ (H("A") ^ H("C")) =
// (H("A") ^ H("A")) ^ (H("B") ^ H(C)) =
// H(B) ^ H(C) =
// hashUpdateUnordered(hashUpdateUnordered("Z", "B"), hashUpdateUnordered("Z", "C"))
//
// hashFinishUnordered "hardens" the result, so that encountering partially
// overlapping input data later on in a different context won't cancel out.
func hashFinishUnordered(h hash.Hash64, a uint64) uint64 {
h.Reset()
// We just panic if the writes fail
e1 := binary.Write(h, binary.LittleEndian, a)
if e1 != nil {
panic(e1)
}
return h.Sum64()
}
// visitFlag is used as a bitmask for affecting visit behavior
type visitFlag uint

View File

@ -13,3 +13,10 @@ type Includable interface {
type IncludableMap interface {
HashIncludeMap(field string, k, v interface{}) (bool, error)
}
// Hashable is an interface that can optionally be implemented by a struct
// to override the hash value. This value will override the hash value for
// the entire struct. Entries in the struct will not be hashed.
type Hashable interface {
Hash() (uint64, error)
}

6
vendor/modules.txt vendored
View File

@ -365,9 +365,9 @@ github.com/klauspost/compress/zstd/internal/xxhash
# github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369
## explicit; go 1.9
github.com/matttproud/golang_protobuf_extensions/pbutil
# github.com/mitchellh/hashstructure v1.0.0
## explicit
github.com/mitchellh/hashstructure
# github.com/mitchellh/hashstructure/v2 v2.0.2
## explicit; go 1.14
github.com/mitchellh/hashstructure/v2
# github.com/moby/locker v1.0.1
## explicit; go 1.13
github.com/moby/locker