sundowndev
/
driftctl
mirror of https://github.com/sundowndev/driftctl.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #859 from cloudskiff/fea/fail_from_incorrect 

do not fail if one multiple from is incorrect
main
Elie 2021-09-16 10:41:02 +02:00 committed by GitHub
parent 3459a17545 a8918f7df1
commit f8a4ba9968
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 252 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/cmd/scan.go
View File

@ -224,7 +224,7 @@ func scanRun(opts *pkg.ScanOptions) error {
scanner := remote.NewScanner(remoteLibrary, alerter, remote.ScannerOptions{Deep: opts.Deep}, driftIgnore)
iacSupplier, err := supplier.GetIACSupplier(opts.From, providerLibrary, opts.BackendOptions, iacProgress, resFactory, driftIgnore)
iacSupplier, err := supplier.GetIACSupplier(opts.From, providerLibrary, opts.BackendOptions, iacProgress, alerter, resFactory, driftIgnore)
if err != nil {
return err
}

27
pkg/iac/errors.go Normal file
View File

@ -0,0 +1,27 @@
package iac
import (
"fmt"
"strings"
)
type StateReadingError struct {
errors []error
}
func NewStateReadingError() *StateReadingError {
return &StateReadingError{}
}
func (s *StateReadingError) Add(err error) {
s.errors = append(s.errors, err)
}
func (s *StateReadingError) Error() string {
var err strings.Builder
_, _ = fmt.Fprint(&err, "There were errors reading your states files : \n")
for _, e := range s.errors {
_, _ = fmt.Fprintf(&err, " - %s\n", e.Error())
}
return err.String()
}

77
pkg/iac/supplier/IacChainSupplier.go Normal file
View File

@ -0,0 +1,77 @@
package supplier
import (
"context"
"runtime"
"github.com/cloudskiff/driftctl/pkg/iac"
"github.com/cloudskiff/driftctl/pkg/parallel"
"github.com/cloudskiff/driftctl/pkg/resource"
)
type IacChainSupplier struct {
suppliers []resource.Supplier
runner *parallel.ParallelRunner
}
func NewIacChainSupplier() *IacChainSupplier {
return &IacChainSupplier{
runner: parallel.NewParallelRunner(context.TODO(), int64(runtime.NumCPU())),
}
}
func (r *IacChainSupplier) AddSupplier(supplier resource.Supplier) {
r.suppliers = append(r.suppliers, supplier)
}
func (r *IacChainSupplier) Resources() ([]*resource.Resource, error) {
for _, supplier := range r.suppliers {
sup := supplier
r.runner.Run(func() (interface{}, error) {
resources, err := sup.Resources()
return &result{err, resources}, nil
})
}
results := make([]*resource.Resource, 0)
isSuccess := false
retrieveError := iac.NewStateReadingError()
ReadLoop:
for {
select {
case supplierResult, ok := <-r.runner.Read():
if !ok || supplierResult == nil {
break ReadLoop
}
// Type cannot be invalid as return type is enforced
// in run function on top
result, _ := supplierResult.(*result)
if result.err != nil {
retrieveError.Add(result.err)
continue
}
isSuccess = true
results = append(results, result.res...)
case <-r.runner.DoneChan():
break ReadLoop
}
}
if r.runner.Err() != nil {
return nil, r.runner.Err()
}
if !isSuccess {
// only fail if all suppliers failed
return nil, retrieveError
}
return results, nil
}
type result struct {
err error
res []*resource.Resource
}

87
pkg/iac/supplier/IacChainSupplier_test.go Normal file
View File

@ -0,0 +1,87 @@
package supplier
import (
"reflect"
"testing"
"github.com/cloudskiff/driftctl/pkg/resource"
"github.com/pkg/errors"
)
func TestIacChainSupplier_Resources(t *testing.T) {
tests := []struct {
name string
initSuppliers func(suppliers *[]resource.Supplier)
want []*resource.Resource
wantErr bool
}{
{
name: "All failed",
initSuppliers: func(suppliers *[]resource.Supplier) {
sup := &resource.MockSupplier{}
sup.On("Resources").Return(nil, errors.New("1"))
*suppliers = append(*suppliers, sup)
sup = &resource.MockSupplier{}
sup.On("Resources").Return(nil, errors.New("2"))
*suppliers = append(*suppliers, sup)
sup = &resource.MockSupplier{}
sup.On("Resources").Return(nil, errors.New("3"))
*suppliers = append(*suppliers, sup)
},
want: nil,
wantErr: true,
},
{
name: "Partial failed",
initSuppliers: func(suppliers *[]resource.Supplier) {
sup := &resource.MockSupplier{}
sup.On("Resources").Return(nil, errors.New("1"))
*suppliers = append(*suppliers, sup)
sup = &resource.MockSupplier{}
sup.On("Resources").Return(nil, errors.New("2"))
*suppliers = append(*suppliers, sup)
sup = &resource.MockSupplier{}
sup.On("Resources").Return([]*resource.Resource{
&resource.Resource{
Id: "ID",
Type: "TYPE",
Attrs: nil,
},
}, nil)
*suppliers = append(*suppliers, sup)
},
want: []*resource.Resource{
&resource.Resource{
Id: "ID",
Type: "TYPE",
Attrs: nil,
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r := NewIacChainSupplier()
suppliers := make([]resource.Supplier, 0)
tt.initSuppliers(&suppliers)
for _, supplier := range suppliers {
r.AddSupplier(supplier)
}
got, err := r.Resources()
if (err != nil) != tt.wantErr {
t.Errorf("Resources() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("Resources() got = %v, want %v", got, tt.want)
}
})
}
}

6
pkg/iac/supplier/supplier.go
View File

@ -3,6 +3,7 @@ package supplier
import (
"fmt"
"github.com/cloudskiff/driftctl/pkg/alerter"
"github.com/cloudskiff/driftctl/pkg/filter"
"github.com/cloudskiff/driftctl/pkg/iac/terraform/state/backend"
"github.com/cloudskiff/driftctl/pkg/output"
@ -34,10 +35,11 @@ func GetIACSupplier(configs []config.SupplierConfig,
library *terraform.ProviderLibrary,
backendOpts *backend.Options,
progress output.Progress,
alerter *alerter.Alerter,
factory resource.ResourceFactory,
filter filter.Filter) (resource.Supplier, error) {
chainSupplier := resource.NewChainSupplier()
chainSupplier := NewIacChainSupplier()
for _, config := range configs {
if !IsSupplierSupported(config.Key) {
return nil, errors.Errorf("Unsupported supplier '%s'", config.Key)
@ -49,7 +51,7 @@ func GetIACSupplier(configs []config.SupplierConfig,
var err error
switch config.Key {
case state.TerraformStateReaderSupplier:
supplier, err = state.NewReader(config, library, backendOpts, progress, deserializer, filter)
supplier, err = state.NewReader(config, library, backendOpts, progress, alerter, deserializer, filter)
default:
return nil, errors.Errorf("Unsupported supplier '%s'", config.Key)
}

5
pkg/iac/supplier/supplier_test.go
View File

@ -5,6 +5,7 @@ import (
"reflect"
"testing"
"github.com/cloudskiff/driftctl/pkg/alerter"
"github.com/cloudskiff/driftctl/pkg/filter"
"github.com/cloudskiff/driftctl/pkg/iac/config"
"github.com/cloudskiff/driftctl/pkg/iac/terraform/state/backend"
@ -90,10 +91,12 @@ func TestGetIACSupplier(t *testing.T) {
repo := resource.InitFakeSchemaRepository("aws", "3.19.0")
factory := terraform.NewTerraformResourceFactory(repo)
alerter := alerter.NewAlerter()
testFilter := &filter.MockFilter{}
_, err := GetIACSupplier(tt.args.config, terraform.NewProviderLibrary(), tt.args.options, progress, factory, testFilter)
_, err := GetIACSupplier(tt.args.config, terraform.NewProviderLibrary(), tt.args.options, progress, alerter, factory, testFilter)
if tt.wantErr != nil && err.Error() != tt.wantErr.Error() {
t.Errorf("GetIACSupplier() error = %v, wantErr %v", err, tt.wantErr)
return

20
pkg/iac/terraform/state/alerts.go Normal file
View File

@ -0,0 +1,20 @@
package state
import "fmt"
type StateReadingAlert struct {
key string
err string
}
func NewStateReadingAlert(key string, err error) *StateReadingAlert {
return &StateReadingAlert{key: key, err: err.Error()}
}
func (s *StateReadingAlert) Message() string {
return fmt.Sprintf("Your analysis may be incomplete. There was an error reading state file '%s': %s", s.key, s.err)
}
func (s *StateReadingAlert) ShouldIgnoreResource() bool {
return false
}

4
pkg/iac/terraform/state/enumerator/file.go
View File

@ -23,6 +23,10 @@ func NewFileEnumerator(config config.SupplierConfig) *FileEnumerator {
}
}
func (s *FileEnumerator) Origin() string {
return s.config.String()
}
func (s *FileEnumerator) Enumerate() ([]string, error) {
path := s.config.Path

4
pkg/iac/terraform/state/enumerator/s3.go
View File

@ -33,6 +33,10 @@ func NewS3Enumerator(config config.SupplierConfig) *S3Enumerator {
}
}
func (s *S3Enumerator) Origin() string {
return s.config.String()
}
func (s *S3Enumerator) Enumerate() ([]string, error) {
bucketPath := strings.Split(s.config.Path, "/")
if len(bucketPath) < 2 {

1
pkg/iac/terraform/state/enumerator/state_enumerator.go
View File

@ -7,6 +7,7 @@ import (
)
type StateEnumerator interface {
Origin() string
Enumerate() ([]string, error)
}

35
pkg/iac/terraform/state/terraform_state_reader.go
View File

@ -4,9 +4,10 @@ import (
"fmt"
"strings"
"github.com/cloudskiff/driftctl/pkg/alerter"
"github.com/cloudskiff/driftctl/pkg/filter"
"github.com/cloudskiff/driftctl/pkg/iac"
"github.com/cloudskiff/driftctl/pkg/output"
"github.com/fatih/color"
"github.com/hashicorp/terraform/addrs"
"github.com/hashicorp/terraform/states"
"github.com/hashicorp/terraform/states/statefile"
@ -39,6 +40,7 @@ type TerraformStateReader struct {
backendOptions *backend.Options
progress output.Progress
filter filter.Filter
alerter *alerter.Alerter
}
func (r *TerraformStateReader) initReader() error {
@ -46,8 +48,8 @@ func (r *TerraformStateReader) initReader() error {
return nil
}
func NewReader(config config.SupplierConfig, library *terraform.ProviderLibrary, backendOpts *backend.Options, progress output.Progress, deserializer *resource.Deserializer, filter filter.Filter) (*TerraformStateReader, error) {
reader := TerraformStateReader{library: library, config: config, deserializer: deserializer, backendOptions: backendOpts, progress: progress, filter: filter}
func NewReader(config config.SupplierConfig, library *terraform.ProviderLibrary, backendOpts *backend.Options, progress output.Progress, alerter *alerter.Alerter, deserializer *resource.Deserializer, filter filter.Filter) (*TerraformStateReader, error) {
reader := TerraformStateReader{library: library, config: config, deserializer: deserializer, backendOptions: backendOpts, progress: progress, alerter: alerter, filter: filter}
err := reader.initReader()
if err != nil {
return nil, err
@ -221,34 +223,43 @@ func (r *TerraformStateReader) retrieveForState(path string) ([]*resource.Resour
r.progress.Inc()
values, err := r.retrieve()
if err != nil {
return nil, err
return nil, errors.Wrap(err, r.config.String())
}
return r.decode(values)
decode, err := r.decode(values)
return decode, errors.Wrap(err, r.config.String())
}
func (r *TerraformStateReader) retrieveMultiplesStates() ([]*resource.Resource, error) {
keys, err := r.enumerator.Enumerate()
if err != nil {
return nil, err
r.alerter.SendAlert("", NewStateReadingAlert(r.enumerator.Origin(), err))
return nil, errors.Wrap(err, r.config.String())
}
logrus.WithFields(logrus.Fields{
"keys": keys,
}).Debug("Enumerated keys")
results := make([]*resource.Resource, 0)
isSuccess := false
readingError := iac.NewStateReadingError()
for _, key := range keys {
resources, err := r.retrieveForState(key)
if err != nil {
if _, ok := err.(*UnsupportedVersionError); ok {
color.New(color.Bold, color.FgYellow).Printf("WARNING: %s\n", err)
continue
}
return nil, err
readingError.Add(err)
r.alerter.SendAlert("", NewStateReadingAlert(key, err))
continue
}
isSuccess = true
results = append(results, resources...)
}
if !isSuccess {
// all key failed, throw an error
return results, readingError
}
return results, nil
}

57
pkg/resource/chain_supplier.go
View File

@ -1,57 +0,0 @@
package resource
import (
"context"
"runtime"
"github.com/cloudskiff/driftctl/pkg/parallel"
)
type ChainSupplier struct {
suppliers []Supplier
runner *parallel.ParallelRunner
}
func NewChainSupplier() *ChainSupplier {
return &ChainSupplier{
runner: parallel.NewParallelRunner(context.TODO(), int64(runtime.NumCPU())),
}
}
func (r *ChainSupplier) AddSupplier(supplier Supplier) {
r.suppliers = append(r.suppliers, supplier)
}
func (r *ChainSupplier) Resources() ([]*Resource, error) {
for _, supplier := range r.suppliers {
sup := supplier
r.runner.Run(func() (interface{}, error) {
return sup.Resources()
})
}
results := make([]*Resource, 0)
ReadLoop:
for {
select {
case supplierResult, ok := <-r.runner.Read():
if !ok || supplierResult == nil {
break ReadLoop
}
// Type cannot be invalid as return type is enforced
// by Supplier interface
resources, _ := supplierResult.([]*Resource)
results = append(results, resources...)
case <-r.runner.DoneChan():
break ReadLoop
}
}
if r.runner.Err() != nil {
return nil, r.runner.Err()
}
return results, nil
}

89
pkg/resource/chain_supplier_test.go
View File

@ -1,89 +0,0 @@
package resource_test
import (
"errors"
"testing"
"github.com/cloudskiff/driftctl/pkg/resource"
"github.com/stretchr/testify/assert"
)
func TestChainSupplier_Resources(t *testing.T) {
assert := assert.New(t)
fakeTestSupplier := resource.MockSupplier{}
fakeTestSupplier.On("Resources").Return(
[]*resource.Resource{
&resource.Resource{
Id: "fake-supplier-1_fake-resource-1",
},
&resource.Resource{
Id: "fake-supplier-1_fake-resource-2",
},
},
nil,
).Once()
anotherFakeTestSupplier := resource.MockSupplier{}
anotherFakeTestSupplier.On("Resources").Return(
[]*resource.Resource{
&resource.Resource{
Id: "fake-supplier-2_fake-resource-1",
},
&resource.Resource{
Id: "fake-supplier-2_fake-resource-2",
},
},
nil,
).Once()
chain := resource.NewChainSupplier()
chain.AddSupplier(&fakeTestSupplier)
chain.AddSupplier(&anotherFakeTestSupplier)
res, err := chain.Resources()
if err != nil {
t.Fatal(err)
}
anotherFakeTestSupplier.AssertExpectations(t)
fakeTestSupplier.AssertExpectations(t)
assert.Len(res, 4)
}
func TestChainSupplier_Resources_WithError(t *testing.T) {
assert := assert.New(t)
fakeTestSupplier := resource.MockSupplier{}
fakeTestSupplier.
On("Resources").
Return([]*resource.Resource{
&resource.Resource{
Id: "fake-supplier-1_fake-resource-1",
},
&resource.Resource{
Id: "fake-supplier-1_fake-resource-2",
},
},
nil,
)
anotherFakeTestSupplier := resource.MockSupplier{}
anotherFakeTestSupplier.
On("Resources").
Return(nil, errors.New("error from another supplier")).
Once()
chain := resource.NewChainSupplier()
chain.AddSupplier(&fakeTestSupplier)
chain.AddSupplier(&anotherFakeTestSupplier)
res, err := chain.Resources()
anotherFakeTestSupplier.AssertExpectations(t)
assert.Nil(res)
assert.Equal("error from another supplier", err.Error())
}