From 8036b7a7028858e16b5d27721601b69559e2e27a Mon Sep 17 00:00:00 2001 From: sundowndev Date: Wed, 21 Jul 2021 17:57:05 +0200 Subject: [PATCH 1/6] feat: allow multiple output flags --- pkg/cmd/scan.go | 36 ++++++++++++---- pkg/cmd/scan_test.go | 97 ++++++++++++++++++++++++++++++++------------ pkg/driftctl.go | 2 +- 3 files changed, 98 insertions(+), 37 deletions(-) diff --git a/pkg/cmd/scan.go b/pkg/cmd/scan.go index cca47895..67ba3de8 100644 --- a/pkg/cmd/scan.go +++ b/pkg/cmd/scan.go @@ -61,12 +61,16 @@ func NewScanCmd() *cobra.Command { ) } - outputFlag, _ := cmd.Flags().GetString("output") - out, err := parseOutputFlag(outputFlag) + outputFlag, _ := cmd.Flags().GetStringSlice("output") + if len(outputFlag) == 0 { + outputFlag = append(outputFlag, output.Example(output.ConsoleOutputType)) + } + + out, err := parseOutputFlags(outputFlag) if err != nil { return err } - opts.Output = *out + opts.Output = out filterFlag, _ := cmd.Flags().GetStringArray("filter") @@ -117,10 +121,10 @@ func NewScanCmd() *cobra.Command { " - Type =='aws_s3_bucket && Id != 'my_bucket' (excludes s3 bucket 'my_bucket')\n"+ " - Attr.Tags.Terraform == 'true' (include only resources that have Tag Terraform equal to 'true')\n", ) - fl.StringP( + fl.StringSliceP( "output", "o", - output.Example(output.ConsoleOutputType), + []string{}, "Output format, by default it will write to the console\n"+ "Accepted formats are: "+strings.Join(output.SupportedOutputsExample(), ",")+"\n", ) @@ -190,7 +194,6 @@ func NewScanCmd() *cobra.Command { func scanRun(opts *pkg.ScanOptions) error { store := memstore.New() - selectedOutput := output.GetOutput(opts.Output, opts.Quiet) c := make(chan os.Signal) signal.Notify(c, os.Interrupt, syscall.SIGTERM) @@ -256,9 +259,12 @@ func scanRun(opts *pkg.ScanOptions) error { analysis.ProviderVersion = resourceSchemaRepository.ProviderVersion.String() analysis.ProviderName = resourceSchemaRepository.ProviderName - err = selectedOutput.Write(analysis) - if err != nil { - return err + for _, o := range opts.Output { + selectedOutput := output.GetOutput(o, opts.Quiet) + err = selectedOutput.Write(analysis) + if err != nil { + return err + } } globaloutput.Printf(color.WhiteString("Scan duration: %s\n", analysis.Duration.Round(time.Second))) @@ -352,6 +358,18 @@ func parseFromFlag(from []string) ([]config.SupplierConfig, error) { return configs, nil } +func parseOutputFlags(out []string) ([]output.OutputConfig, error) { + result := make([]output.OutputConfig, 0, len(out)) + for _, v := range out { + o, err := parseOutputFlag(v) + if err != nil { + return result, err + } + result = append(result, *o) + } + return result, nil +} + func parseOutputFlag(out string) (*output.OutputConfig, error) { schemeOpts := strings.Split(out, "://") if len(schemeOpts) < 2 || schemeOpts[0] == "" { diff --git a/pkg/cmd/scan_test.go b/pkg/cmd/scan_test.go index 9a72985f..1825d625 100644 --- a/pkg/cmd/scan_test.go +++ b/pkg/cmd/scan_test.go @@ -48,6 +48,7 @@ func TestScanCmd_Valid(t *testing.T) { {args: []string{"scan", "--tf-provider-version", "3.30.2"}}, {args: []string{"scan", "--driftignore", "./path/to/driftignore.s3"}}, {args: []string{"scan", "--driftignore", ".driftignore"}}, + {args: []string{"scan", "-o", "html://result.html", "-o", "json://result.json"}}, } for _, tt := range cases { @@ -164,98 +165,140 @@ func Test_parseFromFlag(t *testing.T) { func Test_parseOutputFlag(t *testing.T) { type args struct { - out string + out []string } tests := []struct { name string args args - want *output.OutputConfig + want []output.OutputConfig err error }{ { - name: "test empty", + name: "test empty output", args: args{ - out: "", + out: []string{""}, }, - want: nil, + want: []output.OutputConfig{}, err: fmt.Errorf("Unable to parse output flag '': \nAccepted formats are: console://,html://PATH/TO/FILE.html,json://PATH/TO/FILE.json,plan://PATH/TO/FILE.json"), }, + { + name: "test empty array", + args: args{ + out: []string{}, + }, + want: []output.OutputConfig{}, + err: nil, + }, { name: "test invalid", args: args{ - out: "sdgjsdgjsdg", + out: []string{"sdgjsdgjsdg"}, }, - want: nil, + want: []output.OutputConfig{}, err: fmt.Errorf("Unable to parse output flag 'sdgjsdgjsdg': \nAccepted formats are: console://,html://PATH/TO/FILE.html,json://PATH/TO/FILE.json,plan://PATH/TO/FILE.json"), }, { name: "test invalid", args: args{ - out: "://", + out: []string{"://"}, }, - want: nil, + want: []output.OutputConfig{}, err: fmt.Errorf("Unable to parse output flag '://': \nAccepted formats are: console://,html://PATH/TO/FILE.html,json://PATH/TO/FILE.json,plan://PATH/TO/FILE.json"), }, { name: "test unsupported", args: args{ - out: "foobar://", + out: []string{"foobar://"}, }, - want: nil, + want: []output.OutputConfig{}, err: fmt.Errorf("Unsupported output 'foobar': \nValid formats are: console://,html://PATH/TO/FILE.html,json://PATH/TO/FILE.json,plan://PATH/TO/FILE.json"), }, { name: "test empty json", args: args{ - out: "json://", + out: []string{"json://"}, }, - want: nil, + want: []output.OutputConfig{}, err: fmt.Errorf("Invalid json output 'json://': \nMust be of kind: json://PATH/TO/FILE.json"), }, { name: "test valid console", args: args{ - out: "console://", + out: []string{"console://"}, }, - want: &output.OutputConfig{ - Key: "console", + want: []output.OutputConfig{ + { + Key: "console", + }, }, err: nil, }, { name: "test valid json", args: args{ - out: "json:///tmp/foobar.json", + out: []string{"json:///tmp/foobar.json"}, }, - want: &output.OutputConfig{ - Key: "json", - Path: "/tmp/foobar.json", + want: []output.OutputConfig{ + { + Key: "json", + Path: "/tmp/foobar.json", + }, }, err: nil, }, { name: "test empty jsonplan", args: args{ - out: "plan://", + out: []string{"plan://"}, }, - want: nil, + want: []output.OutputConfig{}, err: fmt.Errorf("Invalid plan output 'plan://': \nMust be of kind: plan://PATH/TO/FILE.json"), }, { name: "test valid jsonplan", args: args{ - out: "plan:///tmp/foobar.json", + out: []string{"plan:///tmp/foobar.json"}, }, - want: &output.OutputConfig{ - Key: "plan", - Path: "/tmp/foobar.json", + want: []output.OutputConfig{ + { + Key: "plan", + Path: "/tmp/foobar.json", + }, }, err: nil, }, + { + name: "test multiple output values", + args: args{ + out: []string{"console:///dev/stdout", "json://result.json"}, + }, + want: []output.OutputConfig{ + { + Key: "console", + }, + { + Key: "json", + Path: "result.json", + }, + }, + err: nil, + }, + { + name: "test multiple output values with invalid value", + args: args{ + out: []string{"console:///dev/stdout", "invalid://result.json"}, + }, + want: []output.OutputConfig{ + { + Key: "console", + }, + }, + err: fmt.Errorf("Unsupported output 'invalid': \nValid formats are: console://,html://PATH/TO/FILE.html,json://PATH/TO/FILE.json,plan://PATH/TO/FILE.json"), + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parseOutputFlag(tt.args.out) + got, err := parseOutputFlags(tt.args.out) if err != nil && err.Error() != tt.err.Error() { t.Fatalf("got error = '%v', expected '%v'", err, tt.err) } diff --git a/pkg/driftctl.go b/pkg/driftctl.go index 7d871ff2..d691ebbc 100644 --- a/pkg/driftctl.go +++ b/pkg/driftctl.go @@ -24,7 +24,7 @@ type ScanOptions struct { Detect bool From []config.SupplierConfig To string - Output output.OutputConfig + Output []output.OutputConfig Filter *jmespath.JMESPath Quiet bool BackendOptions *backend.Options From 8e7af5891ad612166b5181f506cc29a6873e18ed Mon Sep 17 00:00:00 2001 From: sundowndev Date: Thu, 29 Jul 2021 12:19:31 +0200 Subject: [PATCH 2/6] refactor: do not break on output error --- pkg/cmd/scan.go | 12 ++++++++++++ pkg/cmd/scan/output/output.go | 4 ---- pkg/cmd/scan_test.go | 27 +++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/pkg/cmd/scan.go b/pkg/cmd/scan.go index 67ba3de8..48790b2d 100644 --- a/pkg/cmd/scan.go +++ b/pkg/cmd/scan.go @@ -259,9 +259,21 @@ func scanRun(opts *pkg.ScanOptions) error { analysis.ProviderVersion = resourceSchemaRepository.ProviderVersion.String() analysis.ProviderName = resourceSchemaRepository.ProviderName + validOutput := false for _, o := range opts.Output { selectedOutput := output.GetOutput(o, opts.Quiet) err = selectedOutput.Write(analysis) + if err != nil { + logrus.Errorf("Computing output %s://%s failed: %v", o.Key, o.Options["path"], err.Error()) + continue + } + validOutput = true + } + + // Fallback to console output if all output failed + if !validOutput { + logrus.Debug("All outputs failed to compute, fallback to console output") + err = output.NewConsole().Write(analysis) if err != nil { return err } diff --git a/pkg/cmd/scan/output/output.go b/pkg/cmd/scan/output/output.go index 25545646..48190ff7 100644 --- a/pkg/cmd/scan/output/output.go +++ b/pkg/cmd/scan/output/output.go @@ -25,10 +25,6 @@ var supportedOutputExample = map[string]string{ PlanOutputType: PlanOutputExample, } -func SupportedOutputs() []string { - return supportedOutputTypes -} - func SupportedOutputsExample() []string { examples := make([]string, 0, len(supportedOutputExample)) for _, ex := range supportedOutputExample { diff --git a/pkg/cmd/scan_test.go b/pkg/cmd/scan_test.go index 1825d625..d3a336bf 100644 --- a/pkg/cmd/scan_test.go +++ b/pkg/cmd/scan_test.go @@ -295,6 +295,33 @@ func Test_parseOutputFlag(t *testing.T) { }, err: fmt.Errorf("Unsupported output 'invalid': \nValid formats are: console://,html://PATH/TO/FILE.html,json://PATH/TO/FILE.json,plan://PATH/TO/FILE.json"), }, + { + name: "test multiple valid output values", + args: args{ + out: []string{"json://result1.json", "json://result2.json", "json://result3.json"}, + }, + want: []output.OutputConfig{ + { + Key: "json", + Options: map[string]string{ + "path": "result1.json", + }, + }, + { + Key: "json", + Options: map[string]string{ + "path": "result2.json", + }, + }, + { + Key: "json", + Options: map[string]string{ + "path": "result3.json", + }, + }, + }, + err: nil, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 1ae88df550519c4a3ce9bb41f5fbf12c9bc05353 Mon Sep 17 00:00:00 2001 From: sundowndev Date: Thu, 29 Jul 2021 12:43:01 +0200 Subject: [PATCH 3/6] refactor: simplify code --- pkg/cmd/scan.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/cmd/scan.go b/pkg/cmd/scan.go index 48790b2d..314591b1 100644 --- a/pkg/cmd/scan.go +++ b/pkg/cmd/scan.go @@ -261,9 +261,7 @@ func scanRun(opts *pkg.ScanOptions) error { validOutput := false for _, o := range opts.Output { - selectedOutput := output.GetOutput(o, opts.Quiet) - err = selectedOutput.Write(analysis) - if err != nil { + if err = output.GetOutput(o, opts.Quiet).Write(analysis); err != nil { logrus.Errorf("Computing output %s://%s failed: %v", o.Key, o.Options["path"], err.Error()) continue } @@ -273,8 +271,7 @@ func scanRun(opts *pkg.ScanOptions) error { // Fallback to console output if all output failed if !validOutput { logrus.Debug("All outputs failed to compute, fallback to console output") - err = output.NewConsole().Write(analysis) - if err != nil { + if err = output.NewConsole().Write(analysis); err != nil { return err } } From 1157138af102505143ca42ad1c8049b6e846e974 Mon Sep 17 00:00:00 2001 From: sundowndev Date: Mon, 23 Aug 2021 15:45:07 +0200 Subject: [PATCH 4/6] refactor: simplify code --- pkg/cmd/scan.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/cmd/scan.go b/pkg/cmd/scan.go index 314591b1..6cf9909d 100644 --- a/pkg/cmd/scan.go +++ b/pkg/cmd/scan.go @@ -62,9 +62,6 @@ func NewScanCmd() *cobra.Command { } outputFlag, _ := cmd.Flags().GetStringSlice("output") - if len(outputFlag) == 0 { - outputFlag = append(outputFlag, output.Example(output.ConsoleOutputType)) - } out, err := parseOutputFlags(outputFlag) if err != nil { @@ -124,7 +121,7 @@ func NewScanCmd() *cobra.Command { fl.StringSliceP( "output", "o", - []string{}, + []string{output.Example(output.ConsoleOutputType)}, "Output format, by default it will write to the console\n"+ "Accepted formats are: "+strings.Join(output.SupportedOutputsExample(), ",")+"\n", ) From 199259eb3d6012debeeee8914c5f2c3fe1bf82d0 Mon Sep 17 00:00:00 2001 From: sundowndev Date: Tue, 31 Aug 2021 10:59:45 +0200 Subject: [PATCH 5/6] refactor: improve error logs --- pkg/cmd/scan.go | 4 ++-- pkg/cmd/scan/output/config.go | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/cmd/scan.go b/pkg/cmd/scan.go index 6cf9909d..456949a2 100644 --- a/pkg/cmd/scan.go +++ b/pkg/cmd/scan.go @@ -259,7 +259,7 @@ func scanRun(opts *pkg.ScanOptions) error { validOutput := false for _, o := range opts.Output { if err = output.GetOutput(o, opts.Quiet).Write(analysis); err != nil { - logrus.Errorf("Computing output %s://%s failed: %v", o.Key, o.Options["path"], err.Error()) + logrus.Errorf("Error writing to output %s: %v", o.String(), err.Error()) continue } validOutput = true @@ -267,7 +267,7 @@ func scanRun(opts *pkg.ScanOptions) error { // Fallback to console output if all output failed if !validOutput { - logrus.Debug("All outputs failed to compute, fallback to console output") + logrus.Debug("All outputs failed, fallback to console output") if err = output.NewConsole().Write(analysis); err != nil { return err } diff --git a/pkg/cmd/scan/output/config.go b/pkg/cmd/scan/output/config.go index 634c045c..2a08a5eb 100644 --- a/pkg/cmd/scan/output/config.go +++ b/pkg/cmd/scan/output/config.go @@ -1,6 +1,12 @@ package output +import "fmt" + type OutputConfig struct { Key string Path string } + +func (o *OutputConfig) String() string { + return fmt.Sprintf("%s://%s", o.Key, o.Options["path"]) +} From 6e48f177c33d3111159c91c0c97b56a550c27d5b Mon Sep 17 00:00:00 2001 From: sundowndev Date: Tue, 7 Sep 2021 15:43:40 +0200 Subject: [PATCH 6/6] refactor: output config usages --- pkg/cmd/scan/output/config.go | 2 +- pkg/cmd/scan_test.go | 18 ++++++------------ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/pkg/cmd/scan/output/config.go b/pkg/cmd/scan/output/config.go index 2a08a5eb..895a1216 100644 --- a/pkg/cmd/scan/output/config.go +++ b/pkg/cmd/scan/output/config.go @@ -8,5 +8,5 @@ type OutputConfig struct { } func (o *OutputConfig) String() string { - return fmt.Sprintf("%s://%s", o.Key, o.Options["path"]) + return fmt.Sprintf("%s://%s", o.Key, o.Path) } diff --git a/pkg/cmd/scan_test.go b/pkg/cmd/scan_test.go index d3a336bf..b374f0ac 100644 --- a/pkg/cmd/scan_test.go +++ b/pkg/cmd/scan_test.go @@ -302,22 +302,16 @@ func Test_parseOutputFlag(t *testing.T) { }, want: []output.OutputConfig{ { - Key: "json", - Options: map[string]string{ - "path": "result1.json", - }, + Key: "json", + Path: "result1.json", }, { - Key: "json", - Options: map[string]string{ - "path": "result2.json", - }, + Key: "json", + Path: "result2.json", }, { - Key: "json", - Options: map[string]string{ - "path": "result3.json", - }, + Key: "json", + Path: "result3.json", }, }, err: nil,