From c90da70b56762de340e478d84e01a6bd3ac9c478 Mon Sep 17 00:00:00 2001 From: Elie Date: Wed, 24 Feb 2021 11:58:05 +0100 Subject: [PATCH] Allow to use a whole local directory as IaC source --- doc/cmd/scan/iac_source.md | 7 ++ pkg/iac/terraform/state/enumerator/file.go | 81 +++++++++++++++++ .../terraform/state/enumerator/file_test.go | 90 +++++++++++++++++++ .../state/enumerator/state_enumerator.go | 5 +- .../testdata/invalid_symlink/invalid | 1 + .../states/route53/directory/route53.state | 0 .../testdata/states/s3/terraform.tfstate | 0 .../testdata/states/symlink-to-s3-folder | 1 + .../testdata/states/symlink.tfstate | 1 + .../testdata/states/terraform.tfstate | 0 .../testdata/states/terraform.tfstate.backup | 0 .../state/enumerator/testdata/symlink | 1 + .../state/terraform_state_reader_acc_test.go | 27 ++++++ .../route53/.terraform.lock.hcl | 0 .../route53/terraform.tf | 0 .../s3/.terraform.lock.hcl | 0 .../s3/terraform.tf | 0 .../acc/multiple_states_local/.gitignore | 1 + .../route53/.terraform.lock.hcl | 38 ++++++++ .../route53/terraform.tf | 25 ++++++ .../s3/.terraform.lock.hcl | 38 ++++++++ .../acc/multiple_states_local/s3/terraform.tf | 25 ++++++ 22 files changed, 340 insertions(+), 1 deletion(-) create mode 100644 pkg/iac/terraform/state/enumerator/file.go create mode 100644 pkg/iac/terraform/state/enumerator/file_test.go create mode 120000 pkg/iac/terraform/state/enumerator/testdata/invalid_symlink/invalid create mode 100644 pkg/iac/terraform/state/enumerator/testdata/states/route53/directory/route53.state create mode 100644 pkg/iac/terraform/state/enumerator/testdata/states/s3/terraform.tfstate create mode 120000 pkg/iac/terraform/state/enumerator/testdata/states/symlink-to-s3-folder create mode 120000 pkg/iac/terraform/state/enumerator/testdata/states/symlink.tfstate create mode 100644 pkg/iac/terraform/state/enumerator/testdata/states/terraform.tfstate create mode 100644 pkg/iac/terraform/state/enumerator/testdata/states/terraform.tfstate.backup create mode 120000 pkg/iac/terraform/state/enumerator/testdata/symlink rename pkg/iac/terraform/state/testdata/acc/{multiples_states => multiple_states}/route53/.terraform.lock.hcl (100%) rename pkg/iac/terraform/state/testdata/acc/{multiples_states => multiple_states}/route53/terraform.tf (100%) rename pkg/iac/terraform/state/testdata/acc/{multiples_states => multiple_states}/s3/.terraform.lock.hcl (100%) rename pkg/iac/terraform/state/testdata/acc/{multiples_states => multiple_states}/s3/terraform.tf (100%) create mode 100644 pkg/iac/terraform/state/testdata/acc/multiple_states_local/.gitignore create mode 100755 pkg/iac/terraform/state/testdata/acc/multiple_states_local/route53/.terraform.lock.hcl create mode 100644 pkg/iac/terraform/state/testdata/acc/multiple_states_local/route53/terraform.tf create mode 100755 pkg/iac/terraform/state/testdata/acc/multiple_states_local/s3/.terraform.lock.hcl create mode 100644 pkg/iac/terraform/state/testdata/acc/multiple_states_local/s3/terraform.tf diff --git a/doc/cmd/scan/iac_source.md b/doc/cmd/scan/iac_source.md index 5c51f06a..81657a5a 100644 --- a/doc/cmd/scan/iac_source.md +++ b/doc/cmd/scan/iac_source.md @@ -14,6 +14,13 @@ driftctl scan \ # You can also use every file under a given prefix for S3 driftctl scan --from tfstate+s3://statebucketdriftctl/states + +# ... or in a given local folder +# driftctl will recursively use all files under this folder. +# +# N.B. Symlinks under the root folder will be ignored. +# If the folder itself is a symlink it will be followed. +driftctl scan --from tfstate://my-states/directory ``` diff --git a/pkg/iac/terraform/state/enumerator/file.go b/pkg/iac/terraform/state/enumerator/file.go new file mode 100644 index 00000000..f600d308 --- /dev/null +++ b/pkg/iac/terraform/state/enumerator/file.go @@ -0,0 +1,81 @@ +package enumerator + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/cloudskiff/driftctl/pkg/iac/config" +) + +type FileEnumeratorConfig struct { + Bucket *string + Prefix *string +} + +type FileEnumerator struct { + config config.SupplierConfig +} + +func NewFileEnumerator(config config.SupplierConfig) *FileEnumerator { + return &FileEnumerator{ + config, + } +} + +// File enumeration does not follow symlinks. +// We may use something like this https://pkg.go.dev/github.com/facebookgo/symwalk +// to follow links, but it allows infinite loop so be careful ! +// If a symlink is given as root path, we will follow it, but symlinks under this path +// will not be resolved. +func (s *FileEnumerator) Enumerate() ([]string, error) { + path := s.config.Path + info, err := os.Lstat(path) + if err != nil { + return nil, err + } + + // if we got a symlink, use its destination + if info.Mode()&os.ModeSymlink != 0 { + destination, err := filepath.EvalSymlinks(path) + if err != nil { + return nil, err + } + path = destination + info, err = os.Stat(destination) + if err != nil { + return nil, err + } + } + + if info != nil && !info.IsDir() { + return []string{path}, nil + } + + keys := make([]string, 0) + + err = filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { + // Not tested, we should remove a file or folder after WalkDir has enumerated the whole tree in memory + // This edge case does not really need to be covered by tests + if err != nil { + return err + } + // Skip symlinks + if d.Type()&os.ModeSymlink != 0 { + return nil + } + + // Ignore .backup files generated by terraform + if strings.HasSuffix(path, ".backup") { + return nil + } + + if !d.IsDir() { + keys = append(keys, path) + } + return nil + }) + + return keys, err +} diff --git a/pkg/iac/terraform/state/enumerator/file_test.go b/pkg/iac/terraform/state/enumerator/file_test.go new file mode 100644 index 00000000..c9eaa006 --- /dev/null +++ b/pkg/iac/terraform/state/enumerator/file_test.go @@ -0,0 +1,90 @@ +package enumerator + +import ( + "reflect" + "testing" + + "github.com/cloudskiff/driftctl/pkg/iac/config" +) + +func TestFileEnumerator_Enumerate(t *testing.T) { + tests := []struct { + name string + config config.SupplierConfig + want []string + err string + }{ + { + name: "subfolder nesting", + config: config.SupplierConfig{ + Path: "testdata/states", + }, + want: []string{ + "testdata/states/route53/directory/route53.state", + "testdata/states/s3/terraform.tfstate", + "testdata/states/terraform.tfstate", + }, + }, + { + name: "symlinked folder", + config: config.SupplierConfig{ + Path: "testdata/symlink", + }, + want: []string{ + "testdata/states/route53/directory/route53.state", + "testdata/states/s3/terraform.tfstate", + "testdata/states/terraform.tfstate", + }, + }, + { + name: "single state file", + config: config.SupplierConfig{ + Path: "testdata/states/terraform.tfstate", + }, + want: []string{ + "testdata/states/terraform.tfstate", + }, + }, + { + name: "single symlink state file", + config: config.SupplierConfig{ + Path: "testdata/states/symlink.tfstate", + }, + want: []string{ + "testdata/states/terraform.tfstate", + }, + }, + { + name: "invalid folder", + config: config.SupplierConfig{ + Path: "/tmp/dummy-folder/that/does/not/exist", + }, + want: nil, + err: "lstat /tmp/dummy-folder/that/does/not/exist: no such file or directory", + }, + { + name: "invalid symlink", + config: config.SupplierConfig{ + Path: "testdata/invalid_symlink/invalid", + }, + want: nil, + err: "lstat testdata/invalid_symlink/test: no such file or directory", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewFileEnumerator(tt.config) + got, err := s.Enumerate() + if err != nil && err.Error() != tt.err { + t.Fatalf("Expected error '%s', got '%s'", tt.err, err.Error()) + } + if err != nil && tt.err == "" { + t.Fatalf("Expected error '%s' but got nil", tt.err) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Enumerate() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/iac/terraform/state/enumerator/state_enumerator.go b/pkg/iac/terraform/state/enumerator/state_enumerator.go index 14d0cca4..16bda7c2 100644 --- a/pkg/iac/terraform/state/enumerator/state_enumerator.go +++ b/pkg/iac/terraform/state/enumerator/state_enumerator.go @@ -12,7 +12,10 @@ type StateEnumerator interface { func GetEnumerator(config config.SupplierConfig) StateEnumerator { - if config.Backend == backend.BackendKeyS3 { + switch config.Backend { + case backend.BackendKeyFile: + return NewFileEnumerator(config) + case backend.BackendKeyS3: return NewS3Enumerator(config) } diff --git a/pkg/iac/terraform/state/enumerator/testdata/invalid_symlink/invalid b/pkg/iac/terraform/state/enumerator/testdata/invalid_symlink/invalid new file mode 120000 index 00000000..30d74d25 --- /dev/null +++ b/pkg/iac/terraform/state/enumerator/testdata/invalid_symlink/invalid @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/pkg/iac/terraform/state/enumerator/testdata/states/route53/directory/route53.state b/pkg/iac/terraform/state/enumerator/testdata/states/route53/directory/route53.state new file mode 100644 index 00000000..e69de29b diff --git a/pkg/iac/terraform/state/enumerator/testdata/states/s3/terraform.tfstate b/pkg/iac/terraform/state/enumerator/testdata/states/s3/terraform.tfstate new file mode 100644 index 00000000..e69de29b diff --git a/pkg/iac/terraform/state/enumerator/testdata/states/symlink-to-s3-folder b/pkg/iac/terraform/state/enumerator/testdata/states/symlink-to-s3-folder new file mode 120000 index 00000000..c8da893b --- /dev/null +++ b/pkg/iac/terraform/state/enumerator/testdata/states/symlink-to-s3-folder @@ -0,0 +1 @@ +s3 \ No newline at end of file diff --git a/pkg/iac/terraform/state/enumerator/testdata/states/symlink.tfstate b/pkg/iac/terraform/state/enumerator/testdata/states/symlink.tfstate new file mode 120000 index 00000000..63a0ae93 --- /dev/null +++ b/pkg/iac/terraform/state/enumerator/testdata/states/symlink.tfstate @@ -0,0 +1 @@ +terraform.tfstate \ No newline at end of file diff --git a/pkg/iac/terraform/state/enumerator/testdata/states/terraform.tfstate b/pkg/iac/terraform/state/enumerator/testdata/states/terraform.tfstate new file mode 100644 index 00000000..e69de29b diff --git a/pkg/iac/terraform/state/enumerator/testdata/states/terraform.tfstate.backup b/pkg/iac/terraform/state/enumerator/testdata/states/terraform.tfstate.backup new file mode 100644 index 00000000..e69de29b diff --git a/pkg/iac/terraform/state/enumerator/testdata/symlink b/pkg/iac/terraform/state/enumerator/testdata/symlink new file mode 120000 index 00000000..17734456 --- /dev/null +++ b/pkg/iac/terraform/state/enumerator/testdata/symlink @@ -0,0 +1 @@ +states \ No newline at end of file diff --git a/pkg/iac/terraform/state/terraform_state_reader_acc_test.go b/pkg/iac/terraform/state/terraform_state_reader_acc_test.go index d7de05d3..14a8ca13 100644 --- a/pkg/iac/terraform/state/terraform_state_reader_acc_test.go +++ b/pkg/iac/terraform/state/terraform_state_reader_acc_test.go @@ -9,6 +9,33 @@ import ( "github.com/cloudskiff/driftctl/test/acceptance/awsutils" ) +func TestAcc_StateReader_WithMultipleStatesInDirectory(t *testing.T) { + acceptance.Run(t, acceptance.AccTestCase{ + Paths: []string{ + "./testdata/acc/multiple_states_local/s3", + "./testdata/acc/multiple_states_local/route53", + }, + Args: []string{ + "scan", + "--from", "tfstate://testdata/acc/multiple_states_local/states", + "--filter", "Type=='aws_s3_bucket' || Type=='aws_route53_zone'", + }, + Checks: []acceptance.AccCheck{ + { + Check: func(result *acceptance.ScanResult, stdout string, err error) { + if err != nil { + t.Fatal(err) + } + result.AssertInfrastructureIsInSync() + result.AssertManagedCount(2) + result.Equal("aws_route53_zone", result.Managed()[0].TerraformType()) + result.Equal("aws_s3_bucket", result.Managed()[1].TerraformType()) + }, + }, + }, + }) +} + func TestAcc_StateReader_WithMultiplesStatesInS3(t *testing.T) { stateBucketName := "driftctl-acc-test-only" acceptance.Run(t, acceptance.AccTestCase{ diff --git a/pkg/iac/terraform/state/testdata/acc/multiples_states/route53/.terraform.lock.hcl b/pkg/iac/terraform/state/testdata/acc/multiple_states/route53/.terraform.lock.hcl similarity index 100% rename from pkg/iac/terraform/state/testdata/acc/multiples_states/route53/.terraform.lock.hcl rename to pkg/iac/terraform/state/testdata/acc/multiple_states/route53/.terraform.lock.hcl diff --git a/pkg/iac/terraform/state/testdata/acc/multiples_states/route53/terraform.tf b/pkg/iac/terraform/state/testdata/acc/multiple_states/route53/terraform.tf similarity index 100% rename from pkg/iac/terraform/state/testdata/acc/multiples_states/route53/terraform.tf rename to pkg/iac/terraform/state/testdata/acc/multiple_states/route53/terraform.tf diff --git a/pkg/iac/terraform/state/testdata/acc/multiples_states/s3/.terraform.lock.hcl b/pkg/iac/terraform/state/testdata/acc/multiple_states/s3/.terraform.lock.hcl similarity index 100% rename from pkg/iac/terraform/state/testdata/acc/multiples_states/s3/.terraform.lock.hcl rename to pkg/iac/terraform/state/testdata/acc/multiple_states/s3/.terraform.lock.hcl diff --git a/pkg/iac/terraform/state/testdata/acc/multiples_states/s3/terraform.tf b/pkg/iac/terraform/state/testdata/acc/multiple_states/s3/terraform.tf similarity index 100% rename from pkg/iac/terraform/state/testdata/acc/multiples_states/s3/terraform.tf rename to pkg/iac/terraform/state/testdata/acc/multiple_states/s3/terraform.tf diff --git a/pkg/iac/terraform/state/testdata/acc/multiple_states_local/.gitignore b/pkg/iac/terraform/state/testdata/acc/multiple_states_local/.gitignore new file mode 100644 index 00000000..2dc04f43 --- /dev/null +++ b/pkg/iac/terraform/state/testdata/acc/multiple_states_local/.gitignore @@ -0,0 +1 @@ +states diff --git a/pkg/iac/terraform/state/testdata/acc/multiple_states_local/route53/.terraform.lock.hcl b/pkg/iac/terraform/state/testdata/acc/multiple_states_local/route53/.terraform.lock.hcl new file mode 100755 index 00000000..1045bc9c --- /dev/null +++ b/pkg/iac/terraform/state/testdata/acc/multiple_states_local/route53/.terraform.lock.hcl @@ -0,0 +1,38 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "3.19.0" + constraints = "~> 3.19.0" + hashes = [ + "h1:+7Vi7p13+cnrxjXbfJiTimGSFR97xCaQwkkvWcreLns=", + "zh:185a5259153eb9ee4699d4be43b3d509386b473683392034319beee97d470c3b", + "zh:2d9a0a01f93e8d16539d835c02b8b6e1927b7685f4076e96cb07f7dd6944bc6c", + "zh:703f6da36b1b5f3497baa38fccaa7765fb8a2b6440344e4c97172516b49437dd", + "zh:770855565462abadbbddd98cb357d2f1a8f30f68a358cb37cbd5c072cb15b377", + "zh:8008db43149fe4345301f81e15e6d9ddb47aa5e7a31648f9b290af96ad86e92a", + "zh:8cdd27d375da6dcb7687f1fed126b7c04efce1671066802ee876dbbc9c66ec79", + "zh:be22ae185005690d1a017c1b909e0d80ab567e239b4f06ecacdba85080667c1c", + "zh:d2d02e72dbd80f607636cd6237a6c862897caabc635c7b50c0cb243d11246723", + "zh:d8f125b66a1eda2555c0f9bbdf12036a5f8d073499a22ca9e4812b68067fea31", + "zh:f5a98024c64d5d2973ff15b093725a074c0cb4afde07ef32c542e69f17ac90bc", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.1.0" + hashes = [ + "h1:BZMEPucF+pbu9gsPk0G0BHx7YP04+tKdq2MrRDF1EDM=", + "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc", + "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626", + "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff", + "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2", + "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992", + "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427", + "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc", + "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f", + "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b", + "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7", + "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a", + ] +} diff --git a/pkg/iac/terraform/state/testdata/acc/multiple_states_local/route53/terraform.tf b/pkg/iac/terraform/state/testdata/acc/multiple_states_local/route53/terraform.tf new file mode 100644 index 00000000..02b10338 --- /dev/null +++ b/pkg/iac/terraform/state/testdata/acc/multiple_states_local/route53/terraform.tf @@ -0,0 +1,25 @@ +provider "aws" { + region = "us-east-1" +} + +terraform { + required_providers { + aws = { + version = "~> 3.19.0" + } + } + + backend "local" { + path = "../states/route53/terraform.tfstate" + } +} + +resource "random_string" "prefix" { + length = 6 + upper = false + special = false +} + +resource "aws_route53_zone" "foobar" { + name = "${random_string.prefix.result}-example.com" +} diff --git a/pkg/iac/terraform/state/testdata/acc/multiple_states_local/s3/.terraform.lock.hcl b/pkg/iac/terraform/state/testdata/acc/multiple_states_local/s3/.terraform.lock.hcl new file mode 100755 index 00000000..1045bc9c --- /dev/null +++ b/pkg/iac/terraform/state/testdata/acc/multiple_states_local/s3/.terraform.lock.hcl @@ -0,0 +1,38 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "3.19.0" + constraints = "~> 3.19.0" + hashes = [ + "h1:+7Vi7p13+cnrxjXbfJiTimGSFR97xCaQwkkvWcreLns=", + "zh:185a5259153eb9ee4699d4be43b3d509386b473683392034319beee97d470c3b", + "zh:2d9a0a01f93e8d16539d835c02b8b6e1927b7685f4076e96cb07f7dd6944bc6c", + "zh:703f6da36b1b5f3497baa38fccaa7765fb8a2b6440344e4c97172516b49437dd", + "zh:770855565462abadbbddd98cb357d2f1a8f30f68a358cb37cbd5c072cb15b377", + "zh:8008db43149fe4345301f81e15e6d9ddb47aa5e7a31648f9b290af96ad86e92a", + "zh:8cdd27d375da6dcb7687f1fed126b7c04efce1671066802ee876dbbc9c66ec79", + "zh:be22ae185005690d1a017c1b909e0d80ab567e239b4f06ecacdba85080667c1c", + "zh:d2d02e72dbd80f607636cd6237a6c862897caabc635c7b50c0cb243d11246723", + "zh:d8f125b66a1eda2555c0f9bbdf12036a5f8d073499a22ca9e4812b68067fea31", + "zh:f5a98024c64d5d2973ff15b093725a074c0cb4afde07ef32c542e69f17ac90bc", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.1.0" + hashes = [ + "h1:BZMEPucF+pbu9gsPk0G0BHx7YP04+tKdq2MrRDF1EDM=", + "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc", + "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626", + "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff", + "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2", + "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992", + "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427", + "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc", + "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f", + "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b", + "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7", + "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a", + ] +} diff --git a/pkg/iac/terraform/state/testdata/acc/multiple_states_local/s3/terraform.tf b/pkg/iac/terraform/state/testdata/acc/multiple_states_local/s3/terraform.tf new file mode 100644 index 00000000..79e4e4bb --- /dev/null +++ b/pkg/iac/terraform/state/testdata/acc/multiple_states_local/s3/terraform.tf @@ -0,0 +1,25 @@ +provider "aws" { + region = "us-east-1" +} + +terraform { + required_providers { + aws = { + version = "~> 3.19.0" + } + } + + backend "local" { + path = "../states/s3/terraform.tfstate" + } +} + +resource "random_string" "prefix" { + length = 6 + upper = false + special = false +} + +resource "aws_s3_bucket" "foobar" { + bucket = "${random_string.prefix.result}.driftctl-test.com" +}