Allow to use a whole local directory as IaC source

doc/cmd/scan/iac_source.md

@@ -14,6 +14,13 @@ driftctl scan \
 
 # You can also use every file under a given prefix for S3
 driftctl scan --from tfstate+s3://statebucketdriftctl/states
+
+# ... or in a given local folder
+# driftctl will recursively use all files under this folder.
+#
+# N.B. Symlinks under the root folder will be ignored.
+#      If the folder itself is a symlink it will be followed.
+driftctl scan --from tfstate://my-states/directory
 ```
 
 

pkg/iac/terraform/state/enumerator/file.go

@@ -0,0 +1,81 @@
+package enumerator
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/cloudskiff/driftctl/pkg/iac/config"
+)
+
+type FileEnumeratorConfig struct {
+	Bucket *string
+	Prefix *string
+}
+
+type FileEnumerator struct {
+	config config.SupplierConfig
+}
+
+func NewFileEnumerator(config config.SupplierConfig) *FileEnumerator {
+	return &FileEnumerator{
+		config,
+	}
+}
+
+// File enumeration does not follow symlinks.
+// We may use something like this https://pkg.go.dev/github.com/facebookgo/symwalk
+// to follow links, but it allows infinite loop so be careful !
+// If a symlink is given as root path, we will follow it, but symlinks under this path
+// will not be resolved.
+func (s *FileEnumerator) Enumerate() ([]string, error) {
+	path := s.config.Path
+	info, err := os.Lstat(path)
+	if err != nil {
+		return nil, err
+	}
+
+	// if we got a symlink, use its destination
+	if info.Mode()&os.ModeSymlink != 0 {
+		destination, err := filepath.EvalSymlinks(path)
+		if err != nil {
+			return nil, err
+		}
+		path = destination
+		info, err = os.Stat(destination)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if info != nil && !info.IsDir() {
+		return []string{path}, nil
+	}
+
+	keys := make([]string, 0)
+
+	err = filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error {
+		// Not tested, we should remove a file or folder after WalkDir has enumerated the whole tree in memory
+		// This edge case does not really need to be covered by tests
+		if err != nil {
+			return err
+		}
+		// Skip symlinks
+		if d.Type()&os.ModeSymlink != 0 {
+			return nil
+		}
+
+		// Ignore .backup files generated by terraform
+		if strings.HasSuffix(path, ".backup") {
+			return nil
+		}
+
+		if !d.IsDir() {
+			keys = append(keys, path)
+		}
+		return nil
+	})
+
+	return keys, err
+}

pkg/iac/terraform/state/enumerator/file_test.go

@@ -0,0 +1,90 @@
+package enumerator
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/cloudskiff/driftctl/pkg/iac/config"
+)
+
+func TestFileEnumerator_Enumerate(t *testing.T) {
+	tests := []struct {
+		name   string
+		config config.SupplierConfig
+		want   []string
+		err    string
+	}{
+		{
+			name: "subfolder nesting",
+			config: config.SupplierConfig{
+				Path: "testdata/states",
+			},
+			want: []string{
+				"testdata/states/route53/directory/route53.state",
+				"testdata/states/s3/terraform.tfstate",
+				"testdata/states/terraform.tfstate",
+			},
+		},
+		{
+			name: "symlinked folder",
+			config: config.SupplierConfig{
+				Path: "testdata/symlink",
+			},
+			want: []string{
+				"testdata/states/route53/directory/route53.state",
+				"testdata/states/s3/terraform.tfstate",
+				"testdata/states/terraform.tfstate",
+			},
+		},
+		{
+			name: "single state file",
+			config: config.SupplierConfig{
+				Path: "testdata/states/terraform.tfstate",
+			},
+			want: []string{
+				"testdata/states/terraform.tfstate",
+			},
+		},
+		{
+			name: "single symlink state file",
+			config: config.SupplierConfig{
+				Path: "testdata/states/symlink.tfstate",
+			},
+			want: []string{
+				"testdata/states/terraform.tfstate",
+			},
+		},
+		{
+			name: "invalid folder",
+			config: config.SupplierConfig{
+				Path: "/tmp/dummy-folder/that/does/not/exist",
+			},
+			want: nil,
+			err:  "lstat /tmp/dummy-folder/that/does/not/exist: no such file or directory",
+		},
+		{
+			name: "invalid symlink",
+			config: config.SupplierConfig{
+				Path: "testdata/invalid_symlink/invalid",
+			},
+			want: nil,
+			err:  "lstat testdata/invalid_symlink/test: no such file or directory",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := NewFileEnumerator(tt.config)
+			got, err := s.Enumerate()
+			if err != nil && err.Error() != tt.err {
+				t.Fatalf("Expected error '%s', got '%s'", tt.err, err.Error())
+			}
+			if err != nil && tt.err == "" {
+				t.Fatalf("Expected error '%s' but got nil", tt.err)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Enumerate() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/iac/terraform/state/enumerator/state_enumerator.go

@@ -12,7 +12,10 @@ type StateEnumerator interface {
 
 func GetEnumerator(config config.SupplierConfig) StateEnumerator {
 
-	if config.Backend == backend.BackendKeyS3 {
+	switch config.Backend {
+	case backend.BackendKeyFile:
+		return NewFileEnumerator(config)
+	case backend.BackendKeyS3:
 		return NewS3Enumerator(config)
 	}
 

pkg/iac/terraform/state/enumerator/testdata/invalid_symlink/invalid

@@ -0,0 +1 @@
+test

pkg/iac/terraform/state/enumerator/testdata/states/symlink-to-s3-folder

@@ -0,0 +1 @@
+s3

pkg/iac/terraform/state/enumerator/testdata/states/symlink.tfstate

@@ -0,0 +1 @@
+terraform.tfstate

pkg/iac/terraform/state/enumerator/testdata/symlink

@@ -0,0 +1 @@
+states

pkg/iac/terraform/state/terraform_state_reader_acc_test.go

@@ -9,6 +9,33 @@ import (
 	"github.com/cloudskiff/driftctl/test/acceptance/awsutils"
 )
 
+func TestAcc_StateReader_WithMultipleStatesInDirectory(t *testing.T) {
+	acceptance.Run(t, acceptance.AccTestCase{
+		Paths: []string{
+			"./testdata/acc/multiple_states_local/s3",
+			"./testdata/acc/multiple_states_local/route53",
+		},
+		Args: []string{
+			"scan",
+			"--from", "tfstate://testdata/acc/multiple_states_local/states",
+			"--filter", "Type=='aws_s3_bucket' || Type=='aws_route53_zone'",
+		},
+		Checks: []acceptance.AccCheck{
+			{
+				Check: func(result *acceptance.ScanResult, stdout string, err error) {
+					if err != nil {
+						t.Fatal(err)
+					}
+					result.AssertInfrastructureIsInSync()
+					result.AssertManagedCount(2)
+					result.Equal("aws_route53_zone", result.Managed()[0].TerraformType())
+					result.Equal("aws_s3_bucket", result.Managed()[1].TerraformType())
+				},
+			},
+		},
+	})
+}
+
 func TestAcc_StateReader_WithMultiplesStatesInS3(t *testing.T) {
 	stateBucketName := "driftctl-acc-test-only"
 	acceptance.Run(t, acceptance.AccTestCase{

pkg/iac/terraform/state/testdata/acc/multiple_states_local/.gitignore

@@ -0,0 +1 @@
+states

pkg/iac/terraform/state/testdata/acc/multiple_states_local/route53/.terraform.lock.hcl

@@ -0,0 +1,38 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/aws" {
+  version     = "3.19.0"
+  constraints = "~> 3.19.0"
+  hashes = [
+    "h1:+7Vi7p13+cnrxjXbfJiTimGSFR97xCaQwkkvWcreLns=",
+    "zh:185a5259153eb9ee4699d4be43b3d509386b473683392034319beee97d470c3b",
+    "zh:2d9a0a01f93e8d16539d835c02b8b6e1927b7685f4076e96cb07f7dd6944bc6c",
+    "zh:703f6da36b1b5f3497baa38fccaa7765fb8a2b6440344e4c97172516b49437dd",
+    "zh:770855565462abadbbddd98cb357d2f1a8f30f68a358cb37cbd5c072cb15b377",
+    "zh:8008db43149fe4345301f81e15e6d9ddb47aa5e7a31648f9b290af96ad86e92a",
+    "zh:8cdd27d375da6dcb7687f1fed126b7c04efce1671066802ee876dbbc9c66ec79",
+    "zh:be22ae185005690d1a017c1b909e0d80ab567e239b4f06ecacdba85080667c1c",
+    "zh:d2d02e72dbd80f607636cd6237a6c862897caabc635c7b50c0cb243d11246723",
+    "zh:d8f125b66a1eda2555c0f9bbdf12036a5f8d073499a22ca9e4812b68067fea31",
+    "zh:f5a98024c64d5d2973ff15b093725a074c0cb4afde07ef32c542e69f17ac90bc",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/random" {
+  version = "3.1.0"
+  hashes = [
+    "h1:BZMEPucF+pbu9gsPk0G0BHx7YP04+tKdq2MrRDF1EDM=",
+    "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc",
+    "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626",
+    "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff",
+    "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2",
+    "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992",
+    "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427",
+    "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc",
+    "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f",
+    "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b",
+    "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7",
+    "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a",
+  ]
+}

pkg/iac/terraform/state/testdata/acc/multiple_states_local/route53/terraform.tf

@@ -0,0 +1,25 @@
+provider "aws" {
+  region  = "us-east-1"
+}
+
+terraform {
+  required_providers {
+    aws = {
+      version = "~> 3.19.0"
+    }
+  }
+
+  backend "local" {
+    path = "../states/route53/terraform.tfstate"
+  }
+}
+
+resource "random_string" "prefix" {
+  length  = 6
+  upper   = false
+  special = false
+}
+
+resource "aws_route53_zone" "foobar" {
+  name = "${random_string.prefix.result}-example.com"
+}

pkg/iac/terraform/state/testdata/acc/multiple_states_local/s3/.terraform.lock.hcl

@@ -0,0 +1,38 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/aws" {
+  version     = "3.19.0"
+  constraints = "~> 3.19.0"
+  hashes = [
+    "h1:+7Vi7p13+cnrxjXbfJiTimGSFR97xCaQwkkvWcreLns=",
+    "zh:185a5259153eb9ee4699d4be43b3d509386b473683392034319beee97d470c3b",
+    "zh:2d9a0a01f93e8d16539d835c02b8b6e1927b7685f4076e96cb07f7dd6944bc6c",
+    "zh:703f6da36b1b5f3497baa38fccaa7765fb8a2b6440344e4c97172516b49437dd",
+    "zh:770855565462abadbbddd98cb357d2f1a8f30f68a358cb37cbd5c072cb15b377",
+    "zh:8008db43149fe4345301f81e15e6d9ddb47aa5e7a31648f9b290af96ad86e92a",
+    "zh:8cdd27d375da6dcb7687f1fed126b7c04efce1671066802ee876dbbc9c66ec79",
+    "zh:be22ae185005690d1a017c1b909e0d80ab567e239b4f06ecacdba85080667c1c",
+    "zh:d2d02e72dbd80f607636cd6237a6c862897caabc635c7b50c0cb243d11246723",
+    "zh:d8f125b66a1eda2555c0f9bbdf12036a5f8d073499a22ca9e4812b68067fea31",
+    "zh:f5a98024c64d5d2973ff15b093725a074c0cb4afde07ef32c542e69f17ac90bc",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/random" {
+  version = "3.1.0"
+  hashes = [
+    "h1:BZMEPucF+pbu9gsPk0G0BHx7YP04+tKdq2MrRDF1EDM=",
+    "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc",
+    "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626",
+    "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff",
+    "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2",
+    "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992",
+    "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427",
+    "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc",
+    "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f",
+    "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b",
+    "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7",
+    "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a",
+  ]
+}

pkg/iac/terraform/state/testdata/acc/multiple_states_local/s3/terraform.tf

@@ -0,0 +1,25 @@
+provider "aws" {
+  region  = "us-east-1"
+}
+
+terraform {
+  required_providers {
+    aws = {
+      version = "~> 3.19.0"
+    }
+  }
+
+  backend "local" {
+    path = "../states/s3/terraform.tfstate"
+  }
+}
+
+resource "random_string" "prefix" {
+  length  = 6
+  upper   = false
+  special = false
+}
+
+resource "aws_s3_bucket" "foobar" {
+  bucket = "${random_string.prefix.result}.driftctl-test.com"
+}