Merge pull request #897 from cloudskiff/add_state_source_in_resources

Add resource source for terraform state

pkg/iac/config/config.go

@@ -1,7 +1,23 @@
 package config
 
+import "fmt"
+
 type SupplierConfig struct {
 	Key     string
 	Backend string
 	Path    string
 }
+
+func (c *SupplierConfig) String() string {
+	str := c.Key
+	if c.Backend != "" {
+		str += fmt.Sprintf("+%s", c.Backend)
+	}
+	if str != "" {
+		str += "://"
+	}
+	if c.Path != "" {
+		str += c.Path
+	}
+	return str
+}

pkg/iac/config/config_test.go

@@ -0,0 +1,42 @@
+package config
+
+import "testing"
+
+func TestSupplierConfig_String(t *testing.T) {
+	tests := []struct {
+		name   string
+		config SupplierConfig
+		want   string
+	}{
+		{
+			name:   "test with empty config",
+			config: SupplierConfig{},
+			want:   "",
+		},
+		{
+			name: "test with empty path",
+			config: SupplierConfig{
+				Key:     "tfstate",
+				Backend: "s3",
+				Path:    "",
+			},
+			want: "tfstate+s3://",
+		},
+		{
+			name: "test valid config",
+			config: SupplierConfig{
+				Key:     "tfstate",
+				Backend: "s3",
+				Path:    "my-bucket/terraform.tfstate",
+			},
+			want: "tfstate+s3://my-bucket/terraform.tfstate",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.config.String(); got != tt.want {
+				t.Errorf("String() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/iac/terraform/state/terraform_state_reader.go

@@ -25,6 +25,11 @@ import (
 
 const TerraformStateReaderSupplier = "tfstate"
 
+type decodedRes struct {
+	source resource.Source
+	val    cty.Value
+}
+
 type TerraformStateReader struct {
 	library        *terraform.ProviderLibrary
 	config         config.SupplierConfig
@@ -50,7 +55,7 @@ func NewReader(config config.SupplierConfig, library *terraform.ProviderLibrary,
 	return &reader, nil
 }
 
-func (r *TerraformStateReader) retrieve() (map[string][]cty.Value, error) {
+func (r *TerraformStateReader) retrieve() (map[string][]decodedRes, error) {
 	b, err := backend.GetBackend(r.config, r.backendOptions)
 	if err != nil {
 		return nil, err
@@ -63,7 +68,7 @@ func (r *TerraformStateReader) retrieve() (map[string][]cty.Value, error) {
 		return nil, err
 	}
 
-	resMap := make(map[string][]cty.Value)
+	resMap := make(map[string][]decodedRes)
 	for moduleName, module := range state.Modules {
 		logrus.WithFields(logrus.Fields{
 			"module":        moduleName,
@@ -133,12 +138,14 @@ func (r *TerraformStateReader) retrieve() (map[string][]cty.Value, error) {
 					}
 				}
 				_, exists := resMap[stateRes.Addr.Resource.Type]
+				val := decodedRes{
+					source: resource.NewTerraformStateSource(r.config.String(), moduleName, resName),
+					val:    decodedVal.Value,
+				}
 				if !exists {
-					resMap[stateRes.Addr.Resource.Type] = []cty.Value{
-						decodedVal.Value,
-					}
+					resMap[stateRes.Addr.Resource.Type] = []decodedRes{val}
 				} else {
-					resMap[stateRes.Addr.Resource.Type] = append(resMap[stateRes.Addr.Resource.Type], decodedVal.Value)
+					resMap[stateRes.Addr.Resource.Type] = append(resMap[stateRes.Addr.Resource.Type], val)
 				}
 			}
 		}
@@ -175,16 +182,24 @@ func (r *TerraformStateReader) convertInstance(instance *states.ResourceInstance
 	return instanceObj, nil
 }
 
-func (r *TerraformStateReader) decode(values map[string][]cty.Value) ([]resource.Resource, error) {
+func (r *TerraformStateReader) decode(valFromState map[string][]decodedRes) ([]resource.Resource, error) {
 	results := make([]resource.Resource, 0)
 
-	for ty, val := range values {
-		decodedResources, err := r.deserializer.Deserialize(ty, val)
-		if err != nil {
-			logrus.WithField("ty", ty).Warnf("Could not read from state: %+v", err)
-			continue
+	for ty, val := range valFromState {
+		for _, stateVal := range val {
+			res, err := r.deserializer.DeserializeOne(ty, stateVal.val)
+			if err != nil {
+				logrus.WithFields(logrus.Fields{
+					"type":  ty,
+					"name":  stateVal.source.InternalName(),
+					"state": stateVal.source.Source(),
+				}).Warnf("Could not read from state: %+v", err)
+				continue
+			}
+			stateResource, _ := res.(*resource.AbstractResource)
+			stateResource.Source = stateVal.source
+			results = append(results, stateResource)
 		}
-		results = append(results, decodedResources...)
 	}
 
 	return results, nil

pkg/iac/terraform/state/terraform_state_reader_test.go

@@ -43,6 +43,52 @@ func TestReadStateInvalid(t *testing.T) {
 	}
 }
 
+// Check that resource sources are properly set
+func TestTerraformStateReader_Source(t *testing.T) {
+	progress := &output.MockProgress{}
+	progress.On("Inc").Return().Times(1)
+	progress.On("Stop").Return().Times(1)
+
+	provider := mocks.NewMockedGoldenTFProvider("source", nil, false)
+	library := terraform.NewProviderLibrary()
+	library.AddProvider(terraform.AWS, provider)
+
+	repo := testresource.InitFakeSchemaRepository(terraform.AWS, "3.19.0")
+	resourceaws.InitResourcesMetadata(repo)
+
+	factory := terraform.NewTerraformResourceFactory(repo)
+
+	r := &TerraformStateReader{
+		config: config.SupplierConfig{
+			Key:  "tfstate",
+			Path: path.Join(goldenfile.GoldenFilePath, "source", "terraform.tfstate"),
+		},
+		library:      library,
+		progress:     progress,
+		deserializer: resource.NewDeserializer(factory),
+	}
+
+	got, err := r.Resources()
+	assert.Nil(t, err)
+	assert.Len(t, got, 2)
+	for _, res := range got {
+		if res.TerraformType() == resourceaws.AwsS3BucketResourceType {
+			assert.Equal(t, &resource.TerraformStateSource{
+				State:  "tfstate://test/source/terraform.tfstate",
+				Module: "",
+				Name:   "bucket",
+			}, res.(*resource.AbstractResource).Source)
+		}
+		if res.TerraformType() == resourceaws.AwsIamUserResourceType {
+			assert.Equal(t, &resource.TerraformStateSource{
+				State:  "tfstate://test/source/terraform.tfstate",
+				Module: "module.iam_iam-user",
+				Name:   "this_no_pgp",
+			}, res.(*resource.AbstractResource).Source)
+		}
+	}
+}
+
 func TestTerraformStateReader_AWS_Resources(t *testing.T) {
 	tests := []struct {
 		name    string

pkg/iac/terraform/state/test/source/terraform.tfstate

@@ -0,0 +1,81 @@
+{
+  "version": 4,
+  "terraform_version": "0.14.4",
+  "serial": 88,
+  "lineage": "dcb149dc-5e8b-bb81-e690-3980485675f5",
+  "outputs": {},
+  "resources": [
+    {
+      "module": "module.iam_iam-user",
+      "mode": "managed",
+      "type": "aws_iam_access_key",
+      "name": "this_no_pgp",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "index_key": 0,
+          "schema_version": 0,
+          "attributes": {
+            "encrypted_secret": null,
+            "id": "AKIA5QYBVVD2VIEMMUIQ",
+            "key_fingerprint": null,
+            "pgp_key": null,
+            "secret": "",
+            "ses_smtp_password_v4": "",
+            "status": "Active",
+            "user": "MODULE-USER"
+          },
+          "sensitive_attributes": [],
+          "private": "bnVsbA==",
+          "dependencies": [
+            "module.iam_iam-user.aws_iam_user.this"
+          ]
+        }
+      ]
+    },
+    {
+      "mode": "managed",
+      "type": "aws_s3_bucket",
+      "name": "bucket",
+      "provider": "provider.aws",
+      "instances": [
+          {
+              "schema_version": 0,
+              "attributes": {
+                  "acceleration_status": "",
+                  "acl": "private",
+                  "arn": "arn:aws:s3:::bucket-martin-test-drift",
+                  "bucket": "bucket-martin-test-drift",
+                  "bucket_domain_name": "bucket-martin-test-drift.s3.amazonaws.com",
+                  "bucket_prefix": null,
+                  "bucket_regional_domain_name": "bucket-martin-test-drift.s3.eu-west-3.amazonaws.com",
+                  "cors_rule": [],
+                  "force_destroy": false,
+                  "grant": [],
+                  "hosted_zone_id": "Z3R1K369G5AVDG",
+                  "id": "bucket-martin-test-drift",
+                  "lifecycle_rule": [],
+                  "logging": [],
+                  "object_lock_configuration": [],
+                  "policy": null,
+                  "region": "eu-west-3",
+                  "replication_configuration": [],
+                  "request_payer": "BucketOwner",
+                  "server_side_encryption_configuration": [],
+                  "tags": {},
+                  "versioning": [
+                      {
+                          "enabled": false,
+                          "mfa_delete": false
+                      }
+                  ],
+                  "website": [],
+                  "website_domain": null,
+                  "website_endpoint": null
+              },
+              "private": "bnVsbA=="
+          }
+      ]
+    }
+  ]
+}

pkg/resource/resource.go

@@ -17,11 +17,40 @@ type Resource interface {
 	Schema() *Schema
 }
 
+type Source interface {
+	Source() string
+	Namespace() string
+	InternalName() string
+}
+
+type TerraformStateSource struct {
+	State  string
+	Module string
+	Name   string
+}
+
+func NewTerraformStateSource(state, module, name string) *TerraformStateSource {
+	return &TerraformStateSource{state, module, name}
+}
+
+func (s *TerraformStateSource) Source() string {
+	return s.State
+}
+
+func (s *TerraformStateSource) Namespace() string {
+	return s.Module
+}
+
+func (s *TerraformStateSource) InternalName() string {
+	return s.Name
+}
+
 type AbstractResource struct {
-	Id    string
-	Type  string
-	Attrs *Attributes
-	Sch   *Schema `json:"-" diff:"-"`
+	Id     string
+	Type   string
+	Attrs  *Attributes
+	Sch    *Schema `json:"-" diff:"-"`
+	Source Source  `json:"-"`
 }
 
 func (a *AbstractResource) Schema() *Schema {