From 916c458760bb36a3ccff58e0f3578ce9e16fe5cd Mon Sep 17 00:00:00 2001 From: Louis TOUSSAINT Date: Thu, 24 Jun 2021 14:21:03 +0200 Subject: [PATCH 1/6] Issue 615: Create struct to override env config for state_enumerator --- pkg/envproxy/env_proxy.go | 53 +++++++++++++ pkg/envproxy/env_proxy_test.go | 98 ++++++++++++++++++++++++ pkg/iac/terraform/state/enumerator/s3.go | 10 +-- 3 files changed, 156 insertions(+), 5 deletions(-) create mode 100644 pkg/envproxy/env_proxy.go create mode 100644 pkg/envproxy/env_proxy_test.go diff --git a/pkg/envproxy/env_proxy.go b/pkg/envproxy/env_proxy.go new file mode 100644 index 00000000..06817d33 --- /dev/null +++ b/pkg/envproxy/env_proxy.go @@ -0,0 +1,53 @@ +package envproxy + +import ( + "os" + "strings" +) + +type EnvProxy struct { + varPrefix string + varPattern string + DefaultEnv map[string]string +} + +func NewEnvProxy() *EnvProxy { + envMap := map[string]string{} + for _, variable := range os.Environ() { + tmp := strings.SplitN(variable, "=", 2) + envMap[tmp[0]] = tmp[1] + } + return &EnvProxy{ + DefaultEnv: envMap, + } +} + +func (s *EnvProxy) SetProxy(prefix, pattern string) { + s.varPrefix = prefix + s.varPattern = pattern +} + +func (s *EnvProxy) Apply() { + if s.varPrefix == "" || s.varPattern == "" { + return + } + for key, value := range s.DefaultEnv { + if strings.HasPrefix(key, s.varPrefix) { + key = strings.Replace(key, s.varPrefix, s.varPattern, 1) + os.Setenv(key, value) + } + } +} + +func (s *EnvProxy) Restore() { + if s.varPrefix == "" || s.varPattern == "" { + return + } + for key, value := range s.DefaultEnv { + if strings.HasPrefix(key, s.varPrefix) { + key = strings.Replace(key, s.varPrefix, s.varPattern, 1) + value = s.DefaultEnv[key] + } + os.Setenv(key, value) + } +} diff --git a/pkg/envproxy/env_proxy_test.go b/pkg/envproxy/env_proxy_test.go new file mode 100644 index 00000000..2c115860 --- /dev/null +++ b/pkg/envproxy/env_proxy_test.go @@ -0,0 +1,98 @@ +package envproxy + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEnvProxy_Apply(t *testing.T) { + tests := []struct { + name string + proxyArgs []string + modifier bool + }{ + { + name: "Without args on SetProxy", + proxyArgs: []string{"", ""}, + modifier: false, + }, + { + name: "With args on SetProxy", + proxyArgs: []string{"TEST_DCTL_S3_", "TEST_AWS_"}, + modifier: true, + }, + { + name: "With no pattern on SetProxy", + proxyArgs: []string{"TEST_DCTL_S3_", ""}, + modifier: false, + }, + { + name: "With no prefix on SetProxy", + proxyArgs: []string{"", "TEST_AWS_"}, + modifier: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + os.Setenv("TEST_DCTL_S3_PROFILE", "dctl_env") + os.Setenv("TEST_AWS_PROFILE", "aws_env") + expectedEnv := os.Environ() + + envProxy := NewEnvProxy() + envProxy.SetProxy(tt.proxyArgs[0], tt.proxyArgs[1]) + + envProxy.Apply() + + newEnv := os.Environ() + for index, value := range expectedEnv { + if tt.modifier && value == "TEST_AWS_PROFILE=aws_env" { + expectedEnv[index] = strings.Replace( + value, + "TEST_AWS_PROFILE=aws_env", + "TEST_AWS_PROFILE=dctl_env", + 1, + ) + } + } + + if !assert.Equal(t, newEnv, expectedEnv) { + t.Errorf("Expected %v, got %v", expectedEnv, newEnv) + } + }) + } +} + +func TestEnvProxy_Restore(t *testing.T) { + tests := []struct { + name string + proxyArgs []string + }{ + { + name: "With args on SetProxy", + proxyArgs: []string{"TEST_DCTL_S3_", "TEST_AWS_"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + os.Setenv("TEST_DCTL_S3_PROFILE", "dctl_env") + os.Setenv("TEST_AWS_PROFILE", "aws_env") + expectedEnv := os.Environ() + + envProxy := NewEnvProxy() + envProxy.SetProxy(tt.proxyArgs[0], tt.proxyArgs[1]) + os.Setenv("TEST_AWS_PROFILE", "new_aws_env") + + envProxy.Restore() + + newEnv := os.Environ() + if !assert.Equal(t, newEnv, expectedEnv) { + t.Errorf("Expected %v, got %v", expectedEnv, newEnv) + } + }) + } +} diff --git a/pkg/iac/terraform/state/enumerator/s3.go b/pkg/iac/terraform/state/enumerator/s3.go index b2fa0e1a..cf4932a8 100644 --- a/pkg/iac/terraform/state/enumerator/s3.go +++ b/pkg/iac/terraform/state/enumerator/s3.go @@ -8,26 +8,26 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3iface" + "github.com/cloudskiff/driftctl/pkg/envproxy" "github.com/pkg/errors" "github.com/bmatcuk/doublestar/v4" "github.com/cloudskiff/driftctl/pkg/iac/config" ) -type S3EnumeratorConfig struct { - Bucket *string - Prefix *string -} - type S3Enumerator struct { config config.SupplierConfig client s3iface.S3API } func NewS3Enumerator(config config.SupplierConfig) *S3Enumerator { + envProxy := envproxy.NewEnvProxy() + envProxy.SetProxy("DCTL_S3_", "AWS_") + envProxy.Apply() sess := session.Must(session.NewSessionWithOptions(session.Options{ SharedConfigState: session.SharedConfigEnable, })) + envProxy.Restore() return &S3Enumerator{ config, s3.New(sess), From cdf87ac1a6bed9bf79a1b976f796e57286a299ed Mon Sep 17 00:00:00 2001 From: Louis TOUSSAINT Date: Tue, 29 Jun 2021 16:47:42 +0200 Subject: [PATCH 2/6] Issue 615: Modify the env_proxy_test logic and env_proxy variable typo --- pkg/envproxy/env_proxy.go | 32 +++++----- pkg/envproxy/env_proxy_test.go | 106 ++++++++++++++------------------- 2 files changed, 60 insertions(+), 78 deletions(-) diff --git a/pkg/envproxy/env_proxy.go b/pkg/envproxy/env_proxy.go index 06817d33..5e01cb11 100644 --- a/pkg/envproxy/env_proxy.go +++ b/pkg/envproxy/env_proxy.go @@ -6,9 +6,9 @@ import ( ) type EnvProxy struct { - varPrefix string - varPattern string - DefaultEnv map[string]string + fromPrefix string + toPrefix string + defaultEnv map[string]string } func NewEnvProxy() *EnvProxy { @@ -18,35 +18,35 @@ func NewEnvProxy() *EnvProxy { envMap[tmp[0]] = tmp[1] } return &EnvProxy{ - DefaultEnv: envMap, + defaultEnv: envMap, } } -func (s *EnvProxy) SetProxy(prefix, pattern string) { - s.varPrefix = prefix - s.varPattern = pattern +func (s *EnvProxy) SetProxy(fromPrefix, toPrefix string) { + s.fromPrefix = fromPrefix + s.toPrefix = toPrefix } func (s *EnvProxy) Apply() { - if s.varPrefix == "" || s.varPattern == "" { + if s.fromPrefix == "" || s.toPrefix == "" { return } - for key, value := range s.DefaultEnv { - if strings.HasPrefix(key, s.varPrefix) { - key = strings.Replace(key, s.varPrefix, s.varPattern, 1) + for key, value := range s.defaultEnv { + if strings.HasPrefix(key, s.fromPrefix) { + key = strings.Replace(key, s.fromPrefix, s.toPrefix, 1) os.Setenv(key, value) } } } func (s *EnvProxy) Restore() { - if s.varPrefix == "" || s.varPattern == "" { + if s.fromPrefix == "" || s.toPrefix == "" { return } - for key, value := range s.DefaultEnv { - if strings.HasPrefix(key, s.varPrefix) { - key = strings.Replace(key, s.varPrefix, s.varPattern, 1) - value = s.DefaultEnv[key] + for key, value := range s.defaultEnv { + if strings.HasPrefix(key, s.fromPrefix) { + key = strings.Replace(key, s.fromPrefix, s.toPrefix, 1) + value = s.defaultEnv[key] } os.Setenv(key, value) } diff --git a/pkg/envproxy/env_proxy_test.go b/pkg/envproxy/env_proxy_test.go index 2c115860..22f0a5cb 100644 --- a/pkg/envproxy/env_proxy_test.go +++ b/pkg/envproxy/env_proxy_test.go @@ -4,95 +4,77 @@ import ( "os" "strings" "testing" - - "github.com/stretchr/testify/assert" ) -func TestEnvProxy_Apply(t *testing.T) { +func TestEnvProxy(t *testing.T) { tests := []struct { - name string - proxyArgs []string - modifier bool + name string + proxyArgs []string + initialEnv []string + modifiedEnv []string }{ { - name: "Without args on SetProxy", - proxyArgs: []string{"", ""}, - modifier: false, + name: "Without args on SetProxy", + proxyArgs: []string{"", ""}, + initialEnv: []string{"TEST_DCTL_S3_PROFILE=test_dctl_s3_profile", "TEST_AWS_PROFILE=test_aws_profile"}, + modifiedEnv: []string{"TEST_DCTL_S3_PROFILE=test_dctl_s3_profile", "TEST_AWS_PROFILE=test_aws_profile"}, }, { - name: "With args on SetProxy", - proxyArgs: []string{"TEST_DCTL_S3_", "TEST_AWS_"}, - modifier: true, + name: "With args on SetProxy", + proxyArgs: []string{"TEST_DCTL_S3_", "TEST_AWS_"}, + initialEnv: []string{"TEST_DCTL_S3_PROFILE=test_dctl_s3_profile", "TEST_AWS_PROFILE=test_aws_profile"}, + modifiedEnv: []string{"TEST_DCTL_S3_PROFILE=test_dctl_s3_profile", "TEST_AWS_PROFILE=test_dctl_s3_profile"}, }, { - name: "With no pattern on SetProxy", - proxyArgs: []string{"TEST_DCTL_S3_", ""}, - modifier: false, + name: "Without toPrefix on SetProxy", + proxyArgs: []string{"TEST_DCTL_S3_", ""}, + initialEnv: []string{"TEST_DCTL_S3_PROFILE=test_dctl_s3_profile", "TEST_AWS_PROFILE=test_aws_profile"}, + modifiedEnv: []string{"TEST_DCTL_S3_PROFILE=test_dctl_s3_profile", "TEST_AWS_PROFILE=test_aws_profile"}, }, { - name: "With no prefix on SetProxy", - proxyArgs: []string{"", "TEST_AWS_"}, - modifier: false, + name: "Without fromPrefix on SetProxy", + proxyArgs: []string{"", "TEST_AWS_"}, + initialEnv: []string{"TEST_DCTL_S3_PROFILE=test_dctl_s3_profile", "TEST_AWS_PROFILE=test_aws_profile"}, + modifiedEnv: []string{"TEST_DCTL_S3_PROFILE=test_dctl_s3_profile", "TEST_AWS_PROFILE=test_aws_profile"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - os.Setenv("TEST_DCTL_S3_PROFILE", "dctl_env") - os.Setenv("TEST_AWS_PROFILE", "aws_env") - expectedEnv := os.Environ() + + for _, value := range tt.initialEnv { + tmp := strings.SplitN(value, "=", 2) + os.Setenv(tmp[0], tmp[1]) + } envProxy := NewEnvProxy() envProxy.SetProxy(tt.proxyArgs[0], tt.proxyArgs[1]) envProxy.Apply() - newEnv := os.Environ() - for index, value := range expectedEnv { - if tt.modifier && value == "TEST_AWS_PROFILE=aws_env" { - expectedEnv[index] = strings.Replace( - value, - "TEST_AWS_PROFILE=aws_env", - "TEST_AWS_PROFILE=dctl_env", - 1, - ) - } + currentEnv := os.Environ() + if !compareEnv(currentEnv, tt.modifiedEnv) { + t.Errorf("Expected %v, got %v", tt.modifiedEnv, currentEnv) } - if !assert.Equal(t, newEnv, expectedEnv) { - t.Errorf("Expected %v, got %v", expectedEnv, newEnv) - } - }) - } -} - -func TestEnvProxy_Restore(t *testing.T) { - tests := []struct { - name string - proxyArgs []string - }{ - { - name: "With args on SetProxy", - proxyArgs: []string{"TEST_DCTL_S3_", "TEST_AWS_"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - os.Setenv("TEST_DCTL_S3_PROFILE", "dctl_env") - os.Setenv("TEST_AWS_PROFILE", "aws_env") - expectedEnv := os.Environ() - - envProxy := NewEnvProxy() - envProxy.SetProxy(tt.proxyArgs[0], tt.proxyArgs[1]) - os.Setenv("TEST_AWS_PROFILE", "new_aws_env") - envProxy.Restore() - newEnv := os.Environ() - if !assert.Equal(t, newEnv, expectedEnv) { - t.Errorf("Expected %v, got %v", expectedEnv, newEnv) + currentEnv = os.Environ() + if !compareEnv(currentEnv, tt.initialEnv) { + t.Errorf("Expected %v, got %v", tt.initialEnv, currentEnv) } }) } } + +func compareEnv(currentEnv, testEnv []string) bool { + isValid := 0 + for _, initialValue := range testEnv { + for _, value := range currentEnv { + if initialValue == value { + isValid++ + } + } + } + return isValid == len(testEnv) +} From b78aeb5d06efee44f3ff574086fb0b28e95b892c Mon Sep 17 00:00:00 2001 From: Louis TOUSSAINT Date: Tue, 29 Jun 2021 17:44:42 +0200 Subject: [PATCH 3/6] Issue 615: Add test where env is empty in env_proxy_test --- pkg/envproxy/env_proxy_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/envproxy/env_proxy_test.go b/pkg/envproxy/env_proxy_test.go index 22f0a5cb..80c3793b 100644 --- a/pkg/envproxy/env_proxy_test.go +++ b/pkg/envproxy/env_proxy_test.go @@ -37,6 +37,12 @@ func TestEnvProxy(t *testing.T) { initialEnv: []string{"TEST_DCTL_S3_PROFILE=test_dctl_s3_profile", "TEST_AWS_PROFILE=test_aws_profile"}, modifiedEnv: []string{"TEST_DCTL_S3_PROFILE=test_dctl_s3_profile", "TEST_AWS_PROFILE=test_aws_profile"}, }, + { + name: "Without initialEnv", + proxyArgs: []string{"TEST_DCTL_S3_", "TEST_AWS_"}, + initialEnv: []string{}, + modifiedEnv: []string{}, + }, } for _, tt := range tests { From c882c769e815b1db94dc7499d8a271daf1d98fcb Mon Sep 17 00:00:00 2001 From: Louis TOUSSAINT Date: Thu, 1 Jul 2021 17:47:03 +0200 Subject: [PATCH 4/6] Issue 615: Remove setProxy and add parameter to NewEnvProxy directly --- pkg/envproxy/env_proxy.go | 9 +++------ pkg/envproxy/env_proxy_test.go | 15 +++++++-------- pkg/iac/terraform/state/enumerator/s3.go | 3 +-- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/pkg/envproxy/env_proxy.go b/pkg/envproxy/env_proxy.go index 5e01cb11..4fce1e2d 100644 --- a/pkg/envproxy/env_proxy.go +++ b/pkg/envproxy/env_proxy.go @@ -11,22 +11,19 @@ type EnvProxy struct { defaultEnv map[string]string } -func NewEnvProxy() *EnvProxy { +func NewEnvProxy(fromPrefix, toPrefix string) *EnvProxy { envMap := map[string]string{} for _, variable := range os.Environ() { tmp := strings.SplitN(variable, "=", 2) envMap[tmp[0]] = tmp[1] } return &EnvProxy{ + fromPrefix: fromPrefix, + toPrefix: toPrefix, defaultEnv: envMap, } } -func (s *EnvProxy) SetProxy(fromPrefix, toPrefix string) { - s.fromPrefix = fromPrefix - s.toPrefix = toPrefix -} - func (s *EnvProxy) Apply() { if s.fromPrefix == "" || s.toPrefix == "" { return diff --git a/pkg/envproxy/env_proxy_test.go b/pkg/envproxy/env_proxy_test.go index 80c3793b..358a0e99 100644 --- a/pkg/envproxy/env_proxy_test.go +++ b/pkg/envproxy/env_proxy_test.go @@ -37,12 +37,12 @@ func TestEnvProxy(t *testing.T) { initialEnv: []string{"TEST_DCTL_S3_PROFILE=test_dctl_s3_profile", "TEST_AWS_PROFILE=test_aws_profile"}, modifiedEnv: []string{"TEST_DCTL_S3_PROFILE=test_dctl_s3_profile", "TEST_AWS_PROFILE=test_aws_profile"}, }, - { - name: "Without initialEnv", - proxyArgs: []string{"TEST_DCTL_S3_", "TEST_AWS_"}, - initialEnv: []string{}, - modifiedEnv: []string{}, - }, + { + name: "Without initialEnv", + proxyArgs: []string{"TEST_DCTL_S3_", "TEST_AWS_"}, + initialEnv: []string{}, + modifiedEnv: []string{}, + }, } for _, tt := range tests { @@ -53,8 +53,7 @@ func TestEnvProxy(t *testing.T) { os.Setenv(tmp[0], tmp[1]) } - envProxy := NewEnvProxy() - envProxy.SetProxy(tt.proxyArgs[0], tt.proxyArgs[1]) + envProxy := NewEnvProxy(tt.proxyArgs[0], tt.proxyArgs[1]) envProxy.Apply() diff --git a/pkg/iac/terraform/state/enumerator/s3.go b/pkg/iac/terraform/state/enumerator/s3.go index cf4932a8..7a1c74aa 100644 --- a/pkg/iac/terraform/state/enumerator/s3.go +++ b/pkg/iac/terraform/state/enumerator/s3.go @@ -21,8 +21,7 @@ type S3Enumerator struct { } func NewS3Enumerator(config config.SupplierConfig) *S3Enumerator { - envProxy := envproxy.NewEnvProxy() - envProxy.SetProxy("DCTL_S3_", "AWS_") + envProxy := envproxy.NewEnvProxy("DCTL_S3_", "AWS_") envProxy.Apply() sess := session.Must(session.NewSessionWithOptions(session.Options{ SharedConfigState: session.SharedConfigEnable, From 973b67f6d59f5009476fcb78b3bfb3b4ff357296 Mon Sep 17 00:00:00 2001 From: Louis TOUSSAINT Date: Thu, 1 Jul 2021 17:47:38 +0200 Subject: [PATCH 5/6] Issue 615: Add a new test to s3_test.go to check if NewS3Enumerator take DCTL_S3_envVar in charge --- pkg/iac/terraform/state/enumerator/s3_test.go | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/pkg/iac/terraform/state/enumerator/s3_test.go b/pkg/iac/terraform/state/enumerator/s3_test.go index 2dba2104..b9fb1a5b 100644 --- a/pkg/iac/terraform/state/enumerator/s3_test.go +++ b/pkg/iac/terraform/state/enumerator/s3_test.go @@ -2,6 +2,7 @@ package enumerator import ( "errors" + "os" "reflect" "testing" @@ -12,6 +13,52 @@ import ( "github.com/stretchr/testify/mock" ) +func TestS3Enumerator_NewS3Enumerator(t *testing.T) { + tests := []struct { + name string + config config.SupplierConfig + setEnv map[string]string + want string + }{ + { + name: "test with no proxy env var", + config: config.SupplierConfig{ + Key: "tfstate", + Backend: "s3", + Path: "terraform.tfstate", + }, + setEnv: map[string]string{ + "AWS_DEFAULT_REGION": "us-east-1", + }, + want: "us-east-1", + }, + { + name: "test with proxy env var", + config: config.SupplierConfig{ + Key: "tfstate", + Backend: "s3", + Path: "terraform.tfstate", + }, + setEnv: map[string]string{ + "AWS_DEFAULT_REGION": "us-east-1", + "DCTL_S3_DEFAULT_REGION": "eu-west-3", + }, + want: "eu-west-3", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for key, value := range tt.setEnv { + os.Setenv(key, value) + } + got := NewS3Enumerator(tt.config).client.(*s3.S3).Config.Region + if awssdk.StringValue(got) != tt.want { + t.Errorf("NewS3Enumerator().client.Config.Region got = %v, want %v", got, tt.want) + } + }) + } +} + func TestS3Enumerator_Enumerate(t *testing.T) { tests := []struct { name string From 8fb10e0b3a902135faabfd8511412d9d4d74bc98 Mon Sep 17 00:00:00 2001 From: Louis TOUSSAINT Date: Fri, 2 Jul 2021 14:48:25 +0200 Subject: [PATCH 6/6] Issue 615: Add envProxy and tests to s3Reader --- pkg/iac/terraform/state/backend/s3_reader.go | 4 +++ .../terraform/state/backend/s3_reader_test.go | 25 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/pkg/iac/terraform/state/backend/s3_reader.go b/pkg/iac/terraform/state/backend/s3_reader.go index e71276fb..e4c657e3 100644 --- a/pkg/iac/terraform/state/backend/s3_reader.go +++ b/pkg/iac/terraform/state/backend/s3_reader.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/aws/aws-sdk-go/aws/session" + "github.com/cloudskiff/driftctl/pkg/envproxy" "github.com/pkg/errors" "github.com/aws/aws-sdk-go/service/s3" @@ -33,9 +34,12 @@ func NewS3Reader(path string) (*S3Backend, error) { Key: &key, Bucket: &bucket, } + envProxy := envproxy.NewEnvProxy("DCTL_S3_", "AWS_") + envProxy.Apply() sess := session.Must(session.NewSessionWithOptions(session.Options{ SharedConfigState: session.SharedConfigEnable, })) + envProxy.Restore() backend.S3Client = s3.New(sess) return &backend, nil } diff --git a/pkg/iac/terraform/state/backend/s3_reader_test.go b/pkg/iac/terraform/state/backend/s3_reader_test.go index 064fdea2..503f209a 100644 --- a/pkg/iac/terraform/state/backend/s3_reader_test.go +++ b/pkg/iac/terraform/state/backend/s3_reader_test.go @@ -67,6 +67,31 @@ func TestNewS3Reader(t *testing.T) { ) } +func TestNewS3ReaderWithEnvProxy(t *testing.T) { + assert := assert.New(t) + os.Setenv("AWS_DEFAULT_REGION", "us-east-1") + os.Setenv("DCTL_S3_DEFAULT_REGION", "eu-west-3") + reader, err := NewS3Reader("sample_bucket/path/to/state.tfstate") + + got := reader.S3Client.(*s3.S3).Config.Region + if aws.StringValue(got) != "eu-west-3" { + t.Errorf("NewS3Reader().S3Client.Config.Region got = %v, want %v", aws.StringValue(got), "eu-west-3") + } + + if err != nil { + t.Error(err) + } + + assert.Equal( + "path/to/state.tfstate", + *reader.input.Key, + ) + assert.Equal( + "sample_bucket", + *reader.input.Bucket, + ) +} + func TestS3Backend_ReadWithError(t *testing.T) { assert := assert.New(t) fakeS3 := &awstest.MockFakeS3{}