Merge pull request #578 from cloudskiff/feat/cacheS3repository

Implement cache in AWS S3 repository
main
Elie 2021-06-09 16:58:52 +02:00 committed by GitHub
commit b95b550c84
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 134 additions and 38 deletions

View File

@ -36,7 +36,7 @@ func Init(version string, alerter *alerter.Alerter,
repositoryCache := cache.New(100)
s3Repository := repository.NewS3Repository(client.NewAWSClientFactory(provider.session))
s3Repository := repository.NewS3Repository(client.NewAWSClientFactory(provider.session), repositoryCache)
ec2repository := repository.NewEC2Repository(provider.session, repositoryCache)
route53repository := repository.NewRoute53Repository(provider.session, repositoryCache)
lambdaRepository := repository.NewLambdaRepository(provider.session, repositoryCache)

View File

@ -1,10 +1,13 @@
package repository
import (
"fmt"
awssdk "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/cloudskiff/driftctl/pkg/remote/aws/client"
"github.com/cloudskiff/driftctl/pkg/remote/cache"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
@ -19,33 +22,44 @@ type S3Repository interface {
type s3Repository struct {
clientFactory client.AwsClientFactoryInterface
cache cache.Cache
}
func NewS3Repository(factory client.AwsClientFactoryInterface) *s3Repository {
func NewS3Repository(factory client.AwsClientFactoryInterface, c cache.Cache) *s3Repository {
return &s3Repository{
factory,
c,
}
}
func (s *s3Repository) ListAllBuckets() ([]*s3.Bucket, error) {
if v := s.cache.Get("s3ListAllBuckets"); v != nil {
return v.([]*s3.Bucket), nil
}
out, err := s.clientFactory.GetS3Client(nil).ListBuckets(&s3.ListBucketsInput{})
if err != nil {
return nil, err
}
s.cache.Put("s3ListAllBuckets", out.Buckets)
return out.Buckets, nil
}
func (s *s3Repository) ListBucketInventoryConfigurations(bucket *s3.Bucket, region string) ([]*s3.InventoryConfiguration, error) {
cacheKey := fmt.Sprintf("s3ListBucketInventoryConfigurations_%s_%s", *bucket.Name, region)
if v := s.cache.Get(cacheKey); v != nil {
return v.([]*s3.InventoryConfiguration), nil
}
inventoryConfigurations := make([]*s3.InventoryConfiguration, 0)
client := s.clientFactory.GetS3Client(&awssdk.Config{Region: &region})
s3client := s.clientFactory.GetS3Client(&awssdk.Config{Region: &region})
request := &s3.ListBucketInventoryConfigurationsInput{
Bucket: bucket.Name,
ContinuationToken: nil,
}
for {
configurations, err := client.ListBucketInventoryConfigurations(request)
configurations, err := s3client.ListBucketInventoryConfigurations(request)
if err != nil {
return nil, errors.Wrapf(
err,
@ -61,19 +75,25 @@ func (s *s3Repository) ListBucketInventoryConfigurations(bucket *s3.Bucket, regi
}
}
s.cache.Put(cacheKey, inventoryConfigurations)
return inventoryConfigurations, nil
}
func (s *s3Repository) ListBucketMetricsConfigurations(bucket *s3.Bucket, region string) ([]*s3.MetricsConfiguration, error) {
cacheKey := fmt.Sprintf("s3ListBucketMetricsConfigurations_%s_%s", *bucket.Name, region)
if v := s.cache.Get(cacheKey); v != nil {
return v.([]*s3.MetricsConfiguration), nil
}
metricsConfigurationList := make([]*s3.MetricsConfiguration, 0)
client := s.clientFactory.GetS3Client(&awssdk.Config{Region: &region})
s3client := s.clientFactory.GetS3Client(&awssdk.Config{Region: &region})
request := &s3.ListBucketMetricsConfigurationsInput{
Bucket: bucket.Name,
ContinuationToken: nil,
}
for {
configurations, err := client.ListBucketMetricsConfigurations(request)
configurations, err := s3client.ListBucketMetricsConfigurations(request)
if err != nil {
return nil, errors.Wrapf(
err,
@ -88,19 +108,26 @@ func (s *s3Repository) ListBucketMetricsConfigurations(bucket *s3.Bucket, region
break
}
}
s.cache.Put(cacheKey, metricsConfigurationList)
return metricsConfigurationList, nil
}
func (s *s3Repository) ListBucketAnalyticsConfigurations(bucket *s3.Bucket, region string) ([]*s3.AnalyticsConfiguration, error) {
cacheKey := fmt.Sprintf("s3ListBucketAnalyticsConfigurations_%s_%s", *bucket.Name, region)
if v := s.cache.Get(cacheKey); v != nil {
return v.([]*s3.AnalyticsConfiguration), nil
}
analyticsConfigurationList := make([]*s3.AnalyticsConfiguration, 0)
client := s.clientFactory.GetS3Client(&awssdk.Config{Region: &region})
s3client := s.clientFactory.GetS3Client(&awssdk.Config{Region: &region})
request := &s3.ListBucketAnalyticsConfigurationsInput{
Bucket: bucket.Name,
ContinuationToken: nil,
}
for {
configurations, err := client.ListBucketAnalyticsConfigurations(request)
configurations, err := s3client.ListBucketAnalyticsConfigurations(request)
if err != nil {
return nil, errors.Wrapf(
err,
@ -117,10 +144,16 @@ func (s *s3Repository) ListBucketAnalyticsConfigurations(bucket *s3.Bucket, regi
}
}
s.cache.Put(cacheKey, analyticsConfigurationList)
return analyticsConfigurationList, nil
}
func (s *s3Repository) GetBucketLocation(bucket *s3.Bucket) (string, error) {
cacheKey := fmt.Sprintf("s3GetBucketLocation_%s", *bucket.Name)
if v := s.cache.Get(cacheKey); v != nil {
return v.(string), nil
}
bucketLocationRequest := s3.GetBucketLocationInput{Bucket: bucket.Name}
bucketLocationResponse, err := s.clientFactory.GetS3Client(nil).GetBucketLocation(&bucketLocationRequest)
if err != nil {
@ -134,15 +167,20 @@ func (s *s3Repository) GetBucketLocation(bucket *s3.Bucket) (string, error) {
return "", err
}
var location string
// Buckets in Region us-east-1 have a LocationConstraint of null.
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLocation.html#API_GetBucketLocation_ResponseSyntax
if bucketLocationResponse.LocationConstraint == nil {
return "us-east-1", err
location = "us-east-1"
} else {
location = *bucketLocationResponse.LocationConstraint
}
if *bucketLocationResponse.LocationConstraint == "EU" {
return "eu-west-1", err
if location == "EU" {
location = "eu-west-1"
}
return *bucketLocationResponse.LocationConstraint, nil
s.cache.Put(cacheKey, location)
return location, nil
}

View File

@ -1,6 +1,7 @@
package repository
import (
"fmt"
"strings"
"testing"
@ -9,6 +10,7 @@ import (
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/cloudskiff/driftctl/pkg/remote/aws/client"
"github.com/cloudskiff/driftctl/pkg/remote/cache"
awstest "github.com/cloudskiff/driftctl/test/aws"
"github.com/pkg/errors"
"github.com/r3labs/diff/v2"
@ -35,7 +37,7 @@ func Test_s3Repository_ListAllBuckets(t *testing.T) {
},
},
nil,
)
).Once()
},
want: []*s3.Bucket{
{Name: aws.String("bucket1")},
@ -49,7 +51,7 @@ func Test_s3Repository_ListAllBuckets(t *testing.T) {
client.On("ListBuckets", &s3.ListBucketsInput{}).Return(
nil,
awserr.NewRequestFailure(nil, 403, ""),
)
).Once()
},
want: nil,
wantErr: awserr.NewRequestFailure(nil, 403, ""),
@ -57,14 +59,24 @@ func Test_s3Repository_ListAllBuckets(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
store := cache.New(1)
mockedClient := &awstest.MockFakeS3{}
tt.mocks(mockedClient)
factory := client.MockAwsClientFactoryInterface{}
factory.On("GetS3Client", (*aws.Config)(nil)).Return(mockedClient).Once()
r := NewS3Repository(&factory)
r := NewS3Repository(&factory, store)
got, err := r.ListAllBuckets()
factory.AssertExpectations(t)
assert.Equal(t, tt.wantErr, err)
if err == nil {
// Check that results were cached
cachedData, err := r.ListAllBuckets()
assert.NoError(t, err)
assert.Equal(t, got, cachedData)
assert.IsType(t, []*s3.Bucket{}, store.Get("s3ListAllBuckets"))
}
changelog, err := diff.Diff(got, tt.want)
assert.Nil(t, err)
if len(changelog) > 0 {
@ -117,7 +129,7 @@ func Test_s3Repository_ListBucketInventoryConfigurations(t *testing.T) {
NextContinuationToken: awssdk.String("nexttoken"),
},
nil,
)
).Once()
client.On(
"ListBucketInventoryConfigurations",
&s3.ListBucketInventoryConfigurationsInput{
@ -134,7 +146,7 @@ func Test_s3Repository_ListBucketInventoryConfigurations(t *testing.T) {
IsTruncated: awssdk.Bool(false),
},
nil,
)
).Once()
},
want: []*s3.InventoryConfiguration{
{Id: awssdk.String("config1")},
@ -165,7 +177,7 @@ func Test_s3Repository_ListBucketInventoryConfigurations(t *testing.T) {
).Return(
nil,
errors.New("aws error"),
)
).Once()
},
want: nil,
wantErr: "Error listing bucket inventory configuration test-bucket: aws error",
@ -173,11 +185,12 @@ func Test_s3Repository_ListBucketInventoryConfigurations(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
store := cache.New(1)
mockedClient := &awstest.MockFakeS3{}
tt.mocks(mockedClient)
factory := client.MockAwsClientFactoryInterface{}
factory.On("GetS3Client", &aws.Config{Region: awssdk.String(tt.input.region)}).Return(mockedClient).Once()
r := NewS3Repository(&factory)
r := NewS3Repository(&factory, store)
got, err := r.ListBucketInventoryConfigurations(&tt.input.bucket, tt.input.region)
factory.AssertExpectations(t)
if err != nil && tt.wantErr == "" {
@ -186,6 +199,15 @@ func Test_s3Repository_ListBucketInventoryConfigurations(t *testing.T) {
if err != nil {
assert.Equal(t, tt.wantErr, err.Error())
}
if err == nil {
// Check that results were cached
cachedData, err := r.ListBucketInventoryConfigurations(&tt.input.bucket, tt.input.region)
assert.NoError(t, err)
assert.Equal(t, got, cachedData)
assert.IsType(t, []*s3.InventoryConfiguration{}, store.Get(fmt.Sprintf("s3ListBucketInventoryConfigurations_%s_%s", *tt.input.bucket.Name, tt.input.region)))
}
changelog, err := diff.Diff(got, tt.want)
assert.Nil(t, err)
if len(changelog) > 0 {
@ -238,7 +260,7 @@ func Test_s3Repository_ListBucketMetricsConfigurations(t *testing.T) {
NextContinuationToken: awssdk.String("nexttoken"),
},
nil,
)
).Once()
client.On(
"ListBucketMetricsConfigurations",
&s3.ListBucketMetricsConfigurationsInput{
@ -255,7 +277,7 @@ func Test_s3Repository_ListBucketMetricsConfigurations(t *testing.T) {
IsTruncated: awssdk.Bool(false),
},
nil,
)
).Once()
},
want: []*s3.MetricsConfiguration{
{Id: awssdk.String("metric1")},
@ -286,7 +308,7 @@ func Test_s3Repository_ListBucketMetricsConfigurations(t *testing.T) {
).Return(
nil,
errors.New("aws error"),
)
).Once()
},
want: nil,
wantErr: "Error listing bucket metrics configuration test-bucket: aws error",
@ -294,11 +316,12 @@ func Test_s3Repository_ListBucketMetricsConfigurations(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
store := cache.New(1)
mockedClient := &awstest.MockFakeS3{}
tt.mocks(mockedClient)
factory := client.MockAwsClientFactoryInterface{}
factory.On("GetS3Client", &aws.Config{Region: awssdk.String(tt.input.region)}).Return(mockedClient).Once()
r := NewS3Repository(&factory)
r := NewS3Repository(&factory, store)
got, err := r.ListBucketMetricsConfigurations(&tt.input.bucket, tt.input.region)
factory.AssertExpectations(t)
if err != nil && tt.wantErr == "" {
@ -307,6 +330,15 @@ func Test_s3Repository_ListBucketMetricsConfigurations(t *testing.T) {
if err != nil {
assert.Equal(t, tt.wantErr, err.Error())
}
if err == nil {
// Check that results were cached
cachedData, err := r.ListBucketMetricsConfigurations(&tt.input.bucket, tt.input.region)
assert.NoError(t, err)
assert.Equal(t, got, cachedData)
assert.IsType(t, []*s3.MetricsConfiguration{}, store.Get(fmt.Sprintf("s3ListBucketMetricsConfigurations_%s_%s", *tt.input.bucket.Name, tt.input.region)))
}
changelog, err := diff.Diff(got, tt.want)
assert.Nil(t, err)
if len(changelog) > 0 {
@ -359,7 +391,7 @@ func Test_s3Repository_ListBucketAnalyticsConfigurations(t *testing.T) {
NextContinuationToken: awssdk.String("nexttoken"),
},
nil,
)
).Once()
client.On(
"ListBucketAnalyticsConfigurations",
&s3.ListBucketAnalyticsConfigurationsInput{
@ -376,7 +408,7 @@ func Test_s3Repository_ListBucketAnalyticsConfigurations(t *testing.T) {
IsTruncated: awssdk.Bool(false),
},
nil,
)
).Once()
},
want: []*s3.AnalyticsConfiguration{
{Id: awssdk.String("analytic1")},
@ -407,7 +439,7 @@ func Test_s3Repository_ListBucketAnalyticsConfigurations(t *testing.T) {
).Return(
nil,
errors.New("aws error"),
)
).Once()
},
want: nil,
wantErr: "Error listing bucket analytics configuration test-bucket: aws error",
@ -415,11 +447,12 @@ func Test_s3Repository_ListBucketAnalyticsConfigurations(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
store := cache.New(1)
mockedClient := &awstest.MockFakeS3{}
tt.mocks(mockedClient)
factory := client.MockAwsClientFactoryInterface{}
factory.On("GetS3Client", &aws.Config{Region: awssdk.String(tt.input.region)}).Return(mockedClient).Once()
r := NewS3Repository(&factory)
r := NewS3Repository(&factory, store)
got, err := r.ListBucketAnalyticsConfigurations(&tt.input.bucket, tt.input.region)
factory.AssertExpectations(t)
if err != nil && tt.wantErr == "" {
@ -428,6 +461,15 @@ func Test_s3Repository_ListBucketAnalyticsConfigurations(t *testing.T) {
if err != nil {
assert.Equal(t, tt.wantErr, err.Error())
}
if err == nil {
// Check that results were cached
cachedData, err := r.ListBucketAnalyticsConfigurations(&tt.input.bucket, tt.input.region)
assert.NoError(t, err)
assert.Equal(t, got, cachedData)
assert.IsType(t, []*s3.AnalyticsConfiguration{}, store.Get(fmt.Sprintf("s3ListBucketAnalyticsConfigurations_%s_%s", *tt.input.bucket.Name, tt.input.region)))
}
changelog, err := diff.Diff(got, tt.want)
assert.Nil(t, err)
if len(changelog) > 0 {
@ -462,7 +504,7 @@ func Test_s3Repository_GetBucketLocation(t *testing.T) {
LocationConstraint: awssdk.String("eu-east-1"),
},
nil,
)
).Once()
},
want: "eu-east-1",
},
@ -477,7 +519,7 @@ func Test_s3Repository_GetBucketLocation(t *testing.T) {
}).Return(
&s3.GetBucketLocationOutput{},
nil,
)
).Once()
},
want: "us-east-1",
},
@ -492,7 +534,7 @@ func Test_s3Repository_GetBucketLocation(t *testing.T) {
}).Return(
nil,
awserr.New(s3.ErrCodeNoSuchBucket, "", nil),
)
).Once()
},
want: "",
},
@ -507,18 +549,19 @@ func Test_s3Repository_GetBucketLocation(t *testing.T) {
}).Return(
nil,
awserr.New("UnknownError", "aws error", nil),
)
).Once()
},
wantErr: "UnknownError: aws error",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
store := cache.New(1)
mockedClient := &awstest.MockFakeS3{}
tt.mocks(mockedClient)
factory := client.MockAwsClientFactoryInterface{}
factory.On("GetS3Client", (*aws.Config)(nil)).Return(mockedClient).Once()
r := NewS3Repository(&factory)
r := NewS3Repository(&factory, store)
got, err := r.GetBucketLocation(tt.bucket)
factory.AssertExpectations(t)
if err != nil && tt.wantErr == "" {
@ -527,6 +570,15 @@ func Test_s3Repository_GetBucketLocation(t *testing.T) {
if err != nil {
assert.Equal(t, tt.wantErr, err.Error())
}
if err == nil && tt.want != "" {
// Check that results were cached
cachedData, err := r.GetBucketLocation(tt.bucket)
assert.NoError(t, err)
assert.Equal(t, got, cachedData)
assert.IsType(t, "", store.Get(fmt.Sprintf("s3GetBucketLocation_%s", *tt.bucket.Name)))
}
changelog, err := diff.Diff(got, tt.want)
assert.Nil(t, err)
if len(changelog) > 0 {

View File

@ -10,6 +10,7 @@ import (
"github.com/cloudskiff/driftctl/pkg/parallel"
"github.com/cloudskiff/driftctl/pkg/remote/aws/client"
"github.com/cloudskiff/driftctl/pkg/remote/aws/repository"
"github.com/cloudskiff/driftctl/pkg/remote/cache"
remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error"
tf "github.com/cloudskiff/driftctl/pkg/remote/terraform"
"github.com/cloudskiff/driftctl/pkg/resource"
@ -135,7 +136,7 @@ func TestS3BucketAnalyticSupplier_Resources(t *testing.T) {
if err != nil {
t.Fatal(err)
}
repository := repository.NewS3Repository(client.NewAWSClientFactory(provider.session))
repository := repository.NewS3Repository(client.NewAWSClientFactory(provider.session), cache.New(0))
supplierLibrary.AddSupplier(NewS3BucketAnalyticSupplier(provider, repository, deserializer))
}

View File

@ -10,6 +10,7 @@ import (
"github.com/cloudskiff/driftctl/pkg/parallel"
"github.com/cloudskiff/driftctl/pkg/remote/aws/client"
"github.com/cloudskiff/driftctl/pkg/remote/aws/repository"
"github.com/cloudskiff/driftctl/pkg/remote/cache"
remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error"
tf "github.com/cloudskiff/driftctl/pkg/remote/terraform"
"github.com/cloudskiff/driftctl/pkg/resource"
@ -131,7 +132,7 @@ func TestS3BucketInventorySupplier_Resources(t *testing.T) {
t.Fatal(err)
}
repository := repository.NewS3Repository(client.NewAWSClientFactory(provider.session))
repository := repository.NewS3Repository(client.NewAWSClientFactory(provider.session), cache.New(0))
supplierLibrary.AddSupplier(NewS3BucketInventorySupplier(provider, repository, deserializer))
}

View File

@ -10,6 +10,7 @@ import (
"github.com/cloudskiff/driftctl/pkg/parallel"
"github.com/cloudskiff/driftctl/pkg/remote/aws/client"
"github.com/cloudskiff/driftctl/pkg/remote/aws/repository"
"github.com/cloudskiff/driftctl/pkg/remote/cache"
remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error"
tf "github.com/cloudskiff/driftctl/pkg/remote/terraform"
"github.com/cloudskiff/driftctl/pkg/resource"
@ -132,7 +133,7 @@ func TestS3BucketMetricSupplier_Resources(t *testing.T) {
if err != nil {
t.Fatal(err)
}
repository := repository.NewS3Repository(client.NewAWSClientFactory(provider.session))
repository := repository.NewS3Repository(client.NewAWSClientFactory(provider.session), cache.New(0))
supplierLibrary.AddSupplier(NewS3BucketMetricSupplier(provider, repository, deserializer))
}

View File

@ -10,6 +10,7 @@ import (
"github.com/cloudskiff/driftctl/pkg/parallel"
"github.com/cloudskiff/driftctl/pkg/remote/aws/client"
"github.com/cloudskiff/driftctl/pkg/remote/aws/repository"
"github.com/cloudskiff/driftctl/pkg/remote/cache"
remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error"
tf "github.com/cloudskiff/driftctl/pkg/remote/terraform"
"github.com/cloudskiff/driftctl/pkg/resource"
@ -110,7 +111,7 @@ func TestS3BucketNotificationSupplier_Resources(t *testing.T) {
if err != nil {
t.Fatal(err)
}
repository := repository.NewS3Repository(client.NewAWSClientFactory(provider.session))
repository := repository.NewS3Repository(client.NewAWSClientFactory(provider.session), cache.New(0))
supplierLibrary.AddSupplier(NewS3BucketNotificationSupplier(provider, repository, deserializer))
}

View File

@ -10,6 +10,7 @@ import (
"github.com/cloudskiff/driftctl/pkg/parallel"
"github.com/cloudskiff/driftctl/pkg/remote/aws/client"
"github.com/cloudskiff/driftctl/pkg/remote/aws/repository"
"github.com/cloudskiff/driftctl/pkg/remote/cache"
remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error"
tf "github.com/cloudskiff/driftctl/pkg/remote/terraform"
"github.com/cloudskiff/driftctl/pkg/resource"
@ -111,7 +112,7 @@ func TestS3BucketPolicySupplier_Resources(t *testing.T) {
if err != nil {
t.Fatal(err)
}
repository := repository.NewS3Repository(client.NewAWSClientFactory(provider.session))
repository := repository.NewS3Repository(client.NewAWSClientFactory(provider.session), cache.New(0))
supplierLibrary.AddSupplier(NewS3BucketPolicySupplier(provider, repository, deserializer))
}

View File

@ -7,6 +7,7 @@ import (
awssdk "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/cloudskiff/driftctl/pkg/remote/cache"
testresource "github.com/cloudskiff/driftctl/test/resource"
"github.com/cloudskiff/driftctl/pkg/parallel"
@ -92,7 +93,7 @@ func TestS3BucketSupplier_Resources(t *testing.T) {
if err != nil {
t.Fatal(err)
}
repository := repository.NewS3Repository(client.NewAWSClientFactory(provider.session))
repository := repository.NewS3Repository(client.NewAWSClientFactory(provider.session), cache.New(0))
supplierLibrary.AddSupplier(NewS3BucketSupplier(provider, repository, deserializer))
}