From 44a67075237026b0145bbc39fc5ee739b32091fe Mon Sep 17 00:00:00 2001 From: sundowndev Date: Tue, 25 May 2021 16:45:59 +0200 Subject: [PATCH] refactor: use a global cache for all repositories --- pkg/remote/aws/ec2_ami_supplier.go | 5 +++-- pkg/remote/aws/ec2_ami_supplier_test.go | 3 ++- pkg/remote/aws/ec2_ebs_snapshot_supplier.go | 5 +++-- .../aws/ec2_ebs_snapshot_supplier_test.go | 3 ++- pkg/remote/aws/ec2_ebs_volume_supplier.go | 5 +++-- pkg/remote/aws/ec2_ebs_volume_supplier_test.go | 3 ++- pkg/remote/aws/ec2_eip_association_supplier.go | 5 +++-- .../aws/ec2_eip_association_supplier_test.go | 3 ++- pkg/remote/aws/ec2_eip_supplier.go | 5 +++-- pkg/remote/aws/ec2_eip_supplier_test.go | 3 ++- pkg/remote/aws/ec2_instance_supplier.go | 5 +++-- pkg/remote/aws/ec2_instance_supplier_test.go | 3 ++- pkg/remote/aws/ec2_key_pair_supplier.go | 5 +++-- pkg/remote/aws/ec2_key_pair_supplier_test.go | 3 ++- pkg/remote/aws/init.go | 17 ++++++++++------- pkg/remote/aws/repository/ec2_repository.go | 4 ++-- pkg/remote/cache/cache.go | 9 ++++----- 17 files changed, 51 insertions(+), 35 deletions(-) diff --git a/pkg/remote/aws/ec2_ami_supplier.go b/pkg/remote/aws/ec2_ami_supplier.go index 4760ea7f..3181c705 100644 --- a/pkg/remote/aws/ec2_ami_supplier.go +++ b/pkg/remote/aws/ec2_ami_supplier.go @@ -2,6 +2,7 @@ package aws import ( "github.com/cloudskiff/driftctl/pkg/remote/aws/repository" + "github.com/cloudskiff/driftctl/pkg/remote/cache" remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error" "github.com/cloudskiff/driftctl/pkg/resource" @@ -21,11 +22,11 @@ type EC2AmiSupplier struct { runner *terraform.ParallelResourceReader } -func NewEC2AmiSupplier(provider *AWSTerraformProvider, deserializer *resource.Deserializer) *EC2AmiSupplier { +func NewEC2AmiSupplier(provider *AWSTerraformProvider, deserializer *resource.Deserializer, c cache.Cache) *EC2AmiSupplier { return &EC2AmiSupplier{ provider, deserializer, - repository.NewEC2Repository(provider.session), + repository.NewEC2Repository(provider.session, c), terraform.NewParallelResourceReader(provider.Runner().SubRunner()), } } diff --git a/pkg/remote/aws/ec2_ami_supplier_test.go b/pkg/remote/aws/ec2_ami_supplier_test.go index 053035ab..38a6375a 100644 --- a/pkg/remote/aws/ec2_ami_supplier_test.go +++ b/pkg/remote/aws/ec2_ami_supplier_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/aws/aws-sdk-go/aws" + "github.com/cloudskiff/driftctl/pkg/remote/cache" testresource "github.com/cloudskiff/driftctl/test/resource" "github.com/aws/aws-sdk-go/service/ec2" @@ -78,7 +79,7 @@ func TestEC2AmiSupplier_Resources(t *testing.T) { if err != nil { t.Fatal(err) } - supplierLibrary.AddSupplier(NewEC2AmiSupplier(provider, deserializer)) + supplierLibrary.AddSupplier(NewEC2AmiSupplier(provider, deserializer, cache.New(0))) } t.Run(tt.test, func(t *testing.T) { diff --git a/pkg/remote/aws/ec2_ebs_snapshot_supplier.go b/pkg/remote/aws/ec2_ebs_snapshot_supplier.go index bf32a92a..b332c3fd 100644 --- a/pkg/remote/aws/ec2_ebs_snapshot_supplier.go +++ b/pkg/remote/aws/ec2_ebs_snapshot_supplier.go @@ -2,6 +2,7 @@ package aws import ( "github.com/cloudskiff/driftctl/pkg/remote/aws/repository" + "github.com/cloudskiff/driftctl/pkg/remote/cache" remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error" "github.com/cloudskiff/driftctl/pkg/resource" @@ -22,11 +23,11 @@ type EC2EbsSnapshotSupplier struct { runner *terraform.ParallelResourceReader } -func NewEC2EbsSnapshotSupplier(provider *AWSTerraformProvider, deserializer *resource.Deserializer) *EC2EbsSnapshotSupplier { +func NewEC2EbsSnapshotSupplier(provider *AWSTerraformProvider, deserializer *resource.Deserializer, c cache.Cache) *EC2EbsSnapshotSupplier { return &EC2EbsSnapshotSupplier{ provider, deserializer, - repository.NewEC2Repository(provider.session), + repository.NewEC2Repository(provider.session, c), terraform.NewParallelResourceReader(provider.Runner().SubRunner()), } } diff --git a/pkg/remote/aws/ec2_ebs_snapshot_supplier_test.go b/pkg/remote/aws/ec2_ebs_snapshot_supplier_test.go index c4dc1021..79867680 100644 --- a/pkg/remote/aws/ec2_ebs_snapshot_supplier_test.go +++ b/pkg/remote/aws/ec2_ebs_snapshot_supplier_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/cloudskiff/driftctl/pkg/remote/aws/repository" + "github.com/cloudskiff/driftctl/pkg/remote/cache" testresource "github.com/cloudskiff/driftctl/test/resource" remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error" @@ -83,7 +84,7 @@ func TestEC2EbsSnapshotSupplier_Resources(t *testing.T) { if err != nil { t.Fatal(err) } - supplierLibrary.AddSupplier(NewEC2EbsSnapshotSupplier(provider, deserializer)) + supplierLibrary.AddSupplier(NewEC2EbsSnapshotSupplier(provider, deserializer, cache.New(0))) } t.Run(tt.test, func(t *testing.T) { diff --git a/pkg/remote/aws/ec2_ebs_volume_supplier.go b/pkg/remote/aws/ec2_ebs_volume_supplier.go index f9e0229f..f17994a0 100644 --- a/pkg/remote/aws/ec2_ebs_volume_supplier.go +++ b/pkg/remote/aws/ec2_ebs_volume_supplier.go @@ -2,6 +2,7 @@ package aws import ( "github.com/cloudskiff/driftctl/pkg/remote/aws/repository" + "github.com/cloudskiff/driftctl/pkg/remote/cache" remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error" "github.com/cloudskiff/driftctl/pkg/resource" @@ -22,11 +23,11 @@ type EC2EbsVolumeSupplier struct { runner *terraform.ParallelResourceReader } -func NewEC2EbsVolumeSupplier(provider *AWSTerraformProvider, deserializer *resource.Deserializer) *EC2EbsVolumeSupplier { +func NewEC2EbsVolumeSupplier(provider *AWSTerraformProvider, deserializer *resource.Deserializer, c cache.Cache) *EC2EbsVolumeSupplier { return &EC2EbsVolumeSupplier{ provider, deserializer, - repository.NewEC2Repository(provider.session), + repository.NewEC2Repository(provider.session, c), terraform.NewParallelResourceReader(provider.Runner().SubRunner()), } } diff --git a/pkg/remote/aws/ec2_ebs_volume_supplier_test.go b/pkg/remote/aws/ec2_ebs_volume_supplier_test.go index 3100b4f3..ff7cadeb 100644 --- a/pkg/remote/aws/ec2_ebs_volume_supplier_test.go +++ b/pkg/remote/aws/ec2_ebs_volume_supplier_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/cloudskiff/driftctl/pkg/remote/aws/repository" + "github.com/cloudskiff/driftctl/pkg/remote/cache" testresource "github.com/cloudskiff/driftctl/test/resource" remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error" @@ -82,7 +83,7 @@ func TestEC2EbsVolumeSupplier_Resources(t *testing.T) { if err != nil { t.Fatal(err) } - supplierLibrary.AddSupplier(NewEC2EbsVolumeSupplier(provider, deserializer)) + supplierLibrary.AddSupplier(NewEC2EbsVolumeSupplier(provider, deserializer, cache.New(0)) } t.Run(tt.test, func(t *testing.T) { diff --git a/pkg/remote/aws/ec2_eip_association_supplier.go b/pkg/remote/aws/ec2_eip_association_supplier.go index 0a007062..b5943285 100644 --- a/pkg/remote/aws/ec2_eip_association_supplier.go +++ b/pkg/remote/aws/ec2_eip_association_supplier.go @@ -2,6 +2,7 @@ package aws import ( "github.com/cloudskiff/driftctl/pkg/remote/aws/repository" + "github.com/cloudskiff/driftctl/pkg/remote/cache" remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error" "github.com/cloudskiff/driftctl/pkg/resource" @@ -20,11 +21,11 @@ type EC2EipAssociationSupplier struct { runner *terraform.ParallelResourceReader } -func NewEC2EipAssociationSupplier(provider *AWSTerraformProvider, deserializer *resource.Deserializer) *EC2EipAssociationSupplier { +func NewEC2EipAssociationSupplier(provider *AWSTerraformProvider, deserializer *resource.Deserializer, c cache.Cache) *EC2EipAssociationSupplier { return &EC2EipAssociationSupplier{ provider, deserializer, - repository.NewEC2Repository(provider.session), + repository.NewEC2Repository(provider.session, c), terraform.NewParallelResourceReader(provider.Runner().SubRunner())} } diff --git a/pkg/remote/aws/ec2_eip_association_supplier_test.go b/pkg/remote/aws/ec2_eip_association_supplier_test.go index 9417e019..3ef2b899 100644 --- a/pkg/remote/aws/ec2_eip_association_supplier_test.go +++ b/pkg/remote/aws/ec2_eip_association_supplier_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/cloudskiff/driftctl/pkg/remote/aws/repository" + "github.com/cloudskiff/driftctl/pkg/remote/cache" testresource "github.com/cloudskiff/driftctl/test/resource" remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error" @@ -74,7 +75,7 @@ func TestEC2EipAssociationSupplier_Resources(t *testing.T) { if err != nil { t.Fatal(err) } - supplierLibrary.AddSupplier(NewEC2EipAssociationSupplier(provider, deserializer)) + supplierLibrary.AddSupplier(NewEC2EipAssociationSupplier(provider, deserializer, cache.New(0))) } t.Run(tt.test, func(t *testing.T) { diff --git a/pkg/remote/aws/ec2_eip_supplier.go b/pkg/remote/aws/ec2_eip_supplier.go index 6530fb60..2a9bdf3f 100644 --- a/pkg/remote/aws/ec2_eip_supplier.go +++ b/pkg/remote/aws/ec2_eip_supplier.go @@ -2,6 +2,7 @@ package aws import ( "github.com/cloudskiff/driftctl/pkg/remote/aws/repository" + "github.com/cloudskiff/driftctl/pkg/remote/cache" remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error" "github.com/cloudskiff/driftctl/pkg/resource" @@ -22,11 +23,11 @@ type EC2EipSupplier struct { runner *terraform.ParallelResourceReader } -func NewEC2EipSupplier(provider *AWSTerraformProvider, deserializer *resource.Deserializer) *EC2EipSupplier { +func NewEC2EipSupplier(provider *AWSTerraformProvider, deserializer *resource.Deserializer, c cache.Cache) *EC2EipSupplier { return &EC2EipSupplier{ provider, deserializer, - repository.NewEC2Repository(provider.session), + repository.NewEC2Repository(provider.session, c), terraform.NewParallelResourceReader(provider.Runner().SubRunner()), } } diff --git a/pkg/remote/aws/ec2_eip_supplier_test.go b/pkg/remote/aws/ec2_eip_supplier_test.go index 70a0432e..2435e03f 100644 --- a/pkg/remote/aws/ec2_eip_supplier_test.go +++ b/pkg/remote/aws/ec2_eip_supplier_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/cloudskiff/driftctl/pkg/remote/aws/repository" + "github.com/cloudskiff/driftctl/pkg/remote/cache" testresource "github.com/cloudskiff/driftctl/test/resource" remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error" @@ -82,7 +83,7 @@ func TestEC2EipSupplier_Resources(t *testing.T) { if err != nil { t.Fatal(err) } - supplierLibrary.AddSupplier(NewEC2EipSupplier(provider, deserializer)) + supplierLibrary.AddSupplier(NewEC2EipSupplier(provider, deserializer, cache.New(0))) } t.Run(tt.test, func(t *testing.T) { diff --git a/pkg/remote/aws/ec2_instance_supplier.go b/pkg/remote/aws/ec2_instance_supplier.go index fc50350c..fa501a97 100644 --- a/pkg/remote/aws/ec2_instance_supplier.go +++ b/pkg/remote/aws/ec2_instance_supplier.go @@ -2,6 +2,7 @@ package aws import ( "github.com/cloudskiff/driftctl/pkg/remote/aws/repository" + "github.com/cloudskiff/driftctl/pkg/remote/cache" remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error" "github.com/cloudskiff/driftctl/pkg/resource" @@ -21,11 +22,11 @@ type EC2InstanceSupplier struct { runner *terraform.ParallelResourceReader } -func NewEC2InstanceSupplier(provider *AWSTerraformProvider, deserializer *resource.Deserializer) *EC2InstanceSupplier { +func NewEC2InstanceSupplier(provider *AWSTerraformProvider, deserializer *resource.Deserializer, c cache.Cache) *EC2InstanceSupplier { return &EC2InstanceSupplier{ provider, deserializer, - repository.NewEC2Repository(provider.session), + repository.NewEC2Repository(provider.session, c), terraform.NewParallelResourceReader(provider.Runner().SubRunner()), } } diff --git a/pkg/remote/aws/ec2_instance_supplier_test.go b/pkg/remote/aws/ec2_instance_supplier_test.go index dfe5c9b1..0176d585 100644 --- a/pkg/remote/aws/ec2_instance_supplier_test.go +++ b/pkg/remote/aws/ec2_instance_supplier_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/cloudskiff/driftctl/pkg/remote/aws/repository" + "github.com/cloudskiff/driftctl/pkg/remote/cache" testresource "github.com/cloudskiff/driftctl/test/resource" remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error" @@ -97,7 +98,7 @@ func TestEC2InstanceSupplier_Resources(t *testing.T) { if err != nil { t.Fatal(err) } - supplierLibrary.AddSupplier(NewEC2InstanceSupplier(provider, deserializer)) + supplierLibrary.AddSupplier(NewEC2InstanceSupplier(provider, deserializer, cache.New(0)) } t.Run(tt.test, func(t *testing.T) { diff --git a/pkg/remote/aws/ec2_key_pair_supplier.go b/pkg/remote/aws/ec2_key_pair_supplier.go index 2d9dd6d9..850e0c2a 100644 --- a/pkg/remote/aws/ec2_key_pair_supplier.go +++ b/pkg/remote/aws/ec2_key_pair_supplier.go @@ -2,6 +2,7 @@ package aws import ( "github.com/cloudskiff/driftctl/pkg/remote/aws/repository" + "github.com/cloudskiff/driftctl/pkg/remote/cache" remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error" "github.com/cloudskiff/driftctl/pkg/resource" @@ -21,11 +22,11 @@ type EC2KeyPairSupplier struct { runner *terraform.ParallelResourceReader } -func NewEC2KeyPairSupplier(provider *AWSTerraformProvider, deserializer *resource.Deserializer) *EC2KeyPairSupplier { +func NewEC2KeyPairSupplier(provider *AWSTerraformProvider, deserializer *resource.Deserializer, c cache.Cache) *EC2KeyPairSupplier { return &EC2KeyPairSupplier{ provider, deserializer, - repository.NewEC2Repository(provider.session), + repository.NewEC2Repository(provider.session, c), terraform.NewParallelResourceReader(provider.Runner().SubRunner()), } } diff --git a/pkg/remote/aws/ec2_key_pair_supplier_test.go b/pkg/remote/aws/ec2_key_pair_supplier_test.go index cba1637d..26ccb031 100644 --- a/pkg/remote/aws/ec2_key_pair_supplier_test.go +++ b/pkg/remote/aws/ec2_key_pair_supplier_test.go @@ -6,6 +6,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/cloudskiff/driftctl/pkg/remote/aws/repository" + "github.com/cloudskiff/driftctl/pkg/remote/cache" testresource "github.com/cloudskiff/driftctl/test/resource" remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error" @@ -80,7 +81,7 @@ func TestEC2KeyPairSupplier_Resources(t *testing.T) { if err != nil { t.Fatal(err) } - supplierLibrary.AddSupplier(NewEC2KeyPairSupplier(provider, deserializer)) + supplierLibrary.AddSupplier(NewEC2KeyPairSupplier(provider, deserializer, cache.New(0))) } t.Run(tt.test, func(t *testing.T) { diff --git a/pkg/remote/aws/init.go b/pkg/remote/aws/init.go index e1b14ce1..d4b58963 100644 --- a/pkg/remote/aws/init.go +++ b/pkg/remote/aws/init.go @@ -5,6 +5,7 @@ import ( "github.com/cloudskiff/driftctl/pkg/output" "github.com/cloudskiff/driftctl/pkg/remote/aws/client" "github.com/cloudskiff/driftctl/pkg/remote/aws/repository" + "github.com/cloudskiff/driftctl/pkg/remote/cache" "github.com/cloudskiff/driftctl/pkg/resource" "github.com/cloudskiff/driftctl/pkg/resource/aws" "github.com/cloudskiff/driftctl/pkg/terraform" @@ -33,6 +34,8 @@ func Init(alerter *alerter.Alerter, return err } + repositoryCache := cache.New(5 * 1024) + s3Repository := repository.NewS3Repository(client.NewAWSClientFactory(provider.session)) deserializer := resource.NewDeserializer(factory) providerLibrary.AddProvider(terraform.AWS, provider) @@ -43,15 +46,15 @@ func Init(alerter *alerter.Alerter, supplierLibrary.AddSupplier(NewS3BucketMetricSupplier(provider, s3Repository, deserializer)) supplierLibrary.AddSupplier(NewS3BucketNotificationSupplier(provider, s3Repository, deserializer)) supplierLibrary.AddSupplier(NewS3BucketPolicySupplier(provider, s3Repository, deserializer)) - supplierLibrary.AddSupplier(NewEC2EipSupplier(provider, deserializer)) - supplierLibrary.AddSupplier(NewEC2EipAssociationSupplier(provider, deserializer)) - supplierLibrary.AddSupplier(NewEC2EbsVolumeSupplier(provider, deserializer)) - supplierLibrary.AddSupplier(NewEC2EbsSnapshotSupplier(provider, deserializer)) + supplierLibrary.AddSupplier(NewEC2EipSupplier(provider, deserializer, repositoryCache)) + supplierLibrary.AddSupplier(NewEC2EipAssociationSupplier(provider, deserializer, repositoryCache)) + supplierLibrary.AddSupplier(NewEC2EbsVolumeSupplier(provider, deserializer, repositoryCache)) + supplierLibrary.AddSupplier(NewEC2EbsSnapshotSupplier(provider, deserializer, repositoryCache)) supplierLibrary.AddSupplier(NewRoute53ZoneSupplier(provider, deserializer)) supplierLibrary.AddSupplier(NewRoute53RecordSupplier(provider, deserializer)) - supplierLibrary.AddSupplier(NewEC2InstanceSupplier(provider, deserializer)) - supplierLibrary.AddSupplier(NewEC2AmiSupplier(provider, deserializer)) - supplierLibrary.AddSupplier(NewEC2KeyPairSupplier(provider, deserializer)) + supplierLibrary.AddSupplier(NewEC2InstanceSupplier(provider, deserializer, repositoryCache)) + supplierLibrary.AddSupplier(NewEC2AmiSupplier(provider, deserializer, repositoryCache)) + supplierLibrary.AddSupplier(NewEC2KeyPairSupplier(provider, deserializer, repositoryCache)) supplierLibrary.AddSupplier(NewLambdaFunctionSupplier(provider, deserializer)) supplierLibrary.AddSupplier(NewDBSubnetGroupSupplier(provider, deserializer)) supplierLibrary.AddSupplier(NewDBInstanceSupplier(provider, deserializer)) diff --git a/pkg/remote/aws/repository/ec2_repository.go b/pkg/remote/aws/repository/ec2_repository.go index 03c68aa5..4744641b 100644 --- a/pkg/remote/aws/repository/ec2_repository.go +++ b/pkg/remote/aws/repository/ec2_repository.go @@ -27,10 +27,10 @@ type ec2Repository struct { cache cache.Cache } -func NewEC2Repository(session *session.Session) *ec2Repository { +func NewEC2Repository(session *session.Session, c cache.Cache) *ec2Repository { return &ec2Repository{ ec2.New(session), - cache.New(5 * 1024), + c, } } diff --git a/pkg/remote/cache/cache.go b/pkg/remote/cache/cache.go index 00afefe0..9b19740b 100644 --- a/pkg/remote/cache/cache.go +++ b/pkg/remote/cache/cache.go @@ -45,7 +45,7 @@ func (c *LRUCache) Get(key string) interface{} { return nil } -func (c *LRUCache) Put(key string, value interface{}) (overridden bool) { +func (c *LRUCache) Put(key string, value interface{}) bool { c.mu.Lock() defer c.mu.Unlock() @@ -53,11 +53,10 @@ func (c *LRUCache) Put(key string, value interface{}) (overridden bool) { if node, ok := c.m[key]; ok { c.l.MoveToFront(node) node.Value.(*list.Element).Value = pair{key: key, value: value} - overridden = true - return + return true } - // delete the last list node if the list is full + // if the list is full, delete the last element if c.l.Len() == c.cap { idx := c.l.Back().Value.(*list.Element).Value.(pair).key delete(c.m, idx) @@ -74,7 +73,7 @@ func (c *LRUCache) Put(key string, value interface{}) (overridden bool) { element := c.l.PushFront(node) c.m[key] = element - return + return false } func (c *LRUCache) Len() int {