refactor: split vpc supplier

main
sundowndev 2021-06-30 17:30:12 +02:00
parent 9612ab7785
commit a160963596
18 changed files with 321 additions and 304 deletions

View File

@ -73,6 +73,9 @@ func Init(version string, alerter *alerter.Alerter,
remoteLibrary.AddEnumerator(NewEC2EipEnumerator(ec2repository, factory, provider.Config))
remoteLibrary.AddDetailsFetcher(aws.AwsEipResourceType, common.NewGenericDetailFetcher(aws.AwsEipResourceType, provider, deserializer))
remoteLibrary.AddEnumerator(NewVPCEnumerator(ec2repository, factory))
remoteLibrary.AddDetailsFetcher(aws.AwsVpcResourceType, NewVPCDetailsFetcher(provider, deserializer))
supplierLibrary.AddSupplier(NewS3BucketAnalyticSupplier(provider, s3Repository, deserializer))
supplierLibrary.AddSupplier(NewS3BucketMetricSupplier(provider, s3Repository, deserializer))
supplierLibrary.AddSupplier(NewS3BucketPolicySupplier(provider, s3Repository, deserializer))
@ -95,7 +98,6 @@ func Init(version string, alerter *alerter.Alerter,
supplierLibrary.AddSupplier(NewIamRolePolicySupplier(provider, deserializer, iamRepository))
supplierLibrary.AddSupplier(NewIamRolePolicyAttachmentSupplier(provider, deserializer, iamRepository))
supplierLibrary.AddSupplier(NewVPCSecurityGroupRuleSupplier(provider, deserializer, ec2repository))
supplierLibrary.AddSupplier(NewVPCSupplier(provider, deserializer, ec2repository))
supplierLibrary.AddSupplier(NewSubnetSupplier(provider, deserializer, ec2repository))
supplierLibrary.AddSupplier(NewRouteTableSupplier(provider, deserializer, ec2repository))
supplierLibrary.AddSupplier(NewRouteSupplier(provider, deserializer, ec2repository))

View File

@ -1,24 +0,0 @@
[
{
"arn": "arn:aws:ec2:eu-west-3:929327065333:vpc/vpc-a8c5d4c1",
"assign_generated_ipv6_cidr_block": false,
"cidr_block": "172.31.0.0/16",
"default_network_acl_id": "acl-3d435954",
"default_route_table_id": "rtb-b0ba5bd8",
"default_security_group_id": "sg-a74815c8",
"dhcp_options_id": "dopt-9becf2f2",
"enable_classiclink": null,
"enable_classiclink_dns_support": null,
"enable_dns_hostnames": true,
"enable_dns_support": true,
"id": "vpc-a8c5d4c1",
"instance_tenancy": "default",
"ipv6_association_id": "",
"ipv6_cidr_block": "",
"main_route_table_id": "rtb-b0ba5bd8",
"owner_id": "929327065333",
"tags": {
"Name": "Default VPC"
}
}
]

View File

@ -1,62 +0,0 @@
[
{
"arn": "arn:aws:ec2:eu-west-3:929327065333:vpc/vpc-02c50896b59598761",
"assign_generated_ipv6_cidr_block": false,
"cidr_block": "10.0.0.0/16",
"default_network_acl_id": "acl-0acd66b145f6f2102",
"default_route_table_id": "rtb-0bbd95c413323e90a",
"default_security_group_id": "sg-038e08bdd892ccae1",
"dhcp_options_id": "dopt-9becf2f2",
"enable_classiclink": null,
"enable_classiclink_dns_support": null,
"enable_dns_hostnames": false,
"enable_dns_support": true,
"id": "vpc-02c50896b59598761",
"instance_tenancy": "default",
"ipv6_association_id": "",
"ipv6_cidr_block": "",
"main_route_table_id": "rtb-0bbd95c413323e90a",
"owner_id": "929327065333",
"tags": {}
},
{
"arn": "arn:aws:ec2:eu-west-3:929327065333:vpc/vpc-0768e1fd0029e3fc3",
"assign_generated_ipv6_cidr_block": false,
"cidr_block": "10.1.0.0/16",
"default_network_acl_id": "acl-0d17964d633baf4b3",
"default_route_table_id": "rtb-05ef1030c15a0527c",
"default_security_group_id": "sg-00914edefc3bf8bc5",
"dhcp_options_id": "dopt-9becf2f2",
"enable_classiclink": null,
"enable_classiclink_dns_support": null,
"enable_dns_hostnames": false,
"enable_dns_support": true,
"id": "vpc-0768e1fd0029e3fc3",
"instance_tenancy": "default",
"ipv6_association_id": "",
"ipv6_cidr_block": "",
"main_route_table_id": "rtb-05ef1030c15a0527c",
"owner_id": "929327065333",
"tags": {}
},
{
"arn": "arn:aws:ec2:eu-west-3:929327065333:vpc/vpc-020b072316a95b97f",
"assign_generated_ipv6_cidr_block": false,
"cidr_block": "10.2.0.0/16",
"default_network_acl_id": "acl-0ff5d175a1aa8f51a",
"default_route_table_id": "rtb-04d42e3cb1ec4f500",
"default_security_group_id": "sg-048230c519e0c2345",
"dhcp_options_id": "dopt-9becf2f2",
"enable_classiclink": null,
"enable_classiclink_dns_support": null,
"enable_dns_hostnames": false,
"enable_dns_support": true,
"id": "vpc-020b072316a95b97f",
"instance_tenancy": "default",
"ipv6_association_id": "",
"ipv6_cidr_block": "",
"main_route_table_id": "rtb-04d42e3cb1ec4f500",
"owner_id": "929327065333",
"tags": {}
}
]

View File

@ -0,0 +1,35 @@
package aws
import (
"github.com/cloudskiff/driftctl/pkg/resource"
"github.com/cloudskiff/driftctl/pkg/resource/aws"
"github.com/cloudskiff/driftctl/pkg/terraform"
)
type VPCDetailsFetcher struct {
reader terraform.ResourceReader
deserializer *resource.Deserializer
}
func NewVPCDetailsFetcher(provider terraform.ResourceReader, deserializer *resource.Deserializer) *VPCDetailsFetcher {
return &VPCDetailsFetcher{
reader: provider,
deserializer: deserializer,
}
}
func (r *VPCDetailsFetcher) ReadDetails(res resource.Resource) (resource.Resource, error) {
ctyVal, err := r.reader.ReadResource(terraform.ReadResourceArgs{
Ty: resource.ResourceType(res.TerraformType()),
ID: res.TerraformId(),
})
if err != nil {
return nil, err
}
deserializedRes, err := r.deserializer.DeserializeOne(aws.AwsVpcResourceType, *ctyVal)
if err != nil {
return nil, err
}
return deserializedRes, nil
}

View File

@ -0,0 +1,67 @@
package aws
import (
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/cloudskiff/driftctl/pkg/remote/aws/repository"
remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error"
"github.com/cloudskiff/driftctl/pkg/resource/aws"
"github.com/cloudskiff/driftctl/pkg/resource"
)
type VPCEnumerator struct {
repo repository.EC2Repository
factory resource.ResourceFactory
}
func NewVPCEnumerator(repo repository.EC2Repository, factory resource.ResourceFactory) *VPCEnumerator {
return &VPCEnumerator{
repo,
factory,
}
}
func (e *VPCEnumerator) SupportedType() resource.ResourceType {
return aws.AwsVpcResourceType
}
func (e *VPCEnumerator) Enumerate() ([]resource.Resource, error) {
VPCs, defaultVPCs, err := e.repo.ListAllVPCs()
if err != nil {
return nil, remoteerror.NewResourceEnumerationError(err, aws.AwsVpcResourceType)
}
results := make([]resource.Resource, 0, len(VPCs)+len(defaultVPCs))
for _, item := range VPCs {
results = append(
results,
e.readVPC(*item),
)
}
for _, item := range defaultVPCs {
results = append(
results,
e.readVPC(*item),
)
}
return results, nil
}
func (e *VPCEnumerator) readVPC(vpc ec2.Vpc) resource.Resource {
var Ty = e.SupportedType()
if vpc.IsDefault != nil && *vpc.IsDefault {
Ty = aws.AwsDefaultVpcResourceType
}
res := e.factory.CreateAbstractResource(
string(Ty),
*vpc.VpcId,
map[string]interface{}{},
)
return res
}

View File

@ -1,99 +0,0 @@
package aws
import (
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/cloudskiff/driftctl/pkg/remote/aws/repository"
remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error"
"github.com/cloudskiff/driftctl/pkg/resource/aws"
"github.com/zclconf/go-cty/cty"
"github.com/cloudskiff/driftctl/pkg/resource"
"github.com/cloudskiff/driftctl/pkg/terraform"
"github.com/sirupsen/logrus"
)
type VPCSupplier struct {
reader terraform.ResourceReader
deserializer *resource.Deserializer
repository repository.EC2Repository
defaultVPCRunner *terraform.ParallelResourceReader
vpcRunner *terraform.ParallelResourceReader
}
func NewVPCSupplier(provider *AWSTerraformProvider, deserializer *resource.Deserializer, repository repository.EC2Repository) *VPCSupplier {
return &VPCSupplier{
provider,
deserializer,
repository,
terraform.NewParallelResourceReader(provider.Runner().SubRunner()),
terraform.NewParallelResourceReader(provider.Runner().SubRunner()),
}
}
func (s *VPCSupplier) Resources() ([]resource.Resource, error) {
VPCs, defaultVPCs, err := s.repository.ListAllVPCs()
if err != nil {
return nil, remoteerror.NewResourceEnumerationError(err, aws.AwsVpcResourceType)
}
for _, item := range VPCs {
res := *item
s.vpcRunner.Run(func() (cty.Value, error) {
return s.readVPC(res)
})
}
VPCResources, err := s.vpcRunner.Wait()
if err != nil {
return nil, err
}
for _, item := range defaultVPCs {
res := *item
s.defaultVPCRunner.Run(func() (cty.Value, error) {
return s.readVPC(res)
})
}
// Retrieve results from terraform provider
defaultVPCResources, err := s.defaultVPCRunner.Wait()
if err != nil {
return nil, err
}
// Deserialize
deserializedDefaultVPCs, err := s.deserializer.Deserialize(aws.AwsDefaultVpcResourceType, defaultVPCResources)
if err != nil {
return nil, err
}
deserializedVPCs, err := s.deserializer.Deserialize(aws.AwsVpcResourceType, VPCResources)
if err != nil {
return nil, err
}
resources := make([]resource.Resource, 0, len(VPCResources)+len(deserializedDefaultVPCs))
resources = append(resources, deserializedDefaultVPCs...)
resources = append(resources, deserializedVPCs...)
return resources, nil
}
func (s *VPCSupplier) readVPC(vpc ec2.Vpc) (cty.Value, error) {
var Ty resource.ResourceType = aws.AwsVpcResourceType
if vpc.IsDefault != nil && *vpc.IsDefault {
Ty = aws.AwsDefaultVpcResourceType
}
val, err := s.reader.ReadResource(terraform.ReadResourceArgs{
ID: *vpc.VpcId,
Ty: Ty,
})
if err != nil {
logrus.Error(err)
return cty.NilVal, err
}
return *val, nil
}

View File

@ -1,117 +0,0 @@
package aws
import (
"context"
"testing"
"github.com/cloudskiff/driftctl/pkg/remote/aws/repository"
"github.com/cloudskiff/driftctl/pkg/remote/cache"
remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error"
testresource "github.com/cloudskiff/driftctl/test/resource"
"github.com/aws/aws-sdk-go/aws/awserr"
resourceaws "github.com/cloudskiff/driftctl/pkg/resource/aws"
"github.com/cloudskiff/driftctl/pkg/parallel"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/aws"
"github.com/cloudskiff/driftctl/test/goldenfile"
mocks2 "github.com/cloudskiff/driftctl/test/mocks"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/cloudskiff/driftctl/pkg/resource"
"github.com/cloudskiff/driftctl/pkg/terraform"
"github.com/cloudskiff/driftctl/test"
)
func TestVPCSupplier_Resources(t *testing.T) {
cases := []struct {
test string
dirName string
mocks func(client *repository.MockEC2Repository)
err error
}{
{
test: "no VPC",
dirName: "vpc_empty",
mocks: func(client *repository.MockEC2Repository) {
client.On("ListAllVPCs").Once().Return([]*ec2.Vpc{}, []*ec2.Vpc{}, nil)
},
err: nil,
},
{
test: "mixed default VPC and VPC",
dirName: "vpc",
mocks: func(client *repository.MockEC2Repository) {
client.On("ListAllVPCs").Once().Return([]*ec2.Vpc{
{
VpcId: aws.String("vpc-0768e1fd0029e3fc3"),
},
{
VpcId: aws.String("vpc-020b072316a95b97f"),
IsDefault: aws.Bool(false),
},
{
VpcId: aws.String("vpc-02c50896b59598761"),
IsDefault: aws.Bool(false),
},
}, []*ec2.Vpc{
{
VpcId: aws.String("vpc-a8c5d4c1"),
IsDefault: aws.Bool(true),
},
}, nil)
},
err: nil,
},
{
test: "cannot list VPC",
dirName: "vpc_empty",
mocks: func(client *repository.MockEC2Repository) {
client.On("ListAllVPCs").Once().Return(nil, nil, awserr.NewRequestFailure(nil, 403, ""))
},
err: remoteerror.NewResourceEnumerationError(awserr.NewRequestFailure(nil, 403, ""), resourceaws.AwsVpcResourceType),
},
}
for _, c := range cases {
shouldUpdate := c.dirName == *goldenfile.Update
providerLibrary := terraform.NewProviderLibrary()
supplierLibrary := resource.NewSupplierLibrary()
repo := testresource.InitFakeSchemaRepository("aws", "3.19.0")
resourceaws.InitResourcesMetadata(repo)
factory := terraform.NewTerraformResourceFactory(repo)
deserializer := resource.NewDeserializer(factory)
if shouldUpdate {
provider, err := InitTestAwsProvider(providerLibrary)
if err != nil {
t.Fatal(err)
}
supplierLibrary.AddSupplier(NewVPCSupplier(provider, deserializer, repository.NewEC2Repository(provider.session, cache.New(0))))
}
t.Run(c.test, func(tt *testing.T) {
fakeEC2 := repository.MockEC2Repository{}
c.mocks(&fakeEC2)
provider := mocks2.NewMockedGoldenTFProvider(c.dirName, providerLibrary.Provider(terraform.AWS), shouldUpdate)
s := &VPCSupplier{
provider,
deserializer,
&fakeEC2,
terraform.NewParallelResourceReader(parallel.NewParallelRunner(context.TODO(), 10)),
terraform.NewParallelResourceReader(parallel.NewParallelRunner(context.TODO(), 10)),
}
got, err := s.Resources()
assert.Equal(tt, c.err, err)
mock.AssertExpectationsForObjects(tt)
test.CtyTestDiff(got, c.dirName, provider, deserializer, shouldUpdate, tt)
})
}
}

View File

@ -0,0 +1,84 @@
[
{
"arn": "arn:aws:ec2:eu-west-3:929327065333:vpc/vpc-02c50896b59598761",
"assign_generated_ipv6_cidr_block": false,
"cidr_block": "10.0.0.0/16",
"default_network_acl_id": "acl-0acd66b145f6f2102",
"default_route_table_id": "rtb-0bbd95c413323e90a",
"default_security_group_id": "sg-038e08bdd892ccae1",
"dhcp_options_id": "dopt-9becf2f2",
"enable_classiclink": null,
"enable_classiclink_dns_support": null,
"enable_dns_hostnames": false,
"enable_dns_support": true,
"id": "vpc-02c50896b59598761",
"instance_tenancy": "default",
"ipv6_association_id": "",
"ipv6_cidr_block": "",
"main_route_table_id": "rtb-0bbd95c413323e90a",
"owner_id": "929327065333",
"tags": {}
},
{
"arn": "arn:aws:ec2:eu-west-3:929327065333:vpc/vpc-0768e1fd0029e3fc3",
"assign_generated_ipv6_cidr_block": false,
"cidr_block": "10.1.0.0/16",
"default_network_acl_id": "acl-0d17964d633baf4b3",
"default_route_table_id": "rtb-05ef1030c15a0527c",
"default_security_group_id": "sg-00914edefc3bf8bc5",
"dhcp_options_id": "dopt-9becf2f2",
"enable_classiclink": null,
"enable_classiclink_dns_support": null,
"enable_dns_hostnames": false,
"enable_dns_support": true,
"id": "vpc-0768e1fd0029e3fc3",
"instance_tenancy": "default",
"ipv6_association_id": "",
"ipv6_cidr_block": "",
"main_route_table_id": "rtb-05ef1030c15a0527c",
"owner_id": "929327065333",
"tags": {}
},
{
"arn": "arn:aws:ec2:eu-west-3:929327065333:vpc/vpc-020b072316a95b97f",
"assign_generated_ipv6_cidr_block": false,
"cidr_block": "10.2.0.0/16",
"default_network_acl_id": "acl-0ff5d175a1aa8f51a",
"default_route_table_id": "rtb-04d42e3cb1ec4f500",
"default_security_group_id": "sg-048230c519e0c2345",
"dhcp_options_id": "dopt-9becf2f2",
"enable_classiclink": null,
"enable_classiclink_dns_support": null,
"enable_dns_hostnames": false,
"enable_dns_support": true,
"id": "vpc-020b072316a95b97f",
"instance_tenancy": "default",
"ipv6_association_id": "",
"ipv6_cidr_block": "",
"main_route_table_id": "rtb-04d42e3cb1ec4f500",
"owner_id": "929327065333",
"tags": {}
},
{
"arn": "arn:aws:ec2:eu-west-3:929327065333:vpc/vpc-a8c5d4c1",
"assign_generated_ipv6_cidr_block": false,
"cidr_block": "172.31.0.0/16",
"default_network_acl_id": "acl-3d435954",
"default_route_table_id": "rtb-b0ba5bd8",
"default_security_group_id": "sg-a74815c8",
"dhcp_options_id": "dopt-9becf2f2",
"enable_classiclink": null,
"enable_classiclink_dns_support": null,
"enable_dns_hostnames": true,
"enable_dns_support": true,
"id": "vpc-a8c5d4c1",
"instance_tenancy": "default",
"ipv6_association_id": "",
"ipv6_cidr_block": "",
"main_route_table_id": "rtb-b0ba5bd8",
"owner_id": "929327065333",
"tags": {
"Name": "Default VPC"
}
}
]

View File

@ -0,0 +1,132 @@
package remote
import (
"testing"
awssdk "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/cloudskiff/driftctl/mocks"
"github.com/cloudskiff/driftctl/pkg/remote/aws"
"github.com/cloudskiff/driftctl/pkg/remote/cache"
"github.com/cloudskiff/driftctl/pkg/remote/common"
remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error"
testresource "github.com/cloudskiff/driftctl/test/resource"
terraform2 "github.com/cloudskiff/driftctl/test/terraform"
"github.com/cloudskiff/driftctl/pkg/remote/aws/repository"
"github.com/cloudskiff/driftctl/pkg/resource"
resourceaws "github.com/cloudskiff/driftctl/pkg/resource/aws"
"github.com/cloudskiff/driftctl/pkg/terraform"
"github.com/cloudskiff/driftctl/test"
"github.com/cloudskiff/driftctl/test/goldenfile"
"github.com/stretchr/testify/assert"
)
func TestVPC(t *testing.T) {
tests := []struct {
test string
dirName string
mocks func(repository *repository.MockEC2Repository)
wantErr error
}{
{
test: "no VPC",
dirName: "vpc_empty",
mocks: func(client *repository.MockEC2Repository) {
client.On("ListAllVPCs").Once().Return([]*ec2.Vpc{}, []*ec2.Vpc{}, nil)
},
wantErr: nil,
},
{
test: "mixed default VPC and VPC",
dirName: "vpc",
mocks: func(client *repository.MockEC2Repository) {
client.On("ListAllVPCs").Once().Return([]*ec2.Vpc{
{
VpcId: awssdk.String("vpc-0768e1fd0029e3fc3"),
},
{
VpcId: awssdk.String("vpc-020b072316a95b97f"),
IsDefault: awssdk.Bool(false),
},
{
VpcId: awssdk.String("vpc-02c50896b59598761"),
IsDefault: awssdk.Bool(false),
},
}, []*ec2.Vpc{
{
VpcId: awssdk.String("vpc-a8c5d4c1"),
IsDefault: awssdk.Bool(true),
},
}, nil)
},
wantErr: nil,
},
{
test: "cannot list VPC",
dirName: "vpc_empty",
mocks: func(client *repository.MockEC2Repository) {
client.On("ListAllVPCs").Once().Return(nil, nil, awserr.NewRequestFailure(nil, 403, ""))
},
wantErr: remoteerror.NewResourceEnumerationError(awserr.NewRequestFailure(nil, 403, ""), resourceaws.AwsVpcResourceType),
},
}
schemaRepository := testresource.InitFakeSchemaRepository("aws", "3.19.0")
resourceaws.InitResourcesMetadata(schemaRepository)
factory := terraform.NewTerraformResourceFactory(schemaRepository)
deserializer := resource.NewDeserializer(factory)
alerter := &mocks.AlerterInterface{}
for _, c := range tests {
t.Run(c.test, func(tt *testing.T) {
shouldUpdate := c.dirName == *goldenfile.Update
session := session.Must(session.NewSessionWithOptions(session.Options{
SharedConfigState: session.SharedConfigEnable,
}))
scanOptions := ScannerOptions{Deep: true}
providerLibrary := terraform.NewProviderLibrary()
remoteLibrary := common.NewRemoteLibrary()
// Initialize mocks
fakeRepo := &repository.MockEC2Repository{}
c.mocks(fakeRepo)
var repo repository.EC2Repository = fakeRepo
providerVersion := "3.19.0"
realProvider, err := terraform2.InitTestAwsProvider(providerLibrary, providerVersion)
if err != nil {
t.Fatal(err)
}
provider := terraform2.NewFakeTerraformProvider(realProvider)
provider.WithResponse(c.dirName)
// Replace mock by real resources if we are in update mode
if shouldUpdate {
err := realProvider.Init()
if err != nil {
t.Fatal(err)
}
provider.ShouldUpdate()
repo = repository.NewEC2Repository(session, cache.New(0))
}
remoteLibrary.AddEnumerator(aws.NewVPCEnumerator(repo, factory))
remoteLibrary.AddDetailsFetcher(resourceaws.AwsVpcResourceType, aws.NewVPCDetailsFetcher(provider, deserializer))
remoteLibrary.AddDetailsFetcher(resourceaws.AwsDefaultVpcResourceType, aws.NewVPCDetailsFetcher(provider, deserializer))
s := NewScanner(nil, remoteLibrary, alerter, scanOptions)
got, err := s.Resources()
assert.Equal(tt, err, c.wantErr)
if err != nil {
return
}
test.TestAgainstGoldenFile(got, resourceaws.AwsVpcResourceType, c.dirName, provider, deserializer, shouldUpdate, tt)
})
}
}