refactor: split security group rule supplier

pkg/remote/aws/init.go

@@ -91,7 +91,9 @@ func Init(version string, alerter *alerter.Alerter,
 	remoteLibrary.AddDetailsFetcher(aws.AwsVpcResourceType, common.NewGenericDetailsFetcher(aws.AwsVpcResourceType, provider, deserializer))
 	remoteLibrary.AddEnumerator(NewDefaultVPCEnumerator(ec2repository, factory))
 	remoteLibrary.AddDetailsFetcher(aws.AwsDefaultVpcResourceType, common.NewGenericDetailsFetcher(aws.AwsDefaultVpcResourceType, provider, deserializer))
-	remoteLibrary.AddEnumerator(NewEC2RouteTableEnumerator(ec2repository, factory))
+    remoteLibrary.AddEnumerator(NewVPCSecurityGroupRuleEnumerator(ec2repository, factory))
+    remoteLibrary.AddDetailsFetcher(aws.AwsSecurityGroupRuleResourceType, common.NewGenericDetailsFetcher(aws.AwsSecurityGroupRuleResourceType, provider, deserializer))
+    remoteLibrary.AddEnumerator(NewEC2RouteTableEnumerator(ec2repository, factory))
 	remoteLibrary.AddDetailsFetcher(aws.AwsRouteTableResourceType, common.NewGenericDetailsFetcher(aws.AwsRouteTableResourceType, provider, deserializer))
 	remoteLibrary.AddEnumerator(NewEC2DefaultRouteTableEnumerator(ec2repository, factory))
 	remoteLibrary.AddDetailsFetcher(aws.AwsDefaultRouteTableResourceType, NewEC2DefaultRouteTableDetailsFetcher(provider, deserializer))
@@ -171,8 +173,6 @@ func Init(version string, alerter *alerter.Alerter,
 	remoteLibrary.AddEnumerator(NewECRRepositoryEnumerator(ecrRepository, factory))
 	remoteLibrary.AddDetailsFetcher(aws.AwsEcrRepositoryResourceType, common.NewGenericDetailsFetcher(aws.AwsEcrRepositoryResourceType, provider, deserializer))
 
-	supplierLibrary.AddSupplier(NewVPCSecurityGroupRuleSupplier(provider, deserializer, ec2repository))
-
 	err = resourceSchemaRepository.Init(version, provider.Schema())
 	if err != nil {
 		return err

pkg/remote/aws/vpc_security_group_rule_enumerator.go

@@ -7,13 +7,8 @@ import (
 	"github.com/cloudskiff/driftctl/pkg/resource"
 	resourceaws "github.com/cloudskiff/driftctl/pkg/resource/aws"
 
-	"github.com/cloudskiff/driftctl/pkg/terraform"
-
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/hashicorp/terraform/flatmap"
-	"github.com/sirupsen/logrus"
-	"github.com/zclconf/go-cty/cty"
 )
 
 const (
@@ -21,11 +16,9 @@ const (
 	sgRuleTypeEgress  = "egress"
 )
 
-type VPCSecurityGroupRuleSupplier struct {
-	reader       terraform.ResourceReader
-	deserializer *resource.Deserializer
-	repository   repository.EC2Repository
-	runner       *terraform.ParallelResourceReader
+type VPCSecurityGroupRuleEnumerator struct {
+	repository repository.EC2Repository
+	factory    resource.ResourceFactory
 }
 
 type securityGroupRule struct {
@@ -66,74 +59,62 @@ func toInterfaceSlice(val []string) []interface{} {
 	return res
 }
 
-func NewVPCSecurityGroupRuleSupplier(provider *AWSTerraformProvider, deserializer *resource.Deserializer, repository repository.EC2Repository) *VPCSecurityGroupRuleSupplier {
-	return &VPCSecurityGroupRuleSupplier{
-		provider,
-		deserializer,
+func NewVPCSecurityGroupRuleEnumerator(repository repository.EC2Repository, factory resource.ResourceFactory) *VPCSecurityGroupRuleEnumerator {
+	return &VPCSecurityGroupRuleEnumerator{
 		repository,
-		terraform.NewParallelResourceReader(provider.Runner().SubRunner()),
+		factory,
 	}
 }
 
-func (s *VPCSecurityGroupRuleSupplier) Resources() ([]resource.Resource, error) {
-	securityGroups, defaultSecurityGroups, err := s.repository.ListAllSecurityGroups()
+func (e *VPCSecurityGroupRuleEnumerator) SupportedType() resource.ResourceType {
+	return resourceaws.AwsSecurityGroupRuleResourceType
+}
+
+func (e *VPCSecurityGroupRuleEnumerator) Enumerate() ([]resource.Resource, error) {
+	securityGroups, defaultSecurityGroups, err := e.repository.ListAllSecurityGroups()
 	if err != nil {
 		return nil, remoteerror.NewResourceEnumerationError(err, resourceaws.AwsSecurityGroupRuleResourceType)
 	}
+
 	secGroups := make([]*ec2.SecurityGroup, 0, len(securityGroups)+len(defaultSecurityGroups))
 	secGroups = append(secGroups, securityGroups...)
 	secGroups = append(secGroups, defaultSecurityGroups...)
-	securityGroupsRules := s.listSecurityGroupsRules(secGroups)
-	results := make([]cty.Value, 0)
-	if len(securityGroupsRules) > 0 {
-		for _, securityGroupsRule := range securityGroupsRules {
-			rule := securityGroupsRule
-			s.runner.Run(func() (cty.Value, error) {
-				return s.readSecurityGroupRule(rule)
-			})
-		}
-		results, err = s.runner.Wait()
-		if err != nil {
-			return nil, err
-		}
+	securityGroupsRules := e.listSecurityGroupsRules(secGroups)
+
+	results := make([]resource.Resource, 0, len(securityGroupsRules))
+	for _, rule := range securityGroupsRules {
+		results = append(
+			results,
+			e.factory.CreateAbstractResource(
+				string(e.SupportedType()),
+				rule.getId(),
+				map[string]interface{}{
+					"type":                     rule.Type,
+					"security_group_id":        rule.SecurityGroupId,
+					"protocol":                 rule.Protocol,
+					"from_port":                rule.FromPort,
+					"to_port":                  rule.ToPort,
+					"self":                     rule.Self,
+					"source_security_group_id": rule.SourceSecurityGroupId,
+					"cidr_blocks":              rule.CidrBlocks,
+					"ipv6_cidr_blocks":         rule.Ipv6CidrBlocks,
+					"prefix_list_ids":          rule.PrefixListIds,
+				},
+			),
+		)
 	}
-	return s.deserializer.Deserialize(resourceaws.AwsSecurityGroupRuleResourceType, results)
+
+	return results, nil
 }
 
-func (s *VPCSecurityGroupRuleSupplier) readSecurityGroupRule(rule securityGroupRule) (cty.Value, error) {
-	id := rule.getId()
-
-	resSgRule, err := s.reader.ReadResource(terraform.ReadResourceArgs{
-		Ty: resourceaws.AwsSecurityGroupRuleResourceType,
-		ID: id,
-		Attributes: flatmap.Flatten(map[string]interface{}{
-			"type":                     rule.Type,
-			"security_group_id":        rule.SecurityGroupId,
-			"protocol":                 rule.Protocol,
-			"from_port":                rule.FromPort,
-			"to_port":                  rule.ToPort,
-			"self":                     rule.Self,
-			"source_security_group_id": rule.SourceSecurityGroupId,
-			"cidr_blocks":              rule.CidrBlocks,
-			"ipv6_cidr_blocks":         rule.Ipv6CidrBlocks,
-			"prefix_list_ids":          rule.PrefixListIds,
-		}),
-	})
-	if err != nil {
-		logrus.Warnf("Error reading rule from security group %s: %+v", id, err)
-		return cty.NilVal, err
-	}
-	return *resSgRule, nil
-}
-
-func (s *VPCSecurityGroupRuleSupplier) listSecurityGroupsRules(securityGroups []*ec2.SecurityGroup) []securityGroupRule {
+func (e *VPCSecurityGroupRuleEnumerator) listSecurityGroupsRules(securityGroups []*ec2.SecurityGroup) []securityGroupRule {
 	var securityGroupsRules []securityGroupRule
 	for _, sg := range securityGroups {
 		for _, rule := range sg.IpPermissions {
-			securityGroupsRules = append(securityGroupsRules, s.addSecurityGroupRule(sgRuleTypeIngress, rule, sg)...)
+			securityGroupsRules = append(securityGroupsRules, e.addSecurityGroupRule(sgRuleTypeIngress, rule, sg)...)
 		}
 		for _, rule := range sg.IpPermissionsEgress {
-			securityGroupsRules = append(securityGroupsRules, s.addSecurityGroupRule(sgRuleTypeEgress, rule, sg)...)
+			securityGroupsRules = append(securityGroupsRules, e.addSecurityGroupRule(sgRuleTypeEgress, rule, sg)...)
 		}
 	}
 	return securityGroupsRules
@@ -141,7 +122,7 @@ func (s *VPCSecurityGroupRuleSupplier) listSecurityGroupsRules(securityGroups []
 
 // addSecurityGroupRule will iterate through each "Source" as per Aws definition and create a
 // rule with custom attributes
-func (s *VPCSecurityGroupRuleSupplier) addSecurityGroupRule(ruleType string, rule *ec2.IpPermission, sg *ec2.SecurityGroup) []securityGroupRule {
+func (e *VPCSecurityGroupRuleEnumerator) addSecurityGroupRule(ruleType string, rule *ec2.IpPermission, sg *ec2.SecurityGroup) []securityGroupRule {
 	var rules []securityGroupRule
 	for _, groupPair := range rule.UserIdGroupPairs {
 		r := securityGroupRule{

pkg/remote/aws/vpc_security_group_rule_supplier_test.go

@@ -1,193 +0,0 @@
-package aws
-
-import (
-	"context"
-	"testing"
-
-	"github.com/cloudskiff/driftctl/pkg/remote/aws/repository"
-	"github.com/cloudskiff/driftctl/pkg/remote/cache"
-	remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error"
-	testresource "github.com/cloudskiff/driftctl/test/resource"
-
-	"github.com/aws/aws-sdk-go/aws/awserr"
-	resourceaws "github.com/cloudskiff/driftctl/pkg/resource/aws"
-
-	"github.com/cloudskiff/driftctl/pkg/parallel"
-
-	"github.com/aws/aws-sdk-go/service/ec2"
-
-	"github.com/aws/aws-sdk-go/aws"
-
-	"github.com/cloudskiff/driftctl/test/goldenfile"
-	mocks2 "github.com/cloudskiff/driftctl/test/mocks"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/mock"
-
-	"github.com/cloudskiff/driftctl/pkg/resource"
-	"github.com/cloudskiff/driftctl/pkg/terraform"
-	"github.com/cloudskiff/driftctl/test"
-)
-
-func TestVPCSecurityGroupRuleSupplier_Resources(t *testing.T) {
-	cases := []struct {
-		test    string
-		dirName string
-		mocks   func(client *repository.MockEC2Repository)
-		err     error
-	}{
-		{
-			test:    "no security group rules",
-			dirName: "vpc_security_group_rule_empty",
-			mocks: func(client *repository.MockEC2Repository) {
-				client.On("ListAllSecurityGroups").Once().Return([]*ec2.SecurityGroup{
-					{
-						GroupId:             aws.String("sg-0254c038e32f25530"),
-						IpPermissions:       []*ec2.IpPermission{},
-						IpPermissionsEgress: []*ec2.IpPermission{},
-					},
-				}, nil, nil)
-			},
-			err: nil,
-		},
-		{
-			test:    "with security group rules",
-			dirName: "vpc_security_group_rule_multiple",
-			mocks: func(client *repository.MockEC2Repository) {
-				client.On("ListAllSecurityGroups").Once().Return([]*ec2.SecurityGroup{
-					{
-						GroupId: aws.String("sg-0254c038e32f25530"),
-						IpPermissions: []*ec2.IpPermission{
-							{
-								FromPort:   aws.Int64(0),
-								ToPort:     aws.Int64(65535),
-								IpProtocol: aws.String("tcp"),
-								UserIdGroupPairs: []*ec2.UserIdGroupPair{
-									{
-										GroupId: aws.String("sg-0254c038e32f25530"),
-									},
-									{
-										GroupId: aws.String("sg-9e0204ff"),
-									},
-								},
-							},
-							{
-								IpProtocol: aws.String("-1"),
-								IpRanges: []*ec2.IpRange{
-									{
-										CidrIp: aws.String("1.2.0.0/16"),
-									},
-									{
-										CidrIp: aws.String("5.6.7.0/24"),
-									},
-								},
-								Ipv6Ranges: []*ec2.Ipv6Range{
-									{
-										CidrIpv6: aws.String("::/0"),
-									},
-								},
-							},
-						},
-						IpPermissionsEgress: []*ec2.IpPermission{
-							{
-								IpProtocol: aws.String("-1"),
-								IpRanges: []*ec2.IpRange{
-									{
-										CidrIp: aws.String("0.0.0.0/0"),
-									},
-								},
-								Ipv6Ranges: []*ec2.Ipv6Range{
-									{
-										CidrIpv6: aws.String("::/0"),
-									},
-								},
-							},
-						},
-					},
-					{
-						GroupId: aws.String("sg-0cc8b3c3c2851705a"),
-						IpPermissions: []*ec2.IpPermission{
-							{
-								FromPort:   aws.Int64(443),
-								ToPort:     aws.Int64(443),
-								IpProtocol: aws.String("tcp"),
-								IpRanges: []*ec2.IpRange{
-									{
-										CidrIp: aws.String("0.0.0.0/0"),
-									},
-								},
-							},
-						},
-						IpPermissionsEgress: []*ec2.IpPermission{
-							{
-								IpProtocol: aws.String("-1"),
-								IpRanges: []*ec2.IpRange{
-									{
-										CidrIp: aws.String("0.0.0.0/0"),
-									},
-								},
-								Ipv6Ranges: []*ec2.Ipv6Range{
-									{
-										CidrIpv6: aws.String("::/0"),
-									},
-								},
-							},
-							{
-								IpProtocol: aws.String("5"),
-								IpRanges: []*ec2.IpRange{
-									{
-										CidrIp: aws.String("0.0.0.0/0"),
-									},
-								},
-							},
-						},
-					},
-				}, nil, nil)
-			},
-			err: nil,
-		},
-		{
-			test:    "cannot list security group rules",
-			dirName: "vpc_security_group_rule_empty",
-			mocks: func(client *repository.MockEC2Repository) {
-				client.On("ListAllSecurityGroups").Once().Return(nil, nil, awserr.NewRequestFailure(nil, 403, ""))
-			},
-			err: remoteerror.NewResourceEnumerationError(awserr.NewRequestFailure(nil, 403, ""), resourceaws.AwsSecurityGroupRuleResourceType),
-		},
-	}
-	for _, c := range cases {
-		shouldUpdate := c.dirName == *goldenfile.Update
-
-		providerLibrary := terraform.NewProviderLibrary()
-		supplierLibrary := resource.NewSupplierLibrary()
-
-		repo := testresource.InitFakeSchemaRepository("aws", "3.19.0")
-		resourceaws.InitResourcesMetadata(repo)
-		factory := terraform.NewTerraformResourceFactory(repo)
-
-		deserializer := resource.NewDeserializer(factory)
-		if shouldUpdate {
-			provider, err := InitTestAwsProvider(providerLibrary)
-			if err != nil {
-				t.Fatal(err)
-			}
-			supplierLibrary.AddSupplier(NewVPCSecurityGroupRuleSupplier(provider, deserializer, repository.NewEC2Repository(provider.session, cache.New(0))))
-		}
-
-		t.Run(c.test, func(tt *testing.T) {
-			fakeEC2 := repository.MockEC2Repository{}
-			c.mocks(&fakeEC2)
-			provider := mocks2.NewMockedGoldenTFProvider(c.dirName, providerLibrary.Provider(terraform.AWS), shouldUpdate)
-			s := &VPCSecurityGroupRuleSupplier{
-				provider,
-				deserializer,
-				&fakeEC2,
-				terraform.NewParallelResourceReader(parallel.NewParallelRunner(context.TODO(), 10)),
-			}
-			got, err := s.Resources()
-			assert.Equal(tt, c.err, err)
-
-			mock.AssertExpectationsForObjects(tt)
-			test.CtyTestDiff(got, c.dirName, provider, deserializer, shouldUpdate, tt)
-		})
-	}
-}

pkg/remote/ec2_scanner_test.go

@@ -1973,3 +1973,185 @@ func TestEC2Route(t *testing.T) {
 		})
 	}
 }
+
+func TestVpcSecurityGroupRule(t *testing.T) {
+
+	tests := []struct {
+		test    string
+		dirName string
+		mocks   func(repository *repository.MockEC2Repository)
+		wantErr error
+	}{
+		{
+			test:    "no security group rules",
+			dirName: "vpc_security_group_rule_empty",
+			mocks: func(client *repository.MockEC2Repository) {
+				client.On("ListAllSecurityGroups").Once().Return([]*ec2.SecurityGroup{
+					{
+						GroupId:             awssdk.String("sg-0254c038e32f25530"),
+						IpPermissions:       []*ec2.IpPermission{},
+						IpPermissionsEgress: []*ec2.IpPermission{},
+					},
+				}, nil, nil)
+			},
+			wantErr: nil,
+		},
+		{
+			test:    "with security group rules",
+			dirName: "vpc_security_group_rule_multiple",
+			mocks: func(client *repository.MockEC2Repository) {
+				client.On("ListAllSecurityGroups").Once().Return([]*ec2.SecurityGroup{
+					{
+						GroupId: awssdk.String("sg-0254c038e32f25530"),
+						IpPermissions: []*ec2.IpPermission{
+							{
+								FromPort:   awssdk.Int64(0),
+								ToPort:     awssdk.Int64(65535),
+								IpProtocol: awssdk.String("tcp"),
+								UserIdGroupPairs: []*ec2.UserIdGroupPair{
+									{
+										GroupId: awssdk.String("sg-0254c038e32f25530"),
+									},
+									{
+										GroupId: awssdk.String("sg-9e0204ff"),
+									},
+								},
+							},
+							{
+								IpProtocol: awssdk.String("-1"),
+								IpRanges: []*ec2.IpRange{
+									{
+										CidrIp: awssdk.String("1.2.0.0/16"),
+									},
+									{
+										CidrIp: awssdk.String("5.6.7.0/24"),
+									},
+								},
+								Ipv6Ranges: []*ec2.Ipv6Range{
+									{
+										CidrIpv6: awssdk.String("::/0"),
+									},
+								},
+							},
+						},
+						IpPermissionsEgress: []*ec2.IpPermission{
+							{
+								IpProtocol: awssdk.String("-1"),
+								IpRanges: []*ec2.IpRange{
+									{
+										CidrIp: awssdk.String("0.0.0.0/0"),
+									},
+								},
+								Ipv6Ranges: []*ec2.Ipv6Range{
+									{
+										CidrIpv6: awssdk.String("::/0"),
+									},
+								},
+							},
+						},
+					},
+					{
+						GroupId: awssdk.String("sg-0cc8b3c3c2851705a"),
+						IpPermissions: []*ec2.IpPermission{
+							{
+								FromPort:   awssdk.Int64(443),
+								ToPort:     awssdk.Int64(443),
+								IpProtocol: awssdk.String("tcp"),
+								IpRanges: []*ec2.IpRange{
+									{
+										CidrIp: awssdk.String("0.0.0.0/0"),
+									},
+								},
+							},
+						},
+						IpPermissionsEgress: []*ec2.IpPermission{
+							{
+								IpProtocol: awssdk.String("-1"),
+								IpRanges: []*ec2.IpRange{
+									{
+										CidrIp: awssdk.String("0.0.0.0/0"),
+									},
+								},
+								Ipv6Ranges: []*ec2.Ipv6Range{
+									{
+										CidrIpv6: awssdk.String("::/0"),
+									},
+								},
+							},
+							{
+								IpProtocol: awssdk.String("5"),
+								IpRanges: []*ec2.IpRange{
+									{
+										CidrIp: awssdk.String("0.0.0.0/0"),
+									},
+								},
+							},
+						},
+					},
+				}, nil, nil)
+			},
+			wantErr: nil,
+		},
+		{
+			test:    "cannot list security group rules",
+			dirName: "vpc_security_group_rule_empty",
+			mocks: func(client *repository.MockEC2Repository) {
+				client.On("ListAllSecurityGroups").Once().Return(nil, nil, awserr.NewRequestFailure(nil, 403, ""))
+			},
+			wantErr: remoteerror.NewResourceEnumerationError(awserr.NewRequestFailure(nil, 403, ""), resourceaws.AwsSecurityGroupRuleResourceType),
+		},
+	}
+
+	schemaRepository := testresource.InitFakeSchemaRepository("aws", "3.19.0")
+	resourceaws.InitResourcesMetadata(schemaRepository)
+	factory := terraform.NewTerraformResourceFactory(schemaRepository)
+	deserializer := resource.NewDeserializer(factory)
+	alerter := &mocks.AlerterInterface{}
+
+	for _, c := range tests {
+		t.Run(c.test, func(tt *testing.T) {
+			shouldUpdate := c.dirName == *goldenfile.Update
+
+			session := session.Must(session.NewSessionWithOptions(session.Options{
+				SharedConfigState: session.SharedConfigEnable,
+			}))
+
+			scanOptions := ScannerOptions{Deep: true}
+			providerLibrary := terraform.NewProviderLibrary()
+			remoteLibrary := common.NewRemoteLibrary()
+
+			// Initialize mocks
+			fakeRepo := &repository.MockEC2Repository{}
+			c.mocks(fakeRepo)
+			var repo repository.EC2Repository = fakeRepo
+			providerVersion := "3.19.0"
+			realProvider, err := terraform2.InitTestAwsProvider(providerLibrary, providerVersion)
+			if err != nil {
+				t.Fatal(err)
+			}
+			provider := terraform2.NewFakeTerraformProvider(realProvider)
+			provider.WithResponse(c.dirName)
+
+			// Replace mock by real resources if we are in update mode
+			if shouldUpdate {
+				err := realProvider.Init()
+				if err != nil {
+					t.Fatal(err)
+				}
+				provider.ShouldUpdate()
+				repo = repository.NewEC2Repository(session, cache.New(0))
+			}
+
+			remoteLibrary.AddEnumerator(aws.NewVPCSecurityGroupRuleEnumerator(repo, factory))
+			remoteLibrary.AddDetailsFetcher(resourceaws.AwsSecurityGroupRuleResourceType, common.NewGenericDetailsFetcher(resourceaws.AwsSecurityGroupRuleResourceType, provider, deserializer))
+
+			s := NewScanner(nil, remoteLibrary, alerter, scanOptions)
+			got, err := s.Resources()
+			assert.Equal(tt, c.wantErr, err)
+			if err != nil {
+				return
+			}
+			test.TestAgainstGoldenFile(got, resourceaws.AwsSecurityGroupRuleResourceType, c.dirName, provider, deserializer, shouldUpdate, tt)
+		})
+	}
+}