Merge pull request #1145 from cloudskiff/add_google_bigquery_dataset

Add google_bigquery_dataset

pkg/iac/terraform/state/terraform_state_reader_test.go

@@ -338,6 +338,7 @@ func TestTerraformStateReader_Google_Resources(t *testing.T) {
 		{name: "Bucket IAM members", dirName: "google_bucket_iam_member", wantErr: false},
 		{name: "Bucket IAM Policy", dirName: "google_bucket_iam_policy", wantErr: false},
 		{name: "DNS managed zone", dirName: "google_dns_managed_zone", wantErr: false},
+		{name: "bigquery dataset", dirName: "google_bigquery_dataset", wantErr: false},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/iac/terraform/state/test/google_bigquery_dataset/results.golden.json

@@ -0,0 +1,51 @@
+[
+ {
+  "Id": "projects/cloudskiff-dev-elie/datasets/example_dataset",
+  "Type": "google_bigquery_dataset",
+  "Attrs": {
+   "access": [
+    {
+     "domain": "",
+     "group_by_email": "",
+     "role": "OWNER",
+     "special_group": "",
+     "user_by_email": "elie.charra@cloudskiff.com"
+    },
+    {
+     "domain": "",
+     "group_by_email": "",
+     "role": "OWNER",
+     "special_group": "projectOwners",
+     "user_by_email": ""
+    },
+    {
+     "domain": "",
+     "group_by_email": "",
+     "role": "READER",
+     "special_group": "projectReaders",
+     "user_by_email": ""
+    },
+    {
+     "domain": "",
+     "group_by_email": "",
+     "role": "WRITER",
+     "special_group": "projectWriters",
+     "user_by_email": ""
+    }
+   ],
+   "creation_time": 1634647437581,
+   "dataset_id": "example_dataset",
+   "default_partition_expiration_ms": 0,
+   "default_table_expiration_ms": 0,
+   "delete_contents_on_destroy": false,
+   "description": "This is a test description",
+   "etag": "RrsFjBErVHJw1kDByWfKmQ==",
+   "friendly_name": "test",
+   "id": "projects/cloudskiff-dev-elie/datasets/example_dataset",
+   "last_modified_time": 1634647437581,
+   "location": "EU",
+   "project": "cloudskiff-dev-elie",
+   "self_link": "https://bigquery.googleapis.com/bigquery/v2/projects/cloudskiff-dev-elie/datasets/example_dataset"
+  }
+ }
+]

pkg/iac/terraform/state/test/google_bigquery_dataset/terraform.tfstate

@@ -0,0 +1,74 @@
+{
+  "version": 4,
+  "terraform_version": "0.15.5",
+  "serial": 23,
+  "lineage": "80058dbf-c946-85c8-ad1c-165fad8cf85e",
+  "outputs": {},
+  "resources": [
+    {
+      "mode": "managed",
+      "type": "google_bigquery_dataset",
+      "name": "dataset",
+      "provider": "provider[\"registry.terraform.io/hashicorp/google\"]",
+      "instances": [
+        {
+          "schema_version": 0,
+          "attributes": {
+            "access": [
+              {
+                "domain": "",
+                "group_by_email": "",
+                "role": "OWNER",
+                "special_group": "",
+                "user_by_email": "elie.charra@cloudskiff.com",
+                "view": []
+              },
+              {
+                "domain": "",
+                "group_by_email": "",
+                "role": "OWNER",
+                "special_group": "projectOwners",
+                "user_by_email": "",
+                "view": []
+              },
+              {
+                "domain": "",
+                "group_by_email": "",
+                "role": "READER",
+                "special_group": "projectReaders",
+                "user_by_email": "",
+                "view": []
+              },
+              {
+                "domain": "",
+                "group_by_email": "",
+                "role": "WRITER",
+                "special_group": "projectWriters",
+                "user_by_email": "",
+                "view": []
+              }
+            ],
+            "creation_time": 1634647437581,
+            "dataset_id": "example_dataset",
+            "default_encryption_configuration": [],
+            "default_partition_expiration_ms": 0,
+            "default_table_expiration_ms": 0,
+            "delete_contents_on_destroy": false,
+            "description": "This is a test description",
+            "etag": "RrsFjBErVHJw1kDByWfKmQ==",
+            "friendly_name": "test",
+            "id": "projects/cloudskiff-dev-elie/datasets/example_dataset",
+            "labels": null,
+            "last_modified_time": 1634647437581,
+            "location": "EU",
+            "project": "cloudskiff-dev-elie",
+            "self_link": "https://bigquery.googleapis.com/bigquery/v2/projects/cloudskiff-dev-elie/datasets/example_dataset",
+            "timeouts": null
+          },
+          "sensitive_attributes": [],
+          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoyNDAwMDAwMDAwMDAsImRlbGV0ZSI6MjQwMDAwMDAwMDAwLCJ1cGRhdGUiOjI0MDAwMDAwMDAwMH19"
+        }
+      ]
+    }
+  ]
+}

pkg/remote/google/google_bigquery_dataset_enumerator.go

@@ -0,0 +1,49 @@
+package google
+
+import (
+	remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error"
+	"github.com/cloudskiff/driftctl/pkg/remote/google/repository"
+	"github.com/cloudskiff/driftctl/pkg/resource"
+	"github.com/cloudskiff/driftctl/pkg/resource/google"
+)
+
+type GoogleBigqueryDatasetEnumerator struct {
+	repository repository.AssetRepository
+	factory    resource.ResourceFactory
+}
+
+func NewGoogleBigqueryDatasetEnumerator(repo repository.AssetRepository, factory resource.ResourceFactory) *GoogleBigqueryDatasetEnumerator {
+	return &GoogleBigqueryDatasetEnumerator{
+		repository: repo,
+		factory:    factory,
+	}
+}
+
+func (e *GoogleBigqueryDatasetEnumerator) SupportedType() resource.ResourceType {
+	return google.GoogleBigqueryDatasetResourceType
+}
+
+func (e *GoogleBigqueryDatasetEnumerator) Enumerate() ([]*resource.Resource, error) {
+	resources, err := e.repository.SearchAllDatasets()
+
+	if err != nil {
+		return nil, remoteerror.NewResourceListingError(err, string(e.SupportedType()))
+	}
+
+	results := make([]*resource.Resource, 0, len(resources))
+
+	for _, res := range resources {
+		results = append(
+			results,
+			e.factory.CreateAbstractResource(
+				string(e.SupportedType()),
+				trimResourceName(res.GetName()),
+				map[string]interface{}{
+					"name": res.DisplayName,
+				},
+			),
+		)
+	}
+
+	return results, err
+}

pkg/remote/google/init.go

@@ -72,6 +72,8 @@ func Init(version string, alerter *alerter.Alerter,
 	remoteLibrary.AddEnumerator(NewGoogleComputeInstanceGroupEnumerator(assetRepository, factory))
 	remoteLibrary.AddDetailsFetcher(google.GoogleComputeInstanceGroupResourceType, common.NewGenericDetailsFetcher(google.GoogleComputeInstanceGroupResourceType, provider, deserializer))
 
+	remoteLibrary.AddEnumerator(NewGoogleBigqueryDatasetEnumerator(assetRepository, factory))
+
 	err = resourceSchemaRepository.Init(terraform.GOOGLE, provider.Version(), provider.Schema())
 	if err != nil {
 		return err

pkg/remote/google/repository/asset.go

@@ -20,6 +20,7 @@ const (
 	computeNetworkAssetType       = "compute.googleapis.com/Network"
 	dnsManagedZoneAssetType       = "dns.googleapis.com/ManagedZone"
 	computeInstanceGroupAssetType = "compute.googleapis.com/InstanceGroup"
+	bigqueryDatasetAssetType      = "bigquery.googleapis.com/Dataset"
 )
 
 type AssetRepository interface {
@@ -30,6 +31,7 @@ type AssetRepository interface {
 	SearchAllNetworks() ([]*assetpb.ResourceSearchResult, error)
 	SearchAllDNSManagedZones() ([]*assetpb.ResourceSearchResult, error)
 	SearchAllInstanceGroups() ([]*assetpb.ResourceSearchResult, error)
+	SearchAllDatasets() ([]*assetpb.ResourceSearchResult, error)
 }
 
 type assetRepository struct {
@@ -57,6 +59,7 @@ func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearc
 			computeNetworkAssetType,
 			dnsManagedZoneAssetType,
 			computeInstanceGroupAssetType,
+			bigqueryDatasetAssetType,
 		},
 	}
 	var results []*assetpb.ResourceSearchResult
@@ -120,3 +123,7 @@ func (s assetRepository) SearchAllDNSManagedZones() ([]*assetpb.ResourceSearchRe
 func (s assetRepository) SearchAllInstanceGroups() ([]*assetpb.ResourceSearchResult, error) {
 	return s.searchAllResources(computeInstanceGroupAssetType)
 }
+
+func (s assetRepository) SearchAllDatasets() ([]*assetpb.ResourceSearchResult, error) {
+	return s.searchAllResources(bigqueryDatasetAssetType)
+}

pkg/remote/google/repository/mock_AssetRepository.go

@@ -58,6 +58,29 @@ func (_m *MockAssetRepository) SearchAllDNSManagedZones() ([]*asset.ResourceSear
 	return r0, r1
 }
 
+// SearchAllDatasets provides a mock function with given fields:
+func (_m *MockAssetRepository) SearchAllDatasets() ([]*asset.ResourceSearchResult, error) {
+	ret := _m.Called()
+
+	var r0 []*asset.ResourceSearchResult
+	if rf, ok := ret.Get(0).(func() []*asset.ResourceSearchResult); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*asset.ResourceSearchResult)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func() error); ok {
+		r1 = rf()
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // SearchAllFirewalls provides a mock function with given fields:
 func (_m *MockAssetRepository) SearchAllFirewalls() ([]*asset.ResourceSearchResult, error) {
 	ret := _m.Called()

pkg/remote/google_bigquery_scanner_test.go

@@ -0,0 +1,128 @@
+package remote
+
+import (
+	"testing"
+
+	"github.com/cloudskiff/driftctl/mocks"
+	"github.com/cloudskiff/driftctl/pkg/filter"
+	"github.com/cloudskiff/driftctl/pkg/remote/alerts"
+	"github.com/cloudskiff/driftctl/pkg/remote/cache"
+	"github.com/cloudskiff/driftctl/pkg/remote/common"
+	remoteerr "github.com/cloudskiff/driftctl/pkg/remote/error"
+	"github.com/cloudskiff/driftctl/pkg/remote/google"
+	"github.com/cloudskiff/driftctl/pkg/remote/google/repository"
+	"github.com/cloudskiff/driftctl/pkg/resource"
+	googleresource "github.com/cloudskiff/driftctl/pkg/resource/google"
+	"github.com/cloudskiff/driftctl/pkg/terraform"
+	testgoogle "github.com/cloudskiff/driftctl/test/google"
+	testresource "github.com/cloudskiff/driftctl/test/resource"
+	terraform2 "github.com/cloudskiff/driftctl/test/terraform"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	assetpb "google.golang.org/genproto/googleapis/cloud/asset/v1"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func TestGoogleBigqueryDataset(t *testing.T) {
+
+	cases := []struct {
+		test             string
+		assertExpected   func(t *testing.T, got []*resource.Resource)
+		response         []*assetpb.ResourceSearchResult
+		responseErr      error
+		setupAlerterMock func(alerter *mocks.AlerterInterface)
+		wantErr          error
+	}{
+		{
+			test:     "no dataset",
+			response: []*assetpb.ResourceSearchResult{},
+			assertExpected: func(t *testing.T, got []*resource.Resource) {
+				assert.Len(t, got, 0)
+			},
+		},
+		{
+			test: "multiples dataset",
+			assertExpected: func(t *testing.T, got []*resource.Resource) {
+				assert.Len(t, got, 1)
+				assert.Equal(t, "projects/cloudskiff-dev-elie/datasets/example_dataset", got[0].ResourceId())
+				assert.Equal(t, "google_bigquery_dataset", got[0].ResourceType())
+			},
+			response: []*assetpb.ResourceSearchResult{
+				{
+					AssetType: "bigquery.googleapis.com/Dataset",
+					Name:      "//bigquery.googleapis.com/projects/cloudskiff-dev-elie/datasets/example_dataset",
+				},
+			},
+		},
+		{
+			test: "cannot list datasets",
+			assertExpected: func(t *testing.T, got []*resource.Resource) {
+				assert.Len(t, got, 0)
+			},
+			responseErr: status.Error(codes.PermissionDenied, "The caller does not have permission"),
+			setupAlerterMock: func(alerter *mocks.AlerterInterface) {
+				alerter.On(
+					"SendAlert",
+					"google_bigquery_dataset",
+					alerts.NewRemoteAccessDeniedAlert(
+						common.RemoteGoogleTerraform,
+						remoteerr.NewResourceListingError(
+							status.Error(codes.PermissionDenied, "The caller does not have permission"),
+							"google_bigquery_dataset",
+						),
+						alerts.EnumerationPhase,
+					),
+				).Once()
+			},
+		},
+	}
+
+	providerVersion := "3.78.0"
+	schemaRepository := testresource.InitFakeSchemaRepository("google", providerVersion)
+	googleresource.InitResourcesMetadata(schemaRepository)
+	factory := terraform.NewTerraformResourceFactory(schemaRepository)
+
+	for _, c := range cases {
+		t.Run(c.test, func(tt *testing.T) {
+			scanOptions := ScannerOptions{Deep: true}
+			providerLibrary := terraform.NewProviderLibrary()
+			remoteLibrary := common.NewRemoteLibrary()
+
+			// Initialize mocks
+			alerter := &mocks.AlerterInterface{}
+			if c.setupAlerterMock != nil {
+				c.setupAlerterMock(alerter)
+			}
+
+			assetClient, err := testgoogle.NewFakeAssetServer(c.response, c.responseErr)
+			if err != nil {
+				tt.Fatal(err)
+			}
+
+			realProvider, err := terraform2.InitTestGoogleProvider(providerLibrary, providerVersion)
+			if err != nil {
+				tt.Fatal(err)
+			}
+
+			repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0))
+
+			remoteLibrary.AddEnumerator(google.NewGoogleBigqueryDatasetEnumerator(repo, factory))
+
+			testFilter := &filter.MockFilter{}
+			testFilter.On("IsTypeIgnored", mock.Anything).Return(false)
+
+			s := NewScanner(remoteLibrary, alerter, scanOptions, testFilter)
+			got, err := s.Resources()
+			assert.Equal(tt, err, c.wantErr)
+			if err != nil {
+				return
+			}
+			alerter.AssertExpectations(tt)
+			testFilter.AssertExpectations(tt)
+			if c.assertExpected != nil {
+				c.assertExpected(t, got)
+			}
+		})
+	}
+}

pkg/resource/google/google_bigquery_dataset.go

@@ -0,0 +1,13 @@
+package google
+
+import "github.com/cloudskiff/driftctl/pkg/resource"
+
+const GoogleBigqueryDatasetResourceType = "google_bigquery_dataset"
+
+func initGoogleBigqueryDatasetMetadata(resourceSchemaRepository resource.SchemaRepositoryInterface) {
+	resourceSchemaRepository.SetHumanReadableAttributesFunc(GoogleBigqueryDatasetResourceType, func(res *resource.Resource) map[string]string {
+		return map[string]string{
+			"name": *res.Attrs.GetString("name"),
+		}
+	})
+}

pkg/resource/google/google_bigquery_dataset_test.go

@@ -0,0 +1,30 @@
+package google_test
+
+import (
+	"testing"
+
+	"github.com/cloudskiff/driftctl/test"
+	"github.com/cloudskiff/driftctl/test/acceptance"
+)
+
+func TestAcc_Google_BigqueryDataset(t *testing.T) {
+	acceptance.Run(t, acceptance.AccTestCase{
+		TerraformVersion: "0.15.5",
+		Paths:            []string{"./testdata/acc/google_bigquery_dataset"},
+		Args: []string{
+			"scan",
+			"--to", "gcp+tf",
+		},
+		Checks: []acceptance.AccCheck{
+			{
+				Check: func(result *test.ScanResult, stdout string, err error) {
+					if err != nil {
+						t.Fatal(err)
+					}
+					result.AssertInfrastructureIsInSync()
+					result.AssertManagedCount(1)
+				},
+			},
+		},
+	})
+}

pkg/resource/google/metadatas.go

@@ -9,4 +9,5 @@ func InitResourcesMetadata(resourceSchemaRepository resource.SchemaRepositoryInt
 	initGoogleComputeNetworkMetadata(resourceSchemaRepository)
 	initGoogleStorageBucketIamBMemberMetadata(resourceSchemaRepository)
 	initGoogleComputeInstanceGroupMetadata(resourceSchemaRepository)
+	initGoogleBigqueryDatasetMetadata(resourceSchemaRepository)
 }

pkg/resource/google/testdata/acc/google_bigquery_dataset/.driftignore

@@ -0,0 +1,2 @@
+*
+!google_bigquery_dataset

pkg/resource/google/testdata/acc/google_bigquery_dataset/.terraform.lock.hcl

@@ -0,0 +1,21 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/google" {
+  version     = "3.78.0"
+  constraints = "3.78.0"
+  hashes = [
+    "h1:iCyTW8BWdr6Bvd5B89wkxlrB8xLxqHvT1CPmGuKembU=",
+    "zh:027971c4689b6130619827fe57ce260aaca060db3446817d3a92869dba7cc07f",
+    "zh:0876dbecc0d441bf2479edd17fe9141d77274b5071ea5f68ac26a2994bff66f3",
+    "zh:2a5363ed6b1b880f5284e604567cfdabecca809584c30bbe7f19ff568d1ea4cd",
+    "zh:2f5af69b70654bda91199f6393253e3e479107deebfeddc3fe5850b3a1e83dfb",
+    "zh:52e6816ef11f5f799a6626dfff384e2153b37450d8320f1ef1eee8f71a2a87b2",
+    "zh:59ae534607db13db35c0015c06d1ae6d4886f01f7e8fd4e07bc120236a01c494",
+    "zh:65ab2ed1746ea02d0b1bbd8a22ff3a95d09dc8bdb3841fbc17e45e9feccfb327",
+    "zh:877a71d24ff65ede3f0c5973168acfeaea0f2fea3757cab5600efcddfd3171d5",
+    "zh:8b10c9643a4a53148f6758bfd60804b33c2b838482f2c39ed210b729e6b1e2e8",
+    "zh:ba682648d9f6c11a6d04a250ac79eec39271f615f3ff60c5ae73ebfcc2cdb450",
+    "zh:e946561921e0279450e9b9f705de9354ce35562ed4cc0d4cd3512aa9eb1f6486",
+  ]
+}

pkg/resource/google/testdata/acc/google_bigquery_dataset/terraform.tf

@@ -0,0 +1,17 @@
+provider "google" {}
+
+terraform {
+    required_version = "~> 0.15.0"
+    required_providers {
+        google = {
+            version = "3.78.0"
+        }
+    }
+}
+
+resource "google_bigquery_dataset" "dataset" {
+    dataset_id                  = "TestAcc_Google_BigqueryDataset"
+    friendly_name               = "TestAcc_Google_BigqueryDataset"
+    description                 = "This is a test description"
+    location                    = "EU"
+}

pkg/resource/resource_types.go

@@ -144,6 +144,7 @@ var supportedTypes = map[string]ResourceTypeMeta{
 	}},
 	"google_dns_managed_zone":       {},
 	"google_compute_instance_group": {},
+	"google_bigquery_dataset":       {},
 
 	"azurerm_storage_account":   {},
 	"azurerm_storage_container": {},