sundowndev
/
driftctl
mirror of https://github.com/sundowndev/driftctl.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1145 from cloudskiff/add_google_bigquery_dataset 

Add google_bigquery_dataset
main
Elie 2021-10-19 17:20:11 +02:00 committed by GitHub
parent e6b3b1256c 70d0d0e6fa
commit 7c653d0500
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 420 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/iac/terraform/state/terraform_state_reader_test.go
View File

@ -338,6 +338,7 @@ func TestTerraformStateReader_Google_Resources(t *testing.T) {
{name: "Bucket IAM members", dirName: "google_bucket_iam_member", wantErr: false},
{name: "Bucket IAM Policy", dirName: "google_bucket_iam_policy", wantErr: false},
{name: "DNS managed zone", dirName: "google_dns_managed_zone", wantErr: false},
{name: "bigquery dataset", dirName: "google_bigquery_dataset", wantErr: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {

51
pkg/iac/terraform/state/test/google_bigquery_dataset/results.golden.json Executable file
View File

@ -0,0 +1,51 @@
[
{
"Id": "projects/cloudskiff-dev-elie/datasets/example_dataset",
"Type": "google_bigquery_dataset",
"Attrs": {
"access": [
{
"domain": "",
"group_by_email": "",
"role": "OWNER",
"special_group": "",
"user_by_email": "elie.charra@cloudskiff.com"
},
{
"domain": "",
"group_by_email": "",
"role": "OWNER",
"special_group": "projectOwners",
"user_by_email": ""
},
{
"domain": "",
"group_by_email": "",
"role": "READER",
"special_group": "projectReaders",
"user_by_email": ""
},
{
"domain": "",
"group_by_email": "",
"role": "WRITER",
"special_group": "projectWriters",
"user_by_email": ""
}
],
"creation_time": 1634647437581,
"dataset_id": "example_dataset",
"default_partition_expiration_ms": 0,
"default_table_expiration_ms": 0,
"delete_contents_on_destroy": false,
"description": "This is a test description",
"etag": "RrsFjBErVHJw1kDByWfKmQ==",
"friendly_name": "test",
"id": "projects/cloudskiff-dev-elie/datasets/example_dataset",
"last_modified_time": 1634647437581,
"location": "EU",
"project": "cloudskiff-dev-elie",
"self_link": "https://bigquery.googleapis.com/bigquery/v2/projects/cloudskiff-dev-elie/datasets/example_dataset"
}
}
]

74
pkg/iac/terraform/state/test/google_bigquery_dataset/terraform.tfstate Normal file
View File

@ -0,0 +1,74 @@
{
"version": 4,
"terraform_version": "0.15.5",
"serial": 23,
"lineage": "80058dbf-c946-85c8-ad1c-165fad8cf85e",
"outputs": {},
"resources": [
{
"mode": "managed",
"type": "google_bigquery_dataset",
"name": "dataset",
"provider": "provider[\"registry.terraform.io/hashicorp/google\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"access": [
{
"domain": "",
"group_by_email": "",
"role": "OWNER",
"special_group": "",
"user_by_email": "elie.charra@cloudskiff.com",
"view": []
},
{
"domain": "",
"group_by_email": "",
"role": "OWNER",
"special_group": "projectOwners",
"user_by_email": "",
"view": []
},
{
"domain": "",
"group_by_email": "",
"role": "READER",
"special_group": "projectReaders",
"user_by_email": "",
"view": []
},
{
"domain": "",
"group_by_email": "",
"role": "WRITER",
"special_group": "projectWriters",
"user_by_email": "",
"view": []
}
],
"creation_time": 1634647437581,
"dataset_id": "example_dataset",
"default_encryption_configuration": [],
"default_partition_expiration_ms": 0,
"default_table_expiration_ms": 0,
"delete_contents_on_destroy": false,
"description": "This is a test description",
"etag": "RrsFjBErVHJw1kDByWfKmQ==",
"friendly_name": "test",
"id": "projects/cloudskiff-dev-elie/datasets/example_dataset",
"labels": null,
"last_modified_time": 1634647437581,
"location": "EU",
"project": "cloudskiff-dev-elie",
"self_link": "https://bigquery.googleapis.com/bigquery/v2/projects/cloudskiff-dev-elie/datasets/example_dataset",
"timeouts": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoyNDAwMDAwMDAwMDAsImRlbGV0ZSI6MjQwMDAwMDAwMDAwLCJ1cGRhdGUiOjI0MDAwMDAwMDAwMH19"
}
]
}
]
}

49
pkg/remote/google/google_bigquery_dataset_enumerator.go Normal file
View File

@ -0,0 +1,49 @@
package google
import (
remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error"
"github.com/cloudskiff/driftctl/pkg/remote/google/repository"
"github.com/cloudskiff/driftctl/pkg/resource"
"github.com/cloudskiff/driftctl/pkg/resource/google"
)
type GoogleBigqueryDatasetEnumerator struct {
repository repository.AssetRepository
factory resource.ResourceFactory
}
func NewGoogleBigqueryDatasetEnumerator(repo repository.AssetRepository, factory resource.ResourceFactory) *GoogleBigqueryDatasetEnumerator {
return &GoogleBigqueryDatasetEnumerator{
repository: repo,
factory: factory,
}
}
func (e *GoogleBigqueryDatasetEnumerator) SupportedType() resource.ResourceType {
return google.GoogleBigqueryDatasetResourceType
}
func (e *GoogleBigqueryDatasetEnumerator) Enumerate() ([]*resource.Resource, error) {
resources, err := e.repository.SearchAllDatasets()
if err != nil {
return nil, remoteerror.NewResourceListingError(err, string(e.SupportedType()))
}
results := make([]*resource.Resource, 0, len(resources))
for _, res := range resources {
results = append(
results,
e.factory.CreateAbstractResource(
string(e.SupportedType()),
trimResourceName(res.GetName()),
map[string]interface{}{
"name": res.DisplayName,
},
),
)
}
return results, err
}

2
pkg/remote/google/init.go
View File

@ -72,6 +72,8 @@ func Init(version string, alerter *alerter.Alerter,
remoteLibrary.AddEnumerator(NewGoogleComputeInstanceGroupEnumerator(assetRepository, factory))
remoteLibrary.AddDetailsFetcher(google.GoogleComputeInstanceGroupResourceType, common.NewGenericDetailsFetcher(google.GoogleComputeInstanceGroupResourceType, provider, deserializer))
remoteLibrary.AddEnumerator(NewGoogleBigqueryDatasetEnumerator(assetRepository, factory))
err = resourceSchemaRepository.Init(terraform.GOOGLE, provider.Version(), provider.Schema())
if err != nil {
return err

7
pkg/remote/google/repository/asset.go
View File

@ -20,6 +20,7 @@ const (
computeNetworkAssetType = "compute.googleapis.com/Network"
dnsManagedZoneAssetType = "dns.googleapis.com/ManagedZone"
computeInstanceGroupAssetType = "compute.googleapis.com/InstanceGroup"
bigqueryDatasetAssetType = "bigquery.googleapis.com/Dataset"
)
type AssetRepository interface {
@ -30,6 +31,7 @@ type AssetRepository interface {
SearchAllNetworks() ([]*assetpb.ResourceSearchResult, error)
SearchAllDNSManagedZones() ([]*assetpb.ResourceSearchResult, error)
SearchAllInstanceGroups() ([]*assetpb.ResourceSearchResult, error)
SearchAllDatasets() ([]*assetpb.ResourceSearchResult, error)
}
type assetRepository struct {
@ -57,6 +59,7 @@ func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearc
computeNetworkAssetType,
dnsManagedZoneAssetType,
computeInstanceGroupAssetType,
bigqueryDatasetAssetType,
},
}
var results []*assetpb.ResourceSearchResult
@ -120,3 +123,7 @@ func (s assetRepository) SearchAllDNSManagedZones() ([]*assetpb.ResourceSearchRe
func (s assetRepository) SearchAllInstanceGroups() ([]*assetpb.ResourceSearchResult, error) {
return s.searchAllResources(computeInstanceGroupAssetType)
}
func (s assetRepository) SearchAllDatasets() ([]*assetpb.ResourceSearchResult, error) {
return s.searchAllResources(bigqueryDatasetAssetType)
}

23
pkg/remote/google/repository/mock_AssetRepository.go
View File

@ -58,6 +58,29 @@ func (_m *MockAssetRepository) SearchAllDNSManagedZones() ([]*asset.ResourceSear
return r0, r1
}
// SearchAllDatasets provides a mock function with given fields:
func (_m *MockAssetRepository) SearchAllDatasets() ([]*asset.ResourceSearchResult, error) {
ret := _m.Called()
var r0 []*asset.ResourceSearchResult
if rf, ok := ret.Get(0).(func() []*asset.ResourceSearchResult); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*asset.ResourceSearchResult)
}
}
var r1 error
if rf, ok := ret.Get(1).(func() error); ok {
r1 = rf()
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// SearchAllFirewalls provides a mock function with given fields:
func (_m *MockAssetRepository) SearchAllFirewalls() ([]*asset.ResourceSearchResult, error) {
ret := _m.Called()

128
pkg/remote/google_bigquery_scanner_test.go Normal file
View File

@ -0,0 +1,128 @@
package remote
import (
"testing"
"github.com/cloudskiff/driftctl/mocks"
"github.com/cloudskiff/driftctl/pkg/filter"
"github.com/cloudskiff/driftctl/pkg/remote/alerts"
"github.com/cloudskiff/driftctl/pkg/remote/cache"
"github.com/cloudskiff/driftctl/pkg/remote/common"
remoteerr "github.com/cloudskiff/driftctl/pkg/remote/error"
"github.com/cloudskiff/driftctl/pkg/remote/google"
"github.com/cloudskiff/driftctl/pkg/remote/google/repository"
"github.com/cloudskiff/driftctl/pkg/resource"
googleresource "github.com/cloudskiff/driftctl/pkg/resource/google"
"github.com/cloudskiff/driftctl/pkg/terraform"
testgoogle "github.com/cloudskiff/driftctl/test/google"
testresource "github.com/cloudskiff/driftctl/test/resource"
terraform2 "github.com/cloudskiff/driftctl/test/terraform"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
assetpb "google.golang.org/genproto/googleapis/cloud/asset/v1"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
func TestGoogleBigqueryDataset(t *testing.T) {
cases := []struct {
test string
assertExpected func(t *testing.T, got []*resource.Resource)
response []*assetpb.ResourceSearchResult
responseErr error
setupAlerterMock func(alerter *mocks.AlerterInterface)
wantErr error
}{
{
test: "no dataset",
response: []*assetpb.ResourceSearchResult{},
assertExpected: func(t *testing.T, got []*resource.Resource) {
assert.Len(t, got, 0)
},
},
{
test: "multiples dataset",
assertExpected: func(t *testing.T, got []*resource.Resource) {
assert.Len(t, got, 1)
assert.Equal(t, "projects/cloudskiff-dev-elie/datasets/example_dataset", got[0].ResourceId())
assert.Equal(t, "google_bigquery_dataset", got[0].ResourceType())
},
response: []*assetpb.ResourceSearchResult{
{
AssetType: "bigquery.googleapis.com/Dataset",
Name: "//bigquery.googleapis.com/projects/cloudskiff-dev-elie/datasets/example_dataset",
},
},
},
{
test: "cannot list datasets",
assertExpected: func(t *testing.T, got []*resource.Resource) {
assert.Len(t, got, 0)
},
responseErr: status.Error(codes.PermissionDenied, "The caller does not have permission"),
setupAlerterMock: func(alerter *mocks.AlerterInterface) {
alerter.On(
"SendAlert",
"google_bigquery_dataset",
alerts.NewRemoteAccessDeniedAlert(
common.RemoteGoogleTerraform,
remoteerr.NewResourceListingError(
status.Error(codes.PermissionDenied, "The caller does not have permission"),
"google_bigquery_dataset",
),
alerts.EnumerationPhase,
),
).Once()
},
},
}
providerVersion := "3.78.0"
schemaRepository := testresource.InitFakeSchemaRepository("google", providerVersion)
googleresource.InitResourcesMetadata(schemaRepository)
factory := terraform.NewTerraformResourceFactory(schemaRepository)
for _, c := range cases {
t.Run(c.test, func(tt *testing.T) {
scanOptions := ScannerOptions{Deep: true}
providerLibrary := terraform.NewProviderLibrary()
remoteLibrary := common.NewRemoteLibrary()
// Initialize mocks
alerter := &mocks.AlerterInterface{}
if c.setupAlerterMock != nil {
c.setupAlerterMock(alerter)
}
assetClient, err := testgoogle.NewFakeAssetServer(c.response, c.responseErr)
if err != nil {
tt.Fatal(err)
}
realProvider, err := terraform2.InitTestGoogleProvider(providerLibrary, providerVersion)
if err != nil {
tt.Fatal(err)
}
repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0))
remoteLibrary.AddEnumerator(google.NewGoogleBigqueryDatasetEnumerator(repo, factory))
testFilter := &filter.MockFilter{}
testFilter.On("IsTypeIgnored", mock.Anything).Return(false)
s := NewScanner(remoteLibrary, alerter, scanOptions, testFilter)
got, err := s.Resources()
assert.Equal(tt, err, c.wantErr)
if err != nil {
return
}
alerter.AssertExpectations(tt)
testFilter.AssertExpectations(tt)
if c.assertExpected != nil {
c.assertExpected(t, got)
}
})
}
}

13
pkg/resource/google/google_bigquery_dataset.go Normal file
View File

@ -0,0 +1,13 @@
package google
import "github.com/cloudskiff/driftctl/pkg/resource"
const GoogleBigqueryDatasetResourceType = "google_bigquery_dataset"
func initGoogleBigqueryDatasetMetadata(resourceSchemaRepository resource.SchemaRepositoryInterface) {
resourceSchemaRepository.SetHumanReadableAttributesFunc(GoogleBigqueryDatasetResourceType, func(res *resource.Resource) map[string]string {
return map[string]string{
"name": *res.Attrs.GetString("name"),
}
})
}

30
pkg/resource/google/google_bigquery_dataset_test.go Normal file
View File

@ -0,0 +1,30 @@
package google_test
import (
"testing"
"github.com/cloudskiff/driftctl/test"
"github.com/cloudskiff/driftctl/test/acceptance"
)
func TestAcc_Google_BigqueryDataset(t *testing.T) {
acceptance.Run(t, acceptance.AccTestCase{
TerraformVersion: "0.15.5",
Paths: []string{"./testdata/acc/google_bigquery_dataset"},
Args: []string{
"scan",
"--to", "gcp+tf",
},
Checks: []acceptance.AccCheck{
{
Check: func(result *test.ScanResult, stdout string, err error) {
if err != nil {
t.Fatal(err)
}
result.AssertInfrastructureIsInSync()
result.AssertManagedCount(1)
},
},
},
})
}

1
pkg/resource/google/metadatas.go
View File

@ -9,4 +9,5 @@ func InitResourcesMetadata(resourceSchemaRepository resource.SchemaRepositoryInt
initGoogleComputeNetworkMetadata(resourceSchemaRepository)
initGoogleStorageBucketIamBMemberMetadata(resourceSchemaRepository)
initGoogleComputeInstanceGroupMetadata(resourceSchemaRepository)
initGoogleBigqueryDatasetMetadata(resourceSchemaRepository)
}

2
pkg/resource/google/testdata/acc/google_bigquery_dataset/.driftignore vendored Normal file
View File

@ -0,0 +1,2 @@
*
!google_bigquery_dataset

21
pkg/resource/google/testdata/acc/google_bigquery_dataset/.terraform.lock.hcl vendored Normal file
View File

@ -0,0 +1,21 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/hashicorp/google" {
version = "3.78.0"
constraints = "3.78.0"
hashes = [
"h1:iCyTW8BWdr6Bvd5B89wkxlrB8xLxqHvT1CPmGuKembU=",
"zh:027971c4689b6130619827fe57ce260aaca060db3446817d3a92869dba7cc07f",
"zh:0876dbecc0d441bf2479edd17fe9141d77274b5071ea5f68ac26a2994bff66f3",
"zh:2a5363ed6b1b880f5284e604567cfdabecca809584c30bbe7f19ff568d1ea4cd",
"zh:2f5af69b70654bda91199f6393253e3e479107deebfeddc3fe5850b3a1e83dfb",
"zh:52e6816ef11f5f799a6626dfff384e2153b37450d8320f1ef1eee8f71a2a87b2",
"zh:59ae534607db13db35c0015c06d1ae6d4886f01f7e8fd4e07bc120236a01c494",
"zh:65ab2ed1746ea02d0b1bbd8a22ff3a95d09dc8bdb3841fbc17e45e9feccfb327",
"zh:877a71d24ff65ede3f0c5973168acfeaea0f2fea3757cab5600efcddfd3171d5",
"zh:8b10c9643a4a53148f6758bfd60804b33c2b838482f2c39ed210b729e6b1e2e8",
"zh:ba682648d9f6c11a6d04a250ac79eec39271f615f3ff60c5ae73ebfcc2cdb450",
"zh:e946561921e0279450e9b9f705de9354ce35562ed4cc0d4cd3512aa9eb1f6486",
]
}

17
pkg/resource/google/testdata/acc/google_bigquery_dataset/terraform.tf vendored Normal file
View File

@ -0,0 +1,17 @@
provider "google" {}
terraform {
required_version = "~> 0.15.0"
required_providers {
google = {
version = "3.78.0"
}
}
}
resource "google_bigquery_dataset" "dataset" {
dataset_id = "TestAcc_Google_BigqueryDataset"
friendly_name = "TestAcc_Google_BigqueryDataset"
description = "This is a test description"
location = "EU"
}

1
pkg/resource/resource_types.go
View File

@ -144,6 +144,7 @@ var supportedTypes = map[string]ResourceTypeMeta{
}},
"google_dns_managed_zone": {},
"google_compute_instance_group": {},
"google_bigquery_dataset": {},
"azurerm_storage_account": {},
"azurerm_storage_container": {},