Merge branch 'main' into res/api_gtw_method

main
William BEUIL 2021-10-22 14:15:26 +02:00 committed by GitHub
commit 3a3fe7ae4a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 511 additions and 9 deletions

View File

@ -343,6 +343,7 @@ func TestTerraformStateReader_Google_Resources(t *testing.T) {
{name: "bigquery dataset", dirName: "google_bigquery_dataset", wantErr: false},
{name: "bigquery table", dirName: "google_bigquery_table", wantErr: false},
{name: "compute address", dirName: "google_compute_address", wantErr: false},
{name: "cloudfunctions function", dirName: "google_cloudfunctions_function", wantErr: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {

View File

@ -0,0 +1,26 @@
[
{
"Id": "projects/cloudskiff-dev-elie/locations/us-central1/functions/function-test",
"Type": "google_cloudfunctions_function",
"Attrs": {
"available_memory_mb": 128,
"description": "My function",
"entry_point": "helloHttp",
"https_trigger_url": "https://us-central1-cloudskiff-dev-elie.cloudfunctions.net/function-test",
"id": "projects/cloudskiff-dev-elie/locations/us-central1/functions/function-test",
"ingress_settings": "ALLOW_ALL",
"max_instances": 0,
"name": "function-test",
"project": "cloudskiff-dev-elie",
"region": "us-central1",
"runtime": "nodejs14",
"service_account_email": "cloudskiff-dev-elie@appspot.gserviceaccount.com",
"source_archive_bucket": "test-bucket-driftctl",
"source_archive_object": "index.zip",
"timeout": 60,
"trigger_http": true,
"vpc_connector": "",
"vpc_connector_egress_settings": ""
}
}
]

View File

@ -0,0 +1,52 @@
{
"version": 4,
"terraform_version": "0.15.5",
"serial": 22,
"lineage": "0738cef4-9d69-9ccc-aebd-1177cafa0aa9",
"outputs": {},
"resources": [
{
"mode": "managed",
"type": "google_cloudfunctions_function",
"name": "function",
"provider": "provider[\"registry.terraform.io/hashicorp/google\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"available_memory_mb": 128,
"build_environment_variables": null,
"description": "My function",
"entry_point": "helloHttp",
"environment_variables": null,
"event_trigger": [],
"https_trigger_url": "https://us-central1-cloudskiff-dev-elie.cloudfunctions.net/function-test",
"id": "projects/cloudskiff-dev-elie/locations/us-central1/functions/function-test",
"ingress_settings": "ALLOW_ALL",
"labels": null,
"max_instances": 0,
"name": "function-test",
"project": "cloudskiff-dev-elie",
"region": "us-central1",
"runtime": "nodejs14",
"service_account_email": "cloudskiff-dev-elie@appspot.gserviceaccount.com",
"source_archive_bucket": "test-bucket-driftctl",
"source_archive_object": "index.zip",
"source_repository": [],
"timeout": 60,
"timeouts": null,
"trigger_http": true,
"vpc_connector": "",
"vpc_connector_egress_settings": ""
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJyZWFkIjozMDAwMDAwMDAwMDAsInVwZGF0ZSI6MzAwMDAwMDAwMDAwfX0=",
"dependencies": [
"google_storage_bucket.bucket",
"google_storage_bucket_object.archive"
]
}
]
}
]
}

View File

@ -0,0 +1,53 @@
package google
import (
remoteerror "github.com/cloudskiff/driftctl/pkg/remote/error"
"github.com/cloudskiff/driftctl/pkg/remote/google/repository"
"github.com/cloudskiff/driftctl/pkg/resource"
"github.com/cloudskiff/driftctl/pkg/resource/google"
"github.com/sirupsen/logrus"
)
type GoogleCloudFunctionsFunctionEnumerator struct {
repository repository.AssetRepository
factory resource.ResourceFactory
}
func NewGoogleCloudFunctionsFunctionEnumerator(repo repository.AssetRepository, factory resource.ResourceFactory) *GoogleCloudFunctionsFunctionEnumerator {
return &GoogleCloudFunctionsFunctionEnumerator{
repository: repo,
factory: factory,
}
}
func (e *GoogleCloudFunctionsFunctionEnumerator) SupportedType() resource.ResourceType {
return google.GoogleCloudFunctionsFunctionResourceType
}
func (e *GoogleCloudFunctionsFunctionEnumerator) Enumerate() ([]*resource.Resource, error) {
resources, err := e.repository.SearchAllFunctions()
if err != nil {
return nil, remoteerror.NewResourceListingError(err, string(e.SupportedType()))
}
results := make([]*resource.Resource, 0, len(resources))
for _, res := range resources {
name, exist := res.GetResource().GetData().GetFields()["name"]
if !exist || name.GetStringValue() == "" {
logrus.WithField("name", res.GetName()).Warn("Unable to retrieve resource name")
continue
}
results = append(
results,
e.factory.CreateAbstractResource(
string(e.SupportedType()),
name.GetStringValue(),
map[string]interface{}{},
),
)
}
return results, err
}

View File

@ -86,6 +86,7 @@ func Init(version string, alerter *alerter.Alerter,
remoteLibrary.AddEnumerator(NewGoogleBigqueryTableEnumerator(assetRepository, factory))
remoteLibrary.AddEnumerator(NewGoogleComputeAddressEnumerator(assetRepository, factory))
remoteLibrary.AddEnumerator(NewGoogleCloudFunctionsFunctionEnumerator(assetRepository, factory))
err = resourceSchemaRepository.Init(terraform.GOOGLE, provider.Version(), provider.Schema())
if err != nil {

View File

@ -23,6 +23,7 @@ const (
bigqueryDatasetAssetType = "bigquery.googleapis.com/Dataset"
bigqueryTableAssetType = "bigquery.googleapis.com/Table"
computeAddressAssetType = "compute.googleapis.com/Address"
cloudFunctionsFunction = "cloudfunctions.googleapis.com/CloudFunction"
)
type AssetRepository interface {
@ -36,6 +37,7 @@ type AssetRepository interface {
SearchAllDatasets() ([]*assetpb.ResourceSearchResult, error)
SearchAllTables() ([]*assetpb.ResourceSearchResult, error)
SearchAllAddresses() ([]*assetpb.ResourceSearchResult, error)
SearchAllFunctions() ([]*assetpb.Asset, error)
}
type assetRepository struct {
@ -52,6 +54,48 @@ func NewAssetRepository(client *asset.Client, config config.GCPTerraformConfig,
}
}
func (s assetRepository) listAllResources(ty string) ([]*assetpb.Asset, error) {
req := &assetpb.ListAssetsRequest{
Parent: fmt.Sprintf("projects/%s", s.config.Project),
ContentType: assetpb.ContentType_RESOURCE,
AssetTypes: []string{
cloudFunctionsFunction,
},
}
var results []*assetpb.Asset
cacheKey := "listAllResources"
cachedResults := s.cache.GetAndLock(cacheKey)
defer s.cache.Unlock(cacheKey)
if cachedResults != nil {
results = cachedResults.([]*assetpb.Asset)
}
if results == nil {
it := s.client.ListAssets(context.Background(), req)
for {
resource, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
return nil, err
}
results = append(results, resource)
}
s.cache.Put(cacheKey, results)
}
filteredResults := []*assetpb.Asset{}
for _, result := range results {
if result.AssetType == ty {
filteredResults = append(filteredResults, result)
}
}
return filteredResults, nil
}
func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearchResult, error) {
req := &assetpb.SearchAllResourcesRequest{
Scope: fmt.Sprintf("projects/%s", s.config.Project),
@ -141,3 +185,7 @@ func (s assetRepository) SearchAllTables() ([]*assetpb.ResourceSearchResult, err
func (s assetRepository) SearchAllAddresses() ([]*assetpb.ResourceSearchResult, error) {
return s.searchAllResources(computeAddressAssetType)
}
func (s assetRepository) SearchAllFunctions() ([]*assetpb.Asset, error) {
return s.listAllResources(cloudFunctionsFunction)
}

View File

@ -127,6 +127,29 @@ func (_m *MockAssetRepository) SearchAllFirewalls() ([]*asset.ResourceSearchResu
return r0, r1
}
// SearchAllFunctions provides a mock function with given fields:
func (_m *MockAssetRepository) SearchAllFunctions() ([]*asset.Asset, error) {
ret := _m.Called()
var r0 []*asset.Asset
if rf, ok := ret.Get(0).(func() []*asset.Asset); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*asset.Asset)
}
}
var r1 error
if rf, ok := ret.Get(1).(func() error); ok {
r1 = rf()
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// SearchAllInstanceGroups provides a mock function with given fields:
func (_m *MockAssetRepository) SearchAllInstanceGroups() ([]*asset.ResourceSearchResult, error) {
ret := _m.Called()

View File

@ -0,0 +1,152 @@
package remote
import (
"testing"
"github.com/cloudskiff/driftctl/mocks"
"github.com/cloudskiff/driftctl/pkg/filter"
"github.com/cloudskiff/driftctl/pkg/remote/alerts"
"github.com/cloudskiff/driftctl/pkg/remote/cache"
"github.com/cloudskiff/driftctl/pkg/remote/common"
remoteerr "github.com/cloudskiff/driftctl/pkg/remote/error"
"github.com/cloudskiff/driftctl/pkg/remote/google"
"github.com/cloudskiff/driftctl/pkg/remote/google/repository"
"github.com/cloudskiff/driftctl/pkg/resource"
googleresource "github.com/cloudskiff/driftctl/pkg/resource/google"
"github.com/cloudskiff/driftctl/pkg/terraform"
testgoogle "github.com/cloudskiff/driftctl/test/google"
testresource "github.com/cloudskiff/driftctl/test/resource"
terraform2 "github.com/cloudskiff/driftctl/test/terraform"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
assetpb "google.golang.org/genproto/googleapis/cloud/asset/v1"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/structpb"
)
func TestGoogleCloudFunctionsFunction(t *testing.T) {
cases := []struct {
test string
assertExpected func(t *testing.T, got []*resource.Resource)
response []*assetpb.Asset
responseErr error
setupAlerterMock func(alerter *mocks.AlerterInterface)
wantErr error
}{
{
test: "no compute instance",
response: []*assetpb.Asset{},
assertExpected: func(t *testing.T, got []*resource.Resource) {
assert.Len(t, got, 0)
},
},
{
test: "one cloud function returned",
assertExpected: func(t *testing.T, got []*resource.Resource) {
assert.Len(t, got, 1)
assert.Equal(t, "projects/cloudskiff-dev-elie/locations/us-central1/functions/function-test", got[0].ResourceId())
assert.Equal(t, "google_cloudfunctions_function", got[0].ResourceType())
},
response: []*assetpb.Asset{
{
AssetType: "cloudfunctions.googleapis.com/CloudFunction",
Name: "//cloudfunctions.googleapis.com/projects/cloudskiff-dev-elie/locations/us-central1/functions/function-test",
Resource: &assetpb.Resource{
Data: func() *structpb.Struct {
v, err := structpb.NewStruct(map[string]interface{}{
"name": "projects/cloudskiff-dev-elie/locations/us-central1/functions/function-test",
})
if err != nil {
t.Fatal(err)
}
return v
}(),
},
},
},
},
{
test: "one cloud function without resource data",
assertExpected: func(t *testing.T, got []*resource.Resource) {
assert.Len(t, got, 0)
},
response: []*assetpb.Asset{
{
AssetType: "cloudfunctions.googleapis.com/CloudFunction",
Name: "//cloudfunctions.googleapis.com/projects/cloudskiff-dev-elie/locations/us-central1/functions/function-test",
},
},
},
{
test: "cannot list cloud functions",
assertExpected: func(t *testing.T, got []*resource.Resource) {
assert.Len(t, got, 0)
},
responseErr: status.Error(codes.PermissionDenied, "The caller does not have permission"),
setupAlerterMock: func(alerter *mocks.AlerterInterface) {
alerter.On(
"SendAlert",
"google_cloudfunctions_function",
alerts.NewRemoteAccessDeniedAlert(
common.RemoteGoogleTerraform,
remoteerr.NewResourceListingError(
status.Error(codes.PermissionDenied, "The caller does not have permission"),
"google_cloudfunctions_function",
),
alerts.EnumerationPhase,
),
).Once()
},
},
}
providerVersion := "3.78.0"
schemaRepository := testresource.InitFakeSchemaRepository("google", providerVersion)
googleresource.InitResourcesMetadata(schemaRepository)
factory := terraform.NewTerraformResourceFactory(schemaRepository)
for _, c := range cases {
t.Run(c.test, func(tt *testing.T) {
scanOptions := ScannerOptions{}
providerLibrary := terraform.NewProviderLibrary()
remoteLibrary := common.NewRemoteLibrary()
// Initialize mocks
alerter := &mocks.AlerterInterface{}
if c.setupAlerterMock != nil {
c.setupAlerterMock(alerter)
}
assetClient, err := testgoogle.NewFakeAssertServerWithList(c.response, c.responseErr)
if err != nil {
tt.Fatal(err)
}
realProvider, err := terraform2.InitTestGoogleProvider(providerLibrary, providerVersion)
if err != nil {
tt.Fatal(err)
}
repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0))
remoteLibrary.AddEnumerator(google.NewGoogleCloudFunctionsFunctionEnumerator(repo, factory))
testFilter := &filter.MockFilter{}
testFilter.On("IsTypeIgnored", mock.Anything).Return(false)
s := NewScanner(remoteLibrary, alerter, scanOptions, testFilter)
got, err := s.Resources()
assert.Equal(tt, err, c.wantErr)
if err != nil {
return
}
alerter.AssertExpectations(tt)
testFilter.AssertExpectations(tt)
if c.assertExpected != nil {
c.assertExpected(t, got)
}
})
}
}

View File

@ -0,0 +1,3 @@
package google
const GoogleCloudFunctionsFunctionResourceType = "google_cloudfunctions_function"

View File

@ -0,0 +1,36 @@
package google_test
import (
"testing"
"time"
"github.com/cloudskiff/driftctl/test"
"github.com/cloudskiff/driftctl/test/acceptance"
)
func TestAcc_Google_CloudFunctionsFunction(t *testing.T) {
acceptance.Run(t, acceptance.AccTestCase{
TerraformVersion: "0.15.5",
Paths: []string{"./testdata/acc/google_cloudfunctions_function"},
Args: []string{
"scan",
"--to", "gcp+tf",
},
Checks: []acceptance.AccCheck{
{
// New cloud function resources are not visible immediatly on GCP api after an apply
// Logic below retry driftctl scan until we can retrieve the results (infra will be in sync) and for maximum 60 seconds
ShouldRetry: func(result *test.ScanResult, retryDuration time.Duration, retryCount uint8) bool {
return !result.IsSync() && retryDuration < time.Minute
},
Check: func(result *test.ScanResult, stdout string, err error) {
if err != nil {
t.Fatal(err)
}
result.AssertInfrastructureIsInSync()
result.AssertManagedCount(1)
},
},
},
})
}

View File

@ -0,0 +1,2 @@
*
!google_cloudfunctions_function

View File

@ -0,0 +1,39 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/hashicorp/google" {
version = "3.78.0"
constraints = "3.78.0"
hashes = [
"h1:iCyTW8BWdr6Bvd5B89wkxlrB8xLxqHvT1CPmGuKembU=",
"zh:027971c4689b6130619827fe57ce260aaca060db3446817d3a92869dba7cc07f",
"zh:0876dbecc0d441bf2479edd17fe9141d77274b5071ea5f68ac26a2994bff66f3",
"zh:2a5363ed6b1b880f5284e604567cfdabecca809584c30bbe7f19ff568d1ea4cd",
"zh:2f5af69b70654bda91199f6393253e3e479107deebfeddc3fe5850b3a1e83dfb",
"zh:52e6816ef11f5f799a6626dfff384e2153b37450d8320f1ef1eee8f71a2a87b2",
"zh:59ae534607db13db35c0015c06d1ae6d4886f01f7e8fd4e07bc120236a01c494",
"zh:65ab2ed1746ea02d0b1bbd8a22ff3a95d09dc8bdb3841fbc17e45e9feccfb327",
"zh:877a71d24ff65ede3f0c5973168acfeaea0f2fea3757cab5600efcddfd3171d5",
"zh:8b10c9643a4a53148f6758bfd60804b33c2b838482f2c39ed210b729e6b1e2e8",
"zh:ba682648d9f6c11a6d04a250ac79eec39271f615f3ff60c5ae73ebfcc2cdb450",
"zh:e946561921e0279450e9b9f705de9354ce35562ed4cc0d4cd3512aa9eb1f6486",
]
}
provider "registry.terraform.io/hashicorp/random" {
version = "3.1.0"
hashes = [
"h1:BZMEPucF+pbu9gsPk0G0BHx7YP04+tKdq2MrRDF1EDM=",
"zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc",
"zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626",
"zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff",
"zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2",
"zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992",
"zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427",
"zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc",
"zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f",
"zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b",
"zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7",
"zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a",
]
}

View File

@ -0,0 +1,40 @@
provider "google" {
region = "us-central1"
}
terraform {
required_version = "~> 0.15.0"
required_providers {
google = {
version = "3.78.0"
}
}
}
resource "random_string" "postfix" {
length = 6
upper = false
special = false
}
resource "google_storage_bucket" "bucket" {
name = "driftctl-qa-${random_string.postfix.result}"
}
resource "google_storage_bucket_object" "archive" {
name = "index.zip"
bucket = google_storage_bucket.bucket.name
source = "./index.zip"
}
resource "google_cloudfunctions_function" "function" {
name = "function-test"
description = "My function"
runtime = "nodejs14"
available_memory_mb = 128
source_archive_bucket = google_storage_bucket.bucket.name
source_archive_object = google_storage_bucket_object.archive.name
trigger_http = true
entry_point = "helloHttp"
}

View File

@ -158,7 +158,8 @@ var supportedTypes = map[string]ResourceTypeMeta{
"google_project_iam_policy": {children: []ResourceType{
"google_project_iam_member",
}},
"google_compute_address": {},
"google_compute_address": {},
"google_cloudfunctions_function": {},
"azurerm_storage_account": {},
"azurerm_storage_container": {},

View File

@ -33,11 +33,12 @@ import (
)
type AccCheck struct {
PreExec func()
PostExec func()
Env map[string]string
Args func() []string
Check func(result *test.ScanResult, stdout string, err error)
PreExec func()
PostExec func()
Env map[string]string
Args func() []string
ShouldRetry func(result *test.ScanResult, retryDuration time.Duration, retryCount uint8) bool
Check func(result *test.ScanResult, stdout string, err error)
}
type RetryConfig struct {
@ -431,6 +432,18 @@ func Run(t *testing.T, c AccTestCase) {
}
logrus.WithField("args", fmt.Sprintf("%+v", os.Args)).Debug("Running driftctl")
_, out, cmdErr := runDriftCtlCmd(driftctlCmd)
result := c.getResult(t)
var retryCount uint8 = 0
timeBeforeRetry := time.Now()
for {
if check.ShouldRetry == nil || !check.ShouldRetry(result, time.Since(timeBeforeRetry), retryCount) {
break
}
logrus.WithField("count", fmt.Sprintf("%d", retryCount)).Debug("Retrying scan ...")
_, _, _ = runDriftCtlCmd(driftctlCmd)
result = c.getResult(t)
retryCount++
}
// Restore original working directory
if c.WorkingDir != "" {
err = os.Chdir(wd)
@ -447,7 +460,7 @@ func Run(t *testing.T, c AccTestCase) {
_ = os.Unsetenv(key)
}
}
check.Check(c.getResult(t), out, cmdErr)
check.Check(result, out, cmdErr)
if check.PostExec != nil {
check.PostExec()
}

View File

@ -12,6 +12,7 @@ import (
type FakeAssetServer struct {
SearchAllResourcesResults []*assetpb.ResourceSearchResult
ListAssetsResults []*assetpb.Asset
err error
assetpb.UnimplementedAssetServiceServer
}
@ -20,9 +21,20 @@ func (s *FakeAssetServer) SearchAllResources(context.Context, *assetpb.SearchAll
return &assetpb.SearchAllResourcesResponse{Results: s.SearchAllResourcesResults}, s.err
}
func NewFakeAssetServer(results []*assetpb.ResourceSearchResult, err error) (*asset.Client, error) {
func (s *FakeAssetServer) ListAssets(context.Context, *assetpb.ListAssetsRequest) (*assetpb.ListAssetsResponse, error) {
return &assetpb.ListAssetsResponse{Assets: s.ListAssetsResults}, s.err
}
func NewFakeAssertServerWithList(listResults []*assetpb.Asset, err error) (*asset.Client, error) {
return newAssetClient(&FakeAssetServer{ListAssetsResults: listResults, err: err})
}
func NewFakeAssetServer(searchResults []*assetpb.ResourceSearchResult, err error) (*asset.Client, error) {
return newAssetClient(&FakeAssetServer{SearchAllResourcesResults: searchResults, err: err})
}
func newAssetClient(fakeServer *FakeAssetServer) (*asset.Client, error) {
ctx := context.Background()
fakeServer := &FakeAssetServer{SearchAllResourcesResults: results, err: err}
l, err := net.Listen("tcp", "localhost:0")
if err != nil {
return nil, err