From bdf0fdb000b5bc57bb8c8f9c52b0c9c788dddbaa Mon Sep 17 00:00:00 2001 From: James Kwon <96548424+hongil0316@users.noreply.github.com> Date: Tue, 30 Apr 2024 17:26:36 -0400 Subject: [PATCH] Handle Global Cluster Nuke Failure (#691) * Handle global DB cluster nuking failure * Handle Global Cluster --- aws/resource_registry.go | 2 + aws/resources/rds_cluster.go | 4 +- aws/resources/rds_global_cluster.go | 106 ++++++++++++ .../rds_global_cluster_membership.go | 151 ++++++++++++++++++ .../rds_global_cluster_membership_test.go | 92 +++++++++++ .../rds_global_cluster_membership_types.go | 56 +++++++ aws/resources/rds_global_cluster_test.go | 87 ++++++++++ aws/resources/rds_global_cluster_types_.go | 56 +++++++ config/config.go | 2 + config/config_test.go | 2 + 10 files changed, 556 insertions(+), 2 deletions(-) create mode 100644 aws/resources/rds_global_cluster.go create mode 100644 aws/resources/rds_global_cluster_membership.go create mode 100644 aws/resources/rds_global_cluster_membership_test.go create mode 100644 aws/resources/rds_global_cluster_membership_types.go create mode 100644 aws/resources/rds_global_cluster_test.go create mode 100644 aws/resources/rds_global_cluster_types_.go diff --git a/aws/resource_registry.go b/aws/resource_registry.go index 1f646b4d..0d25ae6f 100644 --- a/aws/resource_registry.go +++ b/aws/resource_registry.go @@ -33,6 +33,7 @@ func GetAndInitRegisteredResources(session *session.Session, region string) []*A // GetRegisteredGlobalResources - returns a list of registered global resources. func getRegisteredGlobalResources() []AwsResource { return []AwsResource{ + &resources.DBGlobalClusters{}, &resources.IAMUsers{}, &resources.IAMGroups{}, &resources.IAMPolicies{}, @@ -99,6 +100,7 @@ func getRegisteredRegionalResources() []AwsResource { &resources.MSKCluster{}, &resources.NatGateways{}, &resources.OpenSearchDomains{}, + &resources.DBGlobalClusterMemberships{}, &resources.DBInstances{}, &resources.DBSubnetGroups{}, &resources.DBClusters{}, diff --git a/aws/resources/rds_cluster.go b/aws/resources/rds_cluster.go index 6e0cd226..a1826c58 100644 --- a/aws/resources/rds_cluster.go +++ b/aws/resources/rds_cluster.go @@ -2,9 +2,10 @@ package resources import ( "context" - "github.com/gruntwork-io/cloud-nuke/util" "time" + "github.com/gruntwork-io/cloud-nuke/util" + awsgo "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws" @@ -90,7 +91,6 @@ func (instance *DBClusters) nukeAll(names []*string) error { if len(deletedNames) > 0 { for _, name := range deletedNames { - err := instance.waitUntilRdsClusterDeleted(&rds.DescribeDBClustersInput{ DBClusterIdentifier: name, }) diff --git a/aws/resources/rds_global_cluster.go b/aws/resources/rds_global_cluster.go new file mode 100644 index 00000000..a3e264fc --- /dev/null +++ b/aws/resources/rds_global_cluster.go @@ -0,0 +1,106 @@ +package resources + +import ( + "context" + "time" + + awsgo "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/rds" + "github.com/gruntwork-io/cloud-nuke/config" + "github.com/gruntwork-io/cloud-nuke/logging" + "github.com/gruntwork-io/cloud-nuke/report" + "github.com/gruntwork-io/go-commons/errors" +) + +// wait up to 15 minutes +const ( + dbGlobalClusterDeletionRetryDelay = 10 * time.Second + dbGlobalClusterDeletionRetryCount = 90 +) + +func (instance *DBGlobalClusters) getAll(c context.Context, configObj config.Config) ([]*string, error) { + result, err := instance.Client.DescribeGlobalClustersWithContext(c, &rds.DescribeGlobalClustersInput{}) + if err != nil { + return nil, errors.WithStackTrace(err) + } + + var names []*string + for _, cluster := range result.GlobalClusters { + if !configObj.DBGlobalClusters.ShouldInclude(config.ResourceValue{ + Name: cluster.GlobalClusterIdentifier, + }) { + continue + } + + names = append(names, cluster.GlobalClusterIdentifier) + } + + return names, nil +} + +func (instance *DBGlobalClusters) nukeAll(names []*string) error { + if len(names) == 0 { + logging.Debugf("No RDS DB Global Cluster Membership to nuke") + return nil + } + + logging.Debugf("Deleting Global Cluster (members)") + deletedNames := []*string{} + + for _, name := range names { + _, err := instance.Client.DeleteGlobalCluster(&rds.DeleteGlobalClusterInput{ + GlobalClusterIdentifier: name, + }) + + // Record status of this resource + e := report.Entry{ + Identifier: aws.StringValue(name), + ResourceType: "RDS Global Cluster Membership", + Error: err, + } + report.Record(e) + + switch { + case err != nil: + logging.Debugf("[Failed] %s: %s", *name, err) + + default: + deletedNames = append(deletedNames, name) + logging.Debugf("Deleted RDS DB Global Cluster Membership: %s", awsgo.StringValue(name)) + } + } + + for _, name := range deletedNames { + err := instance.waitUntilRDSGlobalClusterDeleted(*name) + if err != nil { + logging.Errorf("[Failed] %s", err) + return errors.WithStackTrace(err) + } + } + + logging.Debugf("[OK] %d RDS Global DB Cluster(s) Membership nuked in %s", len(deletedNames), instance.Region) + return nil +} + +func (instance *DBGlobalClusters) waitUntilRDSGlobalClusterDeleted(name string) error { + for i := 0; i < dbGlobalClusterDeletionRetryCount; i++ { + _, err := instance.Client.DescribeGlobalClusters(&rds.DescribeGlobalClustersInput{ + GlobalClusterIdentifier: &name, + }) + if err != nil { + if awsErr, isAwsErr := err.(awserr.Error); isAwsErr && awsErr.Code() == rds.ErrCodeGlobalClusterNotFoundFault { + return nil + } + + return errors.WithStackTrace(err) + } + + time.Sleep(dbGlobalClusterDeletionRetryDelay) + logging.Debug("Waiting for RDS Global Cluster to be deleted") + } + + return RdsDeleteError{name: name} +} diff --git a/aws/resources/rds_global_cluster_membership.go b/aws/resources/rds_global_cluster_membership.go new file mode 100644 index 00000000..28631034 --- /dev/null +++ b/aws/resources/rds_global_cluster_membership.go @@ -0,0 +1,151 @@ +package resources + +import ( + "context" + "fmt" + "strings" + "time" + + awsgo "github.com/aws/aws-sdk-go/aws" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/rds" + "github.com/gruntwork-io/cloud-nuke/config" + "github.com/gruntwork-io/cloud-nuke/logging" + "github.com/gruntwork-io/cloud-nuke/report" + "github.com/gruntwork-io/go-commons/errors" +) + +// wait up to 15 minutes +const ( + dbGlobalClusterMembershipsRemovalRetryDelay = 10 * time.Second + dbGlobalClusterMembershipsRemovalRetryCount = 90 +) + +func (instance *DBGlobalClusterMemberships) getAll(c context.Context, configObj config.Config) ([]*string, error) { + result, err := instance.Client.DescribeGlobalClustersWithContext(c, &rds.DescribeGlobalClustersInput{}) + if err != nil { + return nil, errors.WithStackTrace(err) + } + + var names []*string + for _, cluster := range result.GlobalClusters { + if !configObj.DBGlobalClusterMemberships.ShouldInclude(config.ResourceValue{ + Name: cluster.GlobalClusterIdentifier, + }) { + continue + } + + names = append(names, cluster.GlobalClusterIdentifier) + } + + return names, nil +} + +func (instance *DBGlobalClusterMemberships) nukeAll(names []*string) error { + if len(names) == 0 { + logging.Debugf("No RDS DB Global Cluster Membership to nuke") + return nil + } + + logging.Debugf("Deleting Global Cluster (members)") + deletedNames := []*string{} + + for _, name := range names { + deleted, err := instance.removeGlobalClusterMembership(*name) + + // Record status of this resource + e := report.Entry{ + Identifier: aws.StringValue(name), + ResourceType: "RDS Global Cluster Membership", + Error: err, + } + report.Record(e) + + switch { + case err != nil: + logging.Debugf("[Failed] %s: %s", *name, err) + + case !deleted: + logging.Debugf("No RDS Global Cluster Membership was deleted on %s", *name) + + default: + deletedNames = append(deletedNames, name) + logging.Debugf("Deleted RDS DB Global Cluster Membership: %s", awsgo.StringValue(name)) + } + } + + logging.Debugf("[OK] %d RDS Global DB Cluster(s) Membership nuked in %s", len(deletedNames), instance.Region) + return nil +} + +func (instance *DBGlobalClusterMemberships) removeGlobalClusterMembership(name string) (deleted bool, err error) { + gdbcs, err := instance.Client.DescribeGlobalClusters(&rds.DescribeGlobalClustersInput{ + GlobalClusterIdentifier: &name, + }) + if err != nil { + return deleted, fmt.Errorf("fail to describe global cluster: %w", err) + } + if len(gdbcs.GlobalClusters) != 1 || *gdbcs.GlobalClusters[0].GlobalClusterIdentifier != name { + return deleted, fmt.Errorf("unexpected describe result global cluster") + } + gdbc := gdbcs.GlobalClusters[0] + + deletedNames := []string{} + for _, member := range gdbc.GlobalClusterMembers { + region := strings.Split(*member.DBClusterArn, ":")[3] + if instance.Region != "" && instance.Region != region { + logging.Debugf("Skip removing cluster '%s' from global cluster since it is in different region", *member.DBClusterArn) + continue + } + + logging.Debugf("Removing cluster '%s' from global cluster", *member.DBClusterArn) + _, err := instance.Client.RemoveFromGlobalCluster(&rds.RemoveFromGlobalClusterInput{ + GlobalClusterIdentifier: gdbc.GlobalClusterIdentifier, + DbClusterIdentifier: member.DBClusterArn, + }) + if err != nil { + return deleted, fmt.Errorf("fail to remove cluster '%s' from global cluster :%w", *member, err) + } + deletedNames = append(deletedNames, *member.DBClusterArn) + } + for _, name := range deletedNames { + err = instance.waitUntilRdsClusterRemovedFromGlobalCluster(*gdbc.GlobalClusterIdentifier, name) + if err != nil { + return deleted, fmt.Errorf("fail to remove cluster '%s' from global cluster :%w", name, err) + } + } + + return len(deletedNames) > 0, nil +} + +func (instance *DBGlobalClusterMemberships) waitUntilRdsClusterRemovedFromGlobalCluster(arnGlobalCluster string, arnCluster string) error { + for i := 0; i < dbGlobalClusterMembershipsRemovalRetryCount; i++ { + gcs, err := instance.Client.DescribeGlobalClusters(&rds.DescribeGlobalClustersInput{ + GlobalClusterIdentifier: &arnGlobalCluster, + }) + if err != nil { + return errors.WithStackTrace(err) + } + + found := false + for _, gc := range gcs.GlobalClusters { + for _, m := range gc.GlobalClusterMembers { + if *m.DBClusterArn != arnCluster { + continue + } + + found = true + break + } + } + if !found { + return nil + } + + time.Sleep(dbGlobalClusterMembershipsRemovalRetryDelay) + logging.Debug("Waiting for RDS Cluster to be removed from RDS Global Cluster") + } + + return RdsDeleteError{name: arnCluster} +} diff --git a/aws/resources/rds_global_cluster_membership_test.go b/aws/resources/rds_global_cluster_membership_test.go new file mode 100644 index 00000000..165fe292 --- /dev/null +++ b/aws/resources/rds_global_cluster_membership_test.go @@ -0,0 +1,92 @@ +package resources + +import ( + "context" + "regexp" + "strings" + "testing" + + awsgo "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/rds" + "github.com/aws/aws-sdk-go/service/rds/rdsiface" + "github.com/gruntwork-io/cloud-nuke/config" + "github.com/stretchr/testify/assert" +) + +type mockedDBGlobalClusterMemberships struct { + rdsiface.RDSAPI + DescribeGlobalClustersOutput rds.DescribeGlobalClustersOutput + DescribeGlobalClustersError error + RemoveFromGlobalClusterOutput rds.RemoveFromGlobalClusterOutput +} + +func (m mockedDBGlobalClusterMemberships) RemoveFromGlobalCluster(input *rds.RemoveFromGlobalClusterInput) (*rds.RemoveFromGlobalClusterOutput, error) { + return &m.RemoveFromGlobalClusterOutput, nil +} + +func (m mockedDBGlobalClusterMemberships) DescribeGlobalClusters(input *rds.DescribeGlobalClustersInput) (*rds.DescribeGlobalClustersOutput, error) { + return &m.DescribeGlobalClustersOutput, m.DescribeGlobalClustersError +} + +func (m mockedDBGlobalClusterMemberships) DescribeGlobalClustersWithContext(ctx context.Context, input *rds.DescribeGlobalClustersInput, _ ...request.Option) (*rds.DescribeGlobalClustersOutput, error) { + return &m.DescribeGlobalClustersOutput, m.DescribeGlobalClustersError +} + +func TestRDSGlobalClusterMembershipGetAll(t *testing.T) { + t.Parallel() + + testName := "test-db-global-cluster" + dbCluster := DBGlobalClusterMemberships{ + Client: mockedDBGlobalClusterMemberships{ + DescribeGlobalClustersOutput: rds.DescribeGlobalClustersOutput{ + GlobalClusters: []*rds.GlobalCluster{ + { + GlobalClusterIdentifier: &testName, + }, + }, + }, + }, + } + + // Testing empty config + clusters, err := dbCluster.getAll(context.Background(), config.Config{DBGlobalClusterMemberships: config.ResourceType{}}) + assert.NoError(t, err) + assert.Contains(t, awsgo.StringValueSlice(clusters), strings.ToLower(testName)) + + // Testing db cluster exclusion + clusters, err = dbCluster.getAll(context.Background(), config.Config{ + DBGlobalClusterMemberships: config.ResourceType{ + ExcludeRule: config.FilterRule{ + NamesRegExp: []config.Expression{{ + RE: *regexp.MustCompile(testName), + }}, + }, + }, + }) + assert.NoError(t, err) + assert.NotContains(t, awsgo.StringValueSlice(clusters), strings.ToLower(testName)) +} + +func TestRDSGlobalClusterMembershipNukeAll(t *testing.T) { + + t.Parallel() + + testName := "test-db-global-cluster" + dbCluster := DBGlobalClusterMemberships{ + Client: mockedDBGlobalClusterMemberships{ + DescribeGlobalClustersOutput: rds.DescribeGlobalClustersOutput{ + GlobalClusters: []*rds.GlobalCluster{ + { + GlobalClusterIdentifier: &testName, + GlobalClusterMembers: []*rds.GlobalClusterMember{}, + }, + }, + }, + RemoveFromGlobalClusterOutput: rds.RemoveFromGlobalClusterOutput{}, + }, + } + + err := dbCluster.nukeAll([]*string{&testName}) + assert.NoError(t, err) +} diff --git a/aws/resources/rds_global_cluster_membership_types.go b/aws/resources/rds_global_cluster_membership_types.go new file mode 100644 index 00000000..8ee4998d --- /dev/null +++ b/aws/resources/rds_global_cluster_membership_types.go @@ -0,0 +1,56 @@ +package resources + +import ( + "context" + + awsgo "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/rds" + "github.com/aws/aws-sdk-go/service/rds/rdsiface" + "github.com/gruntwork-io/cloud-nuke/config" + "github.com/gruntwork-io/go-commons/errors" +) + +type DBGlobalClusterMemberships struct { + BaseAwsResource + Client rdsiface.RDSAPI + Region string + InstanceNames []string +} + +func (instance *DBGlobalClusterMemberships) Init(session *session.Session) { + instance.Client = rds.New(session) +} + +func (instance *DBGlobalClusterMemberships) ResourceName() string { + return "rds-global-cluster-membership" +} + +// ResourceIdentifiers - The instance names of the rds db instances +func (instance *DBGlobalClusterMemberships) ResourceIdentifiers() []string { + return instance.InstanceNames +} + +func (instance *DBGlobalClusterMemberships) MaxBatchSize() int { + // Tentative batch size to ensure AWS doesn't throttle + return 49 +} + +func (instance *DBGlobalClusterMemberships) GetAndSetIdentifiers(c context.Context, configObj config.Config) ([]string, error) { + identifiers, err := instance.getAll(c, configObj) + if err != nil { + return nil, err + } + + instance.InstanceNames = awsgo.StringValueSlice(identifiers) + return instance.InstanceNames, nil +} + +// Nuke - nuke 'em all!!! +func (instance *DBGlobalClusterMemberships) Nuke(identifiers []string) error { + if err := instance.nukeAll(awsgo.StringSlice(identifiers)); err != nil { + return errors.WithStackTrace(err) + } + + return nil +} diff --git a/aws/resources/rds_global_cluster_test.go b/aws/resources/rds_global_cluster_test.go new file mode 100644 index 00000000..4b154c7e --- /dev/null +++ b/aws/resources/rds_global_cluster_test.go @@ -0,0 +1,87 @@ +package resources + +import ( + "context" + "regexp" + "strings" + "testing" + + awsgo "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/rds" + "github.com/aws/aws-sdk-go/service/rds/rdsiface" + "github.com/gruntwork-io/cloud-nuke/config" + "github.com/stretchr/testify/assert" +) + +type mockedDBGlobalClusters struct { + rdsiface.RDSAPI + DescribeGlobalClustersOutput rds.DescribeGlobalClustersOutput + DescribeGlobalClustersError error + DeleteGlobalClusterOutput rds.DeleteGlobalClusterOutput +} + +func (m mockedDBGlobalClusters) DeleteGlobalCluster(input *rds.DeleteGlobalClusterInput) (*rds.DeleteGlobalClusterOutput, error) { + return &m.DeleteGlobalClusterOutput, nil +} + +func (m mockedDBGlobalClusters) DescribeGlobalClusters(input *rds.DescribeGlobalClustersInput) (*rds.DescribeGlobalClustersOutput, error) { + return &m.DescribeGlobalClustersOutput, m.DescribeGlobalClustersError +} + +func (m mockedDBGlobalClusters) DescribeGlobalClustersWithContext(ctx context.Context, input *rds.DescribeGlobalClustersInput, _ ...request.Option) (*rds.DescribeGlobalClustersOutput, error) { + return &m.DescribeGlobalClustersOutput, m.DescribeGlobalClustersError +} + +func TestRDSGlobalClusterGetAll(t *testing.T) { + t.Parallel() + + testName := "test-db-global-cluster" + dbCluster := DBGlobalClusters{ + Client: mockedDBGlobalClusters{ + DescribeGlobalClustersOutput: rds.DescribeGlobalClustersOutput{ + GlobalClusters: []*rds.GlobalCluster{ + { + GlobalClusterIdentifier: &testName, + }, + }, + }, + }, + } + + // Testing empty config + clusters, err := dbCluster.getAll(context.Background(), config.Config{DBGlobalClusters: config.ResourceType{}}) + assert.NoError(t, err) + assert.Contains(t, awsgo.StringValueSlice(clusters), strings.ToLower(testName)) + + // Testing db cluster exclusion + clusters, err = dbCluster.getAll(context.Background(), config.Config{ + DBGlobalClusters: config.ResourceType{ + ExcludeRule: config.FilterRule{ + NamesRegExp: []config.Expression{{ + RE: *regexp.MustCompile(testName), + }}, + }, + }, + }) + assert.NoError(t, err) + assert.NotContains(t, awsgo.StringValueSlice(clusters), strings.ToLower(testName)) +} + +func TestRDSGlobalClusterNukeAll(t *testing.T) { + + t.Parallel() + + testName := "test-db-global-cluster" + dbCluster := DBGlobalClusters{ + Client: mockedDBGlobalClusters{ + DescribeGlobalClustersOutput: rds.DescribeGlobalClustersOutput{}, + DescribeGlobalClustersError: awserr.New(rds.ErrCodeGlobalClusterNotFoundFault, "", nil), + DeleteGlobalClusterOutput: rds.DeleteGlobalClusterOutput{}, + }, + } + + err := dbCluster.nukeAll([]*string{&testName}) + assert.NoError(t, err) +} diff --git a/aws/resources/rds_global_cluster_types_.go b/aws/resources/rds_global_cluster_types_.go new file mode 100644 index 00000000..b7f2fe67 --- /dev/null +++ b/aws/resources/rds_global_cluster_types_.go @@ -0,0 +1,56 @@ +package resources + +import ( + "context" + + awsgo "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/rds" + "github.com/aws/aws-sdk-go/service/rds/rdsiface" + "github.com/gruntwork-io/cloud-nuke/config" + "github.com/gruntwork-io/go-commons/errors" +) + +type DBGlobalClusters struct { + BaseAwsResource + Client rdsiface.RDSAPI + Region string + InstanceNames []string +} + +func (instance *DBGlobalClusters) Init(session *session.Session) { + instance.Client = rds.New(session) +} + +func (instance *DBGlobalClusters) ResourceName() string { + return "rds-global-cluster" +} + +// ResourceIdentifiers - The instance names of the rds db instances +func (instance *DBGlobalClusters) ResourceIdentifiers() []string { + return instance.InstanceNames +} + +func (instance *DBGlobalClusters) MaxBatchSize() int { + // Tentative batch size to ensure AWS doesn't throttle + return 49 +} + +func (instance *DBGlobalClusters) GetAndSetIdentifiers(c context.Context, configObj config.Config) ([]string, error) { + identifiers, err := instance.getAll(c, configObj) + if err != nil { + return nil, err + } + + instance.InstanceNames = awsgo.StringValueSlice(identifiers) + return instance.InstanceNames, nil +} + +// Nuke - nuke 'em all!!! +func (instance *DBGlobalClusters) Nuke(identifiers []string) error { + if err := instance.nukeAll(awsgo.StringSlice(identifiers)); err != nil { + return errors.WithStackTrace(err) + } + + return nil +} diff --git a/config/config.go b/config/config.go index b20a05ad..b1f7a882 100644 --- a/config/config.go +++ b/config/config.go @@ -30,8 +30,10 @@ type Config struct { CodeDeployApplications ResourceType `yaml:"CodeDeployApplications"` ConfigServiceRecorder ResourceType `yaml:"ConfigServiceRecorder"` ConfigServiceRule ResourceType `yaml:"ConfigServiceRule"` + DBGlobalClusters ResourceType `yaml:"DBGlobalClusters"` DBClusters ResourceType `yaml:"DBClusters"` DBInstances ResourceType `yaml:"DBInstances"` + DBGlobalClusterMemberships ResourceType `yaml:"DBGlobalClusterMemberships"` DBSubnetGroups ResourceType `yaml:"DBSubnetGroups"` DynamoDB ResourceType `yaml:"DynamoDB"` EBSVolume ResourceType `yaml:"EBSVolume"` diff --git a/config/config_test.go b/config/config_test.go index 75c7bc50..bf25053c 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -43,6 +43,8 @@ func emptyConfig() *Config { ResourceType{FilterRule{}, FilterRule{}, ""}, ResourceType{FilterRule{}, FilterRule{}, ""}, ResourceType{FilterRule{}, FilterRule{}, ""}, + ResourceType{FilterRule{}, FilterRule{}, ""}, + ResourceType{FilterRule{}, FilterRule{}, ""}, EC2ResourceType{false, ResourceType{FilterRule{}, FilterRule{}, ""}}, ResourceType{FilterRule{}, FilterRule{}, ""}, ResourceType{FilterRule{}, FilterRule{}, ""},