Skip to content

Commit

Permalink
aws - kafka - migrate to list_clusters_v2 (cloud-custodian#8077)
Browse files Browse the repository at this point in the history
thisisshi authored Jan 27, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 07e6c91 commit a3df118
Showing 28 changed files with 1,399 additions and 623 deletions.
43 changes: 39 additions & 4 deletions c7n/resources/kafka.py
Original file line number Diff line number Diff line change
@@ -14,6 +14,13 @@ class DescribeKafka(DescribeSource):

def augment(self, resources):
for r in resources:
# preserve backwards compat with extant list_clsuters api
if 'Provisioned' in r:
for k, v in r['Provisioned'].items():
# dont overwrite
if k in r:
continue
r[k] = v
if 'Tags' not in r:
continue
tags = []
@@ -28,7 +35,7 @@ class Kafka(QueryResourceManager):

class resource_type(TypeInfo):
service = 'kafka'
enum_spec = ('list_clusters', 'ClusterInfoList', None)
enum_spec = ('list_clusters_v2', 'ClusterInfoList', None)
arn = id = 'ClusterArn'
name = 'ClusterName'
date = 'CreationTime'
@@ -50,9 +57,37 @@ class KafkaSGFilter(SecurityGroupFilter):


@Kafka.filter_registry.register('subnet')
class KafkaSubnetFilter(SubnetFilter):
class KafkaCompoundSubnetFilter(SubnetFilter):

RelatedIdsExpression = "BrokerNodeGroupInfo.ClientSubnets[]"
RelatedIdsExpression = "compound"

def process(self, resources, event=None):
# kafka v2 has both serverless and provisioned resources which have two different
# locations for their subnet info

class ProvisionedSubnetFilter(SubnetFilter):
RelatedIdsExpression = "Provisioned.BrokerNodeGroupInfo.ClientSubnets[]"

class ServerlessSubnetFilter(SubnetFilter):
RelatedIdsExpression = "Serverless.VpcConfigs[].SubnetIds[]"

p = []
s = []

for r in resources:
if r['ClusterType'] == 'PROVISIONED':
p.append(r)
if r['ClusterType'] == 'SERVERLESS':
s.append(r)

result = []
for filtered, fil in ((p, ProvisionedSubnetFilter), (s, ServerlessSubnetFilter), ):
f = fil(self.data, self.manager)
# necessary to validate otherwise the filter wont work
f.validate()
result.extend(f.process(filtered, event))

return result


@Kafka.filter_registry.register('kms-key')
@@ -74,7 +109,7 @@ class KafkaKmsFilter(KmsRelatedFilter):
key: c7n:AliasName
value: alias/aws/kafka
"""
RelatedIdsExpression = 'EncryptionInfo.EncryptionAtRest.DataVolumeKMSKeyId'
RelatedIdsExpression = 'Provisioned.EncryptionInfo.EncryptionAtRest.DataVolumeKMSKeyId'


@Kafka.action_registry.register('set-monitoring')
3 changes: 2 additions & 1 deletion tests/data/iam-actions.json
Original file line number Diff line number Diff line change
@@ -6082,6 +6082,7 @@
"GetCompatibleKafkaVersions",
"ListClusterOperations",
"ListClusters",
"ListClustersV2",
"ListConfigurationRevisions",
"ListConfigurations",
"ListKafkaVersions",
@@ -11166,4 +11167,4 @@
"UpdateGroup",
"UpdateSamplingRule"
]
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
{
"status_code": 200,
"data": {
"ResponseMetadata": {},
"ClusterInfoList": [
{
"ClusterType": "PROVISIONED",
"ClusterArn": "arn:aws:kafka:ap-northeast-2:644160558196:cluster/demo-cluster-1/a4128f17-d0a5-4b03-b858-ea13108e7286-4",
"ClusterName": "demo-cluster-1",
"CreationTime": {
"__class__": "datetime",
"year": 2022,
"month": 12,
"day": 9,
"hour": 23,
"minute": 13,
"second": 39,
"microsecond": 670000
},
"CurrentVersion": "K3P5ROKL5A1OLE",
"State": "ACTIVE",
"Tags": {
"App": "Custodian"
},
"Provisioned": {
"BrokerNodeGroupInfo": {
"BrokerAZDistribution": "DEFAULT",
"ClientSubnets": [
"subnet-04f91d4b",
"subnet-efc87884",
"subnet-2418755f"
],
"InstanceType": "kafka.m5.large",
"SecurityGroups": [
"sg-bd0718c6"
],
"StorageInfo": {
"EbsStorageInfo": {
"VolumeSize": 100
}
},
"ConnectivityInfo": {
"PublicAccess": {
"Type": "DISABLED"
}
}
},
"CurrentBrokerSoftwareInfo": {
"KafkaVersion": "2.8.1"
},
"ClientAuthentication": {
"Sasl": {
"Scram": {
"Enabled": false
},
"Iam": {
"Enabled": true
}
},
"Tls": {
"CertificateAuthorityArnList": [],
"Enabled": false
},
"Unauthenticated": {
"Enabled": false
}
},
"EncryptionInfo": {
"EncryptionAtRest": {
"DataVolumeKMSKeyId": "arn:aws:kms:ap-northeast-2:644160558196:key/d0e564f4-ec4c-43bd-b379-a22f92253dbe"
},
"EncryptionInTransit": {
"ClientBroker": "TLS",
"InCluster": true
}
},
"EnhancedMonitoring": "DEFAULT",
"OpenMonitoring": {
"Prometheus": {
"JmxExporter": {
"EnabledInBroker": false
},
"NodeExporter": {
"EnabledInBroker": false
}
}
},
"NumberOfBrokerNodes": 3,
"ZookeeperConnectString": "z-3.democluster1.ltt504.c4.kafka.ap-northeast-2.amazonaws.com:2181,z-1.democluster1.ltt504.c4.kafka.ap-northeast-2.amazonaws.com:2181,z-2.democluster1.ltt504.c4.kafka.ap-northeast-2.amazonaws.com:2181",
"ZookeeperConnectStringTls": "z-3.democluster1.ltt504.c4.kafka.ap-northeast-2.amazonaws.com:2182,z-1.democluster1.ltt504.c4.kafka.ap-northeast-2.amazonaws.com:2182,z-2.democluster1.ltt504.c4.kafka.ap-northeast-2.amazonaws.com:2182"
}
}
]
}
}

This file was deleted.

Original file line number Diff line number Diff line change
@@ -3,25 +3,26 @@
"data": {
"KeyMetadata": {
"AWSAccountId": "644160558196",
"KeyId": "133b4e81-8447-4c40-83cd-1db75d52b106",
"Arn": "arn:aws:kms:us-east-1:644160558196:key/133b4e81-8447-4c40-83cd-1db75d52b106",
"KeyId": "d0e564f4-ec4c-43bd-b379-a22f92253dbe",
"Arn": "arn:aws:kms:ap-northeast-2:644160558196:key/d0e564f4-ec4c-43bd-b379-a22f92253dbe",
"CreationDate": {
"__class__": "datetime",
"year": 2019,
"month": 1,
"day": 30,
"hour": 10,
"minute": 11,
"second": 4,
"microsecond": 829000
"year": 2022,
"month": 12,
"day": 9,
"hour": 15,
"minute": 13,
"second": 39,
"microsecond": 35000
},
"Enabled": true,
"Description": "Default master key that protects my Kafka data when no other key is defined",
"Description": "Default key that protects my Kafka data when no other key is defined",
"KeyUsage": "ENCRYPT_DECRYPT",
"KeyState": "Enabled",
"Origin": "AWS_KMS",
"KeyManager": "AWS",
"CustomerMasterKeySpec": "SYMMETRIC_DEFAULT",
"KeySpec": "SYMMETRIC_DEFAULT",
"EncryptionAlgorithms": [
"SYMMETRIC_DEFAULT"
],
Loading

0 comments on commit a3df118

Please sign in to comment.