Aws waf region (ansible#48953)

* Add waiter for AWSRegional

* Add support for WAF Regional

* Add support for regional waf web acl

* Remove set_trace, pep formatting

* Add paginator for regional_waf

* Change name of param for waf_regional

This is more in line with how AWS refers to the service. Additional
 changes made to how client is called. Used ternary to reduce if
 statements

* Change parameter name to waf_regional

* Add support for removal waf regional condition

* Change parameter from cloudfront to waf_regional

* Added state: absent waf rule

* Remove set_trace

* Add integration tests for waf regional

* WIP: adding region parameter to tests

* Add support for waf facts module

* Add region to waf regional integration tests

* Update security policy for waf regional testing

* Add type to documentation for waf_regional param

hacking/aws_config/testing_policies/security-policy.json

@@ -90,6 +90,12 @@
             "Resource": [
                 "arn:aws:logs:{{aws_region}}:{{aws_account}}:log-group:ansible-testing*"
             ]
+        },
+        {
+            "Sid": "AllowWAFRegionalusage",
+            "Action": "waf-regional:*",
+            "Effect": "Allow",
+            "Resource": "*"
         }
     ]
 }

lib/ansible/module_utils/aws/waf.py

@@ -162,15 +162,38 @@ def list_rules_with_backoff(client):
     return paginator.paginate().build_full_result()['Rules']
 
 
+@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
+def list_regional_rules_with_backoff(client):
+    resp = client.list_rules()
+    rules = []
+    while resp:
+        rules += resp['Rules']
+        resp = client.list_rules(NextMarker=resp['NextMarker']) if 'NextMarker' in resp else None
+    return rules
+
+
 @AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
 def list_web_acls_with_backoff(client):
     paginator = client.get_paginator('list_web_acls')
     return paginator.paginate().build_full_result()['WebACLs']
 
 
+@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
+def list_regional_web_acls_with_backoff(client):
+    resp = client.list_web_acls()
+    acls = []
+    while resp:
+        acls += resp['WebACLs']
+        resp = client.list_web_acls(NextMarker=resp['NextMarker']) if 'NextMarker' in resp else None
+    return acls
+
+
 def list_web_acls(client, module):
     try:
-        return list_web_acls_with_backoff(client)
+        if client.__class__.__name__ == 'WAF':
+            return list_web_acls_with_backoff(client)
+        elif client.__class__.__name__ == 'WAFRegional':
+            return list_regional_web_acls_with_backoff(client)
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         module.fail_json_aws(e, msg="Couldn't obtain web acls")
 

lib/ansible/module_utils/aws/waiters.py

@@ -305,6 +305,12 @@ def rds_model(name):
         core_waiter.NormalizedOperationMethod(
             waf.get_change_token_status
         )),
+    ('WAFRegional', 'change_token_in_sync'): lambda waf: core_waiter.Waiter(
+        'change_token_in_sync',
+        waf_model('ChangeTokenInSync'),
+        core_waiter.NormalizedOperationMethod(
+            waf.get_change_token_status
+        )),
     ('EKS', 'cluster_active'): lambda eks: core_waiter.Waiter(
         'cluster_active',
         eks_model('ClusterActive'),

lib/ansible/modules/cloud/amazon/aws_waf_condition.py

@@ -57,6 +57,12 @@
         - Whether to remove existing filters from a condition if not passed in I(filters).
         default: False
         type: bool
+    waf_regional:
+        description: Whether to use waf_regional module. Defaults to false.
+        default: false
+        required: no
+        type: bool
+        version_added: 2.9
     state:
         description: Whether the condition should be C(present) or C(absent).
         choices:
@@ -335,7 +341,7 @@
 from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, ec2_argument_spec
 from ansible.module_utils.ec2 import camel_dict_to_snake_dict, AWSRetry, compare_policies
 from ansible.module_utils.aws.waf import run_func_with_change_token_backoff, MATCH_LOOKUP
-from ansible.module_utils.aws.waf import get_rule_with_backoff, list_rules_with_backoff
+from ansible.module_utils.aws.waf import get_rule_with_backoff, list_rules_with_backoff, list_regional_rules_with_backoff
 
 
 class Condition(object):
@@ -519,7 +525,10 @@ def tidy_up_regex_patterns(self, regex_match_set):
     def find_condition_in_rules(self, condition_set_id):
         rules_in_use = []
         try:
-            all_rules = list_rules_with_backoff(self.client)
+            if self.client.__class__.__name__ == 'WAF':
+                all_rules = list_rules_with_backoff(self.client)
+            elif self.client.__class__.__name__ == 'WAFRegional':
+                all_rules = list_regional_rules_with_backoff(self.client)
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             self.module.fail_json_aws(e, msg='Could not list rules')
         for rule in all_rules:
@@ -636,6 +645,7 @@ def main():
             type=dict(required=True, choices=['byte', 'geo', 'ip', 'regex', 'size', 'sql', 'xss']),
             filters=dict(type='list'),
             purge_filters=dict(type='bool', default=False),
+            waf_regional=dict(type='bool', default=False),
             state=dict(default='present', choices=['present', 'absent']),
         ),
     )
@@ -644,7 +654,8 @@ def main():
     state = module.params.get('state')
 
     region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module, boto3=True)
-    client = boto3_conn(module, conn_type='client', resource='waf', region=region, endpoint=ec2_url, **aws_connect_kwargs)
+    resource = 'waf' if not module.params['waf_regional'] else 'waf-regional'
+    client = boto3_conn(module, conn_type='client', resource=resource, region=region, endpoint=ec2_url, **aws_connect_kwargs)
 
     condition = Condition(client, module)
 

lib/ansible/modules/cloud/amazon/aws_waf_facts.py

@@ -17,6 +17,12 @@
   name:
     description:
       - The name of a Web Application Firewall
+  waf_regional:
+      description: Wether to use waf_regional module. Defaults to true
+      default: false
+      required: no
+      type: bool
+      version_added: "2.9"
 
 author:
   - Mike Mochan (@mmochan)
@@ -33,6 +39,11 @@
 - name: obtain all facts for a single WAF
   aws_waf_facts:
     name: test_waf
+
+- name: obtain all facts for a single WAF Regional
+  aws_waf_facts:
+    name: test_waf
+    waf_regional: true
 '''
 
 RETURN = '''
@@ -113,13 +124,14 @@ def main():
     argument_spec.update(
         dict(
             name=dict(required=False),
+            waf_regional=dict(type='bool', default=False),
         )
     )
     module = AnsibleAWSModule(argument_spec=argument_spec, supports_check_mode=True)
 
     region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module, boto3=True)
-    client = boto3_conn(module, conn_type='client', resource='waf', region=region, endpoint=ec2_url, **aws_connect_kwargs)
-
+    resource = 'waf' if not module.params['waf_regional'] else 'waf-regional'
+    client = boto3_conn(module, conn_type='client', resource=resource, region=region, endpoint=ec2_url, **aws_connect_kwargs)
     web_acls = list_web_acls(client, module)
     name = module.params['name']
     if name:

lib/ansible/modules/cloud/amazon/aws_waf_rule.py

@@ -47,8 +47,14 @@
     purge_conditions:
         description:
           - Whether or not to remove conditions that are not passed when updating `conditions`.
-        default: False
+        default: false
         type: bool
+    waf_regional:
+        description: Whether to use waf_regional module. Defaults to false
+        default: false
+        required: no
+        type: bool
+        version_added: "2.9"
 '''
 
 EXAMPLES = '''
@@ -127,8 +133,8 @@
 from ansible.module_utils.aws.core import AnsibleAWSModule
 from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, ec2_argument_spec
 from ansible.module_utils.ec2 import camel_dict_to_snake_dict
-from ansible.module_utils.aws.waf import run_func_with_change_token_backoff, list_rules_with_backoff, MATCH_LOOKUP
-from ansible.module_utils.aws.waf import get_web_acl_with_backoff, list_web_acls_with_backoff
+from ansible.module_utils.aws.waf import run_func_with_change_token_backoff, list_rules_with_backoff, list_regional_rules_with_backoff, MATCH_LOOKUP
+from ansible.module_utils.aws.waf import get_web_acl_with_backoff, list_web_acls_with_backoff, list_regional_web_acls_with_backoff
 
 
 def get_rule_by_name(client, module, name):
@@ -145,8 +151,21 @@ def get_rule(client, module, rule_id):
 
 
 def list_rules(client, module):
+    if client.__class__.__name__ == 'WAF':
+        try:
+            return list_rules_with_backoff(client)
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+            module.fail_json_aws(e, msg='Could not list WAF rules')
+    elif client.__class__.__name__ == 'WAFRegional':
+        try:
+            return list_regional_rules_with_backoff(client)
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+            module.fail_json_aws(e, msg='Could not list WAF Regional rules')
+
+
+def list_regional_rules(client, module):
     try:
-        return list_rules_with_backoff(client)
+        return list_regional_rules_with_backoff(client)
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         module.fail_json_aws(e, msg='Could not list WAF rules')
 
@@ -260,7 +279,10 @@ def ensure_rule_present(client, module):
 def find_rule_in_web_acls(client, module, rule_id):
     web_acls_in_use = []
     try:
-        all_web_acls = list_web_acls_with_backoff(client)
+        if client.__class__.__name__ == 'WAF':
+            all_web_acls = list_web_acls_with_backoff(client)
+        elif client.__class__.__name__ == 'WAFRegional':
+            all_web_acls = list_regional_web_acls_with_backoff(client)
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         module.fail_json_aws(e, msg='Could not list Web ACLs')
     for web_acl in all_web_acls:
@@ -297,15 +319,16 @@ def main():
             metric_name=dict(),
             state=dict(default='present', choices=['present', 'absent']),
             conditions=dict(type='list'),
-            purge_conditions=dict(type='bool', default=False)
+            purge_conditions=dict(type='bool', default=False),
+            waf_regional=dict(type='bool', default=False),
         ),
     )
     module = AnsibleAWSModule(argument_spec=argument_spec)
     state = module.params.get('state')
 
     region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module, boto3=True)
-    client = boto3_conn(module, conn_type='client', resource='waf', region=region, endpoint=ec2_url, **aws_connect_kwargs)
-
+    resource = 'waf' if not module.params['waf_regional'] else 'waf-regional'
+    client = boto3_conn(module, conn_type='client', resource=resource, region=region, endpoint=ec2_url, **aws_connect_kwargs)
     if state == 'present':
         (changed, results) = ensure_rule_present(client, module)
     else:

lib/ansible/modules/cloud/amazon/aws_waf_web_acl.py

@@ -49,12 +49,17 @@
         - Each rule must contain I(name), I(action), I(priority) keys.
         - Priorities must be unique, but not necessarily consecutive. Lower numbered priorities are evalauted first.
         - The I(type) key can be passed as C(rate_based), it defaults to C(regular)
-
     purge_rules:
         description:
         - Whether to remove rules that aren't passed with C(rules).
         default: False
         type: bool
+    waf_regional:
+        description: Whether to use waf_regional module. Defaults to false.
+        default: false
+        required: no
+        type: bool
+        version_added: "2.9"
 '''
 
 EXAMPLES = '''
@@ -140,7 +145,8 @@
 from ansible.module_utils.aws.core import AnsibleAWSModule
 from ansible.module_utils.aws.waiters import get_waiter
 from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, ec2_argument_spec, camel_dict_to_snake_dict
-from ansible.module_utils.aws.waf import list_rules_with_backoff, list_web_acls_with_backoff, run_func_with_change_token_backoff
+from ansible.module_utils.aws.waf import list_rules_with_backoff, list_web_acls_with_backoff, list_regional_web_acls_with_backoff, \
+    run_func_with_change_token_backoff, list_regional_rules_with_backoff
 
 
 def get_web_acl_by_name(client, module, name):
@@ -152,11 +158,18 @@ def get_web_acl_by_name(client, module, name):
 
 
 def create_rule_lookup(client, module):
-    try:
-        rules = list_rules_with_backoff(client)
-        return dict((rule['Name'], rule) for rule in rules)
-    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
-        module.fail_json_aws(e, msg='Could not list rules')
+    if client.__class__.__name__ == 'WAF':
+        try:
+            rules = list_rules_with_backoff(client)
+            return dict((rule['Name'], rule) for rule in rules)
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+            module.fail_json_aws(e, msg='Could not list rules')
+    elif client.__class__.__name__ == 'WAFRegional':
+        try:
+            rules = list_regional_rules_with_backoff(client)
+            return dict((rule['Name'], rule) for rule in rules)
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+            module.fail_json_aws(e, msg='Could not list regional rules')
 
 
 def get_web_acl(client, module, web_acl_id):
@@ -167,10 +180,16 @@ def get_web_acl(client, module, web_acl_id):
 
 
 def list_web_acls(client, module,):
-    try:
-        return list_web_acls_with_backoff(client)
-    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
-        module.fail_json_aws(e, msg='Could not get Web ACLs')
+    if client.__class__.__name__ == 'WAF':
+        try:
+            return list_web_acls_with_backoff(client)
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+            module.fail_json_aws(e, msg='Could not get Web ACLs')
+    elif client.__class__.__name__ == 'WAFRegional':
+        try:
+            return list_regional_web_acls_with_backoff(client)
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+            module.fail_json_aws(e, msg='Could not get Web ACLs')
 
 
 def find_and_update_web_acl(client, module, web_acl_id):
@@ -296,16 +315,17 @@ def main():
             metric_name=dict(),
             state=dict(default='present', choices=['present', 'absent']),
             rules=dict(type='list'),
-            purge_rules=dict(type='bool', default=False)
+            purge_rules=dict(type='bool', default=False),
+            waf_regional=dict(type='bool', default=False),
         ),
     )
     module = AnsibleAWSModule(argument_spec=argument_spec,
                               required_if=[['state', 'present', ['default_action', 'rules']]])
     state = module.params.get('state')
 
     region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module, boto3=True)
-    client = boto3_conn(module, conn_type='client', resource='waf', region=region, endpoint=ec2_url, **aws_connect_kwargs)
-
+    resource = 'waf' if not module.params['waf_regional'] else 'waf-regional'
+    client = boto3_conn(module, conn_type='client', resource=resource, region=region, endpoint=ec2_url, **aws_connect_kwargs)
     if state == 'present':
         (changed, results) = ensure_web_acl_present(client, module)
     else: