Implement Serverless Application Model to perform the transformations

README.md

@@ -12,6 +12,12 @@ conditions, and nesting those functions inside each other) so its a best effort
 validate those values but the promise is to not fail if we can't understand or translate
 all the things that could be going on.
 
+#### Serverless Application Model
+The Serverless Application Model (SAM) is supported by the linter. The template is
+transformed using AWS SAM (https://github.com/awslabs/serverless-application-model)
+before the linter processes the template.
+
+
 ## Install
 ### Command line
 From a command prompt run `python setup.py clean --all` then `python setup.py install`

setup.py

@@ -43,10 +43,11 @@
     package_data={'cfnlint': [
         'data/CloudSpecs/*.json',
         'data/AdditionalSpecs/*.json',
+        'data/Serverless/*.json',
     ]},
     packages=find_packages('src'),
     zip_safe=False,
-    install_requires=['pyyaml', 'six', 'requests'],
+    install_requires=['pyyaml', 'six', 'requests', 'aws-sam-translator'],
     entry_points={
         'console_scripts': [
             'cfn-lint = cfnlint.__main__:main'

src/cfnlint/__init__.py

@@ -21,8 +21,7 @@
 from datetime import datetime
 from yaml.parser import ParserError
 import cfnlint.helpers
-import cfnlint.transforms
-
+from cfnlint.transform import Transform
 
 LOGGER = logging.getLogger(__name__)
 
@@ -116,69 +115,6 @@ def matchall_resource_sub_properties(self, filename, cfn, resource_properties, p
         return []
 
 
-class CloudFormationTransform(object):
-    """CloudFormation Transform Support"""
-    type = ''
-
-    resource_transform = None
-
-    def resource_transform_all(self, template):
-        """Generic transform"""
-        if not self.resource_transform:
-            return []
-
-        return self.resource_transform(template)  # pylint: disable=E1102
-
-
-class TransformsCollection(object):
-    """Collection of transforms"""
-
-    def __init__(self):
-        self.transforms = []
-
-    def register(self, obj):
-        """Register transforms"""
-        self.transforms.append(obj)
-
-    def __iter__(self):
-        return iter(self.transforms)
-
-    def __len__(self):
-        return len(self.transforms)
-
-    def extend(self, more):
-        """Extend rules"""
-        self.transforms.extend(more)
-
-    def run(self, filename, cfn, transform_type):
-        """Run the transforms"""
-        matches = list()
-        for transform in self.transforms:
-            if transform_type == transform.type:
-                try:
-                    transform.resource_transform_all(cfn)
-                except cfnlint.transforms.TransformError as err:
-                    rule = CloudFormationLintRule()
-                    rule.id = 'E0001'
-                    rule.shortdesc = 'Transform Error'
-                    rule.description = 'Transform failed to complete'
-                    matches.append(Match(
-                        err.location[0] + 1, err.location[1] + 1,
-                        err.location[2] + 1, err.location[3] + 1,
-                        filename, rule, 'While Performing a Transform got error: %s' % err.value))
-
-        return matches
-
-    @classmethod
-    def create_from_directory(cls, transformdir):
-        """Create transforms from directory"""
-        result = cls()
-        if transformdir != '':
-            result.transforms = cfnlint.helpers.load_plugins(os.path.expanduser(transformdir))
-
-        return result
-
-
 class RulesCollection(object):
     """Collection of rules"""
 
@@ -822,21 +758,25 @@ class Runner(object):
     """Run all the rules"""
 
     def __init__(
-            self, rules, transforms, filename, template, regions, verbosity=0):
+            self, rules, filename, template, regions, verbosity=0):
 
         self.rules = rules
         self.filename = filename
         self.verbosity = verbosity
-        self.transforms = transforms
         self.cfn = Template(template, regions)
 
     def transform(self):
         """Transform logic"""
-        LOGGER.debug('Transform templates if needed')
+        LOGGER.debug('Transform templates if needed using SAM')
+
+        matches = []
         transform_type = self.cfn.template.get('Transform')
-        matches = list()
-        if transform_type:
-            matches = self.transforms.run(self.filename, self.cfn, transform_type)
+
+        # Don't call transformation if Transform is not specified to prevent
+        # useless execution of the transformation
+        if transform_type == 'AWS::Serverless-2016-10-31':
+            transform = Transform(self.filename, self.cfn.template, self.cfn.regions[0])
+            matches = transform.transform_template()
 
         return matches
 
@@ -874,3 +814,10 @@ class ParseError(cfnlint.CloudFormationLintRule):
     shortdesc = 'Parsing error found when parsing the template'
     description = 'Checks for Null values and Duplicate values in resources'
     tags = ['base']
+
+class TransformError(cfnlint.CloudFormationLintRule):
+    """Transform Lint Rule"""
+    id = 'E0001'
+    shortdesc = 'Error found when transforming the template'
+    description = 'Errors found when transforming the template using the Serverless Application Model'
+    tags = ['base']

src/cfnlint/cfn_json.py

@@ -118,6 +118,12 @@ def __new__(self, x, start_mark, end_mark):
 
             return cls.__new__(self, x)
 
+        def __deepcopy__(self, memo):
+            return self
+
+        def __copy__(self):
+            return self
+
     node_class.__name__ = '%s_node' % cls.__name__
     return node_class
 

src/cfnlint/cfn_yaml.py

@@ -76,6 +76,13 @@ def __new__(self, x, start_mark, end_mark):
                 return cls.__new__(self, x.encode('ascii', 'ignore'))
 
             return cls.__new__(self, x)
+
+        def __deepcopy__(self, memo):
+            return self
+
+        def __copy__(self):
+            return self
+
     node_class.__name__ = '%s_node' % cls.__name__
     return node_class
 

src/cfnlint/core.py

@@ -25,7 +25,7 @@
 import argparse
 import six
 from yaml.parser import ParserError, ScannerError
-from cfnlint import RulesCollection, TransformsCollection, Match
+from cfnlint import RulesCollection, Match
 import cfnlint.formatters as formatters
 import cfnlint.cfn_yaml
 import cfnlint.cfn_json
@@ -34,7 +34,6 @@
 
 LOGGER = logging.getLogger('cfnlint')
 DEFAULT_RULESDIR = os.path.join(os.path.dirname(__file__), 'rules')
-DEFAULT_TRANSFORMSDIR = os.path.join(os.path.dirname(__file__), 'transforms')
 
 
 class ArgumentParser(argparse.ArgumentParser):
@@ -50,9 +49,7 @@ def run_cli(filename, template, rules, fmt, regions, override_spec, formatter):
     if override_spec:
         cfnlint.helpers.override_specs(override_spec)
 
-    transforms = get_transforms()
-    matches = run_checks(
-        filename, template, rules, transforms, regions)
+    matches = run_checks(filename, template, rules, regions)
 
     print_matches(matches, fmt, formatter)
 
@@ -73,13 +70,21 @@ def get_exit_code(matches):
 
 def configure_logging(log_level):
     """Setup Logging"""
+
     ch = logging.StreamHandler()
     ch.setLevel(logging.DEBUG)
+
+    # Update our application logger as well as the root logger to prevent
+    # 3rd party Python code to ignore "our" logging configuration:
+    # https://github.com/awslabs/serverless-application-model/pull/466
     if log_level == 'info':
+        logging.getLogger().setLevel(logging.INFO)
         LOGGER.setLevel(logging.INFO)
     elif log_level == 'debug':
+        logging.getLogger().setLevel(logging.DEBUG)
         LOGGER.setLevel(logging.DEBUG)
     else:
+        logging.getLogger().setLevel(logging.ERROR)
         LOGGER.setLevel(logging.ERROR)
     log_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
     ch.setFormatter(log_formatter)
@@ -135,17 +140,6 @@ def get_rules(rulesdir, ignore_rules):
     return rules
 
 
-def get_transforms():
-    """Get Transforms"""
-    transforms = TransformsCollection()
-    transformdirs = [DEFAULT_TRANSFORMSDIR]
-    for transformdir in transformdirs:
-        transforms.extend(
-            TransformsCollection.create_from_directory(transformdir))
-
-    return transforms
-
-
 def append_parser(parser, defaults):
     """Append arguments to parser"""
     parser.add_argument(
@@ -297,7 +291,7 @@ def get_default_args(template):
     return defaults
 
 
-def run_checks(filename, template, rules, transforms, regions):
+def run_checks(filename, template, rules, regions):
     """Run Checks against the template"""
     if regions:
 
@@ -308,8 +302,7 @@ def run_checks(filename, template, rules, transforms, regions):
 
     matches = list()
 
-    runner = cfnlint.Runner(
-        rules, transforms, filename, template, regions)
+    runner = cfnlint.Runner(rules, filename, template, regions)
     matches.extend(runner.transform())
     # Only do rule analysis if Transform was successful
     if not matches: