Support multiple config files (yandex#47)

gixy/cli/main.py

@@ -35,7 +35,7 @@ def _create_plugin_help(option):
 
 def _get_cli_parser():
     parser = create_parser()
-    parser.add_argument('nginx_file', nargs='?', type=str, default='/etc/nginx/nginx.conf', metavar='nginx.conf',
+    parser.add_argument('nginx_files', nargs='*', type=str, default=['/etc/nginx/nginx.conf'], metavar='nginx.conf',
                         help='Path to nginx.conf, e.g. /etc/nginx/nginx.conf')
 
     parser.add_argument(
@@ -96,11 +96,12 @@ def main():
     args = parser.parse_args()
     _init_logger(args.debug)
 
-    path = os.path.expanduser(args.nginx_file)
-    if path != '-' and not os.path.exists(path):
-        sys.stderr.write('Please specify path to Nginx configuration.\n\n')
-        parser.print_help()
-        sys.exit(1)
+    if len(args.nginx_files) == 1 and args.nginx_files[0] != '-':
+        path = os.path.expanduser(args.nginx_files[0])
+        if not os.path.exists(path):
+            sys.stderr.write('File {path!r} was not found.\nPlease specify correct path to configuration.\n'.format(
+                path=path))
+            sys.exit(1)
 
     try:
         severity = gixy.severity.ALL[args.level]
@@ -148,22 +149,32 @@ def main():
             options[opt_key] = val
         config.set_for(name, options)
 
-    with Gixy(config=config) as yoda:
-        if path == '-':
-            with os.fdopen(sys.stdin.fileno(), 'rb') as fdata:
-                yoda.audit('<stdin>', fdata, is_stdin=True)
-        else:
-            with open(path, mode='rb') as fdata:
-                yoda.audit(path, fdata, is_stdin=False)
-
-        formatted = formatters()[config.output_format]().format(yoda)
-        if args.output_file:
-            with open(config.output_file, 'w') as f:
-                f.write(formatted)
-        else:
-            print(formatted)
-
-        if sum(yoda.stats.values()) > 0:
-            # If something found - exit code must be 1, otherwise 0
-            sys.exit(1)
-        sys.exit(0)
+    formatter = formatters()[config.output_format]()
+    failed = False
+    for input_path in args.nginx_files:
+        path = os.path.abspath(os.path.expanduser(input_path))
+        if not os.path.exists(path):
+            LOG.error('File %s was not found', path)
+            continue
+
+        with Gixy(config=config) as yoda:
+            if path == '-':
+                with os.fdopen(sys.stdin.fileno(), 'rb') as fdata:
+                    yoda.audit('<stdin>', fdata, is_stdin=True)
+            else:
+                with open(path, mode='rb') as fdata:
+                    yoda.audit(path, fdata, is_stdin=False)
+
+            formatter.feed(path, yoda)
+            failed = failed or sum(yoda.stats.values()) > 0
+
+    if args.output_file:
+        with open(config.output_file, 'w') as f:
+            f.write(formatter.flush())
+    else:
+        print(formatter.flush())
+
+    if failed:
+        # If something found - exit code must be 1, otherwise 0
+        sys.exit(1)
+    sys.exit(0)

gixy/core/manager.py

@@ -15,10 +15,6 @@ def __init__(self, config=None):
         self.root = None
         self.config = config or Config()
         self.auditor = PluginsManager(config=self.config)
-        self.stats = {gixy.severity.UNSPECIFIED: 0,
-                      gixy.severity.LOW: 0,
-                      gixy.severity.MEDIUM: 0,
-                      gixy.severity.HIGH: 0}
 
     def audit(self, file_path, file_data, is_stdin=False):
         LOG.debug("Audit config file: {fname}".format(fname=file_path))
@@ -30,12 +26,20 @@ def audit(self, file_path, file_data, is_stdin=False):
         push_context(self.root)
         self._audit_recursive(self.root.children)
 
-    def get_results(self):
+    @property
+    def results(self):
         for plugin in self.auditor.plugins:
             if plugin.issues:
-                self.stats[plugin.severity] += len(plugin.issues)
                 yield plugin
 
+    @property
+    def stats(self):
+        stats = dict.fromkeys(gixy.severity.ALL, 0)
+        for plugin in self.auditor.plugins:
+            if plugin.issues:
+                stats[plugin.severity] += len(plugin.issues)
+        return stats
+
     def _audit_recursive(self, tree):
         for directive in tree:
             self._update_variables(directive)

gixy/formatters/base.py

@@ -1,27 +1,36 @@
 from __future__ import absolute_import
 
+import gixy
 from gixy.directives import block
 
 
 class BaseFormatter(object):
     skip_parents = set([block.Root, block.HttpBlock])
 
+    def __init__(self):
+        self.reports = {}
+        self.stats = dict.fromkeys(gixy.severity.ALL, 0)
+
     def format_reports(self, reports, stats):
         raise NotImplementedError("Formatter must override format_reports function")
 
-    def format(self, manager):
-        reports = []
-        for result in manager.get_results():
+    def feed(self, path, manager):
+        for severity in gixy.severity.ALL:
+            self.stats[severity] += manager.stats[severity]
+
+        self.reports[path] = []
+        for result in manager.results:
             report = self._prepare_result(manager.root,
                                           summary=result.summary,
                                           severity=result.severity,
                                           description=result.description,
                                           issues=result.issues,
                                           plugin=result.name,
                                           help_url=result.help_url)
-            reports.extend(report)
+            self.reports[path].extend(report)
 
-        return self.format_reports(reports, manager.stats)
+    def flush(self):
+        return self.format_reports(self.reports, self.stats)
 
     def _prepare_result(self, root, issues, severity, summary, description, plugin, help_url):
         result = {}

gixy/formatters/console.py

@@ -6,8 +6,9 @@
 
 class ConsoleFormatter(BaseFormatter):
     def __init__(self):
+        super(ConsoleFormatter, self).__init__()
         env = Environment(loader=PackageLoader('gixy', 'formatters/templates'), trim_blocks=True, lstrip_blocks=True)
         self.template = env.get_template('console.j2')
 
     def format_reports(self, reports, stats):
-        return self.template.render(issues=reports, stats=stats)
+        return self.template.render(reports=reports, stats=stats)

gixy/formatters/json.py

@@ -7,4 +7,18 @@
 
 class JsonFormatter(BaseFormatter):
     def format_reports(self, reports, stats):
-        return json.dumps(reports, sort_keys=True, indent=2, separators=(',', ': '))
+        result = []
+        for path, issues in reports.items():
+            for issue in issues:
+                result.append(dict(
+                    path=path,
+                    plugin=issue['plugin'],
+                    summary=issue['summary'],
+                    severity=issue['severity'],
+                    description=issue['description'],
+                    reference=issue['help_url'],
+                    reason=issue['reason'],
+                    config=issue['config']
+                ))
+
+        return json.dumps(result, sort_keys=True, indent=2, separators=(',', ': '))

gixy/formatters/templates/console.j2

@@ -1,35 +1,43 @@
 {% set colors = {'DEF': '\033[0m', 'TITLE': '\033[95m', 'UNSPECIFIED': '\033[0m', 'LOW': '\033[94m', 'MEDIUM': '\033[93m', 'HIGH': '\033[91m'} %}
 
-{{ colors['TITLE'] }}==================== Results ==================={{ colors['DEF'] }}
+{{ colors.TITLE }}==================== Results ==================={{ colors.DEF }}
+{% for path, issues in reports.items() %}
+File path: {{ path }}
 {% if not issues %}
 No issues found.
+
 {% else %}
 
 {% for issue in issues|sort(attribute='severity') %}
-{{ colors[issue.severity] }}Problem: [{{ issue.plugin }}] {{ issue.summary }}
+{{ colors[issue.severity] }}>> Problem: [{{ issue.plugin }}] {{ issue.summary }}
 {% if issue.description %}
-Description: {{ issue.description }}
+   Description: {{ issue.description }}
 {% endif %}
 {% if issue.help_url %}
-Additional info: {{ issue.help_url }}
+   Additional info: {{ issue.help_url }}
 {% endif %}
 {% if issue.reason %}
-Reason: {{ issue.reason }}
+   Reason: {{ issue.reason }}
 {% endif %}
-{{ colors['DEF'] }}Pseudo config:{{ issue.config }}
+   Pseudo config:{{ colors.DEF }}
+{{ issue.config }}
 
 {% if not loop.last %}
---------8<--------8<--------8<--------8<--------
+------------------------------------------------
 
 {% endif %}
 {% endfor %}
 {% endif %}
+{% if not loop.last %}
+--------8<--------8<--------8<--------8<--------
 
+{% endif %}
+{% endfor %}
 {% if stats %}
-{{ colors['TITLE'] }}==================== Summary ==================={{ colors['DEF'] }}
+{{ colors.TITLE }}==================== Summary ==================={{ colors.DEF }}
 Total issues:
-    Unspecified: {{ stats['UNSPECIFIED'] }}
-    Low: {{ stats['LOW'] }}
-    Medium: {{ stats['MEDIUM'] }}
-    High: {{ stats['HIGH'] }}
+    Unspecified: {{ stats.UNSPECIFIED }}
+    Low: {{ stats.LOW }}
+    Medium: {{ stats.MEDIUM }}
+    High: {{ stats.HIGH }}
 {% endif %}

gixy/formatters/templates/text.j2

@@ -1,35 +1,43 @@
 
 ==================== Results ===================
+{% for path, issues in reports.items() %}
+File path: {{ path }}
 {% if not issues %}
 No issues found.
+
 {% else %}
 
 {% for issue in issues|sort(attribute='severity') %}
-Problem: [{{ issue.plugin }}] {{ issue.summary }}
-Severity: {{ issue.severity }}
+>> Problem: [{{ issue.plugin }}] {{ issue.summary }}
+   Severity: {{ issue.severity }}
 {% if issue.description %}
-Description: {{ issue.description }}
+   Description: {{ issue.description }}
 {% endif %}
 {% if issue.help_url %}
-Additional info: {{ issue.help_url }}
+   Additional info: {{ issue.help_url }}
 {% endif %}
 {% if issue.reason %}
-Reason: {{ issue.reason }}
+   Reason: {{ issue.reason }}
 {% endif %}
-Pseudo config: {{ issue.config }}
+   Pseudo config:
+{{ issue.config }}
 
 {% if not loop.last %}
---------8<--------8<--------8<--------8<--------
+------------------------------------------------
 
 {% endif %}
 {% endfor %}
 {% endif %}
+{% if not loop.last %}
+--------8<--------8<--------8<--------8<--------
 
+{% endif %}
+{% endfor %}
 {% if stats %}
 ==================== Summary ===================
 Total issues:
-    Unspecified: {{ stats['UNSPECIFIED'] }}
-    Low: {{ stats['LOW'] }}
-    Medium: {{ stats['MEDIUM'] }}
-    High: {{ stats['HIGH'] }}
+    Unspecified: {{ stats.UNSPECIFIED }}
+    Low: {{ stats.LOW }}
+    Medium: {{ stats.MEDIUM }}
+    High: {{ stats.HIGH }}
 {% endif %}

gixy/formatters/text.py

@@ -6,8 +6,9 @@
 
 class TextFormatter(BaseFormatter):
     def __init__(self):
+        super(TextFormatter, self).__init__()
         env = Environment(loader=PackageLoader('gixy', 'formatters/templates'), trim_blocks=True, lstrip_blocks=True)
         self.template = env.get_template('text.j2')
 
     def format_reports(self, reports, stats):
-        return self.template.render(issues=reports, stats=stats)
+        return self.template.render(reports=reports, stats=stats)

tests/plugins/test_simply.py

@@ -4,7 +4,6 @@
 from os import path
 import json
 
-import gixy
 from ..utils import *
 from gixy.core.manager import Manager as Gixy
 from gixy.core.plugins_manager import PluginsManager
@@ -83,7 +82,9 @@ def check_configuration(plugin, config_path, test_config):
     plugin_options = parse_plugin_options(config_path)
     with yoda_provider(plugin, plugin_options) as yoda:
         yoda.audit(config_path, open(config_path, mode='r'))
-        results = RawFormatter().format(yoda)
+        formatter = BaseFormatter()
+        formatter.feed(config_path, yoda)
+        _, results = formatter.reports.popitem()
 
         assert_equals(len(results), 1, 'Should have one report')
         result = results[0]
@@ -104,7 +105,5 @@ def check_configuration(plugin, config_path, test_config):
 def check_configuration_fp(plugin, config_path, test_config):
     with yoda_provider(plugin) as yoda:
         yoda.audit(config_path, open(config_path, mode='r'))
-        results = RawFormatter().format(yoda)
-
-        assert_equals(len(results), 0,
+        assert_equals(len([x for x in yoda.results]), 0,
                       'False positive configuration must not trigger any plugins')

tests/utils.py

@@ -62,8 +62,3 @@ def match_value(self, k, dv, v):
         else:
             result = dv.find(v) >= 0
         return result
-
-
-class RawFormatter(BaseFormatter):
-    def format_reports(self, reports, stats):
-        return reports