Python 3 support (elastic#4346)

* Ignore local pyenv file, add more Python 3 compatibility code, fix autopep8 errors
* add six to modules installed during Windows build
* make pep8 happy

.appveyor.yml

@@ -55,7 +55,7 @@ install:
   - set PYTHONPATH=C:\Python27
   - set PATH=%PYTHONPATH%;%PYTHONPATH%\Scripts;%PATH%
   - python --version
-  - pip install jinja2 nose nose-timer PyYAML redis elasticsearch
+  - pip install six jinja2 nose nose-timer PyYAML redis elasticsearch
   - easy_install C:/pywin_inst.exe
 
 # To run your custom scripts instead of automatic MSBuild

.gitignore

@@ -15,6 +15,7 @@
 *.dev.yml
 *.generated.yml
 coverage.out
+.python-version
 
 # Editor swap files
 *.swp

.travis.yml

@@ -87,6 +87,7 @@ addons:
       - libpcap-dev
 
 before_install:
+  - python --version
   - umask 022
   - chmod -R go-w $GOPATH/src/github.com/elastic/beats
   # Docker-compose installation

Makefile

@@ -66,7 +66,7 @@ clean-vendor:
 check: python-env
 	$(foreach var,$(PROJECTS),$(MAKE) -C $(var) check || exit 1;)
 	# Checks also python files which are not part of the beats
-	$(FIND) -name *.py -exec autopep8 -d --max-line-length 120  {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false)
+	. $(PYTHON_ENV)/bin/activate && $(FIND) -name *.py -exec autopep8 -d --max-line-length 120  {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false)
 	# Validate that all updates were committed
 	$(MAKE) update
 	git diff | cat
@@ -83,7 +83,7 @@ misspell:
 fmt: python-env
 	$(foreach var,$(PROJECTS),$(MAKE) -C $(var) fmt || exit 1;)
 	# Cleans also python files which are not part of the beats
-	. $(PYTHON_ENV)/bin/activate && find . -type f -name *.py -not -path "*/vendor/*" -not -path "*/build/*" -not -path "*/.git/*" -exec autopep8 --in-place --max-line-length 120  {} \;
+	. $(PYTHON_ENV)/bin/activate && $(FIND) . -name *.py -exec autopep8 --in-place --max-line-length 120  {} \;
 
 .PHONY: lint
 lint:
@@ -136,11 +136,11 @@ upload-release:
 	aws s3 cp --recursive --acl public-read build/upload s3://download.elasticsearch.org/beats/
 
 .PHONY: notice
-notice:
-	python dev-tools/generate_notice.py .
+notice: python-env
+	@. $(PYTHON_ENV)/bin/activate && python dev-tools/generate_notice.py .
 
 # Sets up the virtual python environment
 .PHONY: python-env
 python-env:
 	@test -d $(PYTHON_ENV) || virtualenv $(VIRTUALENV_PARAMS) $(PYTHON_ENV)
-	@. $(PYTHON_ENV)/bin/activate && pip install -q --upgrade pip autopep8
+	@. $(PYTHON_ENV)/bin/activate && pip install -q --upgrade pip autopep8 six

dev-tools/aggregate_coverage.py

@@ -37,7 +37,7 @@ def main(arguments):
                         stmt = int(stmt)
                         count = int(count)
                         prev_count = 0
-                        if lines.has_key(position):
+                        if position in lines:
                             (_, prev_stmt, prev_count) = lines[position]
                             assert prev_stmt == stmt
                         lines[position] = (position, stmt, prev_count + count)

dev-tools/generate_notice.py

@@ -1,7 +1,10 @@
+from __future__ import print_function
+
 import glob
 import os
 import datetime
 import argparse
+import six
 
 
 def read_file(filename):
@@ -10,7 +13,7 @@ def read_file(filename):
         print("File not found {}".format(filename))
         return ""
 
-    with open(filename, 'r') as f:
+    with open(filename, 'rb') as f:
         file_content = f.read()
         return file_content
 
@@ -22,7 +25,7 @@ def get_library_path(license):
     split = license.split(os.sep)
     for i, word in reversed(list(enumerate(split))):
         if word == "vendor":
-            return "/".join(split[i+1:])
+            return "/".join(split[i + 1:])
     return "/".join(split)
 
 
@@ -41,37 +44,38 @@ def add_licenses(f, vendor_dirs):
     # Sort licenses by package path, ignore upper / lower case
     for key in sorted(licenses, key=str.lower):
         license_file = licenses[key]
-        f.write("\n--------------------------------------------------------------------\n")
-        f.write("{}\n".format(key))
-        f.write("--------------------------------------------------------------------\n")
+        f.write(six.b("\n--------------------------------------------------------------------\n"))
+        f.write(six.b("{}\n".format(key)))
+        f.write(six.b("--------------------------------------------------------------------\n"))
         copyright = read_file(license_file)
-        if "Apache License" not in copyright:
+        if six.b('Apache License') not in copyright:
             f.write(copyright)
         else:
             # it's an Apache License, so include only the NOTICE file
-            f.write("Apache License\n\n")
+            f.write(six.b("Apache License\n\n"))
             for notice_file in glob.glob(os.path.join(os.path.dirname(license_file), "NOTICE*")):
-                f.write("-------{}-----\n".format(os.path.basename(notice_file)))
+                notice_file_hdr = "-------{}-----\n".format(os.path.basename(notice_file))
+                f.write(six.b(notice_file_hdr))
                 f.write(read_file(notice_file))
 
 
 def create_notice(filename, beat, copyright, vendor_dirs):
 
     now = datetime.datetime.now()
 
-    with open(filename, "w+") as f:
+    with open(filename, "wb+") as f:
 
         # Add header
-        f.write("{}\n".format(beat))
-        f.write("Copyright 2014-{0} {1}\n".format(now.year, copyright))
-        f.write("\n")
-        f.write("This product includes software developed by The Apache Software \n" +
-                "Foundation (http://www.apache.org/).\n\n")
+        f.write(six.b("{}\n".format(beat)))
+        f.write(six.b("Copyright 2014-{0} {1}\n".format(now.year, copyright)))
+        f.write(six.b("\n"))
+        f.write(six.b("This product includes software developed by The Apache Software \n") +
+                six.b("Foundation (http://www.apache.org/).\n\n"))
 
         # Add licenses for 3rd party libraries
-        f.write("==========================================================================\n")
-        f.write("Third party libraries used by the Beats project:\n")
-        f.write("==========================================================================\n\n")
+        f.write(six.b("==========================================================================\n"))
+        f.write(six.b("Third party libraries used by the Beats project:\n"))
+        f.write(six.b("==========================================================================\n\n"))
         add_licenses(f, vendor_dirs)
 
 
@@ -92,7 +96,7 @@ def create_notice(filename, beat, copyright, vendor_dirs):
     notice = os.path.join(cwd, "NOTICE")
     vendor_dirs = []
 
-    print args.vendor
+    print(args.vendor)
 
     for root, dirs, files in os.walk(args.vendor):
 

filebeat/scripts/docs_collector.py

@@ -1,6 +1,7 @@
 import os
 import argparse
 import yaml
+import six
 
 # Collects docs for all modules
 
@@ -30,7 +31,7 @@ def collect(beat_name):
         module_file = generated_note
         module_file += "[[filebeat-module-" + module + "]]\n"
 
-        with file(module_doc) as f:
+        with open(module_doc) as f:
             module_file += f.read()
 
         beat_path = path + "/" + module + "/_meta"
@@ -59,12 +60,12 @@ def collect(beat_name):
 
     module_list_output = generated_note
     module_list_output += "  * <<filebeat-modules-overview>>\n"
-    for m, title in sorted(modules_list.iteritems()):
+    for m, title in sorted(six.iteritems(modules_list)):
         module_list_output += "  * <<filebeat-module-" + m + ">>\n"
 
     module_list_output += "\n\n--\n\n"
     module_list_output += "include::modules-overview.asciidoc[]\n"
-    for m, title in sorted(modules_list.iteritems()):
+    for m, title in sorted(six.iteritems(modules_list)):
         module_list_output += "include::modules/" + m + ".asciidoc[]\n"
 
     # Write module link list

filebeat/tests/system/config/filebeat.yml.j2

@@ -34,10 +34,10 @@ filebeat.prospectors:
   {%- if prospector_processors %}
   processors:
     {%- for processor in prospector_processors %}
-    {%- for name, settings in processor.iteritems() %}
+    {%- for name, settings in processor.items() %}
     - {{name}}:
       {%- if settings %}
-      {%- for k, v in settings.iteritems() %}
+      {%- for k, v in settings.items() %}
         {{k}}:
           {{v | default([])}}
       {%- endfor %}
@@ -142,10 +142,10 @@ geoip:
 
 processors:
 {%- for processor in processors %}
-{%- for name, settings in processor.iteritems() %}
+{%- for name, settings in processor.items() %}
 - {{name}}:
   {%- if settings %}
-  {%- for k, v in settings.iteritems() %}
+  {%- for k, v in settings.items() %}
     {{k}}:
       {{v | default([])}}
   {%- endfor %}

filebeat/tests/system/test_json.py

@@ -1,5 +1,6 @@
 from filebeat import BaseTest
 import os
+import six
 
 """
 Tests for the JSON decoding functionality.
@@ -195,7 +196,7 @@ def test_timestamp_in_message(self):
 
         output = self.read_output()
         assert len(output) == 5
-        assert all(isinstance(o["@timestamp"], basestring) for o in output)
+        assert all(isinstance(o["@timestamp"], six.string_types) for o in output)
         assert output[0]["@timestamp"] == "2016-04-05T18:47:18.444Z"
 
         assert output[1]["@timestamp"] != "invalid"
@@ -237,7 +238,7 @@ def test_type_in_message(self):
 
         output = self.read_output()
         assert len(output) == 3
-        assert all(isinstance(o["@timestamp"], basestring) for o in output)
+        assert all(isinstance(o["@timestamp"], six.string_types) for o in output)
         assert output[0]["type"] == "test"
 
         assert "type" not in output[1]

libbeat/tests/system/beat/beat.py

@@ -198,7 +198,7 @@ def render_config_template(self, template_name=None,
         kargs["beat"] = self
         output_str = template.render(**kargs)
         with open(os.path.join(self.working_dir, output), "wb") as f:
-            f.write(output_str)
+            f.write(output_str.encode('utf8'))
 
     # Returns output as JSON object with flattened fields (. notation)
     def read_output(self,

libbeat/tests/system/requirements.txt

@@ -7,3 +7,4 @@ redis
 elasticsearch
 requests
 autopep8
+six

metricbeat/scripts/docs_collector.py

@@ -1,6 +1,7 @@
 import os
 import argparse
 import yaml
+import six
 
 # Collects docs for all modules and metricset
 
@@ -33,7 +34,7 @@ def collect(beat_name):
         module_file = generated_note
         module_file += "[[metricbeat-module-" + module + "]]\n"
 
-        with file(module_doc) as f:
+        with open(module_doc) as f:
             module_file += f.read()
 
         beat_path = path + "/" + module + "/_meta"
@@ -63,7 +64,7 @@ def collect(beat_name):
 """ + beat_name + ".modules:\n"
 
             # Load metricset yaml
-            with file(config_file) as f:
+            with open(config_file) as f:
                 # Add 2 spaces for indentation in front of each line
                 for line in f:
                     module_file += line
@@ -135,11 +136,11 @@ def collect(beat_name):
             f.write(module_file)
 
     module_list_output = generated_note
-    for m, title in sorted(modules_list.iteritems()):
+    for m, title in sorted(six.iteritems(modules_list)):
         module_list_output += "  * <<metricbeat-module-" + m + "," + title + ">>\n"
 
     module_list_output += "\n\n--\n\n"
-    for m, title in sorted(modules_list.iteritems()):
+    for m, title in sorted(six.iteritems(modules_list)):
         module_list_output += "include::modules/" + m + ".asciidoc[]\n"
 
     # Write module link list

metricbeat/scripts/fields_collector.py

@@ -21,7 +21,7 @@ def collect():
             continue
 
         # Load module yaml
-        with file(module_fields) as f:
+        with open(module_fields) as f:
             tmp = f.read()
             fields_yml += tmp
 
@@ -35,7 +35,7 @@ def collect():
                 continue
 
             # Load metricset yaml
-            with file(metricset_fields) as f:
+            with open(metricset_fields) as f:
                 # Add 4 spaces for indentation in front of each line
                 for line in f:
                     if len(line.strip()) > 0:

metricbeat/tests/system/config/metricbeat.yml.j2

@@ -123,10 +123,10 @@ tags: [
 {%- if processors %}
 processors:
 {%- for processor in processors %}
-{%- for name, settings in processor.iteritems() %}
+{%- for name, settings in processor.items() %}
 - {{name}}:
   {%- if settings %}
-  {%- for k, v in settings.iteritems() %}
+  {%- for k, v in settings.items() %}
     {{k}}:
       {{v | default([])}}
   {%- endfor %}

metricbeat/tests/system/test_system.py

@@ -1,4 +1,5 @@
 import re
+import six
 import sys
 import unittest
 import metricbeat
@@ -404,8 +405,8 @@ def test_process_metricbeat(self):
 
         assert re.match("(?i)metricbeat.test(.exe)?", output["system.process.name"])
         assert re.match("(?i).*metricbeat.test(.exe)? -systemTest", output["system.process.cmdline"])
-        assert isinstance(output["system.process.state"], basestring)
-        assert isinstance(output["system.process.cpu.start_time"], basestring)
+        assert isinstance(output["system.process.state"], six.string_types)
+        assert isinstance(output["system.process.cpu.start_time"], six.string_types)
         self.check_username(output["system.process.username"])
 
     def check_username(self, observed, expected=None):

packetbeat/tests/system/config/packetbeat.yml.j2

@@ -176,10 +176,10 @@ packetbeat.shutdown_timeout: {{ shutdown_timeout|default(0) }}
 
 processors:
 {%- for processor in processors %}
-{%- for name, settings in processor.iteritems() %}
+{%- for name, settings in processor.items() %}
 - {{name}}:
   {%- if settings %}
-  {%- for k, v in settings.iteritems() %}
+  {%- for k, v in settings.items() %}
     {{k}}:
       {{v | default([])}}
   {%- endfor %}

packetbeat/tests/system/test_0002_thrift_basics.py

@@ -105,7 +105,8 @@ def test_thrift_tutorial_with_idl(self):
         objs = self.read_output()
         assert len(objs) == 17
         assert all([o["type"] == "thrift" for o in objs])
-        assert all([o["thrift.service"] == "Calculator"])
+        assert all([o["thrift.service"] == "Calculator" or
+                    o["thrift.service"] == "SharedService" for o in objs])
 
         assert objs[0]["method"] == "ping"
         assert objs[0]["thrift.params"] == "()"
@@ -159,7 +160,7 @@ def test_thrift_integration(self):
         objs = self.read_output()
         assert len(objs) == 26
         assert all([o["type"] == "thrift" for o in objs])
-        assert all([o["thrift.service"] == "ThriftTest"])
+        assert all([o["thrift.service"] == "ThriftTest" for o in objs])
 
         # check a few things
 

packetbeat/tests/system/test_0006_wsgi.py

@@ -1,5 +1,7 @@
 from packetbeat import BaseTest
 
+import six
+
 """
 Tests for parsing WSGI traffic.
 """
@@ -108,7 +110,7 @@ def test_send_headers_options(self):
         assert len(o["http.request.headers"]) > 0
         assert len(o["http.response.headers"]) > 0
         assert isinstance(o["http.response.headers"]["set-cookie"],
-                          basestring)
+                          six.string_types)
 
         self.render_config_template(
             http_ports=[8888],