Remove generated fields.yml from git repositories (elastic#2931)

As some of the generated fields are also needed for the tests, make collect must be run in some cases before running the tests. Testsuite does this automatically. It was not added to each test as it would run multiple times and causes some issue with docker container access rights. * Adapt script to rely on generated fields.generated.yml file if it exists * Add fields.generate.yml to gitignore * Make update dependent on collect call instead of the other way around as update requires now collect. * Make system-tests use fields.generated.yml * Remove obsolete metricbeat.json file NOTE: More fields.yml generated fields can be removed in the future. This PR provides the basic logic for it.
jetgeng · Nov 8, 2016 · 946aa56 · 946aa56
1 parent 9596aaa
commit 946aa56
Show file tree

Hide file tree

Showing 17 changed files with 56 additions and 3,605 deletions.
diff --git a/.appveyor.yml b/.appveyor.yml
@@ -11,7 +11,7 @@ environment:
   PYWIN_DL: https://beats-files.s3.amazonaws.com/deps/pywin32-220.win32-py2.7.exe
   matrix:
   - PROJ: github.com\elastic\beats\metricbeat
-    BEAT: metricbeat 
+    BEAT: metricbeat
   - PROJ: github.com\elastic\beats\filebeat
     BEAT: filebeat
   - PROJ: github.com\elastic\beats\winlogbeat
@@ -72,6 +72,11 @@ test_script:
   # System tests
   - ps: Add-AppveyorTest "System tests" -Outcome Running
   - go test -race -c -cover -covermode=atomic -coverpkg ./...
+  - ps: |
+      if ($env:BEAT -eq "metricbeat") {
+        cp .\etc\fields.common.yml .\etc\fields.generated.yml
+        python .\scripts\fields_collector.py | out-file -append -encoding UTF8 -filepath .\etc\fields.generated.yml
+      }
   - ps: cd tests/system
   - nosetests --with-timer
   - ps: Update-AppveyorTest "System tests" -Outcome Passed

diff --git a/.gitignore b/.gitignore
@@ -10,6 +10,7 @@
 /glide.lock
 /beats.iml
 *.dev.yml
+*.generated.yml
 
 # Editor swap files
 *.swp

diff --git a/Makefile b/Makefile
@@ -34,9 +34,10 @@ coverage-report:
 	-tail -q -n +2 ./libbeat/${COVERAGE_DIR}/*.cov >> ./${COVERAGE_DIR}/full.cov
 	go tool cover -html=./${COVERAGE_DIR}/full.cov -o ${COVERAGE_DIR}/full.html
 
-.PHONY: collect
-collect:
-	$(foreach var,$(PROJECTS),$(MAKE) -C $(var) collect || exit 1;)
+.PHONY: update
+update:
+	$(MAKE) -C libbeat collect
+	$(foreach var,$(BEATS),$(MAKE) -C $(var) update || exit 1;)
 
 .PHONY: clean
 clean:
@@ -54,7 +55,7 @@ clean-vendor:
 check:
 	$(foreach var,$(PROJECTS),$(MAKE) -C $(var) check || exit 1;)
 	# Validate that all updates were commited
-	$(MAKE) collect
+	$(MAKE) update
 	git update-index --refresh
 	git diff-index --exit-code HEAD --
 

diff --git a/filebeat/Makefile b/filebeat/Makefile
@@ -14,4 +14,4 @@ before-build:
 
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect: update
+collect:
diff --git a/generate/beat/{{cookiecutter.beat}}/Makefile b/generate/beat/{{cookiecutter.beat}}/Makefile
@@ -42,4 +42,4 @@ before-build:
 
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect: update
+collect:
diff --git a/heartbeat/Makefile b/heartbeat/Makefile
@@ -12,4 +12,4 @@ before-build:
 
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect: update
+collect:
diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile
@@ -105,7 +105,7 @@ simplify:
 # Cleans up directory and source code with gofmt
 .PHONY: clean
 clean:
-	rm -rf build ${BEATNAME} ${BEATNAME}.test ${BEATNAME}.exe ${BEATNAME}.test.exe
+	rm -rf build ${BEATNAME} ${BEATNAME}.test ${BEATNAME}.exe ${BEATNAME}.test.exe etc/fields.generated.yml
 
 # Shortcut for continuous integration
 # This should always run before merging.
@@ -194,7 +194,7 @@ test: unit
 
 # Runs all tests and generates the coverage reports
 .PHONY: testsuite
-testsuite:
+testsuite: clean collect
 	$(MAKE) unit-tests
 
 	# Setups environment if TEST_ENVIRONMENT is set to true
@@ -227,7 +227,8 @@ coverage-report:
 
 # Update expects the most recent version of libbeat in the GOPATH
 .PHONY: update
-update: python-env
+update: python-env collect
+
 	# Update config
 	echo "Update config file"
 	-rm -f ${BEATNAME}.yml
@@ -242,7 +243,6 @@ update: python-env
 	fi;
 
 	# Update docs
-
 	. ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_fields_docs.py $(PWD) ${BEATNAME} ${ES_BEATS}
 
 	# Generate index templates

diff --git a/libbeat/scripts/generate_fields_docs.py b/libbeat/scripts/generate_fields_docs.py
@@ -1,5 +1,5 @@
 import yaml
-import sys
+import os
 import argparse
 
 def document_fields(output, section, sections, path):
@@ -112,8 +112,14 @@ def fields_to_asciidoc(input, output, beat):
     beat_name = args.beatname
     es_beats = args.es_beats
 
+    fields_yml = beat_path + "/etc/fields.generated.yml"
+
+    # Not all beats have a fields.generated.yml. Fall back to fields.yml
+    if not os.path.isfile(fields_yml):
+        fields_yml = beat_path + "/etc/fields.yml"
+
     # Read fields.yml
-    with open(beat_path + "/etc/fields.yml") as f:
+    with open(fields_yml) as f:
         fields = f.read()
 
     # Prepends beat fields from libbeat

diff --git a/libbeat/scripts/generate_index_pattern.py b/libbeat/scripts/generate_index_pattern.py
@@ -100,8 +100,14 @@ def get_index_pattern_name(index):
 
     args = parser.parse_args()
 
+    fields_yml = args.beat + "/etc/fields.generated.yml"
+
+    # Not all beats have a fields.generated.yml. Fall back to fields.yml
+    if not os.path.isfile(fields_yml):
+        fields_yml = args.beat + "/etc/fields.yml"
+
     # generate the index-pattern content
-    with open(args.beat + "/etc/fields.yml", 'r') as f:
+    with open(fields_yml, 'r') as f:
         fields = f.read()
 
         # Prepend beat fields from libbeat

diff --git a/libbeat/scripts/generate_template.py b/libbeat/scripts/generate_template.py
@@ -12,6 +12,7 @@
 import yaml
 import json
 import argparse
+import os
 
 
 def fields_to_es_template(args, input, output, index, version):
@@ -306,7 +307,13 @@ def fill_field_properties(args, field, defaults, path):
         target += "-es2x"
     target += ".json"
 
-    with open(args.path + "/etc/fields.yml", 'r') as f:
+    fields_yml = args.path + "/etc/fields.generated.yml"
+
+    # Not all beats have a fields.generated.yml. Fall back to fields.yml
+    if not os.path.isfile(fields_yml):
+        fields_yml = args.path + "/etc/fields.yml"
+
+    with open(fields_yml, 'r') as f:
         fields = f.read()
 
         # Prepend beat fields from libbeat

diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py
@@ -376,7 +376,7 @@ def all_fields_are_expected(self, objs, expected_fields,
                     raise Exception("Unexpected key '{}' found"
                                     .format(key))
 
-    def load_fields(self, fields_doc="../../etc/fields.yml"):
+    def load_fields(self, fields_doc="../../etc/fields.generated.yml"):
         """
         Returns a list of fields to expect in the output dictionaries
         and a second list that contains the fields that have a
@@ -405,6 +405,10 @@ def extract_fields(doc_list, name):
                         dictfields.append(newName)
             return fields, dictfields
 
+        # Not all beats have a fields.generated.yml. Fall back to fields.yml
+        if not os.path.isfile(fields_doc):
+            fields_doc = "../../etc/fields.yml"
+
         # TODO: Make fields_doc path more generic to work with beat-generator
         with open(fields_doc, "r") as f:
             # TODO: Make this path more generic to work with beat-generator.

diff --git a/metricbeat/Makefile b/metricbeat/Makefile
@@ -22,21 +22,22 @@ include ${ES_BEATS}/libbeat/scripts/Makefile
 # Collects all module dashboards
 .PHONY: kibana
 kibana:
-	-rm -r etc/kibana
+	# To not remove index-pattern as generated by update
+	-rm -r etc/kibana/dashboard etc/kibana/search etc/kibana/visualization
 	mkdir -p etc/kibana
 	-cp -r module/*/_meta/kibana etc/
 
 # Collects all module and metricset fields
 .PHONY: fields
 fields:
 	mkdir -p etc/
-	cat ${ES_BEATS}/metricbeat/etc/_meta/fields_base.yml > etc/fields.yml
-	. ${PYTHON_ENV}/bin/activate; python ${ES_BEATS}/metricbeat/scripts/fields_collector.py >> etc/fields.yml
+	cat ${ES_BEATS}/metricbeat/etc/fields.common.yml > etc/fields.generated.yml
+	. ${PYTHON_ENV}/bin/activate; python ${ES_BEATS}/metricbeat/scripts/fields_collector.py >> etc/fields.generated.yml
 
 # Collects all module docs
 .PHONY: collect-docs
 collect-docs: python-env
-	-rm -r docs/modules
+	-rm -rf docs/modules
 	mkdir -p docs/modules
 	. ${PYTHON_ENV}/bin/activate; python ${ES_BEATS}/metricbeat/scripts/docs_collector.py --beat ${BEATNAME}
 
@@ -63,7 +64,7 @@ before-build:
 
 # Runs all collection steps and updates afterwards
 .PHONY: collect
-collect: fields collect-docs configs kibana imports update
+collect: fields collect-docs configs kibana imports
 
 # Creates a new metricset. Requires the params MODULE and METRICSET
 .PHONY: create-metricset

diff --git a/metricbeat/etc/_meta/fields_base.yml → metricbeat/etc/fields.common.yml b/metricbeat/etc/_meta/fields_base.yml → metricbeat/etc/fields.common.yml
@@ -28,3 +28,4 @@
       example: metricsets
       description: >
         The document type. Always set to "metricsets".
+
Original file line number	Diff line number	Diff line change
Expand Up		@@ -28,3 +28,4 @@
		example: metricsets
		description: >
		The document type. Always set to "metricsets".