refactoring(properties) Normalize and make it consistent between Stan…

…dalone and Spring - Some keys were not usable with Spring (such as 'ogham.email.subject.first-line-prefix' because 'ogham.email.subject' was String type) - Create all @ConfigurationProperties and @NestedConfigurationProperties for all Ogham properties - Fix some properties (Freemarker template suffix that was not consistent with other properties) - Update samples to use new properties - Update tests to use new properties - Fix variant resolution based on extension (found when running samples) - Add support for .ftlh extension - Add script to automatically search for property definitions and write them in an application.properties (this way the properties are automatically tested by Spring). If a property is malformed, the project can't be built (test failure). If a property is not known by Spring, there is a warning (in the IDE). - Fix resource attachment name error
project-kotinos · Mar 4, 2020 · 092425b · 092425b
1 parent 098068c
commit 092425b
Show file tree

Hide file tree

Showing 129 changed files with 5,041 additions and 1,205 deletions.
diff --git a/.tools/properties-consistency/.default-values b/.tools/properties-consistency/.default-values
@@ -0,0 +1,33 @@
+ogham.email.javamail.body.charset=UTF-8
+ogham.email.sendgrid.unit-testing=false
+ogham.email.subject.extract-html-title.enable=false
+ogham.email.template.cache=false
+ogham.email.thymeleaf.cache=false
+ogham.freemarker.static-method-access.enable=false
+ogham.mimetype.default-mimetype=text/plain
+ogham.mimetype.tika.fail-if-octet-stream=false
+ogham.sms.cloudhopper.bind-type=transmitter
+ogham.sms.cloudhopper.data-coding-scheme.auto.enable=false
+ogham.sms.cloudhopper.encoder.auto-guess.enable=false
+ogham.sms.cloudhopper.interface-version=3.4
+ogham.sms.cloudhopper.split.enable=false
+ogham.sms.cloudhopper.user-data.use-short-message=false
+ogham.sms.cloudhopper.user-data.use-tlv-message-payload=false
+ogham.sms.from.alphanumeric-code-format.enable=false
+ogham.sms.from.international-format.enable=false
+ogham.sms.from.short-code-format.enable=false
+ogham.sms.ovh.options.no-stop=false
+ogham.sms.ovh.options.sms-coding=gsm7
+ogham.sms.ovh.url=http://foo
+ogham.sms.smpp.bind-type=transmitter
+ogham.sms.smpp.data-coding-scheme.auto.enable=false
+ogham.sms.smpp.encoder.auto-guess.enable=false
+ogham.sms.smpp.user-data.use-short-message=false
+ogham.sms.smpp.user-data.use-tlv-message-payload=false
+ogham.sms.smpp.split.enable=false
+ogham.sms.template.cache=false
+ogham.sms.thymeleaf.cache=false
+ogham.sms.to.international-format.enable=false
+ogham.template.cache=false
+ogham.wrap-uncaught-exceptions.enable=false
+ogham.sms.split.enable=false
diff --git a/.tools/properties-consistency/.ignore-props b/.tools/properties-consistency/.ignore-props
@@ -0,0 +1,41 @@
+# in comments
+${custom.high-priority}
+${custom.low-priority}
+${email.sender.address}
+${sms.sender.number}
+${smpp.host}
+${smpp.port}
+${smpp.system-id}
+${smpp.password}
+# in code
+${property.key}
+# in html code
+${value}
+# in Spring tests
+${local.server.port}
+# in tests utils
+${fieldName}
+${messageIndex}
+${name}
+${found}
+${partName}
+${tagName}
+${numberName}
+# in tests
+${nested.name}
+${nested.value}
+${configurer.after-init.high-priority}
+${configurer.after-init.low-priority}
+${configurer.before-build.default.high-priority}
+${configurer.before-build.default.low-priority}
+${configurer.before-build.service-provider.high-priority}
+${configurer.before-build.service-provider.low-priority}
+${custom.property}
+${custom.property.for.api-key}
+${custom.property.for.host}
+${custom.property.for.url}
+${custom.property.high-priority}
+${custom.property.key}
+${custom.property.low-priority}
+# internal property
+${ogham.sms.smsglobal.service-provider.auto-conf.force}
diff --git a/.tools/properties-consistency/list-all-properties.sh b/.tools/properties-consistency/list-all-properties.sh
@@ -0,0 +1,137 @@
+#!/bin/bash
+
+
+# Script that generates the `application-consistency-check.properties` file that will
+# contain all properties defined by Ogham (such as "${propery.key}").
+# Then we can use Spring to automatically check if properties are well-formed or not.
+# This is also useful to detect a property defined by Ogham but not known by
+# Spring Boot configuration processor (so completion won't be available).
+#
+# It searches across all files using regular expression and for each
+# defined property (except excluded ones), it generates a comment to indicate:
+# - where the property is defined
+# - where the property is used
+
+
+set -e
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+OGHAM_DIR="$SCRIPT_DIR/../.."
+
+PROPERTY_FILE="ogham-spring-boot-autoconfigure/src/test/resources/config/application-consistency-check.properties"
+
+DEBUG_FILE="${OGHAM_DIR}/target/properties-debug.log"
+DEBUG_SEPARATOR="\n--------------------------------------------\n"
+echo "" > "${DEBUG_FILE}"
+
+SKIPPED_PROPERTIES_FILE="${OGHAM_DIR}/target/skipped-properties.diff"
+echo "" > "${SKIPPED_PROPERTIES_FILE}"
+
+EXCLUDED_PROPS="$(sed -e 's/^[[:space:]]*#.*// ; /^[[:space:]]*$/d' "${SCRIPT_DIR}/.ignore-props")" 
+DEFAULT_VALUES="$(sed -e 's/^[[:space:]]*#.*// ; /^[[:space:]]*$/d' "${SCRIPT_DIR}/.default-values")" 
+
+
+# 1) find all lines surrounded by "${}" (with associated file and line number)
+echo "Finding all lines that contains defined property..."
+FOUND="$(grep -rnE "$OGHAM_DIR" \
+		--exclude-dir="target" \
+		--exclude-dir=".externalToolBuilders" \
+		--exclude-dir=".tools" \
+		--exclude="site.xml" \
+		--exclude="mvnw" \
+		--exclude="*.sh" \
+		--exclude="README.adoc" \
+		-e '"\$\{[^}]+\}"' \
+	| grep --invert "${PROPERTY_FILE}" \
+	| cat)"
+																						echo -e "${DEBUG_SEPARATOR}FOUND=\n$FOUND\n\n" >> "${DEBUG_FILE}"
+
+# 2) generate list of properties
+#    and filter some properties (defined in comments for example)
+#    and remove surrounding characters
+echo "Generating list of property keys..."
+ALL_POSSIBLE_PROPERTIES="$(echo "$FOUND" | grep -ohE -e '"\$\{[^}]+\}"' \
+	| sort | uniq)"
+ALL_PROPERTIES="$(echo "$FOUND" | grep -ohE -e '"\$\{[a-zA-Z0-9.\-]+\}"' \
+	| sort | uniq)"
+PROPERTIES="$(echo "$ALL_PROPERTIES" \
+	| grep -vF "${EXCLUDED_PROPS}")"
+
+																						echo -e "${DEBUG_SEPARATOR}ALL_POSSIBLE_PROPERTIES=\n$ALL_POSSIBLE_PROPERTIES\n\n" >> "${DEBUG_FILE}"
+																						echo -e "${DEBUG_SEPARATOR}ALL_PROPERTIES=\n$ALL_PROPERTIES\n\n" >> "${DEBUG_FILE}"
+																						echo -e "${DEBUG_SEPARATOR}PROPERTIES=\n$PROPERTIES\n\n" >> "${DEBUG_FILE}"
+
+KEYS="$(echo "$PROPERTIES" | sed 's/"${//' | sed 's/}"//')"
+																						echo -e "${DEBUG_SEPARATOR}FINAL KEYS=\n$KEYS\n\n" >> "${DEBUG_FILE}"
+
+# 3) display skipped keys
+echo -e "Automatically skipped properties:\n" >> "${SKIPPED_PROPERTIES_FILE}"
+diff <(echo "$ALL_POSSIBLE_PROPERTIES") <(echo "$ALL_PROPERTIES") >> "${SKIPPED_PROPERTIES_FILE}" 2>&1 || true
+echo -e "\n\nManually skipped properties:\n" >> "${SKIPPED_PROPERTIES_FILE}"
+diff <(echo "$ALL_PROPERTIES") <(echo "$PROPERTIES") >> "${SKIPPED_PROPERTIES_FILE}" 2>&1 || true
+
+
+# 4) group by key
+echo "Generating comments with property definition locations and usage locations..."
+GROUPED=""
+for KEY in $KEYS; do
+	# find all lines that match the key
+	# remove useless part of the path (make it relative to root)
+	# prefix by comment character
+	MATCHES="$(echo "$FOUND" | grep -F "\${$KEY}" \
+		| sed -e 's#^.*\.tools/properties-consistency/\.\./\.\./##g' \
+		| sed -e 's/^/# /g')"
+																						echo -e "key     = '$KEY'\nmatches = '$MATCHES'" >> "${DEBUG_FILE}"
+	# include files where the key is used
+	USAGES="$(grep -rnF "$OGHAM_DIR" \
+			--exclude-dir="target" \
+			--exclude-dir=".externalToolBuilders" \
+			--exclude-dir=".tools" \
+			--exclude="site.xml" \
+			--exclude="mvnw" \
+			--exclude="*.sh" \
+			--exclude="README.adoc" \
+			-e "\"$KEY\"" \
+		| grep --invert "${PROPERTY_FILE}" \
+		| sed -e 's#^.*\.tools/properties-consistency/\.\./\.\./##g' \
+		| sed -e 's/^/# /g')"
+																						echo -e "usages  = '$USAGES'" >> "${DEBUG_FILE}"
+	GROUPED="$(echo -e "${GROUPED}\n\n#-------------------\n# DEFINITIONS\n#-------------------\n#\n${MATCHES}\n#\n#-------------------\n# USAGES\n#-------------------\n#\n${USAGES}\n${KEY}")"
+done
+																						echo -e "${DEBUG_SEPARATOR}GROUPED=\n$GROUPED\n\n" >> "${DEBUG_FILE}"
+
+# 5) add default value
+echo "Adding a default value for each key..."
+GROUPED_WITH_VALUE="$GROUPED"
+for KEY in $KEYS; do
+	ESCAPED_KEY_PATTERN="$(echo "$KEY" | sed 's/\./\\./g ; s/\-/\\-/g')"
+	DEFAULT_VALUE="$(echo "$DEFAULT_VALUES" | grep -Ee "^${ESCAPED_KEY_PATTERN}=" | sed -re 's/^.+=(.+)$/\1/')"
+	DEFAULT_VALUE="${DEFAULT_VALUE:-0}"
+																						echo -e "key           = '$KEY'\nescaped key   = '$ESCAPED_KEY_PATTERN'\ndefault value = '$DEFAULT_VALUE'\nsed           = sed -re \"s#^${ESCAPED_KEY_PATTERN}\$#${KEY}=${DEFAULT_VALUE}#\"\n" >> "${DEBUG_FILE}"
+
+	GROUPED_WITH_VALUE="$(echo "$GROUPED_WITH_VALUE" | sed -re "s#^${ESCAPED_KEY_PATTERN}\$#${KEY}=${DEFAULT_VALUE}#")"
+done
+
+# 6) write to file
+echo "Generating final file..."
+HEADER=$(cat <<-END
+#=========================================================================================
+#                                         WARNING
+#=========================================================================================
+# This file is auto-generated using '.tools/properties-consistency/list-all-properties.sh' 
+# script.
+#
+# /!\ DO NOT EDIT THIS FILE DIRECTLY /!\ 
+#=========================================================================================
+END
+)
+
+SKIPPED_PROPERTIES="$(cat "${SKIPPED_PROPERTIES_FILE}" \
+	| sed -E -e '/^[[:space:]]*[0-9,]+d[0-9,]+[[:space:]]*$/d' \
+	| sed -e 's/< //g' \
+	| sed -e 's/^/# /g')"
+echo -e "${HEADER}\n\n${SKIPPED_PROPERTIES}\n\n${GROUPED_WITH_VALUE}" > "${OGHAM_DIR}/${PROPERTY_FILE}"
+
+# # 7) print result
+echo "Generated '${PROPERTY_FILE}':"
+cat "${OGHAM_DIR}/${PROPERTY_FILE}"