jenkins-execute-hms-test.sh

#!/bin/bash

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script executes all hive metastore upgrade scripts on an specific
# database server in order to verify that upgrade scripts are working
# properly.

# This script is run on jenkins only, and it creates some LXC containers
# in order to execute the metastore-upgrade-tests for different
# server configurations.

set -x
. jenkins-common.sh

test -n "$BRANCH" || fail "BRANCH must be specified"
test -n "$ISSUE_NUM" || fail "ISSUE_NUM must be specified"
test -n "$SSH_HOST" || fail "SSH_HOST must be specified"
test -n "$SSH_KEY" || fail "SSH_KEY must be specified"
test -n "$BUILD_TAG" || fail "BUILD_TAG must be specified"

test -n "$WORKSPACE" || WORKSPACE="$HOME"

export JIRA_NAME="HIVE-${ISSUE_NUM}"
export ROOT=$PWD
export JIRA_ROOT_URL="https://issues.apache.org"
export BUILD_TAG="${BUILD_TAG##jenkins-}"

process_jira

# Jenkins may call this script with BRANCH=trunk to refer to SVN.
# We use git here, so we change to master.
[[ "$BRANCH" = "trunk" ]] && BRANCH="master"

PUBLISH_VARS=(
	BUILD_STATUS=buildStatus
	BUILD_TAG=buildTag
	LOGS_URL=logsURL
	JENKINS_URL=jenkinsURL
	PATCH_URL=patchUrl
	JIRA_NAME=jiraName
	JIRA_URL=jiraUrl
	BRANCH=branch
	REPO=repository
	REPO_NAME=repositoryName
	REPO_TYPE=repositoryType
	TESTS_EXECUTED=numTestsExecuted
	FAILED_TESTS=failedTests
	MESSAGES=messages
)

BUILD_STATUS=0
PATCH_URL="${JIRA_ROOT_URL}${PATCH_URL}"
TESTS_EXECUTED=0
FAILED_TESTS=()
MESSAGES=()

PROFILE_PROPERTIES_FILE="/usr/local/hiveptest/etc/public/${BUILD_PROFILE}.properties"
[[ -f $PROFILE_PROPERTIES_FILE ]] || fail "$PROFILE_PROPERTIES_FILE file does not exist."

profile_properties_get() {
	grep "^$1 = " "$PROFILE_PROPERTIES_FILE" | awk '{print $3}'
}

REPO_TYPE=$(profile_properties_get "repositoryType")
REPO_NAME=$(profile_properties_get "repository")
REPO=$(profile_properties_get "repositoryName")
JIRA_URL=$(profile_properties_get "jiraUrl")

# Avoid showing this information on Jira
set +x
JIRA_USER=$(profile_properties_get "jiraUser")
JIRA_PASS=$(profile_properties_get "jiraPassword")
set -x

JENKINS_URL=$(profile_properties_get "jenkinsURL")
LOGS_URL=$(profile_properties_get "logsURL")

build_ptest2() {
	local path="$1"
	local curpath="$PWD"

	test -d $path || mkdir -p $path
	rm -rf $path
	git clone --depth 1 -b $BRANCH https://github.com/apache/hive.git $path/ || return 1
	cd $path/testutils/ptest2
	mvn clean package -DskipTests -Drat.numUnapprovedLicenses=1000 -Dmaven.repo.local=$WORKSPACE/.m2 || return 1

	cd $curpath
}

publish_results() {
	local file="$1"

	build_ptest2 "hive/build" || return 1

	# Avoid showing this information on Jira
	set +x
	java -cp "hive/build/testutils/ptest2/target/*:hive/build/testutils/ptest2/target/lib/*" org.apache.hive.ptest.execution.JIRAService \
			--user "$JIRA_USER" \
			--password "$JIRA_PASS" \
			--file "$file"
	set -x
}

add_json_object() {
	echo -n "\"$2\" : \"$3\"" >> $1
}

add_json_array() {
	echo -n "\"$2\" : [ $3 ]" >> $1
}

create_publish_file() {
	local json_file=$(mktemp)
	local arr_length=${#PUBLISH_VARS[@]}

	vars_added=0
	echo "{" > $json_file
	for i in ${PUBLISH_VARS[@]}
	do
		var=${i%=*}
		key=${i#*=}
		val=

		# Treat this as an array
		if [[ $var = "FAILED_TESTS" ]]; then
			if [[ ${#FAILED_TESTS[@]} -gt 0 ]]; then
				val=$(printf "\"%s\","  "${FAILED_TESTS[@]}")
				val=${val%?}
			fi
			add_json_array $json_file $key "$val"
		elif [[ $var = "MESSAGES" ]]; then
			if [[ ${#MESSAGES[@]} -gt 0 ]]; then
				val=$(printf "\"%s\","  "${MESSAGES[@]}")
				val=${val%?}
			fi
			add_json_array $json_file $key "$val"
		else
			val=${!var}
			add_json_object $json_file $key $val
		fi

		vars_added=$((vars_added+1))

		if [[ $vars_added -lt $arr_length ]]; then
			echo "," >> $json_file
		else
			echo >> $json_file
		fi
	done
	echo "}" >> $json_file

	echo $json_file
}

if patch_contains_hms_upgrade "$PATCH_URL"; then
	ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i $SSH_KEY $SSH_HOST "
		rm -rf hive/ &&
		git clone --depth 1 -b $BRANCH https://github.com/apache/hive.git &&
		cd hive/ &&
		curl ${PATCH_URL} | bash -x testutils/ptest2/src/main/resources/smart-apply-patch.sh - &&
		sudo bash -x testutils/metastore/execute-test-on-lxc.sh --patch \"${PATCH_URL}\" --branch $BRANCH
	"
	BUILD_STATUS=$?
	if [[ $BUILD_STATUS = 0 ]]; then
		tmp_test_log="/tmp/execute-test-on-lxc.sh.log"

		scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i $SSH_KEY ${SSH_HOST}:"$tmp_test_log" "$tmp_test_log" || exit 1
		TESTS_EXECUTED=$(cat "$tmp_test_log"  | grep "Executing sql test" | wc -l)

		while read line
		do
			if echo $line | grep 'Test failed' > /dev/null; then
				FAILED_TESTS+=("$line")
			elif echo $line | grep 'Executing sql test' >/dev/null; then
				# Remove 'Executing sql test' line from MESSAGES log to avoid a verbose
				# comment on JIRA
				continue
			fi

			MESSAGES+=("$line")
		done < "$tmp_test_log"

		rm "$tmp_test_log"
	fi

	json_file=$(create_publish_file)
	publish_results "$json_file"
	ret=$?

	rm "$json_file"
	exit $ret
fi