diff --git a/Jenkinsfile b/Jenkinsfile index b88da9bfce..53e4e7d1b4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,188 +23,163 @@ pipeline { options { buildDiscarder(logRotator(numToKeepStr: '5')) - timeout (time: 20, unit: 'HOURS') + timeout(time: 8, unit: 'HOURS') timestamps() checkoutToSubdirectory('src') + // Ensure only one build runs per PR at a time + disableConcurrentBuilds(abortPrevious: true) + } + + parameters { + string(name: 'JIRA_ISSUE_KEY', + defaultValue: '', + description: 'The JIRA issue to comment on. Example: TEZ-1234') } environment { SOURCEDIR = 'src' - // will also need to change notification section below + YETUSDIR = 'yetus' PATCHDIR = 'out' DOCKERFILE = "${SOURCEDIR}/build-tools/docker/Dockerfile" - YETUS='yetus' - // Branch or tag name. Yetus release tags are 'rel/X.Y.Z' - YETUS_VERSION='rel/0.15.1' - } + // Credentials IDs + GITHUB_CRED_ID = 'apache-tez-at-github.com' + JIRA_CRED_ID = 'tez-ci' - parameters { - string(name: 'JIRA_ISSUE_KEY', - defaultValue: '', - description: 'The JIRA issue that has a patch needing pre-commit testing. Example: HADOOP-1234') + // Yetus version + YETUS_VERSION = 'rel/0.15.1' } stages { - stage ('install yetus') { + stage('Install Yetus') { steps { - dir("${WORKSPACE}/${YETUS}") { + dir("${WORKSPACE}/${env.YETUSDIR}") { checkout([ $class: 'GitSCM', branches: [[name: "${env.YETUS_VERSION}"]], - userRemoteConfigs: [[ url: 'https://github.com/apache/yetus']]] + userRemoteConfigs: [[url: 'https://github.com/apache/yetus']]] ) } } } - stage ('precommit-run') { + stage('Yetus Analysis') { steps { - withCredentials( - [usernamePassword(credentialsId: 'apache-tez-at-github.com', - passwordVariable: 'GITHUB_TOKEN', - usernameVariable: 'GITHUB_USER'), - usernamePassword(credentialsId: 'tez-ci', - passwordVariable: 'JIRA_PASSWORD', - usernameVariable: 'JIRA_USER')]) { - sh '''#!/usr/bin/env bash - - set -e - - TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh" - - # this must be clean for every run - if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then - rm -rf "${WORKSPACE}/${PATCHDIR}" - fi - mkdir -p "${WORKSPACE}/${PATCHDIR}" - - # if given a JIRA issue, process it. If CHANGE_URL is set - # (e.g., Github Branch Source plugin), process it. - # otherwise exit, because we don't want Hadoop to do a - # full build. We wouldn't normally do this check for smaller - # projects. :) - if [[ -n "${JIRA_ISSUE_KEY}" ]]; then - YETUS_ARGS+=("${JIRA_ISSUE_KEY}") - elif [[ -z "${CHANGE_URL}" ]]; then - echo "Full build skipped" > "${WORKSPACE}/${PATCHDIR}/report.html" - exit 0 - fi - - YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}") - - # where the source is located - YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}") - - # our project defaults come from a personality file - YETUS_ARGS+=("--project=tez") - - # lots of different output formats - YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt") - YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt") - YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html") - - # enable writing back to Github - YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}") - - # auto-kill any surefire stragglers during unit test runs - YETUS_ARGS+=("--reapermode=kill") - - # set relatively high limits for ASF machines - # changing these to higher values may cause problems - # with other jobs on systemd-enabled machines - YETUS_ARGS+=("--proclimit=5500") - YETUS_ARGS+=("--dockermemlimit=20g") - - # -1 spotbugs issues that show up prior to the patch being applied - # YETUS_ARGS+=("--spotbugs-strict-precheck") - # rsync these files back into the archive dir - YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml") - - # URL for user-side presentation in reports and such to our artifacts - # (needs to match the archive bits below) - YETUS_ARGS+=("--build-url-artifacts=artifact/out") - - # plugins to enable - YETUS_ARGS+=("--plugins=all") - - # use Hadoop's bundled shelldocs - YETUS_ARGS+=("--shelldocs=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/shelldocs") - - # don't let these tests cause -1s because we aren't really paying that - # much attention to them - YETUS_ARGS+=("--tests-filter=checkstyle") - - # run in docker mode and specifically point to our - # Dockerfile since we don't want to use the auto-pulled version. - YETUS_ARGS+=("--docker") - YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") - YETUS_ARGS+=("--mvn-custom-repos") - - # effectively treat dev-suport as a custom maven module - YETUS_ARGS+=("--skip-dirs=dev-support") - - # help keep the ASF boxes clean - YETUS_ARGS+=("--sentinel") - - # test with Java 21 eclipse-temurin docker image path - YETUS_ARGS+=("--java-home=/opt/java/openjdk") - YETUS_ARGS+=("--debug") - - # write Yetus report as GitHub comment (YETUS-1102) - YETUS_ARGS+=("--github-write-comment") - YETUS_ARGS+=("--github-use-emoji-vote") - - "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" - ''' + withCredentials([ + usernamePassword(credentialsId: env.GITHUB_CRED_ID, passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER'), + usernamePassword(credentialsId: env.JIRA_CRED_ID, passwordVariable: 'JIRA_PASSWORD', usernameVariable: 'JIRA_USER') + ]) { + sh '''#!/usr/bin/env bash + set -e + + TESTPATCHBIN="${WORKSPACE}/${YETUSDIR}/precommit/src/main/shell/test-patch.sh" + + rm -rf "${WORKSPACE}/${PATCHDIR}" + mkdir -p "${WORKSPACE}/${PATCHDIR}" + + cd "${WORKSPACE}/${SOURCEDIR}" + # Ensure origin/master is available for diffing + git fetch origin master || true + git diff origin/master...HEAD > "${WORKSPACE}/${PATCHDIR}/local-pr.patch" + cd "${WORKSPACE}" + + YETUS_ARGS=() + + # 1. Target Issue/PR + # Pass the Jira issue or use the local patch for GitHub PRs + if [[ -n "${JIRA_ISSUE_KEY}" ]]; then + YETUS_ARGS+=("${JIRA_ISSUE_KEY}") + elif [[ -n "${CHANGE_ID}" ]]; then + echo "Processing PR: ${CHANGE_ID} using local patch" + else + echo "No PR or JIRA key provided. Skipping Yetus." + exit 0 + fi + + # 2. Core Paths & Project + # Set the base directory, patch output directory, and project name + YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}") + YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}") + YETUS_ARGS+=("--project=tez") + + # 3. Docker & Environment + # Run tests inside Docker, allowing dirty workspace (since Jenkins handles the checkout) + YETUS_ARGS+=("--docker") + YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") + YETUS_ARGS+=("--java-home=/opt/java/openjdk") + YETUS_ARGS+=("--dirty-workspace") + YETUS_ARGS+=("--build-url-artifacts=artifact/out") + + # 4. Plugins & Personality + # Load all plugins eagerly and provide the custom Tez personality file. + # Note: We copy the personality to PATCHDIR because Yetus' robot mode cleans untracked files in the workspace. + cp "${WORKSPACE}/${SOURCEDIR}/dev-support/tez-personality.sh" "${WORKSPACE}/${PATCHDIR}/tez-personality.sh" + YETUS_ARGS+=("--plugins=all") + YETUS_ARGS+=("--personality=${WORKSPACE}/${PATCHDIR}/tez-personality.sh") + YETUS_ARGS+=("--mvn-custom-repos") + YETUS_ARGS+=("--tests-filter=checkstyle") + YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml") + YETUS_ARGS+=("--reapermode=kill") + YETUS_ARGS+=("--sentinel") + + # 5. Reporting & GitHub Integration + # Configure report output files and pass the GitHub token for PR commenting + YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}") + YETUS_ARGS+=("--github-write-comment") + YETUS_ARGS+=("--github-use-emoji-vote") + YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html") + YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt") + YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt") + + echo "Running Yetus with local patch overriding GitHub..." + "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" "${WORKSPACE}/${PATCHDIR}/local-pr.patch" + ''' } } } - } post { always { - script { - // Yetus output - archiveArtifacts "${env.PATCHDIR}/**" - // Publish the HTML report so that it can be looked at - // Has to be relative to WORKSPACE. - publishHTML (target: [ - allowMissing: true, - keepAll: true, - alwaysLinkToLastBuild: true, - // Has to be relative to WORKSPACE - reportDir: "${env.PATCHDIR}", - reportFiles: 'report.html', - reportName: 'Yetus Report' - ]) - // Publish JUnit results - try { - junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml" - } catch(e) { - echo 'junit processing: ' + e.toString() + script { + sh ''' + # Fix permissions before attempting to read/zip directories created by root Docker processes + echo "Cleaning up workspace permissions to prevent HADOOP-13951 and unzipping errors..." + chmod -R u+rxw "${WORKSPACE}" || true + + # Zip test logs for easier debugging and storage efficiency + cd "${SOURCEDIR}" || exit 0 + find . -type d -name "surefire-reports" | while read -r dir; do + rel_path=$(echo "${dir#./}" | tr '/' '_') + zip -r -q "../${PATCHDIR}/test-logs-${rel_path}.zip" "${dir}" || true + done + ''' + + dir(env.SOURCEDIR) { + // Safely publish junit results + junit testResults: "**/target/surefire-reports/*.xml", allowEmptyResults: true + } + + // Archive Yetus output, reports, and zipped test logs + archiveArtifacts artifacts: "${env.PATCHDIR}/**", allowEmptyArchive: true + + // Publish Yetus HTML report to Jenkins UI sidebar only if it exists + if (fileExists("${env.PATCHDIR}/report.html")) { + publishHTML(target: [ + allowMissing: true, + alwaysLinkToLastBuild: true, + keepAll: true, + reportDir: "${env.PATCHDIR}", + reportFiles: 'report.html', + reportName: 'Yetus Report' + ]) + } } - } } - // Jenkins pipeline jobs fill slaves on PRs without this :( - cleanup() { + cleanup { script { - sh ''' - # See YETUS-764 - if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then - echo "test-patch process appears to still be running: killing" - kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true - sleep 10 - fi - if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then - echo "test-patch container appears to still be running: killing" - docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true - fi - # See HADOOP-13951 - chmod -R u+rxw "${WORKSPACE}" - ''' + // Wipe the workspace to prevent Jenkins agent disk exhaustion deleteDir() } } diff --git a/build-tools/smart-apply-patch.sh b/build-tools/smart-apply-patch.sh deleted file mode 100755 index 2a9dbd24dd..0000000000 --- a/build-tools/smart-apply-patch.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env bash -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -# -# Determine if the patch file is a git diff file with prefixes. -# These files are generated via "git diff" *without* the --no-prefix option. -# -# We can apply these patches more easily because we know that the a/ and b/ -# prefixes in the "diff" lines stands for the project root directory. -# So we don't have to hunt for the project root. -# And of course, we know that the patch file was generated using git, so we -# know git apply can handle it properly. -# -# Arguments: file name. -# Return: 0 if it is a git diff; 1 otherwise. -# -is_git_diff_with_prefix() { - DIFF_TYPE="unknown" - while read -r line; do - if [[ "$line" =~ ^diff\ ]]; then - if [[ "$line" =~ ^diff\ \-\-git ]]; then - DIFF_TYPE="git" - else - return 1 # All diff lines must be diff --git lines. - fi - fi - if [[ "$line" =~ ^\+\+\+\ ]] || - [[ "$line" =~ ^\-\-\-\ ]]; then - if ! [[ "$line" =~ ^....[ab]/ ]]; then - return 1 # All +++ and --- lines must start with a/ or b/. - fi - fi - done < $1 - [ x$DIFF_TYPE == x"git" ] || return 1 - return 0 # return true (= 0 in bash) -} - -PATCH_FILE=$1 -DRY_RUN=$2 -if [ -z "$PATCH_FILE" ]; then - echo usage: $0 patch-file - exit 1 -fi - -PATCH=${PATCH:-patch} # allow overriding patch binary - -# Cleanup handler for temporary files -TOCLEAN="" -cleanup() { - rm $TOCLEAN - exit $1 -} -trap "cleanup 1" HUP INT QUIT TERM - -# Allow passing "-" for stdin patches -if [ "$PATCH_FILE" == "-" ]; then - PATCH_FILE=/tmp/tmp.in.$$ - cat /dev/fd/0 > $PATCH_FILE - TOCLEAN="$TOCLEAN $PATCH_FILE" -fi - -# Special case for git-diff patches without --no-prefix -if is_git_diff_with_prefix "$PATCH_FILE"; then - GIT_FLAGS="--binary -p1 -v" - if [[ -z $DRY_RUN ]]; then - GIT_FLAGS="$GIT_FLAGS --stat --apply " - echo Going to apply git patch with: git apply "${GIT_FLAGS}" - else - GIT_FLAGS="$GIT_FLAGS --check " - fi - git apply ${GIT_FLAGS} "${PATCH_FILE}" - exit $? -fi - -# Come up with a list of changed files into $TMP -TMP=/tmp/tmp.paths.$$ -TOCLEAN="$TOCLEAN $TMP" - -if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then - PLEVEL=0 - #if the patch applied at P0 there is the possability that all we are doing - # is adding new files and they would apply anywhere. So try to guess the - # correct place to put those files. - - TMP2=/tmp/tmp.paths.2.$$ - TOCLEAN="$TOCLEAN $TMP2" - - egrep '^patching file |^checking file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort | uniq > $TMP2 - - if [ ! -s $TMP2 ]; then - echo "Error: Patch dryrun couldn't detect changes the patch would make. Exiting." - cleanup 1 - fi - - #first off check that all of the files do not exist - FOUND_ANY=0 - for CHECK_FILE in $(cat $TMP2) - do - if [[ -f $CHECK_FILE ]]; then - FOUND_ANY=1 - fi - done - - if [[ "$FOUND_ANY" = "0" ]]; then - #all of the files are new files so we have to guess where the correct place to put it is. - - # if all of the lines start with a/ or b/, then this is a git patch that - # was generated without --no-prefix - if ! grep -qv '^a/\|^b/' $TMP2 ; then - echo Looks like this is a git patch. Stripping a/ and b/ prefixes - echo and incrementing PLEVEL - PLEVEL=$[$PLEVEL + 1] - sed -i -e 's,^[ab]/,,' $TMP2 - fi - fi -elif $PATCH -p1 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then - PLEVEL=1 -elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then - PLEVEL=2 -else - echo "The patch does not appear to apply with p0 to p2"; - cleanup 1; -fi - -# If this is a dry run then exit instead of applying the patch -if [[ -n $DRY_RUN ]]; then - cleanup 0; -fi - -echo Going to apply patch with: $PATCH -p$PLEVEL -$PATCH -p$PLEVEL -E < $PATCH_FILE - -cleanup $? diff --git a/build-tools/test-patch.sh b/build-tools/test-patch.sh deleted file mode 100755 index feb9644893..0000000000 --- a/build-tools/test-patch.sh +++ /dev/null @@ -1,821 +0,0 @@ -#!/usr/bin/env bash -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -#set -x - -### Setup some variables. -### BUILD_URL is set by Hudson if it is run by patch process -### Read variables from properties file -bindir=$(dirname $0) - -# Defaults -if [ -z "$MAVEN_HOME" ]; then - MVN=mvn -else - MVN=$MAVEN_HOME/bin/mvn -fi - -PROJECT_NAME=Tez -JENKINS=false -PATCH_DIR=/tmp -BASEDIR=$(pwd) -PS=${PS:-ps} -AWK=${AWK:-awk} -WGET=${WGET:-wget} -GIT=${GIT:-git} -GREP=${GREP:-grep} -PATCH=${PATCH:-patch} -DIFF=${DIFF:-diff} -JIRACLI=${JIRA:-jira} -SED=${SED:-sed} -CURL=${CURL:-curl} -SPOTBUGS_HOME=${SPOTBUGS_HOME} - -############################################################################### -printUsage() { - echo "Usage: $0 [options] patch-file | defect-number" - echo - echo "Where:" - echo " patch-file is a local patch file containing the changes to test" - echo " defect-number is a JIRA defect number (e.g. 'TEZ-1234') to test (Jenkins only)" - echo - echo "Options:" - echo "--patch-dir= The directory for working and output files (default '/tmp')" - echo "--basedir= The directory to apply the patch to (default current directory)" - echo "--mvn-cmd= The 'mvn' command to use (default \$MAVEN_HOME/bin/mvn, or 'mvn')" - echo "--ps-cmd= The 'ps' command to use (default 'ps')" - echo "--awk-cmd= The 'awk' command to use (default 'awk')" - echo "--git-cmd= The 'git' command to use (default 'git')" - echo "--grep-cmd= The 'grep' command to use (default 'grep')" - echo "--patch-cmd= The 'patch' command to use (default 'patch')" - echo "--diff-cmd= The 'diff' command to use (default 'diff')" - echo "--spotbugs-home= Spotbugs home directory (default SPOTBUGS_HOME environment variable)" - echo "--dirty-workspace Allow the local git workspace to have uncommitted changes" - echo "--run-tests Run all tests below the base directory" - echo - echo "Jenkins-only options:" - echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)" - echo "--wget-cmd= The 'wget' command to use (default 'wget')" - echo "--jira-cmd= The 'jira' command to use (default 'jira')" - echo "--jira-password= The password for the 'jira' command" -} - -############################################################################### -parseArgs() { - for i in $* - do - case $i in - --jenkins) - JENKINS=true - ;; - --patch-dir=*) - PATCH_DIR=${i#*=} - ;; - --basedir=*) - BASEDIR=${i#*=} - ;; - --mvn-cmd=*) - MVN=${i#*=} - ;; - --ps-cmd=*) - PS=${i#*=} - ;; - --awk-cmd=*) - AWK=${i#*=} - ;; - --wget-cmd=*) - WGET=${i#*=} - ;; - --git-cmd=*) - GIT=${i#*=} - ;; - --grep-cmd=*) - GREP=${i#*=} - ;; - --patch-cmd=*) - PATCH=${i#*=} - ;; - --diff-cmd=*) - DIFF=${i#*=} - ;; - --jira-cmd=*) - JIRACLI=${i#*=} - ;; - --jira-password=*) - JIRA_PASSWD=${i#*=} - ;; - --spotbugs-home=*) - SPOTBUGS_HOME=${i#*=} - ;; - --dirty-workspace) - DIRTY_WORKSPACE=true - ;; - --run-tests) - RUN_TESTS=true - ;; - *) - PATCH_OR_DEFECT=$i - ;; - esac - done - if [ -z "$PATCH_OR_DEFECT" ]; then - printUsage - exit 1 - fi - if [[ $JENKINS == "true" ]] ; then - echo "Running in Jenkins mode" - defect=$PATCH_OR_DEFECT - else - echo "Running in developer mode" - JENKINS=false - ### PATCH_FILE contains the location of the patchfile - PATCH_FILE=$PATCH_OR_DEFECT - if [[ ! -e "$PATCH_FILE" ]] ; then - echo "Unable to locate the patch file $PATCH_FILE" - cleanupAndExit 0 - fi - ### Check if $PATCH_DIR exists. If it does not exist, create a new directory - if [[ ! -e "$PATCH_DIR" ]] ; then - mkdir "$PATCH_DIR" - if [[ $? == 0 ]] ; then - echo "$PATCH_DIR has been created" - else - echo "Unable to create $PATCH_DIR" - cleanupAndExit 0 - fi - fi - ### Obtain the patch filename to append it to the version number - defect=`basename $PATCH_FILE` - fi -} - -############################################################################### -checkout () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Testing patch for ${defect}." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - ### When run by a developer, if the workspace contains modifications, do not continue - ### unless the --dirty-workspace option was set - status=`$GIT status --porcelain` - if [[ $JENKINS == "false" ]] ; then - if [[ "$status" != "" && -z $DIRTY_WORKSPACE ]] ; then - echo "ERROR: can't run in a workspace that contains the following modifications" - echo "$status" - cleanupAndExit 1 - fi - echo - else - cd $BASEDIR - $GIT reset --hard - $GIT clean -xdf - $GIT checkout master - $GIT pull --rebase - fi - GIT_REVISION=`git rev-parse --verify --short HEAD` - return $? -} - -############################################################################### -downloadPatch () { - ### Download latest patch file (ignoring .htm and .html) when run from patch process - if [[ $JENKINS == "true" ]] ; then - $WGET -q -O $PATCH_DIR/jira http://issues.apache.org/jira/browse/$defect - if [[ `$GREP -c 'Patch Available' $PATCH_DIR/jira` == 0 ]] ; then - echo "$defect is not \"Patch Available\". Exiting." - cleanupAndExit 0 - fi - relativePatchURL=`$GREP -o '"/jira/secure/attachment/[0-9]*/[^"]*' $PATCH_DIR/jira | $GREP -v -e 'htm[l]*$' | sort | tail -1 | $GREP -o '/jira/secure/attachment/[0-9]*/[^"]*'` - patchURL="http://issues.apache.org${relativePatchURL}" - patchNum=`echo $patchURL | $GREP -o '[0-9]*/' | $GREP -o '[0-9]*'` - echo "$defect patch is being downloaded at `date` from" - echo "$patchURL" - $WGET -q -O $PATCH_DIR/patch $patchURL - VERSION=${GIT_REVISION}_${defect}_PATCH-${patchNum} - JIRA_COMMENT="Here are the results of testing the latest attachment - $patchURL - against master revision ${GIT_REVISION}." - - ### Copy the patch file to $PATCH_DIR - else - VERSION=PATCH-${defect} - cp $PATCH_FILE $PATCH_DIR/patch - if [[ $? == 0 ]] ; then - echo "Patch file $PATCH_FILE copied to $PATCH_DIR" - else - echo "Could not copy $PATCH_FILE to $PATCH_DIR" - cleanupAndExit 0 - fi - fi -} - -############################################################################### -verifyPatch () { - # Before building, check to make sure that the patch is valid - $bindir/smart-apply-patch.sh $PATCH_DIR/patch dryrun - if [[ $? != 0 ]] ; then - echo "PATCH APPLICATION FAILED" - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 patch{color}. The patch command could not apply the patch." - return 1 - else - return 0 - fi -} - -############################################################################### -prebuildWithoutPatch () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Pre-build master to verify master stability and javac warnings" - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "Compiling $(pwd)" - echo "$MVN clean test -DskipTests -Ptest-patch > $PATCH_DIR/masterJavacWarnings.txt 2>&1" - $MVN clean test -DskipTests -Ptest-patch > $PATCH_DIR/masterJavacWarnings.txt 2>&1 - if [[ $? != 0 ]] ; then - echo "master compilation is broken?" - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 patch{color}. master compilation may be broken." - return 1 - fi - - echo "$MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/masterJavadocWarnings.txt 2>&1" - $MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/masterJavadocWarnings.txt 2>&1 - if [[ $? != 0 ]] ; then - echo "master javadoc compilation is broken?" - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 patch{color}. master compilation may be broken." - return 1 - fi - - return 0 -} - -############################################################################### -### Check for @author tags in the patch -checkAuthor () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Checking there are no @author tags in the patch." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - authorTags=`$GREP -c -i '@author' $PATCH_DIR/patch` - echo "There appear to be $authorTags @author tags in the patch." - if [[ $authorTags != 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 @author{color}. The patch appears to contain $authorTags @author tags which the Tez community has agreed to not allow in code contributions." - return 1 - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 @author{color}. The patch does not contain any @author tags." - return 0 -} - -############################################################################### -### Check for tests in the patch -checkTests () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Checking there are new or changed tests in the patch." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - testReferences=`$GREP -c -i -e '^+++.*/test' $PATCH_DIR/patch` - echo "There appear to be $testReferences test files referenced in the patch." - if [[ $testReferences == 0 ]] ; then - if [[ $JENKINS == "true" ]] ; then - patchIsDoc=`$GREP -c -i 'title="documentation' $PATCH_DIR/jira` - if [[ $patchIsDoc != 0 ]] ; then - echo "The patch appears to be a documentation patch that doesn't require tests." - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+0 tests included{color}. The patch appears to be a documentation patch that doesn't require tests." - return 0 - fi - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 tests included{color}. The patch doesn't appear to include any new or modified tests. - Please justify why no new tests are needed for this patch. - Also please list what manual steps were performed to verify this patch." - return 1 - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 tests included{color}. The patch appears to include $testReferences new or modified test files." - return 0 -} - -cleanUpXml () { - cd $BASEDIR/conf - for file in `ls *.xml.template` - do - rm -f `basename $file .template` - done - cd $BASEDIR -} - -############################################################################### -### Attempt to apply the patch -applyPatch () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Applying patch." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - export PATCH - $bindir/smart-apply-patch.sh $PATCH_DIR/patch - if [[ $? != 0 ]] ; then - echo "PATCH APPLICATION FAILED" - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 patch{color}. The patch command could not apply the patch." - return 1 - fi - return 0 -} - -############################################################################### -calculateJavadocWarnings() { - WARNING_FILE="$1" - RET=$(egrep "^[0-9]+ warnings?$" "$WARNING_FILE" | awk '{sum+=$1} END {print sum}') -} - -### Check there are no javadoc warnings -checkJavadocWarnings () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Determining number of patched javadoc warnings." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "$MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/patchJavadocWarnings.txt 2>&1" - $MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/patchJavadocWarnings.txt 2>&1 - calculateJavadocWarnings "$PATCH_DIR/masterJavadocWarnings.txt" - nummasterJavadocWarnings=$RET - calculateJavadocWarnings "$PATCH_DIR/patchJavadocWarnings.txt" - numPatchJavadocWarnings=$RET - grep -i warning "$PATCH_DIR/masterJavadocWarnings.txt" > "$PATCH_DIR/masterJavadocWarningsFiltered.txt" - grep -i warning "$PATCH_DIR/patchJavadocWarnings.txt" > "$PATCH_DIR/patchJavadocWarningsFiltered.txt" - diff -u "$PATCH_DIR/masterJavadocWarningsFiltered.txt" \ - "$PATCH_DIR/patchJavadocWarningsFiltered.txt" > \ - "$PATCH_DIR/diffJavadocWarnings.txt" - rm -f "$PATCH_DIR/masterJavadocWarningsFiltered.txt" "$PATCH_DIR/patchJavadocWarningsFiltered.txt" - echo "There appear to be $nummasterJavadocWarnings javadoc warnings before the patch and $numPatchJavadocWarnings javadoc warnings after applying the patch." - if [[ $nummasterJavadocWarnings != "" && $numPatchJavadocWarnings != "" ]] ; then - if [[ $numPatchJavadocWarnings -gt $nummasterJavadocWarnings ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 javadoc{color}. The javadoc tool appears to have generated `expr $(($numPatchJavadocWarnings-$nummasterJavadocWarnings))` warning messages. - See $BUILD_URL/artifact/patchprocess/diffJavadocWarnings.txt for details." - return 1 - fi - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 javadoc{color}. There were no new javadoc warning messages." - return 0 -} - -############################################################################### -### Check there are no changes in the number of Javac warnings -checkJavacWarnings () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Determining number of patched javac warnings." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "$MVN clean test -DskipTests -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1" - $MVN clean test -DskipTests -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1 - if [[ $? != 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 javac{color:red}. The patch appears to cause the build to fail." - return 2 - fi - ### Compare master and patch javac warning numbers - if [[ -f $PATCH_DIR/patchJavacWarnings.txt ]] ; then - $GREP '\[WARNING\]' $PATCH_DIR/masterJavacWarnings.txt > $PATCH_DIR/filteredmasterJavacWarnings.txt - $GREP '\[WARNING\]' $PATCH_DIR/patchJavacWarnings.txt > $PATCH_DIR/filteredPatchJavacWarnings.txt - masterJavacWarnings=`cat $PATCH_DIR/filteredmasterJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` - patchJavacWarnings=`cat $PATCH_DIR/filteredPatchJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` - echo "There appear to be $masterJavacWarnings javac compiler warnings before the patch and $patchJavacWarnings javac compiler warnings after applying the patch." - if [[ $patchJavacWarnings != "" && $masterJavacWarnings != "" ]] ; then - if [[ $patchJavacWarnings -gt $masterJavacWarnings ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 javac{color}. The applied patch generated $patchJavacWarnings javac compiler warnings (more than the master's current $masterJavacWarnings warnings)." - - $DIFF $PATCH_DIR/filteredmasterJavacWarnings.txt $PATCH_DIR/filteredPatchJavacWarnings.txt > $PATCH_DIR/diffJavacWarnings.txt - JIRA_COMMENT_FOOTER="Javac warnings: $BUILD_URL/artifact/patchprocess/diffJavacWarnings.txt -$JIRA_COMMENT_FOOTER" - - return 1 - fi - fi - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 javac{color}. The applied patch does not increase the total number of javac compiler warnings." - return 0 -} - -############################################################################### -### Check there are no changes in the number of release audit (RAT) warnings -checkReleaseAuditWarnings () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Determining number of patched release audit warnings." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "$MVN apache-rat:check > $PATCH_DIR/patchReleaseAuditOutput.txt 2>&1" - $MVN apache-rat:check > $PATCH_DIR/patchReleaseAuditOutput.txt 2>&1 - find $BASEDIR -name rat.txt | xargs cat > $PATCH_DIR/patchReleaseAuditWarnings.txt - - ### Compare master and patch release audit warning numbers - if [[ -f $PATCH_DIR/patchReleaseAuditWarnings.txt ]] ; then - patchReleaseAuditWarnings=`$GREP -c '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt` - echo "" - echo "" - echo "There appear to be $patchReleaseAuditWarnings release audit warnings after applying the patch." - if [[ $patchReleaseAuditWarnings != "" ]] ; then - if [[ $patchReleaseAuditWarnings -gt 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 release audit{color}. The applied patch generated $patchReleaseAuditWarnings release audit warnings." - $GREP '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt > $PATCH_DIR/patchReleaseAuditProblems.txt - echo "Lines that start with ????? in the release audit report indicate files that do not have an Apache license header." >> $PATCH_DIR/patchReleaseAuditProblems.txt - JIRA_COMMENT_FOOTER="Release audit warnings: $BUILD_URL/artifact/patchprocess/patchReleaseAuditProblems.txt -$JIRA_COMMENT_FOOTER" - return 1 - fi - fi - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 release audit{color}. The applied patch does not increase the total number of release audit warnings." - return 0 -} - - -############################################################################### -### Install the new jars so tests and spotbugs can find all of the updated jars -buildAndInstall () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Installing all of the jars" - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "$MVN install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess" - $MVN install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess - return $? -} - - -############################################################################### -### Check there are no changes in the number of Spotbugs warnings -checkSpotbugsWarnings () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Determining number of patched Spotbugs warnings." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - - rc=0 - echo " Running spotbugs " - echo "$MVN clean test spotbugs:spotbugs -DskipTests < /dev/null > $PATCH_DIR/patchSpotBugsOutput.txt 2>&1" - $MVN clean test spotbugs:spotbugs -DskipTests < /dev/null > $PATCH_DIR/patchSpotBugsOutput.txt 2>&1 - rc=$? - spotbugs_version=$(${AWK} 'match($0, /spotbugs-maven-plugin:[^:]*:spotbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchSpotBugsOutput.txt") - - if [ $rc != 0 ] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 spotbugs{color}. The patch appears to cause Spotbugs (version ${spotbugs_version}) to fail." - return 1 - fi - - spotbugsWarnings=0 - for file in $(find $BASEDIR -name spotbugsXml.xml) - do - relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix - if [ ! $relative_file == "target/spotbugsXml.xml" ]; then - module_suffix=${relative_file%/target/spotbugsXml.xml} # strip trailing path - module_suffix=`basename ${module_suffix}` - fi - - cp $file $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml - $SPOTBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \ - $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml \ - $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml - newSpotbugsWarnings=`$SPOTBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml \ - $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.xml | $AWK '{print $1}'` - echo "Found $newSpotbugsWarnings Spotbugs warnings ($file)" - spotbugsWarnings=$((spotbugsWarnings+newSpotbugsWarnings)) - $SPOTBUGS_HOME/bin/convertXmlToText -html \ - $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.xml \ - $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.html - if [[ $newSpotbugsWarnings > 0 ]] ; then - JIRA_COMMENT_FOOTER="Spotbugs warnings: $BUILD_URL/artifact/patchprocess/newPatchSpotbugsWarnings${module_suffix}.html -$JIRA_COMMENT_FOOTER" - fi - done - - if [[ $spotbugsWarnings -gt 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 spotbugs{color}. The patch appears to introduce $spotbugsWarnings new Spotbugs (version ${spotbugs_version}) warnings." - return 1 - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 spotbugs{color}. The patch does not introduce any new Spotbugs (version ${spotbugs_version}) warnings." - return 0 -} - -############################################################################### -### Run the tests -runTests () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Running tests." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - - failed_tests="" - failed_test_builds="" - test_timeouts="" - test_logfile=$PATCH_DIR/testrun.txt - echo " Running tests " - echo " $MVN clean install -fn -D${PROJECT_NAME}PatchProcess" - $MVN clean install -fn > $test_logfile 2>&1 - test_build_result=$? - cat $test_logfile - module_test_timeouts=`$AWK '/^Running / { if (last) { print last } last=$2 } /^Tests run: / { last="" }' $test_logfile` - if [[ -n "$module_test_timeouts" ]] ; then - test_timeouts="$test_timeouts $module_test_timeouts" - fi - module_failed_tests=`find . -name 'TEST*.xml' | xargs $GREP -l -E "/dev/null - fi -} - -############################################################################### -### Cleanup files -cleanupAndExit () { - local result=$1 - if [[ $JENKINS == "true" ]] ; then - if [ -e "$PATCH_DIR" ] ; then - mv $PATCH_DIR $BASEDIR - fi - fi - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Finished build." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - exit $result -} - -############################################################################### -############################################################################### -############################################################################### - -JIRA_COMMENT="" -JIRA_COMMENT_FOOTER="Console output: $BUILD_URL/console - -This message is automatically generated." - -### Check if arguments to the script have been specified properly or not -parseArgs $@ -cd $BASEDIR - -checkout -RESULT=$? -if [[ $JENKINS == "true" ]] ; then - if [[ $RESULT != 0 ]] ; then - exit 100 - fi -fi -downloadPatch -verifyPatch -(( RESULT = RESULT + $? )) -if [[ $RESULT != 0 ]] ; then - submitJiraComment 1 - cleanupAndExit 1 -fi -prebuildWithoutPatch -(( RESULT = RESULT + $? )) -if [[ $RESULT != 0 ]] ; then - submitJiraComment 1 - cleanupAndExit 1 -fi -checkAuthor -(( RESULT = RESULT + $? )) - -if [[ $JENKINS == "true" ]] ; then - cleanUpXml -fi -checkTests -(( RESULT = RESULT + $? )) -applyPatch -APPLY_PATCH_RET=$? -(( RESULT = RESULT + $APPLY_PATCH_RET )) -if [[ $APPLY_PATCH_RET != 0 ]] ; then - submitJiraComment 1 - cleanupAndExit 1 -fi -checkJavacWarnings -JAVAC_RET=$? -#2 is returned if the code could not compile -if [[ $JAVAC_RET == 2 ]] ; then - submitJiraComment 1 - cleanupAndExit 1 -fi -(( RESULT = RESULT + $JAVAC_RET )) -checkJavadocWarnings -(( RESULT = RESULT + $? )) -buildAndInstall -checkSpotbugsWarnings -(( RESULT = RESULT + $? )) -checkReleaseAuditWarnings -(( RESULT = RESULT + $? )) -### Run tests for Jenkins or if explictly asked for by a developer -if [[ $JENKINS == "true" || $RUN_TESTS == "true" ]] ; then - runTests - (( RESULT = RESULT + $? )) -fi -JIRA_COMMENT_FOOTER="Test results: $BUILD_URL/testReport/ -$JIRA_COMMENT_FOOTER" - -submitJiraComment $RESULT -cleanupAndExit $RESULT diff --git a/dev-support/tez-personality.sh b/dev-support/tez-personality.sh new file mode 100755 index 0000000000..0d29cb3c9e --- /dev/null +++ b/dev-support/tez-personality.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Apache Yetus personality for Apache Tez + +personality_plugins "all" + +## @description Globals for this personality +## @audience private +## @stability evolving +## @replaceable no +function personality_globals +{ + # shellcheck disable=SC2034 + PROJECT_NAME=tez + # shellcheck disable=SC2034 + BUILDTOOL=maven + # shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + # shellcheck disable=SC2034 + JIRA_ISSUE_RE='^TEZ-[0-9]+$' + # shellcheck disable=SC2034 + GITHUB_REPO="apache/tez" + + # Increase memory for maven + # shellcheck disable=SC2034 + MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx4g -XX:+UseG1GC"}" + + # Default Yetus settings for Tez + # shellcheck disable=SC2034 + DOCKER_MEMORY="20g" + # shellcheck disable=SC2034 + PROC_LIMIT=5500 + + # Unconditionally add compile and unit tests to ensure full build/UT on every PR as requested. + # add_test in personality_globals is the recommended way in 0.15.1 to force-enable plugins. + add_test compile + add_test unit +} + +## @description Module selection +## @audience private +## @stability evolving +## @replaceable no +function personality_modules +{ + local repostatus=$1 + local testtype=$2 + local extra="" + local MODULES=("${CHANGED_MODULES[@]}") + + yetus_debug "Personality: ${repostatus} ${testtype}" + + # Ensure a clean state for every test type + clear_personality_queue + + extra="-Ptest-patch" + + # Always run these tests on the root module to ensure everything is covered. + # Note: Analysis tests (javac, spotbugs, checkstyle, javadoc) are NOT forced to root + # to allow them to run "smartly" only on changed modules. + if [[ ${testtype} == unit || ${testtype} == compile || ${testtype} == mvninstall ]]; then + yetus_debug "Forcing root module and fail-at-end for ${testtype}" + extra="${extra} -fae" + MODULES=(.) + fi + + if [[ ${testtype} == spotbugs ]]; then + extra="${extra} -Pspotbugs" + fi + + for module in "${MODULES[@]}"; do + # shellcheck disable=SC2086 + personality_enqueue_module ${module} ${extra} + done +} + +## @description File filter to determine which tests to run +## @audience private +## @stability evolving +## @replaceable no +function personality_file_filter +{ + local filename=$1 + + yetus_debug "Tez file filter: ${filename}" + + if [[ ${filename} =~ \.java$ || ${filename} =~ \.proto$ || ${filename} =~ pom\.xml$ ]]; then + add_test javac + add_test spotbugs + add_test checkstyle + add_test javadoc + fi + + if [[ ${filename} =~ findbugs-exclude\.xml$ ]]; then + add_test spotbugs + fi + + if [[ ${filename} =~ \.sh$ ]] || [[ ${filename} =~ Jenkinsfile ]]; then + add_test shellcheck + fi + + if [[ ${filename} =~ \.md$ ]] || [[ ${filename} =~ \.txt$ ]]; then + add_test codespell + fi +}