#!/bin/bash -eu

################### TO DO ###################
#
# * Check hg is installed
# * Check hg clone is working (error code 255)
# * Check virtualenv is installed
# * Add an option not to refresh tools repo
# * Check connectivity to an arbitrary master and report nicely
# * Summarize failures from manage_masters.py at end of log
# * Option to checkconfig only without update
# * Only update wiki if there really are changes
# * Only reconfig if there really are changes, or if forced
# * Send console output also to a log file
#
#############################################

# Include our shared function library
if [ -e ../../lib/shell/functions ]; then
    . ../../lib/shell/functions
else
    echo "Shared functions missing: ../../lib/shell/functions" >&2
    exit 1
fi

START_TIME="$(date +%s)"

# Explicitly unset any pre-existing environment variables to avoid variable collision
unset PREPARE_ONLY FORCE_RECONFIG MERGE_TO_PRODUCTION UPDATE_BUGZILLA UPDATE_WIKI RECONFIG_DIR RECONFIG_CREDENTIALS_FILE WIKI_USERNAME WIKI_PASSWORD

# Force python into unbuffered mode, so we get full output, as it happens from the fabric scripts
export PYTHONUNBUFFERED=1

function usage {
    echo
    echo "This script can be used to reconfig interactively, or non-interactively. It will merge"
    echo "buildbotcustom, buildbot-configs from default to production(-0.8)."
    echo "It will then reconfig the buildbot masters. Afterwards, if all was successful, the script will"
    echo "update the maintenance wiki (https://wiki.mozilla.org/ReleaseEngineering/Maintenance) and"
    echo "will additionally post comments to bugzilla for bugs it finds in the commit messages of"
    echo "merged changes, providing a link to the hg web interface of the commits that have been"
    echo "deployed to production."
    echo
    echo "USAGE"
    echo "     1) $(basename "${0}") -h"
    echo "     2) $(basename "${0}") [-b] [-f] [-m] [-n] [-p] [-r RECONFIG_DIR] [-t] [-w RECONFIG_CREDENTIALS_FILE]"
    echo
    echo "OPTIONS"
    echo "    -b:                            No posting of comments to Bugzilla."
    echo "    -f:                            Force reconfig, even if no changes merged."
    echo "    -h:                            Display help."
    echo "    -m:                            No merging of default -> production(-0.8) of hg branches."
    echo "    -n:                            No wiki update."
    echo "    -p:                            Prepare only; does not push changes to hg, nor perform"
    echo "                                   reconfig, nor update wiki, nor post to Bugzilla. Useful"
    echo "                                   for validating setup, or resolving merge conflicts in"
    echo "                                   advance early rather than waiting until real reconfig to"
    echo "                                   resolve conflicts."
    echo "    -r RECONFIG_DIR:               Use directory RECONFIG_DIR for storing temporary files"
    echo "                                   (default is /tmp/reconfig). This directory, and any"
    echo "                                   necessary parent directories will be created if required."
    echo "    -w RECONFIG_CREDENTIALS_FILE:  Source environment variables: BUGZILLA_USERNAME,"
    echo "                                   BUGZILLA_PASSWORD, WIKI_USERNAME, WIKI_PASSWORD from file"
    echo "                                   RECONFIG_CREDENTIALS_FILE (default is ~/.reconfig/config)."
    echo
    echo "EXIT CODES"
    echo "     0        Success"
    echo "     1        Error loading shared functions"
    echo "     2        Bad command line options specified"
    echo "    64        Could not create directory to store results (RECONFIG_DIR)"
    echo "    65        Wiki credentials file not found"
    echo "    66        Python 2.7 not found in PATH"
    echo "    67        Reconfig aborted by user after incomplete reconfig found"
    echo "    68        Aborted due to incomplete reconfig found but non-interactive session, so"
    echo "              could not ask user how to proceed"
    echo "    69        Error during hg merge (most likely due to merge conflict(s))"
    echo "    70        Theoretically not possible - update_maintenance_wiki.sh tries and fails"
    echo "              to create the RECONFIG_DIR when it detects it does not exist (but should"
    echo "              have already been created by end_to_end_reconfig.sh)"
    echo "    71        Theoretically not possible - no wiki markdown file passed to"
    echo "              update_maintenance_wiki.sh"
    echo "    72        Theoretically not possible - non-existing markdown file passed to"
    echo "              update_maintenance_wiki.sh"
    echo "    73        WIKI_USERNAME not specified in reconfig credentials file"
    echo "    74        WIKI_PASSWORD not specified in reconfig credentials file"
    echo "    75        Could not retreieve login token from wiki - probably a connectivity issue"
    echo "    76        Wiki user/password not authorized to update wiki page"
    echo "              https://wiki.mozilla.org/ReleaseEngineering/Maintenance"
    echo "    77        Wiki API provided a reason not to supply an edit token for the wiki page,"
    echo "              which is something other than user/password not authorized (which would"
    echo "              result in exit code 76)"
    echo "    78        Wiki API response to request to provide an edit token produced a"
    echo "              non-parsable response"
    echo "    79        BUGZILLA_USERNAME not specified in reconfig credentials file"
    echo "    80        BUGZILLA_PASSWORD not specified in reconfig credentials file"
    echo "    81        Bugzilla user/password not authorized to update Bugzilla"
    echo "    82        Could not install python packages into a virtualenv"
    echo
}

# Simple function to output the name of this script and the options that were passed to it
function command_called {
    echo -n "Command called:"
    for ((INDEX=0; INDEX<=$#; INDEX+=1))
    do
        echo -n " '${!INDEX}'"
    done
    echo ''
    echo "From directory: '$(pwd)'"
}

# writes hg commands to the hg log, and just summarizes command called in main log
function hg_wrapper {
    if [ $1 == 'clone' ]; then
        command=('hg' "${@}" "${RECONFIG_DIR}/${repo}")
    else
        command=('hg' -R "${RECONFIG_DIR}/${repo}" "${@}")
    fi
    command_full="Running:"
    for ((INDEX=0; INDEX<${#command[@]}; INDEX+=1)); do
        command_full="${command_full} '${command[$INDEX]}'"
    done
    echo "  * ${command_full}"
    {
        echo
        echo "$(date): ${command_full}"
        echo
        HG_START="$(date +%s)"
        set +e
        "${command[@]}"
        HG_RETURN_CODE=$?
        set -e
        HG_STOP="$(date +%s)"
        echo
        echo "$(date): Completed ($((HG_STOP - HG_START))s)"
    } >>"${RECONFIG_DIR}/hg-${START_TIME}.log" 2>&1
    return "${HG_RETURN_CODE}"
}

set +u
command_called "${@}" | sed '1s/^/  * /;2s/^/    /'
set -u

echo "  * Start timestamp: ${START_TIME}"
echo "  * Parsing parameters of $(basename "${0}")..."
# Parse parameters passed to this script
while getopts ":bfhmnpr:w:" opt; do
    case "${opt}" in
        b)  UPDATE_BUGZILLA=0
            ;;
        f)  FORCE_RECONFIG=1
            ;;
        h)  echo "  * Help option requested"
            usage
            exit 0
            ;;
        m)  MERGE_TO_PRODUCTION=0
            ;;
        n)  UPDATE_WIKI=0
            ;;
        p)  PREPARE_ONLY=1
            ;;
        r)  RECONFIG_DIR="${OPTARG}"
            ;;
        w)  RECONFIG_CREDENTIALS_FILE="${OPTARG}"
            ;;
        ?)  usage >&2
            exit 2
            ;;
    esac
done

echo "  * Setting defaults for parameters not provided in command line options..."

PREPARE_ONLY="${PREPARE_ONLY:-0}"
FORCE_RECONFIG="${FORCE_RECONFIG:-0}"
MERGE_TO_PRODUCTION="${MERGE_TO_PRODUCTION:-1}"
UPDATE_BUGZILLA="${UPDATE_BUGZILLA:-1}"
UPDATE_WIKI="${UPDATE_WIKI:-1}"
RECONFIG_DIR="${RECONFIG_DIR:-/tmp/reconfig}"
RECONFIG_CREDENTIALS_FILE="${RECONFIG_CREDENTIALS_FILE:-${HOME}/.reconfig/config}"

RECONFIG_UPDATE_FILE="reconfig_update_for_maintenance.wiki"

##### Now check parsed parameters are valid...

echo "  * Validating parameters..."

if [ "${PREPARE_ONLY}" == 0 ]; then
    echo "  * Will be preparing *and performing* reconfig, all being well"
else
    echo "  * Preparing reconfig only; will not enact changes"
fi

if [ ! -d "${RECONFIG_DIR}" ]; then
    echo "  * Storing reconfig output under '${RECONFIG_DIR}'..."
    if ! mkdir -p "${RECONFIG_DIR}"; then
        echo "ERROR: Directory '${RECONFIG_DIR}' could not be created from directory '$(pwd)'" >&2
        exit 64
    fi
else
    echo "  * Reconfig directory '${RECONFIG_DIR}' exists - OK"
fi

# Convert ${RECONFIG_DIR} to an absolute directory, in case it is relative, by stepping into it...
pushd "${RECONFIG_DIR}" >/dev/null
if [ "${RECONFIG_DIR}" != "$(pwd)" ]; then
    echo "  * Reconfig directory absolute path: '$(pwd)'"
fi
RECONFIG_DIR="$(pwd)"
popd >/dev/null

# To avoid user getting confused about parent directory, tell user the
# absolute path of the credentials file...
echo "  * Checking we have a reconfig credentials file..."
RECONFIG_CREDS_PARENT_DIR="$(dirname "${RECONFIG_CREDENTIALS_FILE}")"
if [ ! -e "${RECONFIG_CREDS_PARENT_DIR}" ]; then
echo "  * Reconfig credentials file parent directory '${RECONFIG_CREDS_PARENT_DIR}' not found; creating..."
    mkdir -p "${RECONFIG_CREDS_PARENT_DIR}"
fi
pushd "${RECONFIG_CREDS_PARENT_DIR}" >/dev/null
ABS_RECONFIG_CREDENTIALS_FILE="$(pwd)/$(basename "${RECONFIG_CREDENTIALS_FILE}")"
popd >/dev/null
if [ "${RECONFIG_CREDENTIALS_FILE}" == "${ABS_RECONFIG_CREDENTIALS_FILE}" ]; then
    echo "  * Reconfig credentials file location: '${RECONFIG_CREDENTIALS_FILE}'"
else
    echo "  * Reconfig credentials file location: '${RECONFIG_CREDENTIALS_FILE}' (absolute path: '${ABS_RECONFIG_CREDENTIALS_FILE}')"
fi
if [ ! -e "${ABS_RECONFIG_CREDENTIALS_FILE}" ]; then
    echo "  * Reconfig credentials file '${ABS_RECONFIG_CREDENTIALS_FILE}' not found; creating..."
    {
        echo "# Needed if updating wiki - note the wiki does *not* use your LDAP credentials..."
        echo "export WIKI_USERNAME='naughtymonkey'"
        echo "export WIKI_PASSWORD='nobananas'"
        echo
        echo "# Needed if updating Bugzilla bugs to mark them as in production - *no* Persona integration - must be a native Bugzilla account..."
        echo "# Details for the 'Release Engineering SlaveAPI Service' <slaveapi@mozilla.releng.tld> Bugzilla user can be found in the RelEng"
        echo "# private repo, in file passwords/slaveapi-bugzilla.txt.gpg (needs decrypting with an approved gpg key)."
        echo "export BUGZILLA_USERNAME='naughtymonkey'"
        echo "export BUGZILLA_PASSWORD='nobananas'"
    } > "${RECONFIG_CREDENTIALS_FILE}"
    echo "  * Created credentials file '${ABS_RECONFIG_CREDENTIALS_FILE}'. Please edit this file, setting appropriate values, then rerun."
    exit 65
else
    echo "  * Loading config from '${ABS_RECONFIG_CREDENTIALS_FILE}'..."
    source "${RECONFIG_CREDENTIALS_FILE}"
fi

# Now step into directory this script is in...
cd "$(dirname "${0}")"

# Test python version, and availability of fabric...
echo "  * Checking python version is 2.7..."
if ! python --version 2>&1 | grep -q '^Python 2\.7'; then
    echo "ERROR: Python version 2.7 not found - please make sure python 2.7 is in your PATH" >&2
    exit 66
fi

run_virtualenv() {
    if hash virtualenv2 2>/dev/null; then
        virtualenv2 "$@"
    else
        virtualenv -p python2 "$@"
    fi
}

for package in fabric requests; do
    installed_package=false
    echo "  * Checking ${package} package is available in python environment..."
    if ! python -c "import ${package}" >/dev/null 2>&1; then
        echo "  * Package ${package} not found"
        if [ ! -e "${RECONFIG_DIR}/reconfig-virtual-env" ]; then
            echo "  * Creating virtualenv directory '${RECONFIG_DIR}/reconfig-virtual-env' for reconfig tool..."
            echo "  * Logging to: '${RECONFIG_DIR}/virtualenv-installation.log'..."
            run_virtualenv "${RECONFIG_DIR}/reconfig-virtual-env" >"${RECONFIG_DIR}/virtualenv-installation.log" 2>&1
            set +u
            source "${RECONFIG_DIR}/reconfig-virtual-env/bin/activate"
            set -u
            echo "  * Installing ${package} under '${RECONFIG_DIR}/reconfig-virtual-env'..."
            pip install "${package}" >>"${RECONFIG_DIR}/virtualenv-installation.log" 2>&1
            installed_package=true
        else
            echo "  * Attempting to use existing virtualenv found in '${RECONFIG_DIR}/reconfig-virtual-env'..."
            set +u
            source "${RECONFIG_DIR}/reconfig-virtual-env/bin/activate"
            set -u
            if ! python -c "import ${package}" >/dev/null 2>&1; then
                echo "  * Package ${package} not found under virtualenv '${RECONFIG_DIR}/reconfig-virtual-env', installing..."
                pip install "${package}" >"${RECONFIG_DIR}/virtualenv-installation.log" 2>&1
                installed_package=true
            else
                echo "  * Using existing package ${package} found in '${RECONFIG_DIR}/reconfig-virtual-env'"
            fi
        fi
    else
        echo "  * Package ${package} found"
    fi
    if "${installed_package}"; then
        echo "  * Re-checking if package ${package} is now available in python environment..."
        if ! python -c "import ${package}" >/dev/null 2>&1; then
            echo "ERROR: Could not successfully install package ${package} into python virtualenv '${RECONFIG_DIR}/reconfig-virtual-env'" >&2
            echo "Exiting..." >&2
            exit 82
        else
            echo "  * Package ${package} installed successfully into python environment"
        fi
    fi
done

if [ "${UPDATE_BUGZILLA}" == '1' ]; then
    echo "  * Updates to Bugzilla enabled"
    echo "  * Testing existence of Bugzilla login credentials..."
    if [ -z "${BUGZILLA_USERNAME}" ]; then
        echo "ERROR: Environment variable BUGZILLA_USERNAME must be set for posting comments to https://bugzilla.mozilla.org/" >&2
        echo "Exiting..." >&2
        exit 79
    else
        echo "  * Environment variable BUGZILLA_USERNAME defined ('${BUGZILLA_USERNAME}')"
    fi
    if [ -z "${BUGZILLA_PASSWORD}" ]; then
        echo "ERROR: Environment variable BUGZILLA_PASSWORD must be set for posting comments to https://bugzilla.mozilla.org/" >&2
        echo "Exiting..." >&2
        exit 80
    else
        echo "  * Environment variable BUGZILLA_PASSWORD defined"
    fi
fi

if [ "${UPDATE_WIKI}" == '1' ]; then
    echo "  * Wiki update enabled"
    # Now validate wiki credentials by performing a dry run...
    echo "  * Testing login credentials for wiki..."
    ./update_maintenance_wiki.sh -d
else
    echo "  * Not updating wiki"
fi

if [ "${UPDATE_BUGZILLA}" == '1' ]; then
    echo "  * Testing Bugzilla login as user '${BUGZILLA_USERNAME}'..."
    BUGZILLA_LOGIN_RESPONSE="$(mktemp -t bugzilla_login.XXXXXXXXXX)"
    curl -s -G --data-urlencode "login=${BUGZILLA_USERNAME}" --data-urlencode "password=${BUGZILLA_PASSWORD}" 'https://bugzilla.mozilla.org/rest/login' > "${BUGZILLA_LOGIN_RESPONSE}"
    if grep -q '"token":' "${BUGZILLA_LOGIN_RESPONSE}"; then
        echo "  * Login to Bugzilla successful"
    else
        echo "ERROR: Login to Bugzilla failed - response given:" >&2
        cat "${BUGZILLA_LOGIN_RESPONSE}" | sed 's/^/    /' >&2
        rm "${BUGZILLA_LOGIN_RESPONSE}"
        echo >&2
        echo "Please check values of BUGZILLA_USERNAME and BUGZILLA_PASSWORD set in this file: '${RECONFIG_CREDENTIALS_FILE}'" >&2
        echo "Exiting..." >&2
        exit 81
    fi
    rm "${BUGZILLA_LOGIN_RESPONSE}"
else
    echo "  * Not updating Bugzilla"
fi

# Check if a previous reconfig did not complete
if [ -f "${RECONFIG_DIR}/pending_changes" ]; then
    echo "  * It looks like a previous reconfig did not complete"
    echo "  * Checking if 'standard in' is connected to a terminal..."
    if [ -t 0 ]; then
        # 'standard in' is connected to a terminal, can ask user a question!
        echo "  * Please select one of the following options:"
        echo "        1) Continue with existing reconfig (e.g. if you have resolved a merge conflict)"
        echo "        2) Delete saved state for existing reconfig, and start from fresh"
        echo "        3) Abort and exit reconfig process"
        choice=''
        while [ "${choice}" != 1 ] && [ "${choice}" != 2 ] && [ "${choice}" != 3 ]; do
            echo -n "    Your choice: "
            read choice
        done
        case "${choice}" in
            1) echo "  * Continuing with stalled reconfig..."
               ;;
            2) echo "  * Cleaning out previous reconfig from '${RECONFIG_DIR}'..."
               rm -rf "${RECONFIG_DIR}"/{buildbot-configs,buildbotcustom,pending_changes,${RECONFIG_UPDATE_FILE}}
               ;;
            3) echo "  * Aborting reconfig..."
               exit 67
               ;;
        esac
    else
        # 'standard in' not connected to a terminal, assume no user connected...
        echo "  * Non-interactive shell detected, cannot ask whether to continue or not, therefore aborting..."
        exit 68
    fi
fi

### If we get this far, all our preflight checks have passed, so now on to business...
echo "  * All preflight checks passed in '$(basename "${0}")'"

if [ "${PREPARE_ONLY}" == '0' ]; then
    # Failing to update IRC is non-fatal.
    IRC_MSG="Merge to production has started for buildbot repos."
    if [ "${FORCE_RECONFIG}" == '1' ]; then
        IRC_MSG="Reconfig has started."
    fi
    ./update_irc.sh "${IRC_MSG}" || true
fi

# Merges buildbot-configs from default -> production.
# Merges buildbostcustom from default -> production-0.8.
# Returns 0 if something got merged, otherwise returns 1.
function merge_to_production {
    [ "${MERGE_TO_PRODUCTION}" == 0 ] && return 0
    echo "  * hg log for this session: '${RECONFIG_DIR}/hg-${START_TIME}.log'"
    # Create an empty merge file to compare against so we don't claim merge success when nothing was merged.
    {
        echo "Merging from default"
        echo
    } > "${RECONFIG_DIR}/empty_merge"
    for repo in buildbot-configs buildbotcustom; do
        if [ -d "${RECONFIG_DIR}/${repo}" ]; then
            echo "  * Existing hg clone of ${repo} found: '${RECONFIG_DIR}/${repo}' - pulling for updates..."
            hg_wrapper pull
        else
            echo "  * Cloning ssh://hg.mozilla.org/build/${repo} into '${RECONFIG_DIR}/${repo}'..."
            hg_wrapper clone "ssh://hg.mozilla.org/build/${repo}"
        fi
        if [ "${repo}" == 'buildbotcustom' ]; then
            branch='production-0.8'
        else
            branch='production'
        fi
        hg_wrapper up -r "${branch}"
        echo "  * Finding ${repo} changesets that would get merged from default to ${branch}..."
        {
            echo "Merging from default"
            echo
            hg -R "${RECONFIG_DIR}/${repo}" merge -P default
        } >> "${RECONFIG_DIR}/${repo}_preview_changes.txt" 2>/dev/null
        # Merging can fail if there are no changes between default and "${branch}"
        set +e
        hg_wrapper merge default
        RETVAL="${?}"
        set -e
        if [ "${RETVAL}" == '255' ]; then
            echo "  * No new changes found in ${repo}"
        elif [ "${RETVAL}" != '0' ]; then
            echo "ERROR: An error occurred during hg merge (exit code was ${RETVAL}). Please resolve conflicts/issues in '${RECONFIG_DIR}/${repo}',"
            echo "       push to ${branch} branch, and run this script again." >&2
            exit 69
        else
            echo "  * ${repo} merge resulted in change"
        fi
        # Still commit and push, even if no new changes, since a merge conflict might have been resolved
        hg_wrapper commit -l "${RECONFIG_DIR}/${repo}_preview_changes.txt"
        if [ "${PREPARE_ONLY}" == '0' ]; then
            echo "  * Pushing '${RECONFIG_DIR}/${repo}' ${branch} branch to ssh://hg.mozilla.org/build/${repo}..."
            hg_wrapper push
        fi
        if ! diff "${RECONFIG_DIR}/empty_merge" "${RECONFIG_DIR}/${repo}_preview_changes.txt" > /dev/null; then
  	    echo "${repo}" >> "${RECONFIG_DIR}/pending_changes"
        fi
    done
    [ -f "${RECONFIG_DIR}/pending_changes" ] && return 0 || return 1
}

# Now we process the commit messages from all the changes we landed. This is handled by the python script
# process_commit_comments.py. We pass options to this script based on what steps are enabled (e.g. wiki
# updates, bugzilla updates). Create an array for this, and if it is empty at the end, we know we don't
# have to do anything (arrays are better than strings since they can contains spaces).
commit_processing_options=()

if [ "${UPDATE_WIKI}" == '1' ]; then
    commit_processing_options+=('--wiki-markup-file' "${RECONFIG_DIR}/${RECONFIG_UPDATE_FILE}")
fi
if [ "${UPDATE_BUGZILLA}" == '1' ] && [ "${PREPARE_ONLY}" == '0' ]; then
    commit_processing_options+=('--update-bugzilla')
fi

# Display the commands in advance in case the reconfig fails part way through.
if [ "${#commit_processing_options[@]}" -gt 0 ]; then
    echo "  * Providing post-processing commands in case reconfig aborts."
    echo -n "  * Will run '$(pwd)/process_commit_comments.py' --logdir '${RECONFIG_DIR}'"
    for ((i=0; i<${#commit_processing_options[@]}; i+=1)); do
        echo -n " '${commit_processing_options[${i}]}'"
    done
    echo
    echo "  * Will run: './update_maintenance_wiki.sh -r \"${RECONFIG_DIR}\" -w \"${RECONFIG_DIR}/${RECONFIG_UPDATE_FILE}\"'"
fi

# Return code of merge_to_production is 0 if merge performed successfully and changes made
if merge_to_production; then
    echo "  * All changes merged to production."
fi

if [ "${FORCE_RECONFIG}" == '1' ]; then
    production_masters_url='https://hg.mozilla.org/build/tools/raw-file/default/buildfarm/maintenance/production-masters.json'
    if [ "${PREPARE_ONLY}" != '0' ]; then
        echo "  * Preparing reconfig only; not running: '$(pwd)/manage_masters.py' -f '${production_masters_url}' -j16 -R scheduler -R build -R try -R tests show_revisions update"
        echo "  * Preparing reconfig only; not running: '$(pwd)/manage_masters.py' -f '${production_masters_url}' -j16 -R scheduler -R build -R try -R tests update_master_config"
        echo "  * Preparing reconfig only; not running: '$(pwd)/manage_masters.py' -f '${production_masters_url}' -j32 -R scheduler -R build -R try -R tests checkconfig reconfig"
        echo "  * Preparing reconfig only; not running: '$(pwd)/manage_masters.py' -f '${production_masters_url}' -j16 -R scheduler -R build -R try -R tests show_revisions"
    else
        echo "  * Fabric log for buildbot masters: '${RECONFIG_DIR}/manage_masters-${START_TIME}.log'"
        # Split into two steps so -j option can be varied between them
        echo "  * Running: '$(pwd)/manage_masters.py' -f '${production_masters_url}' -j16 -R scheduler -R build -R try -R tests show_revisions update"
        ./manage_masters.py -f "${production_masters_url}" -j16 -R scheduler -R build -R try -R tests show_revisions update >>"${RECONFIG_DIR}/manage_masters-${START_TIME}.log" 2>&1
        echo "  * Running: '$(pwd)/manage_masters.py' -f '${production_masters_url}' -j16 -R scheduler -R build -R try -R tests update_master_config"
        ./manage_masters.py -f "${production_masters_url}" -j16 -R scheduler -R build -R try -R tests update_master_config >>"${RECONFIG_DIR}/manage_masters-${START_TIME}.log" 2>&1
        echo "  * Running: '$(pwd)/manage_masters.py' -f '${production_masters_url}' -j32 -R scheduler -R build -R try -R tests checkconfig reconfig"
        ./manage_masters.py -f "${production_masters_url}" -j32 -R scheduler -R build -R try -R tests checkconfig reconfig >>"${RECONFIG_DIR}/manage_masters-${START_TIME}.log" 2>&1
        # delete this now, since changes have been deployed
        [ -f "${RECONFIG_DIR}/pending_changes" ] && mv "${RECONFIG_DIR}/pending_changes" "${RECONFIG_DIR}/pending_changes_${START_TIME}"
        echo "  * Running: '$(pwd)/manage_masters.py' -f '${production_masters_url}' -j16 -R scheduler -R build -R try -R tests show_revisions"
        ./manage_masters.py -f "${production_masters_url}" -j16 -R scheduler -R build -R try -R tests show_revisions >>"${RECONFIG_DIR}/manage_masters-${START_TIME}.log" 2>&1
        echo "  * Reconfig of masters completed"
    fi
fi

if [ "${#commit_processing_options[@]}" -gt 0 ]; then
    echo -n "  * Running '$(pwd)/process_commit_comments.py' --logdir '${RECONFIG_DIR}'"
    for ((i=0; i<${#commit_processing_options[@]}; i+=1)); do echo -n " '${commit_processing_options[${i}]}'"; done
    echo
    ./process_commit_comments.py --logdir "${RECONFIG_DIR}" "${commit_processing_options[@]}"
else
    echo "  * Skipping commit messages processing step, since no wiki update nor Bugzilla update enabled, so not required"
fi

if [ -f "${RECONFIG_DIR}/${RECONFIG_UPDATE_FILE}" ]; then
    if [ "${UPDATE_WIKI}" == "1" ]; then
        if [ "${PREPARE_ONLY}" != '0' ]; then
            echo "  * Running: './update_maintenance_wiki.sh -d -r \"${RECONFIG_DIR}\" -w \"${RECONFIG_DIR}/${RECONFIG_UPDATE_FILE}\"'"
            ./update_maintenance_wiki.sh -d -r "${RECONFIG_DIR}" -w "${RECONFIG_DIR}/${RECONFIG_UPDATE_FILE}"
        else
            echo "  * Running: './update_maintenance_wiki.sh -r \"${RECONFIG_DIR}\" -w \"${RECONFIG_DIR}/${RECONFIG_UPDATE_FILE}\"'"
            ./update_maintenance_wiki.sh -r "${RECONFIG_DIR}" -w "${RECONFIG_DIR}/${RECONFIG_UPDATE_FILE}"
            for file in "${RECONFIG_DIR}"/*_preview_changes.txt
            do
                mv "${file}" "$(echo "${file}" | sed "s/\\.txt\$/_${START_TIME}&/")"
            done 2>/dev/null || true
            RECONFIG_STOP_TIME="$(date +%s)"
            RECONFIG_ELAPSED=$((RECONFIG_STOP_TIME - START_TIME))
            RECONFIG_ELAPSED_DISPLAY=`show_time ${RECONFIG_ELAPSED}`
            IRC_MSG="Merge has finished in ${RECONFIG_ELAPSED_DISPLAY}. See http://bit.ly/reconfigs for details. Masters will reconfig themselves automatically on the hour."
            if [ "${FORCE_RECONFIG}" == '1' ]; then
                IRC_MSG="Reconfig has finished in ${RECONFIG_ELAPSED_DISPLAY}. See http://bit.ly/reconfigs for details."
            fi
            # Failing to update IRC is non-fatal.
            ./update_irc.sh "${IRC_MSG}" || true
        fi
    fi

    echo "  * Summary of changes:"
    cat "${RECONFIG_DIR}/${RECONFIG_UPDATE_FILE}" | sed 's/^/        /'
fi

echo "  * Directory '${RECONFIG_DIR}' contains artifacts from reconfig process"

STOP_TIME="$(date +%s)"
ELAPSED_TIME=$((STOP_TIME - START_TIME))
ELAPSED_DISPLAY=`show_time ${ELAPSED_TIME}`
echo "  * Finish timestamp: ${STOP_TIME}"
echo "  * Time taken: ${ELAPSED_DISPLAY}"
