#!/bin/bash
#
# Copyright (C) 2014 Cloudwatt <libre.licensing@cloudwatt.com>
# Copyright (C) 2014, 2015, 2016, 2017 Red Hat <contact@redhat.com>
#
# Author: Loic Dachary <loic@dachary.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Library Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Library Public License for more details.
#

# ceph-disk.sh is launched by tox which expects tox.ini in current
# directory. so we cannot run ceph-disk.sh in build directory directly,
# and hence not able to use detect-build-env-vars.sh to set the build
# env vars.
if [ -z "$CEPH_ROOT" ] || [ -z "$CEPH_BIN" ] || [ -z "$CEPH_LIB" ]; then
    CEPH_ROOT=`readlink -f $(dirname $0)/../../..`
    CEPH_BIN=$CEPH_ROOT
    CEPH_LIB=$CEPH_ROOT/.libs
fi
source $CEPH_ROOT/qa/standalone/ceph-helpers.sh

set -x

PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}:  '

export PATH=$CEPH_BIN:.:$PATH # make sure program from sources are preferred
export LD_LIBRARY_PATH=$CEPH_LIB
: ${CEPH_DISK:=ceph-disk}
CEPH_DISK_ARGS=
CEPH_DISK_ARGS+=" --verbose"
CEPH_DISK_ARGS+=" --prepend-to-path="
: ${CEPH_DISK_TIMEOUT:=360}
if [ `uname` != FreeBSD ]; then
    PROCDIR=""
else
    PROCDIR="/compat/linux"
fi

cat=$(which cat)
diff=$(which diff)
mkdir=$(which mkdir)
rm=$(which rm)
uuidgen=$(which uuidgen)
if [ `uname` = FreeBSD ]; then
    # for unknown reasons FreeBSD timeout does not return sometimes
    timeout=""
else
    timeout="$(which timeout) $CEPH_DISK_TIMEOUT"
fi

function setup() {
    local dir=$1
    teardown $dir
    mkdir -p $dir/osd
    mkdir -p $(get_asok_dir)
    touch $dir/ceph.conf # so ceph-disk think ceph is the cluster
}

function teardown() {
    local dir=$1
    if ! test -e $dir ; then
        return
    fi
    kill_daemons $dir
    if [ `uname` != FreeBSD ] && \
       [ $(stat -f -c '%T' .) == "btrfs" ]; then
        rm -fr $dir/*/*db
        __teardown_btrfs $dir
    fi
    grep " $(pwd)/$dir/" < ${PROCDIR}/proc/mounts | while read mounted rest ; do
        umount $mounted
    done
    rm -fr $dir
    rm -rf $(get_asok_dir)
}

function command_fixture() {
    local dir=$1
    shift
    local command=$1
    shift
    local fpath=`readlink -f $(which $command)`
    [ "$fpath" = `readlink -f $CEPH_BIN/$command` ] || [ "$fpath" = `readlink -f $(pwd)/$command` ] || return 1

    cat > $dir/$command <<EOF
#!/bin/bash
touch $dir/used-$command
exec $CEPH_BIN/$command "\$@"
EOF
    chmod +x $dir/$command
}

function tweak_path() {
    local dir=$1
    shift
    local tweaker=$1
    shift

    setup $dir

    command_fixture $dir ceph-conf || return 1
    command_fixture $dir ceph-osd || return 1

    test_activate_dir $dir || return 1

    [ ! -f $dir/used-ceph-conf ] || return 1
    [ ! -f $dir/used-ceph-osd ] || return 1

    teardown $dir

    setup $dir

    command_fixture $dir ceph-conf || return 1
    command_fixture $dir ceph-osd || return 1

    $tweaker $dir test_activate_dir || return 1

    [ -f $dir/used-ceph-osd ] || return 1

    teardown $dir
}

function use_prepend_to_path() {
    local dir=$1
    shift

    local ceph_disk_args
    ceph_disk_args+=" --statedir=$dir"
    ceph_disk_args+=" --sysconfdir=$dir"
    ceph_disk_args+=" --prepend-to-path=$dir"
    ceph_disk_args+=" --verbose"
    CEPH_DISK_ARGS="$ceph_disk_args" \
        "$@" $dir || return 1
}

function test_prepend_to_path() {
    local dir=$1
    shift
    tweak_path $dir use_prepend_to_path || return 1
}

function use_path() {
    local dir=$1
    shift
    PATH="$dir:$PATH" \
        "$@" $dir || return 1
}

function test_path() {
    local dir=$1
    shift
    tweak_path $dir use_path || return 1
}

function test_no_path() {
    local dir=$1
    shift
    ( export PATH=../ceph-detect-init/virtualenv/bin:virtualenv/bin:$CEPH_BIN:/usr/bin:/bin:/usr/local/bin ; test_activate_dir $dir) || return 1
}

function test_mark_init() {
    local dir=$1
    shift

    run_mon $dir a
    create_rbd_pool

    local osd_data=$dir/dir
    $mkdir -p $osd_data

    local osd_uuid=$($uuidgen)

    $mkdir -p $osd_data

    ${CEPH_DISK} $CEPH_DISK_ARGS \
        prepare --filestore --osd-uuid $osd_uuid $osd_data || return 1

    ${CEPH_DISK} $CEPH_DISK_ARGS \
        --verbose \
        activate \
        --mark-init=auto \
        --no-start-daemon \
        $osd_data || return 1

    test -f $osd_data/$(ceph-detect-init) || return 1

    if test systemd = $(ceph-detect-init) ; then
        expected=sysvinit
    else
        expected=systemd
    fi
    $timeout ${CEPH_DISK} $CEPH_DISK_ARGS \
        --verbose \
        activate \
        --mark-init=$expected \
        --no-start-daemon \
        $osd_data || return 1

    ! test -f $osd_data/$(ceph-detect-init) || return 1
    test -f $osd_data/$expected || return 1
}

function test_zap() {
    local dir=$1
    local osd_data=$dir/dir
    $mkdir -p $osd_data

    ${CEPH_DISK} $CEPH_DISK_ARGS zap $osd_data 2>&1 | grep -q 'not full block device' || return 1

    $rm -fr $osd_data
}

# ceph-disk prepare returns immediately on success if the magic file
# exists in the --osd-data directory.
function test_activate_dir_magic() {
    local dir=$1
    local uuid=$($uuidgen)
    local osd_data=$dir/osd

    echo a failure to create the fsid file implies the magic file is not created

    mkdir -p $osd_data/fsid
    CEPH_ARGS="--fsid $uuid" \
     ${CEPH_DISK} $CEPH_DISK_ARGS prepare --filestore $osd_data > $dir/out 2>&1
    grep --quiet 'Is a directory' $dir/out || return 1
    ! [ -f $osd_data/magic ] || return 1
    rmdir $osd_data/fsid

    echo successfully prepare the OSD

    CEPH_ARGS="--fsid $uuid" \
     ${CEPH_DISK} $CEPH_DISK_ARGS prepare --filestore $osd_data 2>&1 | tee $dir/out
    grep --quiet 'Preparing osd data dir' $dir/out || return 1
    grep --quiet $uuid $osd_data/ceph_fsid || return 1
    [ -f $osd_data/magic ] || return 1

    echo will not override an existing OSD

    CEPH_ARGS="--fsid $($uuidgen)" \
     ${CEPH_DISK} $CEPH_DISK_ARGS prepare --filestore $osd_data 2>&1 | tee $dir/out
    grep --quiet 'Data dir .* already exists' $dir/out || return 1
    grep --quiet $uuid $osd_data/ceph_fsid || return 1
}

function read_write() {
    local dir=$1
    local file=${2:-$(uuidgen)}
    local pool=rbd

    echo FOO > $dir/$file
    $timeout rados --pool $pool put $file $dir/$file || return 1
    $timeout rados --pool $pool get $file $dir/$file.copy || return 1
    $diff $dir/$file $dir/$file.copy || return 1
}

function test_pool_read_write() {
    local dir=$1
    local pool=rbd

    $timeout ceph osd pool set $pool size 1 || return 1
    read_write $dir || return 1
}

function test_activate() {
    local dir=$1
    shift
    local osd_data=$1
    shift

    mkdir -p $osd_data

    ${CEPH_DISK} $CEPH_DISK_ARGS \
        prepare --filestore "$@" $osd_data || return 1

    $timeout ${CEPH_DISK} $CEPH_DISK_ARGS \
        activate \
        --mark-init=none \
        $osd_data || return 1

    test_pool_read_write $dir || return 1
}

function test_reuse_osd_id() {
    local dir=$1

    run_mon $dir a || return 1
    run_mgr $dir x || return 1
    create_rbd_pool

    test_activate $dir $dir/dir1 --osd-uuid $(uuidgen) || return 1

    #
    # add a new OSD with a given OSD id (13)
    #
    local osd_uuid=$($uuidgen)
    local osd_id=13
    test_activate $dir $dir/dir2 --osd-id $osd_id --osd-uuid $osd_uuid || return 1
    test $osd_id = $(ceph osd new $osd_uuid) || return 1

    #
    # make sure the OSD is in use by the PGs
    #
    wait_osd_id_used_by_pgs $osd_id $PG_NUM || return 1
    read_write $dir SOMETHING || return 1

    #
    # set the OSD out and verify it is no longer used by the PGs
    #
    ceph osd out osd.$osd_id || return 1
    wait_osd_id_used_by_pgs $osd_id 0 || return 1

    #
    # kill the OSD and destroy it (do not purge, retain its place in the crushmap)
    #
    kill_daemons $dir TERM osd.$osd_id || return 1
    ceph osd destroy osd.$osd_id --yes-i-really-mean-it || return 1

    #
    # add a new OSD with the same id as the destroyed OSD
    #
    osd_uuid=$($uuidgen)
    test_activate $dir $dir/dir3 --osd-id $osd_id --osd-uuid $osd_uuid || return 1
    test $osd_id = $(ceph osd new $osd_uuid) || return 1
}

function test_activate_dir() {
    local dir=$1
    shift

    run_mon $dir a || return 1
    run_mgr $dir x || return 1
    create_rbd_pool
    $@

    test_activate $dir $dir/dir || return 1
}

function test_activate_dir_bluestore() {
    local dir=$1

    run_mon $dir a || return 1
    run_mgr $dir x || return 1
    create_rbd_pool

    local osd_data=$dir/dir
    $mkdir -p $osd_data
    local to_prepare=$osd_data
    local to_activate=$osd_data
    local osd_uuid=$($uuidgen)

    CEPH_ARGS=" --bluestore-block-size=10737418240 $CEPH_ARGS" \
      ${CEPH_DISK} $CEPH_DISK_ARGS \
        prepare --bluestore --block-file --osd-uuid $osd_uuid $to_prepare || return 1

    CEPH_ARGS=" --osd-objectstore=bluestore --bluestore-fsck-on-mount=true --bluestore-block-db-size=67108864 --bluestore-block-wal-size=134217728 --bluestore-block-size=10737418240 $CEPH_ARGS" \
      $timeout ${CEPH_DISK} $CEPH_DISK_ARGS \
        activate \
        --mark-init=none \
        $to_activate || return 1

    test_pool_read_write $dir || return 1
}

function test_find_cluster_by_uuid() {
    local dir=$1
    test_activate_dir $dir 2>&1 | tee $dir/test_find
    ! grep "No cluster conf found in $dir" $dir/test_find || return 1
    teardown $dir

    setup $dir
    test_activate_dir $dir "rm $dir/ceph.conf" > $dir/test_find 2>&1
    cp $dir/test_find /tmp
    grep --quiet "No cluster conf found in $dir" $dir/test_find || return 1
}

# http://tracker.ceph.com/issues/9653
function test_keyring_path() {
    local dir=$1
    test_activate_dir $dir 2>&1 | tee $dir/test_keyring
    grep --quiet "keyring $dir/bootstrap-osd/ceph.keyring" $dir/test_keyring || return 1
}

function test_crush_device_class() {
    local dir=$1
    shift

    run_mon $dir a
    create_rbd_pool

    local osd_data=$dir/dir
    $mkdir -p $osd_data

    local osd_uuid=$($uuidgen)

    $mkdir -p $osd_data

    ${CEPH_DISK} $CEPH_DISK_ARGS \
        prepare --filestore --osd-uuid $osd_uuid \
                --crush-device-class CRUSH_CLASS \
                $osd_data || return 1
    test -f $osd_data/crush_device_class || return 1
    test $(cat $osd_data/crush_device_class) = CRUSH_CLASS || return 1

    CEPH_ARGS="--crush-location=root=default $CEPH_ARGS" \
      ${CEPH_DISK} $CEPH_DISK_ARGS \
        --verbose \
        activate \
        --mark-init=none \
        $osd_data || return 1

    ok=false
    for delay in 2 4 8 16 32 64 128 256 ; do
        if ceph osd crush dump | grep --quiet 'CRUSH_CLASS' ; then
            ok=true
            break
        fi
        sleep $delay
        ceph osd crush dump # for debugging purposes
    done
    $ok || return 1
}

function run() {
    local dir=$1
    shift
    CEPH_DISK_ARGS+=" --statedir=$dir"
    CEPH_DISK_ARGS+=" --sysconfdir=$dir"

    export CEPH_MON="127.0.0.1:7451" # git grep '\<7451\>' : there must be only one
    export CEPH_ARGS
    CEPH_ARGS+=" --fsid=$(uuidgen)"
    CEPH_ARGS+=" --auth-supported=none"
    CEPH_ARGS+=" --mon-host=$CEPH_MON"
    CEPH_ARGS+=" --chdir="
    CEPH_ARGS+=" --journal-dio=false"
    CEPH_ARGS+=" --erasure-code-dir=$CEPH_LIB"
    CEPH_ARGS+=" --plugin-dir=$CEPH_LIB"
    CEPH_ARGS+=" --log-file=$dir/\$name.log"
    CEPH_ARGS+=" --pid-file=$dir/\$name.pidfile"
    CEPH_ARGS+=" --osd-class-dir=$CEPH_LIB"
    CEPH_ARGS+=" --run-dir=$dir"
    CEPH_ARGS+=" --osd-failsafe-full-ratio=.99"
    CEPH_ARGS+=" --osd-journal-size=100"
    CEPH_ARGS+=" --debug-osd=20"
    CEPH_ARGS+=" --debug-bdev=20"
    CEPH_ARGS+=" --debug-bluestore=20"
    CEPH_ARGS+=" --osd-max-object-name-len=460"
    CEPH_ARGS+=" --osd-max-object-namespace-len=64 "
    local default_actions
    default_actions+="test_path "
    default_actions+="test_no_path "
    default_actions+="test_find_cluster_by_uuid "
    default_actions+="test_prepend_to_path "
    default_actions+="test_activate_dir_magic "
    default_actions+="test_activate_dir "
    default_actions+="test_keyring_path "
    [ `uname` != FreeBSD ] && \
      default_actions+="test_mark_init "
    default_actions+="test_zap "
    [ `uname` != FreeBSD ] && \
      default_actions+="test_activate_dir_bluestore "
    default_actions+="test_crush_device_class "
    default_actions+="test_reuse_osd_id "
    local actions=${@:-$default_actions}
    for action in $actions  ; do
        setup $dir || return 1
        set -x
        $action $dir || return 1
        set +x
        teardown $dir || return 1
    done
}

main test-ceph-disk "$@"

# Local Variables:
# compile-command: "cd .. ; test/ceph-disk.sh # test_activate_dir"
# End:
