copycomdb2

#!/bin/bash
# Uses comdb2ar to serialise and deserialise a database.

PREFIX=@CMAKE_INSTALL_PREFIX@

arch=`uname`
rsh=ssh
recovery_opts=""

scriptname=${0##*/}
[ -z "$TMPDIR" ] && TMPDIR=/tmp
[ -d "$TMPDIR" ] || TMPDIR=/tmp
export HOSTNAME="$(hostname)"   # export it so bbhotcp gets it too
umask 02                        # we may have been called via rsh

function usage
{
    printf "
$scriptname [-l logfile] [-C strip|preserve|qa] [-R]
           <sourcemachine:fully qualified lrl_file>
           [<destmachine:dest lrl dir> <destmachine:dest db dir>]

PULL MODE examples:

 $ copycomdb2 ibm3:/bb/bin/abc.lrl
  -this pulls database exactly as it is from ibm3.

 $ copycomdb2 ibm3:/bb/bin/abc.lrl /t1/bin/
  -this copies lrl to /t1/bin, and puts data in same location as source

 $ copycomdb2 ibm3:/bb/bin/abc.lrl /t2/bin/ /t2/data/abc/
  -this copies lrl to /t2/bin, and puts data in /t2/data/abc/

PUSH MODE examples:

 $ copycomdb2 /bb/bin/abc.lrl ibm3:
  -this pushes database exactly as it is to ibm3

 $ copycomdb2 /bb/bin/abc.lrl ibm3:/t1/bin/
  -this copies lrl to /t1/bin, and puts data in same location as source

 $ copycomdb2 /bb/bin/abc.lrl ibm3:/t2/bin/ ibm3:/t2/data/abc/
  -this copies lrl to /t2/bin, and puts data in /t2/data/abc/

LOCAL COPY MODE example (NB. you cannot copy a database onto itself):

  $ copycomdb2 /bb/bin/abc.lrl \$HOME \$HOME/mycopy
   -this copies lrl to \$HOME, and puts data in \$HOME/mycopy

LOCAL MOVE MODE example (NB. make sure the db is not running):

  $ copycomdb2 -m /bb/bin/abc.lrl /bb/data/newhome /bb/data/newhome
   -this moves lrl and data to /bb/data/newhome

BACKUP MODE example:

  $ copycomdb2 -b /path/to/abc.lrl > /bb/backup/abc.lz4
   -this serializes abc to /bb/backup/abc.lz4

RESTORE MODE example:

  $ copycomdb2 -r /t1/bin /t1/data/abc < /bb/backup/abc.lz4
   -this unpacks abc's lrl to /t1/bin, and puts data in /t1/data/abc

By default output is logged to $TMPDIR/copycomdb2.<dbname>.log

Other options:
    -l logfile  Log output to the named file.  Specify - to log outut to stdout
    -C strip    Strip cluster information from the copied lrl file
    -C preserve Preserve cluster information in copied lrl file
    -C qa       Strip cluster information and consumers.
    -d          Copy data files only; do not copy lrl or csc2 files
    -s          Copy support files (lrl and csc2) only; do not copy any data
    -m          Move mode - move files rather than copying.  The database
                should not be running if you use move mode.
    -L          Do not disable log file deletion during the copy (dangerous)
    -x tskname  Specify the task to use for full recovery (default is
                whatever is compiled in COMDB2_ROOT)
    -u percent  Set the percent disk full at which copycomdb2 will abort if
                reached on the destination machine.  The default is 95%%.
    -b          Serialize a backup of a database to stdout
    -r          Restore a backup of a database from stdin
    -p          Copy only the incremental differences
    -H db       Creates a physical replicant with name 'db'.
    -y type     Sets physical replicant type-string.
    -P rmt_path Sets the remote path for comdb2ar.
    -z rmt_path Sets the remote path for lz4.
    -R          Does not run full recovery on the copied database.
\n"

   exit 1
}

function newoccurence {
   file=$1
   max=$2
   rm -f $file.$max
   for i in $(seq $max -1 1); do
       mv $file.$i $file.$(($i+1)) 2>/dev/null
   done
   mv $file $file.1 2>/dev/null
}


function die
{
    [ $# != 0 ] && printf "$scriptname: $@\n" >&2
    echo >&2 "FATAL ERROR"
    exit 1
}

function normalize_hostname
{
    host=$1
    echo $host
}

#
# Process command line options
#

copycomdb2log=
cluster_info=default
copy_lrls=yes
copy_data=yes
put_csc2_with_data=yes
disable_log_delete=yes
bbhotcp_opts=
usefastcopy=no
movemode=no
force=no
backupmode=no
restoremode=no
max_du_percent=95
partial=no
partialopts=
outmode=x
phys_repmode=no
phys_name=
phys_copy_type=
phys_copy_flag=no
comdb2ar_rmt=
lz4_rmt=
do_recovery=yes

[ -z "$comdb2ar" ] && comdb2ar="${PREFIX}/bin/comdb2ar"
[ -z "$lz4" ] && lz4="${PREFIX}/bin/lz4"

function nolz4 {
    while [[ $# -gt 0 && $1 = '-*' ]]; do
        shift
    done
    shift 2
    cat
}

if [[ ! -x $comdb2ar ]]; then
   comdb2ar=$(which comdb2ar 2>/dev/null)
fi

if [[ ! -x $lz4 ]]; then
   lz4=$(which lz4 2>/dev/null)
   if [[ -z "$lz4" ]]; then
       lz4=nolz4
   fi
fi

set - `getopt "LC:ht:l:rdsSx:fmaZu:bpH:y:P:z:R" "$@"` || die
for arg in "$@" ; do
    case "$arg" in
        -h)
            usage
            ;;
        -x)
            recovery_opts="-x $2"
            shift 2
            ;;
        -C)
            cluster_info="$2"
            shift 2
            if [ "$cluster_info" != preserve -a "$cluster_info" != strip -a "$cluster_info" != qa ]
            then
                die "argument to -C must be strip, preserve, or qa"
            fi
            ;;
        -t)
            echo >&2 "-t option no longer supported or necessary"
            shift 2
            ;;
        -l)
            copycomdb2log="$2"
            shift 2
            ;;
        -L)
            disable_log_delete=no
            shift
            ;;
        -r)
            restoremode=yes
            shift
            ;;
        -d)
            echo >&2 "-d option is now ignored"
            shift
            ;;
        -s)
            copy_data=no
            shift
            ;;
        -S)
            echo >&2 "-S option is now ignored"
            shift
            ;;
        -f)
            # Several tools specify -f explicitly, so be silent about 
            # ignoring it for now.
            #print -u2 -- "-f option is now ignored"
            shift
            ;;
        -m) 
            movemode=yes
            shift
            ;;
        -u)
            max_du_percent="$2"
            shift 2
            ;;

        -a)
            # Lame, but -f was taken.  this is not documented because
            # I really don't want anyone using it.  It's there in
            # case of an emergency.
            force=yes
            max_du_percent=101
            shift
            ;;

        -b) backupmode=yes
            shift
            ;;

        -p) partial=yes
            shift
            ;;

        -H) phys_repmode=yes
            phys_name="$2"
            shift 2
            ;;

        -y) phys_copy_type="$2"
            phys_copy_flag=yes
            shift 2
            ;;

        -P) comdb2ar_rmt="$2"
            shift 2
            ;;

        -z) lz4_rmt="$2"
            shift 2
            ;;

        -R) do_recovery=no
            shift
            ;;

        --)
            shift
            break
            ;;
    esac
done

# allow comdb2ar_rmt to be changed
[ -z "$comdb2ar_rmt" ] && comdb2ar_rmt="${comdb2ar}"
[ -z "$lz4_rmt" ] && lz4_rmt="${lz4}"

[ $copy_data = no -a $copy_lrls = no ] && die "cannot use -s and -d together"

# restrict physical replicant mode
[ $movemode = yes -a $phys_repmode = yes ] && die "cannot replicate and local move"
[ $backupmode = yes -a $phys_repmode = yes ] && die "cannot replicate and backup"
[ $restoremode = yes -a $phys_repmode = yes ] && die "cannot replicate and restore"

# Get source file and host
if [[ $restoremode = no ]]; then
    source="$1"
    if [[ $source = *:*.lrl ]]; then
        srcmach="${source%%:*}"
        srclrl="${source#*:}"
    elif [[ $source = *.lrl ]]; then
        srcmach="$HOSTNAME"
        srclrl="$source"
    else
        usage
    fi
    [[ $srclrl = /* ]] || die "must specify FULLY QUALIFIED source lrl name"
fi

# Look for hostname overrides in the destination and make sure they're
# consistent (e.g. you can't specify an lrl destination on one machine
# and a data destination on another machine)
if [[ $restoremode = yes ]]; then
    destlrldir=$1
    destdbdir=$2
    destlrlmach=
    destdbmach=
    destmach="$HOSTNAME"
elif [[ $backupmode = no ]]; then
    destlrldir=$2
    destdbdir=$3
    destlrlmach=
    destdbmach=
    destmach="$HOSTNAME"
else
    destlrlmach=
    destdbmach=
    destmach="$HOSTNAME"
fi

if [[ $destlrldir = *:* ]]; then
    destlrlmach="${destlrldir%%:*}"
    destlrldir="${destlrldir#*:}"
    destmach="$destlrlmach"
fi
if [[ $destdbdir = *:* ]]; then
    destdbmach="${destdbdir%%:*}"
    destdbdir="${destdbdir#*:}"
    destmach="$destdbmach"
fi

# Standardise the host names
srcmach=$(normalize_hostname $srcmach) || die
destmach=$(normalize_hostname $destmach) || die

if [ "$destlrlmach" != "$destdbmach" ]; then
    if [ -n "$destdbmach" -a -n "$destlrlmach" ]; then
        die "destination machines inconsistent ($destlrlmach != $destdbmach)"
    fi
fi


# Get the assumed dbname and LRL directory from the source LRL
if [[ $restoremode = yes ]]; then
    dbname=restore.$$
else
    dbname=${srclrl%.lrl}
    dbname=${dbname##*/}
    srclrldir=${srclrl%/*}
fi

# If no dest lrl dir given, use source lrl dir.
# comdb2ar takes care of using source data dir (read from lrl) if
# no data dir provided
[ -z "$destlrldir" ] && destlrldir="$srclrldir"

# Make sure we aren't trying to copy the database over itself.
if [[ $backupmode = "no" ]]; then
    if [[ "$srcmach" = "$destmach" ]]; then
        if [[ "$movemode" = "no" && "$srclrldir" = "$destlrldir" ]]; then
            die "destination same as source (lrl)"
        fi

        srcdir=$(grep -v '[ ]#' $srclrldir/$dbname.lrl | grep -w dir | cut -d' ' -f2)
        if [[ -n "$destdbdir" ]]; then
            if [[ $(dirname ${destdbdir}/x) = $(dirname ${srcdir}/x) ]]; then
                die "destination same as source (dta)"
            fi
        fi
    fi
fi

# Determine mode (PUSH or PULL or LOCAL).  First normalise the hostname
# (so that sdv5 doesn't look different to sundev5, for example).  Also
# determine if we should strip or preserve cluster node information.
if [ "$srcmach" = "$HOSTNAME" -a "$destmach" != "$HOSTNAME" ]; then
    copy_mode=PUSH
    [ $movemode = no ] || die "move mode can only be used locally"
elif [ "$srcmach" != "$HOSTNAME" -a "$destmach" = "$HOSTNAME" ]; then
    copy_mode=PULL
    [ $movemode = no ] || die "move mode can only be used locally"
elif [ "$srcmach" = "$HOSTNAME" -a "$destmach" = "$HOSTNAME" ]; then
    copy_mode=LOCAL
else
    die "cannot work on two remote machines (srcmach=$srcmach, destmach=$destmach)"
fi

# if we want an incremental copy, create the partial files on the
# destination, and get them to the source
partials=$TMPDIR/$dbname.$$.inc.tar
partialsdir=$TMPDIR/$dbname.$$.inc
partiallrl=

if [[ $partial = yes ]]; then
    partiallrl=$(basename $srclrl)
    if [ $copy_mode = PULL ]; then
        if [ "$cluster_info" = strip ]; then
            orig=$destlrldir/$partiallrl.orig.$(date +"%Y%m%dT%H%MZ%3N")
            mv $destlrldir/$partiallrl $orig
            egrep -v "cluster nodes" $orig > $destlrldir/$partiallrl
            cluster_info=default
        fi
        $comdb2ar p $destlrldir/$partiallrl > $partials
        rcp -q $partials $srcmach:$partials
        $rsh $srcmach "mkdir  $TMPDIR/$dbname.$$.inc"
        $rsh $srcmach "tar xvf $partials -C $partialsdir"
    else
        if [ "$cluster_info" = strip ]; then
            orig=$destlrldir/$partiallrl.orig.$(date +"%Y%m%dT%H%MZ%3N")
            $rsh $destmach "mv $destlrldir/$partiallrl $orig"
            $rsh $destmach "egrep -v "cluster nodes" $orig > $destlrldir/$partiallrl"
            cluster_info=default
        fi
        $rsh $destmach $comdb2ar p $destlrldir/$partiallrl > $partials
        rcp -q $destmach:$partials $partials
        mkdir  $TMPDIR/$dbname.$$.inc
        tar xvf $partials -C $partialsdir
    fi
    partialopts="-I inc -b $partialsdir"
    outmode=P
fi

if [ $cluster_info = default ] ; then
    # I use separate vars here so I can easily test
    srchostname="$srcmach"
    desthostname="$destmach"
    #srchostname=sundev9
    #desthostname=pny-comdb2a1
    srchosttest=no
    desthosttest=no
    if [ $srchosttest = yes -a $desthosttest = no ] ; then
        # test -> prod
        cluster_info=strip
    elif [ $srchosttest = no -a $desthosttest = yes ] ; then
        # prod -> test
        cluster_info=strip
    else
        cluster_info=preserve
    fi
fi

logfile="$copycomdb2log"
if [ "$logfile" = "-" ] ; then
    logfile=$TMPDIR/copycomdb2.$dbname.$$.tmplog
else
    if [ -z "$logfile" ] ; then
        logfile=$TMPDIR/copycomdb2.$dbname.log
    fi
    newoccurence $logfile 7
    logfiletext=" see $logfile"
fi
echo "$scriptname $*" >>$logfile
echo "Run at $(date)" >>$logfile

arstatus="$TMPDIR/comdb2ar_ar.$$.txt"
exstatus="$TMPDIR/comdb2ar_ex.$$.txt"

touch $arstatus $exstatus

aropts="-C $cluster_info"
exopts="-C $cluster_info -u $max_du_percent $recovery_opts"

[ $copy_data = no ]             && aropts="$aropts -s"
[ $disable_log_delete = no ]    && aropts="$aropts -L"
[ $phys_repmode = yes ]         && aropts="$aropts -E $phys_name"
[ $phys_copy_flag = yes ]       && aropts="$aropts -T $phys_copy_type"
[ $do_recovery = no ]           && exopts="$exopts -R"

if [ $restoremode = yes ]; then
    $IONICE $lz4 -d stdin stdout | $comdb2ar $COMDB2AR_AROPTS $exopts x $destlrldir $destdbdir 2>$arstatus 
elif [ $backupmode = yes ]; then
    $IONICE $comdb2ar $COMDB2AR_AROPTS $aropts c $srclrl 2>$arstatus | $lz4 stdin stdout
else
    printf "\n\nSetting up pipe\n_______________________________________\n\n" >>$logfile
    (
    rmt=

    if [ $copy_mode = PULL ] ; then
        rmt=$srcmach
    fi

    if [ $copy_mode = PUSH ] ; then
        rmt=$destmach
    fi


    if [[ "$arch" = "Linux" ]]; then 
        ionice -c 3 echo 2>&1 |grep "Operation not permitted" >/dev/null 
        if [[ $? -eq 1 ]]; then
            echo "using local ionice"
            IONICE="ionice -c 3 "
        else
            echo "not using local ionice"
            IONICE=
        fi
    fi


    if [[ -n "$rmt" ]]; then
        rmtarch=$($rsh $rmt uname)

        if [[ "$rmtarch" = "Linux" ]]; then 
            if [[ $($rsh $rmt 'ionice -c 3 echo 2>&1 |grep Operation >/dev/null; echo $?') == 1 ]]; then 
                echo "using remote ionice"
                IONICE_RMT="ionice -c 3 "
            else
                echo "not using remote ionice"
                IONICE_RMT=
            fi
        fi
    fi

    if [ $copy_mode = PULL ] ; then
        # Pull mode.  Serialise the db at the destination and pipe this in to
        # a local deserialise.
        if [[ $partial = yes ]] ; then
            $rsh $srcmach "$IONICE_RMT $comdb2ar_rmt $COMDB2AR_AROPTS $aropts c $partialopts $srclrl 2>$arstatus | $lz4_rmt stdin stdout " | \
                $lz4 -d stdin stdout | $IONICE $comdb2ar $COMDB2AR_EXOPTS $exopts $outmode $destlrldir/$partiallrl >$exstatus 2>&1
            rc=$?
        else
            $rsh $srcmach "$IONICE_RMT $comdb2ar_rmt $COMDB2AR_AROPTS $aropts c $srclrl 2>$arstatus | $lz4_rmt stdin stdout " | \
                $lz4 -d stdin stdout | $IONICE $comdb2ar $COMDB2AR_EXOPTS $exopts $outmode $destlrldir $destdbdir >$exstatus 2>&1
            rc=$?
        fi

    elif [ $copy_mode = PUSH ] ; then
        # Push mode.  Serialise db locally, pipe through o rshd deserialise
        # process on remote machine.
        if [[ $partial = yes ]] ; then
            $IONICE $comdb2ar $COMDB2AR_AROPTS $aropts c $partialopts $srclrl 2>$arstatus | $lz4 stdin stdout | \
                $rsh $destmach "$lz4_rmt -d stdin stdout | $IONICE_RMT $comdb2ar_rmt $COMDB2AR_EXOPTS $exopts $outmode $destlrldir/$partiallrl " >$exstatus 2>&1
            rc=$?
        else
            $IONICE $comdb2ar $COMDB2AR_AROPTS $aropts c $srclrl 2>$arstatus | $lz4 stdin stdout | \
                $rsh $destmach "$lz4_rmt -d stdin stdout | $IONICE_RMT $comdb2ar_rmt $COMDB2AR_EXOPTS $exopts $outmode $destlrldir $destdbdir " >$exstatus 2>&1
            rc=$?
        fi

    elif [ $copy_mode = LOCAL ] ; then

        # Local mode, easy
        if [[ $partial = yes ]] ; then
            $IONICE $comdb2ar $COMDB2AR_AROPTS $aropts c $srclrl 2>$arstatus | \
                $IONICE $comdb2ar $COMDB2AR_EXOPTS $exopts $partialopts $outmode $destlrldir/$partiallrl >$exstatus 2>&1
            rc=$?
        else
            $IONICE $comdb2ar $COMDB2AR_AROPTS $aropts c $srclrl 2>$arstatus | \
                $IONICE $comdb2ar $COMDB2AR_EXOPTS $exopts $partialopts $outmode $destlrldir $destdbdir >$exstatus 2>&1
            rc=$?
        fi
    fi
    exit $rc
    ) >>$logfile 2>&1
fi
error=$?

printf "\n\nSerialise log from $srcmach\n_______________________________________\n\n" >>$logfile
cat $arstatus >>$logfile
printf "\n\nDeserialise log from $destmach\n_______________________________________\n\n" >>$logfile
cat $exstatus >>$logfile

if [ "$copycomdb2log" = "-" ] ; then
    cat $logfile
    rm -f "$logfile"
fi

# Check the status files for an error (last line would be Error)
if tail -1 $arstatus | grep -w Error >/dev/null ; then
    error=1
elif tail -1 $exstatus | grep -w Error >/dev/null ; then
    error=1
fi
rm -f $arstatus $exstatus

if [ $error != 0 ] ; then
    [ -f "$logfile" ] && tail "$logfile"
    die "Fatal error$logfiletext"
fi

exit 0