#!/bin/sh

##
## Initial and continous ZFS filesystems replication
##

PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin

if [ ! `whoami` = "root" ]; then
 echo "You need to be root."
 exit
fi


l_old="@local_replicate.base"
r_old="@remote_replicate.base"
l_new="@local_replicate.delta"
r_new="@remote_replicate.delta"
log="/var/log/replicate.log"
cmd=$(mktemp /tmp/replicate.job.XXXXXX)
ERR=$(mktemp /tmp/replicate.err.XXXXXX)
pid="/var/run/replicate.pid"
path="/usr/local/etc/replicatejobs/"
jobs=`find $path -type f ! -iname sample_* ! -iname noauto_* | sort -d`

##
## Mail settings
##
subject="A replication error has occurred!"
address="storageadm@slu.se"
mail=$(mktemp /tmp/replicate.mail.XXXXXX)

usage()
{
 echo ""
 echo "Usage:        `basename $0` [Options] [Flags] [Job]"
 echo ""
 echo "Options: -c|-h"
 echo "-c:           Clean."
 echo "              removes replicated filesystems and snapshot(s)."
 echo "-h:           Usage."
 echo ""
 echo "Flags: auto"
 echo "auto:         Clean auto."
 echo "              removes all scheduled snapshot(s)."
 echo ""
 echo "Examples:     `basename $0` | `basename $0` jobname"
 echo "              `basename $0` -c | `basename $0` -c jobname | `basename $0` -c auto"
 echo ""
}

l_base()
 {
  ##
  ## Makes local baseline replication
  ##
  
  if zpool status `echo $lfs | cut -f1 -d /` | grep "scan:" | egrep -qo "(scrub in progress|resilver in progress)"; then
   echo "`date`: A Scrub or Resilver is currently in progress on source pool, aborting." >> $log
   echo "" >> $log
   rm ${pid}
   exit 1
  elif zpool status $lp | grep "scan:" | egrep -qo "(scrub in progress|resilver in progress)"; then
   echo "`date`: A Scrub or Resilver is currently in progress on destination pool, aborting." >> $log
   echo "" >> $log
   rm ${pid}
   exit 1
  fi

  if [ "$r" = "yes" ]
   then
    SNAPSHOT="zfs snapshot -r"
    SEND="zfs send -R"
   else
    SNAPSHOT="zfs snapshot"
    SEND="zfs send -p"
  fi

  echo "echo \"\`date\`: Beginning local baseline replication sequence on \\\"$lfs\\\"\" >> ${log}" >> ${cmd}

  # Take initial snapshot(s):
   echo "sudo ${SNAPSHOT} $lfs$l_old 2>> ${ERR}" >> ${cmd}
   echo errorcheck >> ${cmd}
  echo "echo \"\`date\`: Initial snapshot(s) created\" >> $log" >> ${cmd}

  # Replicate data:
   echo "sudo ${SEND} $lfs$l_old 2>> ${ERR} | sudo zfs recv -du $lp 2>> ${ERR}" >> ${cmd}
   echo errorcheck >> ${cmd}
  echo "echo \"\`date\`: Data replicated\" >> $log" >> ${cmd}

  echo "echo \"\`date\`: Local baseline replication sequence finished on \\\"$lfs\\\"\" >> ${log}" >> ${cmd}
  echo "echo \"\" >> $log" >> ${cmd}
  echo "exit 0" >> ${cmd}
  echo "" >> ${cmd}
 }

r_base()
 {
  ##
  ## Makes remote baseline replication
  ##

  SSH="ssh"

  if [ "${compress}" = "yes" ]; then
    SSH="ssh -C"
  fi

  if [ ! -z ${port} ]; then
    SSH="${SSH} -p ${port}"
  else
    SSH="${SSH} -p 22"
  fi

  if [ `ping -c 1 -W 1000 $rh | grep -oe "[0-9] packets received" | awk '{print $1}'` -eq "0" ]; then
   echo "`date`: Remote baseline replication sequence aborted on \"$lfs\"! No response from \"$rh\"." >> $log
   echo "" >> $log
   rm ${pid}
   exit 1
  elif zpool status `echo $lfs | cut -f1 -d /` | grep "scan:" | egrep -qo "(scrub in progress|resilver in progress)"; then
   echo "`date`: A Scrub or Resilver is currently in progress on source pool, aborting." >> $log
   echo "" >> $log
   rm ${pid}
   exit 1
  elif su replicator -c "${SSH} $rh zpool status $rp | grep \"scan:\" | egrep -qo '(scrub in progress|resilver in progress)'"; then
   echo "`date`: A Scrub or Resilver is currently in progress on remote pool, aborting." >> $log
   echo "" >> $log
   rm ${pid}
   exit 1
  fi

  if [ "$r" = "yes" ]
   then
    SNAPSHOT="zfs snapshot -r"
    SEND="zfs send -R"
   else
    SNAPSHOT="zfs snapshot"
    SEND="zfs send -p"
  fi

  echo "echo \"\`date\`: Beginning remote baseline replication sequence on \\\"$lfs\\\"\" >> ${log}" >> ${cmd}

  # Take initial snapshot(s):
   echo "sudo ${SNAPSHOT} $lfs$r_old 2>> ${ERR}" >> ${cmd}
   echo errorcheck >> ${cmd}
  echo "echo \"\`date\`: Initial snapshot(s) created\" >> $log" >> ${cmd}

  # Replicate data:
   echo "sudo ${SEND} $lfs$r_old 2>> ${ERR} | ${SSH} $rh sudo zfs recv -du $rp/$lh 2>> ${ERR}" >> ${cmd}
   echo errorcheck >> ${cmd}
  echo "echo \"\`date\`: Data replicated\" >> $log" >> ${cmd}

  echo "echo \"\`date\`: Remote baseline replication sequence finished on \\\"$lfs\\\"\" >> ${log}" >> ${cmd}
  echo "echo \"\" >> $log" >> ${cmd}
  echo "exit 0" >> ${cmd}
  echo "" >> ${cmd}
 }

l_inc()
 {
  ## 
  ## Makes local incremental replication
  ##

  if zpool status `echo $lfs | cut -f1 -d /` | grep "scan:" | egrep -qo "(scrub in progress|resilver in progress)"; then
   echo "`date`: A Scrub or Resilver is currently in progress on source pool, aborting." >> $log
   echo "" >> $log
   rm ${pid}
   exit 1
  elif zpool status $lp | grep "scan:" | egrep -qo "(scrub in progress|resilver in progress)"; then
   echo "`date`: A Scrub or Resilver is currently in progress on destination pool, aborting." >> $log
   echo "" >> $log
   rm ${pid}
   exit 1
  fi

  tfs=`echo $lfs | cut -f 2-512 -d / | sed "s/^/$lp\//"`

  echo "`date`: Beginning local incremental replication sequence on \"$lfs\"" >> $log

  # Take new source snapshot(s):  

  if [ "$r" = "yes" ]
   then
    SOURCEBASE=`zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep replicate.base | wc -l | sed 's/^[ \t]*//'`
    SOURCEDELTA=`zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep replicate.delta | wc -l | sed 's/^[ \t]*//'`
    DESTBASE=`zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep replicate.base | wc -l | sed 's/^[ \t]*//'`
    DESTDELTA=`zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep replicate.delta | wc -l | sed 's/^[ \t]*//'`
    SNAPSHOT="zfs snapshot -r"
    SEND="zfs send -R"
    DESTROY="zfs destroy -r"
    RENAME="zfs rename -r"
   else
    SOURCEBASE=`zfs list -H -t snapshot -o name $lfs$r_old 2>/dev/null | wc -l | sed 's/^[ \t]*//'`
    SOURCEDELTA=`zfs list -H -t snapshot -o name $lfs$r_new 2>/dev/null | wc -l | sed 's/^[ \t]*//'`
    DESTBASE=`zfs list -H -t snapshot -o name $tfs$r_old 2>/dev/null | wc -l | sed 's/^[ \t]*//'`
    DESTDELTA=`zfs list -H -t snapshot -o name $tfs$r_new 2>/dev/null | wc -l | sed 's/^[ \t]*//'`
    SNAPSHOT="zfs snapshot"
    SEND="zfs send -p"
    DESTROY="zfs destroy"
    RENAME="zfs rename"
  fi

  rebaseline()
   {
    echo "echo \"\`date\`: No appropriate correctional steps where found, have to rebaseline\" >> $log" >> ${cmd}
    if [ ${SOURCEBASE} -gt "0" ]; then
     echo "sudo ${DESTROY} $lfs$l_old 2>> ${ERR}" >> ${cmd}
     echo errorcheck >> ${cmd}
     echo "echo \"\`date\`: Local base snapshot destroyed\" >> $log" >> ${cmd}
    elif [ ${SOURCEDELTA} -gt "0" ]; then
     echo "sudo ${DESTROY} $lfs$l_new 2>> ${ERR}" >> ${cmd}
     echo errorcheck >> ${cmd}
     echo "echo \"\`date\`: Local delta snapshot destroyed\" >> $log" >> ${cmd}
    elif [ ${DESTBASE} -gt "0" ]; then
     echo "sudo ${DESTROY} $tfs$l_old 2>> ${ERR}" >> ${cmd}
     echo errorcheck >> ${cmd}
     echo "echo \"\`date\`: Target base snapshot destroyed\" >> $log" >> ${cmd}
    elif [ ${DESTDELTA} -gt "0" ]; then
     echo "sudo ${DESTROY} $tfs$l_new 2>> ${ERR}" >> ${cmd}
     echo errorcheck >> ${cmd}
     echo "echo \"\`date\`: Target delta snapshot destroyed\" >> $log" >> ${cmd}
    elif [ $(zfs list -H -o name -r $tfs 2>/dev/null | awk 'END{print NR}') -gt "0" ]; then
     echo "sudo ${DESTROY} $tfs 2>> ${ERR}" >> ${cmd}
     echo errorcheck >> ${cmd}
     echo "echo \"\`date\`: Target filesystem destroyed\" >> $log" >> ${cmd}
    fi
    echo "sudo ${SNAPSHOT} $lfs$l_old 2>> ${ERR}" >> ${cmd}
    echo errorcheck >> ${cmd}
    echo "echo \"\`date\`: Base snapshot(s) created\" >> $log" >> ${cmd}
    echo "sudo ${SEND} $lfs$l_old 2>> ${ERR} | sudo zfs recv -du $lp 2>> ${ERR}" >> ${cmd}
    echo errorcheck >> ${cmd}
    echo "echo \"\`date\`: Data replicated\" >> $log" >> ${cmd}
    echo "echo \"\`date\`: Rebaseline complete\" >> $log" >> ${cmd}
    echo "sudo ${SNAPSHOT} $lfs$l_new 2>> ${ERR}" >> ${cmd}
    echo errorcheck >> ${cmd}
    echo "echo \"\`date\`: Delta snapshot(s) created\" >> $log" >> ${cmd}
   }

  dryrun()
   {
    DRYRUNOUT=$(mktemp /tmp/replicate.dryrun.XXXXXX)

    if [ "$r" = "yes" ]
     then
      DODRYRUN=`zfs send -vRi $lfs$l_old $lfs$l_new 2> ${DRYRUNOUT} | zfs recv -vdn $lp 2>> ${DRYRUNOUT}`
     else
      DODRYRUN=`zfs send -vi $lfs$l_old $lfs$l_new 2> ${DRYRUNOUT} | zfs recv -vdn $lp 2>> ${DRYRUNOUT}`
    fi

    DRYRUNOK=`egrep '(could not send|cannot receive)' ${DRYRUNOUT} | wc -l | sed 's/^[ \t]*//'`
    rm ${DRYRUNOUT}; echo ${DRYRUNOK}
   }

  if [ ${SOURCEBASE} -gt "0" ]
   then
    if [ ${SOURCEDELTA} -gt "0" ]
     then
      if [ ${DESTBASE} -gt "0" ]
       then
        if [ ${DESTDELTA} -gt "0" ]
         then
          if [ ${DESTDELTA} = ${DESTBASE} ]
           then
            echo "`date`: Probably interrupted while destroying target .base(s)" >> $log
            ${DESTROY} $tfs$l_old 2>> ${ERR}
            errorcheck
            echo "`date`: Target .base snapshot(s) destroyed" >> $log
            ${RENAME} $tfs$l_new $tfs$l_old 2>> ${ERR}
            errorcheck
            echo "`date`: Target .delta snapshot(s) renamed .base" >> $log
            ${DESTROY} $lfs$l_old 2>> ${ERR}
            errorcheck
            echo "`date`: Source .base snapshot(s) destroyed" >> $log
            ${RENAME} $lfs$l_new $lfs$l_old 2>> ${ERR}
            errorcheck
            echo "`date`: Source .delta snapshot(s) renamed .base" >> $log
            ${SNAPSHOT} $lfs$l_new 2>> ${ERR}
            errorcheck
            echo "`date`: New source .delta snapshot(s) created, proceeding" >> $log
            if [ `dryrun` -gt "0" ]
             then
              echo "`date`: Dry run unsuccessful, rebaselining" >> $log
              rebaseline
             else
              echo "`date`: Dry run successful, OK to resend" >> $log
            fi            
           else
            rebaseline
          fi
         else
          if [ `dryrun` -gt "0" ]
           then
            echo "`date`: Dry run unsuccessful" >> $log
            echo "`date`: Probably interrupted while destroying source .base(s)" >> $log
            ${DESTROY} $lfs$l_old 2>> ${ERR}
            errorcheck
            echo "`date`: Source .base snapshot(s) destroyed" >> $log
            ${RENAME} $lfs$l_new $lfs$l_old 2>> ${ERR}
            errorcheck
            echo "`date`: Source .delta snapshot(s) renamed .base" >> $log
            ${SNAPSHOT} $lfs$l_new 2>> ${ERR}
            errorcheck
            echo "`date`: New source .delta snapshot(s) created, proceeding" >> $log
            if [ `dryrun` -gt "0" ]
             then
              echo "`date`: Dry run unsuccessful, rebaselining" >> $log
              rebaseline
             else
              echo "`date`: Dry run successful, OK to resend" >> $log
            fi
           else
            echo "`date`: Dry run successful" >> $log
            echo "`date`: Probably interrupted while transferring, resending" >> $log
          fi          
        fi
       else
        if [ ${DESTDELTA} -gt "0" ]
         then
          if [ ${SOURCEDELTA} = ${DESTDELTA} ]
           then
            echo "`date`: Probably interrupted while renaming target .delta to .base" >> $log
            ${RENAME} $tfs$l_new $tfs$l_old 2>> ${ERR}
            errorcheck
            echo "`date`: Target .delta snapshot(s) renamed .base" >> $log
            ${DESTROY} $lfs$l_old 2>> ${ERR}
            errorcheck
            echo "`date`: Source .base snapshot(s) destroyed" >> $log
            ${RENAME} $lfs$l_new $lfs$l_old 2>> ${ERR}
            errorcheck
            echo "`date`: Source .delta snapshot(s) renamed .base" >> $log
            ${SNAPSHOT} $lfs$l_new 2>> ${ERR}
            errorcheck
            echo "`date`: New source .delta snapshot(s) created, proceeding" >> $log
            if [ `dryrun` -gt "0" ]
             then
              echo "`date`: Dry run unsuccessful, rebaselining" >> $log
              rebaseline
             else
              echo "`date`: Dry run successful, OK to resend" >> $log
            fi
           else
            rebaseline
          fi
         else
          rebaseline
        fi
      fi
     else
      if [ ${DESTBASE} -gt "0" ]
       then
        if [ ${DESTDELTA} -gt "0" ]
         then
          ${DESTROY} $tfs$l_new 2>> ${ERR}
          errorcheck
          echo "`date`: Target .delta snapshot(s) found and destroyed" >> $log
          ${SNAPSHOT} $lfs$l_new 2>> ${ERR}
          errorcheck
          echo "`date`: New source .delta snapshot(s) created, proceeding" >> $log
         else
          ${SNAPSHOT} $lfs$l_new 2>> ${ERR}
          errorcheck
          echo "`date`: New source .delta snapshot(s) created, proceeding" >> $log
          if [ `dryrun` -gt "0" ]
           then
           echo "`date`: Dry run unsuccessful, rebaselining" >> $log
           rebaseline
          else
           echo "`date`: Dry run successful, OK to resend" >> $log
          fi
        fi
       else
        if [ ${DESTDELTA} -gt "0" ]
         then
          if [ ${SOURCEBASE} = ${DESTDELTA} ]
           then
            echo "`date`: No target .base found, but source .base matches target .delta. Renaming target .delta to .base" >> $log
            ${RENAME} $tfs$l_new $tfs$l_old 2>> ${ERR}
            errorcheck
            echo "`date`: Target .delta snapshot(s) renamed .base" >> $log
            ${SNAPSHOT} $lfs$l_new 2>> ${ERR}
            errorcheck
            echo "`date`: New source .delta snapshot(s) created, proceeding" >> $log
             if [ `dryrun` -gt "0" ]
              then
              echo "`date`: Dry run unsuccessful, rebaselining" >> $log
              rebaseline
             else
              echo "`date`: Dry run successful, OK to resend" >> $log
             fi
           else
            rebaseline
          fi
         else
          rebaseline
        fi
      fi
    fi
   else
    if [ ${SOURCEDELTA} -gt "0" ]
     then
      if [ ${DESTBASE} -gt "0" ]
       then
        if [ ${DESTDELTA} -gt "0" ]
         then
          if [ ${SOURCEDELTA} = ${DESTDELTA} ]
           then
            echo "`date`: Probably interrupted while destroying target .base(s)" >> $log
            ${DESTROY} $tfs$l_old 2>> ${ERR}
            errorcheck
            echo "`date`: Target .base snapshot(s) destroyed" >> $log
            ${RENAME} $tfs$l_new $tfs$l_old 2>> ${ERR}
            errorcheck
            echo "`date`: Target .delta snapshot(s) renamed .base" >> $log
            ${RENAME} $lfs$l_new $lfs$l_old 2>> ${ERR}
            errorcheck
            echo "`date`: Source .delta snapshot(s) renamed .base" >> $log
            ${SNAPSHOT} $lfs$l_new 2>> ${ERR}
            errorcheck
            echo "`date`: New source .delta snapshot(s) created, proceeding" >> $log
            if [ `dryrun` -gt "0" ]
             then
              echo "`date`: Dry run unsuccessful, rebaselining" >> $log
              rebaseline
             else
              echo "`date`: Dry run successful, OK to resend" >> $log
            fi
           else
            rebaseline
          fi
         else
          if [ ${SOURCEDELTA} = ${DESTBASE} ]
           then
            echo "`date`: Probably interrupted while renaming source .delta to .base" >> $log
            ${RENAME} $lfs$l_new $lfs$l_old 2>> ${ERR}
            errorcheck
            echo "`date`: Source .delta snapshot(s) renamed .base" >> $log
            ${SNAPSHOT} $lfs$l_new 2>> ${ERR}
            errorcheck
            echo "`date`: New source .delta snapshot(s) created, proceeding" >> $log
            if [ `dryrun` -gt "0" ]
             then
              echo "`date`: Dry run unsuccessful, rebaselining" >> $log
              rebaseline
             else
              echo "`date`: Dry run successful, OK to resend" >> $log
            fi
           else
            rebaseline
          fi
        fi
       else
        if [ ${DESTDELTA} -gt "0" ]
         then
          if [ ${SOURCEDELTA} = ${DESTDELTA} ]
           then
            echo "`date`: No source or target .base, but source and target .delta matches. Can try to rename them to .base and test a resend"
            ${RENAME} $tfs$l_new $tfs$l_old 2>> ${ERR}
            errorcheck
            echo "`date`: Target .delta snapshot(s) renamed .base" >> $log
            ${RENAME} $lfs$l_new $lfs$l_old 2>> ${ERR}
            errorcheck
            echo "`date`: Source .delta snapshot(s) renamed .base" >> $log
            ${SNAPSHOT} $lfs$l_new 2>> ${ERR}
            errorcheck
            echo "`date`: New source .delta snapshot(s) created, proceeding" >> $log
            if [ `dryrun` -gt "0" ]
             then
              echo "`date`: Dry run unsuccessful, rebaselining" >> $log
              rebaseline
             else
              echo "`date`: Dry run successful, OK to resend" >> $log
            fi
           else
            rebaseline
          fi
         else
          rebaseline
        fi
      fi
     else
      if [ ${DESTBASE} -gt "0" ]
       then
        if [ ${DESTDELTA} -gt "0" ]
         then
          rebaseline
         else
          rebaseline
        fi
       else
        if [ ${DESTDELTA} -gt "0" ]
         then
          rebaseline
         else
          rebaseline
        fi
      fi
    fi
  fi

  # Replicate data:
   echo "sudo ${SEND} -i $lfs$l_old $lfs$l_new 2>> ${ERR} | sudo zfs recv -du $lp 2>> ${ERR}" >> ${cmd}
   echo errorcheck >> ${cmd}
  echo "echo \"\`date\`: Data replicated\" >> $log" >> ${cmd}

  # Destroy target .base snapshot(s):
  if [ "$r" = "yes" ]
   then
    if [ `zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep replicate.base | wc -l | sed 's/^[ \t]*//'` -gt "0" ]; then
     echo "sudo ${DESTROY} $tfs$l_old 2>> ${ERR}" >> ${cmd}
     echo errorcheck >> ${cmd}
     echo "echo \"\`date\`: Target .base snapshot(s) destroyed\" >> $log" >> ${cmd}
    fi
   else
    if [ `zfs list -H -t snapshot -o name $tfs$l_old 2>/dev/null | wc -l | sed 's/^[ \t]*//'` -gt "0" ]; then
     echo "sudo ${DESTROY} $tfs$l_old 2>> ${ERR}" >> ${cmd}
     echo errorcheck >> ${cmd}
     echo "echo \"\`date\`: Target .base snapshot(s) destroyed\" >> $log" >> ${cmd}
    fi
  fi

  # Rename target .delta snapshot(s) .old:
   echo "sudo ${RENAME} $tfs$l_new $tfs$l_old 2>> ${ERR}" >> ${cmd}
   echo errorcheck >> ${cmd}
  echo "echo \"\`date\`: Target .delta snapshot(s) renamed .base\" >> $log" >> ${cmd}

  # Destroy source .base snapshot(s):
   echo "sudo ${DESTROY} $lfs$l_old 2>> ${ERR}" >> ${cmd}
   echo errorcheck >> ${cmd}
  echo "echo \"\`date\`: Source .base snapshot(s) destroyed\" >> $log" >> ${cmd}

  # Rename source .delta snapshot(s) .base:
   echo "sudo ${RENAME} $lfs$l_new $lfs$l_old 2>> ${ERR}" >> ${cmd}
   echo errorcheck >> ${cmd}
  echo "echo \"\`date\`: Source .delta snapshot(s) renamed .base\" >> $log" >> ${cmd}

  echo "echo \"\`date\`: Local incremental replication sequence finished on \\\"$lfs\\\"\" >> ${log}" >> ${cmd}
  echo "echo \"\" >> $log" >> ${cmd}
  echo "exit 0" >> ${cmd}
  echo "" >> ${cmd}
 }

r_inc()
 {
  ##
  ## Makes remote incremental replication
  ##

  SSH="ssh"

  if [ "${compress}" = "yes" ]; then
    SSH="ssh -C"
  fi

  if [ ! -z ${port} ]; then
    SSH="${SSH} -p ${port}"
  else
    SSH="${SSH} -p 22"
  fi

  if [ `ping -c 1 -W 1000 $rh | grep -oe "[0-9] packets received" | awk '{print $1}'` -eq "0" ]; then
   echo "`date`: Remote incremental replication sequence aborted on \"$lfs\"! No response from \"$rh\"." >> $log
   echo "" >> $log
   rm ${pid}
   exit 1
  fi
  if zpool status `echo $lfs | cut -f1 -d /` | grep "scan:" | egrep -qo "(scrub in progress|resilver in progress)"; then
   echo "`date`: A Scrub or Resilver is currently in progress on source pool, aborting." >> $log
   echo "" >> $log
   rm ${pid}
   exit 1
  elif su replicator -c "${SSH} $rh zpool status $rp | grep \"scan:\" | egrep -qo '(scrub in progress|resilver in progress)'"; then
   echo "`date`: A Scrub or Resilver is currently in progress on remote pool, aborting." >> $log
   echo "" >> $log
   rm ${pid}
   exit 1
  fi

  tfs=`echo $lfs | cut -f 2-512 -d / | sed "s/^/$rp\/$lh\//"`

  echo "`date`: Beginning remote incremental replication sequence on \"$lfs\"" >> $log

  # Take new source snapshot(s):

  REPID="`awk -F':' '{print $6}' /etc/passwd | grep replicator`/.ssh/id_rsa"

  if [ "$r" = "yes" ]
   then
    SOURCEBASE=`zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep replicate.base | wc -l | sed 's/^[ \t]*//'`
    SOURCEDELTA=`zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep replicate.delta | wc -l | sed 's/^[ \t]*//'`
    DESTBASE=`su replicator -c "${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep replicate.base | wc -l | sed 's/^[ \t]*//'"`
    DESTDELTA=`su replicator -c "${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep replicate.delta | wc -l | sed 's/^[ \t]*//'"`
    SNAPSHOT="zfs snapshot -r"
    SEND="zfs send -R"
    DESTROY="zfs destroy -r"
    RENAME="zfs rename -r"
   else
    SOURCEBASE=`zfs list -H -t snapshot -o name $lfs$r_old 2>/dev/null | wc -l | sed 's/^[ \t]*//'`
    SOURCEDELTA=`zfs list -H -t snapshot -o name $lfs$r_new 2>/dev/null | wc -l | sed 's/^[ \t]*//'`
    DESTBASE=`su replicator -c "${SSH} $rh zfs list -H -t snapshot -o name $tfs$r_old 2>/dev/null | wc -l | sed 's/^[ \t]*//'"`
    DESTDELTA=`su replicator -c "${SSH} $rh zfs list -H -t snapshot -o name $tfs$r_new 2>/dev/null | wc -l | sed 's/^[ \t]*//'"`
    SNAPSHOT="zfs snapshot"
    SEND="zfs send -p"
    DESTROY="zfs destroy"
    RENAME="zfs rename"
  fi

  rebaseline()
   {
    echo "echo \"\`date\`: No appropriate correctional steps where found, have to rebaseline\" >> $log" >> ${cmd}
    if [ ${SOURCEBASE} -gt "0" ]; then
     echo "sudo ${DESTROY} $lfs$r_old 2>> ${ERR}" >> ${cmd}
     echo errorcheck >> ${cmd}
     echo "echo \"\`date\`: Local base snapshot destroyed\" >> $log" >> ${cmd}
    fi
    if [ ${SOURCEDELTA} -gt "0" ]; then
     echo "sudo ${DESTROY} $lfs$r_new 2>> ${ERR}" >> ${cmd}
     echo errorcheck >> ${cmd}
     echo "echo \"\`date\`: Local delta snapshot destroyed\" >> $log" >> ${cmd}
    fi
    if [ ${DESTBASE} -gt "0" ]; then
     echo "${SSH} $rh sudo ${DESTROY} $tfs$r_old 2>> ${ERR}" >> ${cmd}
     echo errorcheck >> ${cmd}
     echo "echo \"\`date\`: Target base snapshot destroyed\" >> $log" >> ${cmd}
    fi
    if [ ${DESTDELTA} -gt "0" ]; then
     echo "${SSH} $rh sudo ${DESTROY} $tfs$r_new 2>> ${ERR}" >> ${cmd}
     echo errorcheck >> ${cmd}
     echo "echo \"\`date\`: Target delta snapshot destroyed\" >> $log" >> ${cmd}
    fi
    if [ $(su replicator -c "${SSH} $rh zfs list -H -o name -r $tfs 2>/dev/null | awk 'END{print NR}'") -gt "0" ]; then
     echo "${SSH} $rh sudo ${DESTROY} $tfs 2>> ${ERR}" >> ${cmd}
     echo errorcheck >> ${cmd}
     echo "echo \"\`date\`: Target filesystem destroyed\" >> $log" >> ${cmd}
    fi
    echo "sudo ${SNAPSHOT} $lfs$r_old 2>> ${ERR}" >> ${cmd}
    echo errorcheck >> ${cmd}
    echo "echo \"\`date\`: Base snapshot(s) created\" >> $log" >> ${cmd}
    echo "sudo ${SEND} $lfs$r_old 2>> ${ERR} | ${SSH} $rh sudo zfs recv -du $rp/$lh 2>> ${ERR}" >> ${cmd}
    echo errorcheck >> ${cmd}
    echo "echo \"\`date\`: Data replicated\" >> $log" >> ${cmd}
    echo "echo \"\`date\`: Rebaseline complete\" >> $log" >> ${cmd}
    echo "sudo ${SNAPSHOT} $lfs$r_new 2>> ${ERR}" >> ${cmd}
    echo errorcheck >> ${cmd}
    echo "echo \"\`date\`: Delta snapshot(s) created\" >> $log" >> ${cmd}
   }

  dryrun()
   {
    DRYRUNOUT=$(mktemp /tmp/replicate.dryrun.XXXXXX)

    if [ "$r" = "yes" ]
     then
      DODRYRUN=`zfs send -vRi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}`
     else
      DODRYRUN=`zfs send -vi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}`
    fi

    DRYRUNOK=`egrep '(could not send|cannot receive)' ${DRYRUNOUT} | wc -l | sed 's/^[ \t]*//'`
    rm ${DRYRUNOUT}; echo ${DRYRUNOK}
   }

  if [ ${SOURCEBASE} -gt "0" ]
   then
    if [ ${SOURCEDELTA} -gt "0" ]
     then
      if [ ${DESTBASE} -gt "0" ]
       then
        if [ ${DESTDELTA} -gt "0" ]
         then
          if [ ${DESTDELTA} = ${DESTBASE} ]
           then
            echo "`date`: Probably interrupted while destroying target .base(s)" >> $log
            ${SSH} -i ${REPID} replicator@$rh sudo ${DESTROY} $tfs$r_old 2>> ${ERR}
            errorcheck
            echo "`date`: Target .base snapshot(s) destroyed" >> $log
            ${SSH} -i ${REPID} replicator@$rh sudo ${RENAME} $tfs$r_new $tfs$r_old 2>> ${ERR}
            errorcheck
            echo "`date`: Target .delta snapshot(s) renamed .base" >> $log
            ${DESTROY} $lfs$r_old 2>> ${ERR}
            errorcheck
            echo "`date`: Source .base snapshot(s) destroyed" >> $log
            ${RENAME} $lfs$r_new $lfs$r_old 2>> ${ERR}
            errorcheck
            echo "`date`: Source .delta snapshot(s) renamed .base" >> $log
            ${SNAPSHOT} $lfs$r_new 2>> ${ERR}
            errorcheck
            echo "`date`: New source .delta snapshot(s) created, proceeding" >> $log
            if [ `dryrun` -gt "0" ]
             then
              echo "`date`: Dry run unsuccessful, rebaselining" >> $log
              rebaseline
             else
              echo "`date`: Dry run successful, OK to resend" >> $log
            fi            
           else
            rebaseline
          fi
         else
          if [ `dryrun` -gt "0" ]
           then
            echo "`date`: Dry run unsuccessful" >> $log
            echo "`date`: Probably interrupted while destroying source .base(s)" >> $log
            ${DESTROY} $lfs$r_old 2>> ${ERR}
            errorcheck
            echo "`date`: Source .base snapshot(s) destroyed" >> $log
            ${RENAME} $lfs$r_new $lfs$r_old 2>> ${ERR}
            errorcheck
            echo "`date`: Source .delta snapshot(s) renamed .base" >> $log
            ${SNAPSHOT} $lfs$r_new 2>> ${ERR}
            errorcheck
            echo "`date`: New source .delta snapshot(s) created, proceeding" >> $log
            if [ `dryrun` -gt "0" ]
             then
              echo "`date`: Dry run unsuccessful, rebaselining" >> $log
              rebaseline
             else
              echo "`date`: Dry run successful, OK to resend" >> $log
            fi
           else
            echo "`date`: Dry run successful" >> $log
            echo "`date`: Probably interrupted while transferring, resending" >> $log
          fi          
        fi
       else
        if [ ${DESTDELTA} -gt "0" ]
         then
          if [ ${SOURCEDELTA} = ${DESTDELTA} ]
           then
            echo "`date`: Probably interrupted while renaming target .delta to .base" >> $log
            ${SSH} -i ${REPID} replicator@$rh sudo ${RENAME} $tfs$r_new $tfs$r_old 2>> ${ERR}
            errorcheck
            echo "`date`: Target .delta snapshot(s) renamed .base" >> $log
            ${DESTROY} $lfs$r_old 2>> ${ERR}
            errorcheck
            echo "`date`: Source .base snapshot(s) destroyed" >> $log
            ${RENAME} $lfs$r_new $lfs$r_old 2>> ${ERR}
            errorcheck
            echo "`date`: Source .delta snapshot(s) renamed .base" >> $log
            ${SNAPSHOT} $lfs$r_new 2>> ${ERR}
            errorcheck
            echo "`date`: New source .delta snapshot(s) created, proceeding" >> $log
            if [ `dryrun` -gt "0" ]
             then
              echo "`date`: Dry run unsuccessful, rebaselining" >> $log
              rebaseline
             else
              echo "`date`: Dry run successful, OK to resend" >> $log
            fi
           else
            rebaseline
          fi
         else
          rebaseline
        fi
      fi
     else
      if [ ${DESTBASE} -gt "0" ]
       then
        if [ ${DESTDELTA} -gt "0" ]
         then
          ${SSH} -i ${REPID} replicator@$rh sudo ${DESTROY} $tfs$r_new 2>> ${ERR}
          errorcheck
          echo "`date`: Target .delta snapshot(s) found and destroyed" >> $log
          ${SNAPSHOT} $lfs$r_new 2>> ${ERR}
          errorcheck
          echo "`date`: New source .delta snapshot(s) created, proceeding" >> $log
         else
          ${SNAPSHOT} $lfs$r_new 2>> ${ERR}
          errorcheck
          echo "`date`: New source .delta snapshot(s) created, proceeding" >> $log
          if [ `dryrun` -gt "0" ]
           then
           echo "`date`: Dry run unsuccessful, rebaselining" >> $log
           rebaseline
          else
           echo "`date`: Dry run successful, OK to resend" >> $log
          fi
        fi
       else
        if [ ${DESTDELTA} -gt "0" ]
         then
          if [ ${SOURCEBASE} = ${DESTDELTA} ]
           then
            echo "`date`: No target .base found, but source .base matches target delta. Renaming target .delta to .base" >> $log
            ${SSH} -i ${REPID} replicator@$rh sudo ${RENAME} $tfs$r_new $tfs$r_old 2>> ${ERR}
            errorcheck
            echo "`date`: Target .delta snapshot(s) renamed .base" >> $log
            ${SNAPSHOT} $lfs$r_new 2>> ${ERR}
            errorcheck
            echo "`date`: New source .delta snapshot(s) created, proceeding" >> $log
             if [ `dryrun` -gt "0" ]
              then
              echo "`date`: Dry run unsuccessful, rebaselining" >> $log
              rebaseline
             else
              echo "`date`: Dry run successful, OK to resend" >> $log
             fi
           else
            rebaseline
          fi
         else
          rebaseline
        fi
      fi
    fi
   else
    if [ ${SOURCEDELTA} -gt "0" ]
     then
      if [ ${DESTBASE} -gt "0" ]
       then
        if [ ${DESTDELTA} -gt "0" ]
         then
          if [ ${SOURCEDELTA} = ${DESTDELTA} ]
           then
            echo "`date`: Probably interrupted while destroying target .base(s)" >> $log
            ${SSH} -i ${REPID} replicator@$rh sudo ${DESTROY} $tfs$r_old 2>> ${ERR}
            errorcheck
            echo "`date`: Target .base snapshot(s) destroyed" >> $log
            ${SSH} -i ${REPID} replicator@$rh sudo ${RENAME} $tfs$r_new $tfs$r_old 2>> ${ERR}
            errorcheck
            echo "`date`: Target .delta snapshot(s) renamed .base" >> $log
            ${RENAME} $lfs$r_new $lfs$r_old 2>> ${ERR}
            errorcheck
            echo "`date`: Source .delta snapshot(s) renamed .base" >> $log
            ${SNAPSHOT} $lfs$r_new 2>> ${ERR}
            errorcheck
            echo "`date`: New source .delta snapshot(s) created, proceeding" >> $log
            if [ `dryrun` -gt "0" ]
             then
              echo "`date`: Dry run unsuccessful, rebaselining" >> $log
              rebaseline
             else
              echo "`date`: Dry run successful, OK to resend" >> $log
            fi
           else
            rebaseline
          fi
         else
          if [ ${SOURCEDELTA} = ${DESTBASE} ]
           then
            echo "`date`: Probably interrupted while renaming source .delta to .base" >> $log
            ${RENAME} $lfs$r_new $lfs$r_old 2>> ${ERR}
            errorcheck
            echo "`date`: Source .delta snapshot(s) renamed .base" >> $log
            ${SNAPSHOT} $lfs$r_new 2>> ${ERR}
            errorcheck
            echo "`date`: New source .delta snapshot(s) created, proceeding" >> $log
            if [ `dryrun` -gt "0" ]
             then
              echo "`date`: Dry run unsuccessful, rebaselining" >> $log
              rebaseline
             else
              echo "`date`: Dry run successful, OK to resend" >> $log
            fi
           else
            rebaseline
          fi
        fi
       else
        if [ ${DESTDELTA} -gt "0" ]
         then
          if [ ${SOURCEDELTA} = ${DESTDELTA} ]
           then
            echo "`date`: No source or target base, but source and target delta matches. Can try to rename them to base and test a resend"
            ${SSH} -i ${REPID} replicator@$rh sudo ${RENAME} $tfs$r_new $tfs$r_old 2>> ${ERR}
            errorcheck
            echo "`date`: Target .delta snapshot(s) renamed .base" >> $log
            ${RENAME} $lfs$r_new $lfs$r_old 2>> ${ERR}
            errorcheck
            echo "`date`: Source .delta snapshot(s) renamed .base" >> $log
            ${SNAPSHOT} $lfs$r_new 2>> ${ERR}
            errorcheck
            echo "`date`: New source .delta snapshot(s) created, proceeding" >> $log
            if [ `dryrun` -gt "0" ]
             then
              echo "`date`: Dry run unsuccessful, rebaselining" >> $log
              rebaseline
             else
              echo "`date`: Dry run successful, OK to resend" >> $log
            fi
           else
            rebaseline
          fi
         else
          rebaseline
        fi
      fi
     else
      if [ ${DESTBASE} -gt "0" ]
       then
        if [ ${DESTDELTA} -gt "0" ]
         then
          rebaseline
         else
          rebaseline
        fi
       else
        if [ ${DESTDELTA} -gt "0" ]
         then
          rebaseline
         else
          rebaseline
        fi
      fi
    fi
  fi

  # Replicate data:
    echo "sudo ${SEND} -i $lfs$r_old $lfs$r_new 2>> ${ERR} | ${SSH} $rh sudo zfs recv -du $rp/$lh 2>> ${ERR}" >> ${cmd}
    echo errorcheck >> ${cmd}
  echo "echo \"\`date\`: Data replicated\" >> $log" >> ${cmd}

  # Destroy target .base snapshot(s):
  if [ $(su replicator -c "${SSH} $rh zfs list -H -t snapshot -o name -r $tfs 2>/dev/null | grep replicate.base | awk 'END{print NR}'") -gt "0" ]; then
    echo "${SSH} $rh sudo ${DESTROY} $tfs$r_old 2>> ${ERR}" >> ${cmd}
    echo errorcheck >> ${cmd}
    echo "echo \"\`date\`: Target .base snapshot(s) destroyed\" >> $log" >> ${cmd}
  fi

  # Rename target .delta snapshot(s) to .base:
    echo "${SSH} $rh sudo ${RENAME} $tfs$r_new $tfs$r_old 2>> ${ERR}" >> ${cmd}
    echo errorcheck >> ${cmd}
  echo "echo \"\`date\`: Target .delta snapshot(s) renamed .base\" >> $log" >> ${cmd}

  # Destroy source .base snapshot(s):
    echo "sudo ${DESTROY} $lfs$r_old 2>> ${ERR}" >> ${cmd}
    echo errorcheck >> ${cmd}
  echo "echo \"\`date\`: Source .base snapshot(s) destroyed\" >> $log" >> ${cmd}

  # Rename source .delta snapshot(s) to .base:
    echo "sudo ${RENAME} $lfs$r_new $lfs$r_old 2>> ${ERR}" >> ${cmd}
    echo errorcheck >> ${cmd}
  echo "echo \"\`date\`: Source .delta snapshot(s) renamed .base\" >> $log" >> ${cmd}

  echo "echo \"\`date\`: Remote incremental replication sequence finished on \\\"$lfs\\\"\" >> ${log}" >> ${cmd}
  echo "echo \"\" >> $log" >> ${cmd}
  echo "exit 0" >> ${cmd}
 }

l_clean()
 {
  ##
  ## Cleans locally replicated file system
  ##

  if zpool status `echo $lfs | cut -f1 -d /` | grep "scan:" | egrep -qo "(scrub in progress|resilver in progress)"; then
   echo "`date`: A Scrub or Resilver is currently in progress on source pool, aborting." >> $log
   echo "" >> $log
   rm ${pid}
   exit 1
    elif zpool status $lp | grep "scan:" | egrep -qo "(scrub in progress|resilver in progress)"; then
     echo "`date`: A Scrub or Resilver is currently in progress on destination pool, aborting." >> $log
     echo "" >> $log
     rm ${pid}
     exit 1
  fi

  tfs=`echo $lfs | cut -f 2-512 -d / | sed "s/^/$lp\//"`

  if [ "$r" = "yes" ]
   then
    SOURCEBASE=`zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep replicate.base | wc -l | sed 's/^[ \t]*//'`
    SOURCEDELTA=`zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep replicate.delta | wc -l | sed 's/^[ \t]*//'`
    DESTBASE=`zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep replicate.base | wc -l | sed 's/^[ \t]*//'`
    DESTDELTA=`zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep replicate.delta | wc -l | sed 's/^[ \t]*//'`
    SNAPSHOT="zfs snapshot -r"
    SEND="zfs send -R"
    DESTROY="zfs destroy -r"
    RENAME="zfs rename -r"
   else   
    SOURCEBASE=`zfs list -H -t snapshot -o name $lfs$r_old 2>/dev/null | wc -l | sed 's/^[ \t]*//'`
    SOURCEDELTA=`zfs list -H -t snapshot -o name $lfs$r_new 2>/dev/null | wc -l | sed 's/^[ \t]*//'`
    DESTBASE=`zfs list -H -t snapshot -o name $tfs$r_old 2>/dev/null | wc -l | sed 's/^[ \t]*//'`
    DESTDELTA=`zfs list -H -t snapshot -o name $tfs$r_new 2>/dev/null | wc -l | sed 's/^[ \t]*//'`
    SNAPSHOT="zfs snapshot"
    SEND="zfs send -p"
    DESTROY="zfs destroy"
    RENAME="zfs rename"
  fi

  echo "echo \"\`date\`: Beginning local cleaning process on \\\"$lfs\\\"\" >> ${log}" >> ${cmd}

  if [ ${SOURCEBASE} -gt "0" ]; then
   echo "sudo ${DESTROY} $lfs$l_old 2>> ${ERR}" >> ${cmd}
   echo errorcheck >> ${cmd}
   echo "echo \"\`date\`: Local base snapshot destroyed\" >> $log" >> ${cmd}
  fi
  if [ ${SOURCEDELTA} -gt "0" ]; then
   echo "sudo ${DESTROY} $lfs$l_new 2>> ${ERR}" >> ${cmd}
   echo errorcheck >> ${cmd}
   echo "echo \"\`date\`: Local delta snapshot destroyed\" >> $log" >> ${cmd}
  fi
  if [ ${DESTBASE} -gt "0" ]; then
   echo "sudo ${DESTROY} $tfs$l_old 2>> ${ERR}" >> ${cmd}
   echo errorcheck >> ${cmd}
   echo "echo \"\`date\`: Target base snapshot destroyed\" >> $log" >> ${cmd}
  fi
  if [ ${DESTDELTA} -gt "0" ]; then
   echo "sudo ${DESTROY} $tfs$l_new 2>> ${ERR}" >> ${cmd}
   echo errorcheck >> ${cmd}
   echo "echo \"\`date\`: Target delta snapshot destroyed\" >> $log" >> ${cmd}
  fi
  if [ $(zfs list -H -o name -r $tfs 2>/dev/null | awk 'END{print NR}') -gt "0" ]; then
   echo "sudo ${DESTROY} $tfs 2>> ${ERR}" >> ${cmd}
   echo errorcheck >> ${cmd}
   echo "echo \"\`date\`: Target filesystem destroyed\" >> $log" >> ${cmd}
  fi

  echo "echo \"\`date\`: Local cleanup complete on \\\"$lfs\\\"\" >> ${log}" >> ${cmd}
  echo "echo \"\" >> $log" >> ${cmd}
  echo "" >> ${cmd}
 }

r_clean()
 {
  ##
  ## Cleans remotely replicated file systems
  ##

  SSH="ssh"

  if [ "${compress}" = "yes" ]; then
    SSH="ssh -C"
  fi

  if [ ! -z ${port} ]; then
    SSH="${SSH} -p ${port}"
  else
    SSH="${SSH} -p 22"
  fi

  if [ `ping -c 1 -W 1000 $rh | grep -oe "[0-9] packets received" | awk '{print $1}'` -eq "0" ]; then
   echo "`date`: Remote cleanup aborted on \"$lfs\"! No response from \"$rh\"." >> $log
   echo "" >> $log
   rm ${pid}
   exit 1
  elif zpool status `echo $lfs | cut -f1 -d /` | grep "scan:" | egrep -qo "(scrub in progress|resilver in progress)"; then
   echo "`date`: A Scrub or Resilver is currently in progress on source pool, aborting." >> $log
   echo "" >> $log
   rm ${pid}
   exit 1
  elif su replicator -c "${SSH} $rh zpool status $rp | grep \"scan:\" | egrep -qo '(scrub in progress|resilver in progress)'"; then
   echo "`date`: A Scrub or Resilver is currently in progress on destination pool, aborting." >> $log
   echo "" >> $log
   rm ${pid}
   exit 1
  fi

  tfs=`echo $lfs | cut -f 2-512 -d / | sed "s/^/$rp\/$lh\//"`

  if [ "$r" = "yes" ]
   then
    SOURCEBASE=`zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep replicate.base | wc -l | sed 's/^[ \t]*//'`
    SOURCEDELTA=`zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep replicate.delta | wc -l | sed 's/^[ \t]*//'`
    DESTBASE=`su replicator -c "${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep replicate.base | wc -l | sed 's/^[ \t]*//'"`
    DESTDELTA=`su replicator -c "${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep replicate.delta | wc -l | sed 's/^[ \t]*//'"`
    SNAPSHOT="zfs snapshot -r"
    SEND="zfs send -R"
    DESTROY="zfs destroy -r"
    RENAME="zfs rename -r"
   else   
    SOURCEBASE=`zfs list -H -t snapshot -o name $lfs$r_old 2>/dev/null | wc -l | sed 's/^[ \t]*//'`
    SOURCEDELTA=`zfs list -H -t snapshot -o name $lfs$r_new 2>/dev/null | wc -l | sed 's/^[ \t]*//'`
    DESTBASE=`su replicator -c "${SSH} $rh zfs list -H -t snapshot -o name $tfs$r_old 2>/dev/null | wc -l | sed 's/^[ \t]*//'"`
    DESTDELTA=`su replicator -c "${SSH} $rh zfs list -H -t snapshot -o name $tfs$r_new 2>/dev/null | wc -l | sed 's/^[ \t]*//'"`
    SNAPSHOT="zfs snapshot"
    SEND="zfs send -p"
    DESTROY="zfs destroy"
    RENAME="zfs rename"
  fi

  echo "echo \"\`date\`: Beginning remote cleaning process on \\\"$lfs\\\"\" >> ${log}" >> ${cmd}

  if [ ${SOURCEBASE} -gt "0" ]; then
   echo "sudo ${DESTROY} $lfs$r_old 2>> ${ERR}" >> ${cmd}
   echo errorcheck >> ${cmd}
   echo "echo \"\`date\`: Local base snapshot destroyed\" >> $log" >> ${cmd}
  fi
  if [ ${SOURCEDELTA} -gt "0" ]; then
   echo "sudo ${DESTROY} $lfs$r_new 2>> ${ERR}" >> ${cmd}
   echo errorcheck >> ${cmd}
   echo "echo \"\`date\`: Local delta snapshot destroyed\" >> $log" >> ${cmd}
  fi
  if [ ${DESTBASE} -gt "0" ]; then
   echo "${SSH} $rh sudo ${DESTROY} $tfs$r_old 2>> ${ERR}" >> ${cmd}
   echo errorcheck >> ${cmd}
   echo "echo \"\`date\`: Target base snapshot destroyed\" >> $log" >> ${cmd}
  fi
  if [ ${DESTDELTA} -gt "0" ]; then
   echo "${SSH} $rh sudo ${DESTROY} $tfs$r_new 2>> ${ERR}" >> ${cmd}
   echo errorcheck >> ${cmd}
   echo "echo \"\`date\`: Target delta snapshot destroyed\" >> $log" >> ${cmd}
  fi
  if [ $(su replicator -c "${SSH} $rh zfs list -H -o name -r $tfs 2>/dev/null | awk 'END{print NR}'") -gt "0" ]; then
   echo "${SSH} $rh sudo ${DESTROY} $tfs 2>> ${ERR}" >> ${cmd}
   echo errorcheck >> ${cmd}
   echo "echo \"\`date\`: Target filesystem destroyed\" >> $log" >> ${cmd}
  fi

  echo "echo \"\`date\`: Remote cleanup complete on \\\"$lfs\\\"\" >> ${log}" >> ${cmd}
  echo "echo \"\" >> $log" >> ${cmd}
  echo "" >> ${cmd}
 }

errorcheck()
 {
  if [ "$?" -ne "0" ]; then
   tail -2 ${ERR} >> $log
   tail -10 $log > ${mail}
   mail -s "$subject" $address < ${mail}
   cat /dev/null > ${mail}
   exit 1
  fi
 }

usenonecipher()
 {
  sed -i '' -e "s/${SSH} $rh sudo zfs recv/${SSH} -oNoneEnabled=yes -oNoneSwitch=yes $rh sudo zfs recv/" ${cmd}
 }

sanity()
 {
  if [ -e $pid ]; then
   echo "`date`: Started a new process, but the previous \"replicate\" is still running, aborting." >> $log
   echo "" >> $log
   rm ${ERR}; rm ${cmd}; rm ${mail}; exit 1
  fi
  chown replicator ${mail}; chmod 600 ${mail}
  chown replicator ${ERR}; chmod 600 ${ERR}
  chown replicator ${cmd}; chmod 700 ${cmd}
  cat > ${cmd} << EOF
#!/bin/sh
errorcheck()
 {
  if [ "\$?" -ne "0" ]; then
    tail -2 ${ERR} >> $log
    tail -10 $log > ${mail}
    mail -s "$subject" $address < ${mail}
    cat /dev/null > ${mail}
    exit 1
  fi
 }

EOF
 }

autoclean()
 {
  ##
  ## Find and destroy all automatically taken snapshots
  ##

  zfs list -t snapshot -o name | grep auto > ${cmd}
  cat ${cmd}
  sed -i '' 's/^/sudo zfs destroy /' ${cmd}
 }

allow()
 {
  sudoers="/usr/local/etc/sudoers"

  if [ `grep "ssh" ${cmd} | wc -l` -gt "0" ]
   then
    allows=`grep zfs ${cmd} | cut -f 1 -d ">" | tr '|' '\n' | sed -e 's/^ //' -e 's/ 2$//' | sed  -e 's/sudo //' -e 's/^ //' -e 's/^/replicator ALL=(ALL) NOPASSWD: \/sbin\//' | grep -v ssh`
   else
    allows=`grep zfs ${cmd} | cut -f 1 -d ">" | tr '|' '\n' | sed -e 's/^ //' -e 's/ 2$//' | sed  -e 's/sudo //' -e 's/^ //' -e 's/^/replicator ALL=(ALL) NOPASSWD: \/sbin\//'`
  fi
  if [ `grep "replicator" $sudoers | wc -l` -gt "0" ]; then
   sed -i '' '/replicator/d' $sudoers
  fi
  echo "$allows" >> $sudoers
 }

disallow()
 {
  if [ `grep "replicator" $sudoers | wc -l` -gt "0" ]; then
   sed -i '' '/replicator/d' $sudoers
  fi
 }

##
## Initial preparation
##

case "$1" in

 "")
  ##
  ## If no jobname is specified, it will find all jobs in config directory and
  ## process them one by one
  ##

  for job in $jobs
   do
    sanity
    . $job
    touch $pid
    if [ -z "$rh" ]
     then
      if [ `zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep @local_replicate | wc -l | sed 's/^[ \t]*//'` -gt "0" ]
       then
        l_inc
       else
        l_base
      fi
     else
      if [ `zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep @remote_replicate | wc -l | sed 's/^[ \t]*//'` -gt "0" ]
       then
        r_inc
        if [ "$none" = "yes" ]; then
         usenonecipher
        fi
       else
        r_base
        if [ "$none" = "yes" ]; then
         usenonecipher
        fi
      fi
    fi
    allow
    su replicator -c ${cmd}
    disallow
    rm $pid
   done
 ;;

 -c)
  ##
  ## When cleaning, if no jobname is specified, it will find all jobs in
  ## config directory and process them one by one
  ##

  if [ "$2" = "auto" ]
   then
   else
    echo ""
    echo "Starting off by deleting replication snapshot(s) and cleaning the entire"
    echo "target pool zfs filesystem. This can take a while, depending on how much"
    echo "is stored."
    sleep 10
  fi

  if [ "$2" = "" ];  then
   for job in `find $path -type f ! -iname sample_* ! -iname noauto_* | sort -rd`
    do
     sanity
     . $job
     touch $pid
     if [ -z "$rh" ]
      then
       l_clean
      else
       r_clean
     fi
     allow
     su replicator -c ${cmd}
     disallow
     rm $pid
    done
   exit
  fi

  ##
  ## Also when cleaning, if you type in shorthand jobname, it will only clean
  ## specified job
  ##

  if [ -e ${path}$2 ];  then
   sanity
   . $path$2
   touch $pid
   if [ -z "$rh" ]      
    then
     l_clean       
    else
     r_clean
   fi
   allow
   su replicator -c ${cmd}
   disallow
   rm $pid
  else

   ##
   ## If you type "auto" when cleaning, it finds and destroys all snapshots with
   ## the word "auto" in them.
   ##

   if [ "$2" = "auto" ]
    then
     sanity
     touch $pid
     autoclean
     while true
      do
       read -r -p 'These snapshot(s) will destroyed. Confirm? "Y|n"' choice
        case "$choice" in
         y|Y) allow
              su replicator -c ${cmd}
              disallow
              rm $pid
              check=`zfs list -t snapshot`
              result="no datasets available"
               if [ "$check" = "$result" ]; then
                echo "Seems clean enough"
               else
                echo "Seems there are snaps left:"
                zfs list -t snapshot
               fi; break ;;
         n|N) echo "Suit yourself..." ; break ;;
           *) echo "Suit yourself..." ; break ;;
        esac
      done
    else
     echo ""
     echo "### Error! ###"
     echo "Jobname \"$2\" does not exist"
     usage
   fi
  fi
 ;;

 -h)
  usage
 ;;

 *)
  ##
  ## If you type in shorthand jobname, it will only process specified job
  ##

  if [ -e ${path}$1 ]
   then
    sanity
    . $path$1
    touch $pid
    if [ -z "$rh" ]
     then
      if [ `zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep @local_replicate | wc -l | sed 's/^[ \t]*//'` -gt "0" ]
       then
        l_inc        
       else
        l_base
      fi
     else
      if [ `zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep @remote_replicate | wc -l | sed 's/^[ \t]*//'` -gt "0" ]
       then
        r_inc        
        if [ "$none" = "yes" ]; then
         usenonecipher
        fi
       else
        r_base
        if [ "$none" = "yes" ]; then
         usenonecipher
        fi
      fi
    fi
    allow
    su replicator -c ${cmd}
    disallow
    rm $pid
   else
    echo ""
    echo "### Error! ###"
    echo "Jobname \"$1\" does not exist"
    usage
  fi
 ;;

esac

##
## Final cleanup
##

rm -f /tmp/replicate.*
