--- replicate.old 2015-10-05 09:34:59.292597469 +0200 +++ replicate.new 2015-10-23 07:32:35.707976797 +0200 @@ -55,18 +55,6 @@ ## Makes local baseline replication ## - if zpool status `echo $lfs | cut -f1 -d /` | grep "scan:" | egrep -qo "(scrub in progress|resilver in progress)"; then - echo "`date`: A Scrub or Resilver is currently in progress on source pool, aborting." >> $log - echo "" >> $log - rm ${pid} - exit 1 - elif zpool status $lp | grep "scan:" | egrep -qo "(scrub in progress|resilver in progress)"; then - echo "`date`: A Scrub or Resilver is currently in progress on destination pool, aborting." >> $log - echo "" >> $log - rm ${pid} - exit 1 - fi - if [ "$r" = "yes" ] then SNAPSHOT="zfs snapshot -r" @@ -129,12 +117,33 @@ ## Makes remote baseline replication ## - SSH="ssh" - if [ "${compress}" = "yes" ]; then - SSH="ssh -C" + for KNOWN_COMPRESSORS in pigz gzip xz; do + KNOWN_COMPRESSORS_INSTALLED=$(whereis ${KNOWN_COMPRESSORS} | awk '{print $2}') + if [ $(echo ${KNOWN_COMPRESSORS_INSTALLED} | wc -w) -ne 0 ]; then + INSTALLED_COMPRESSORS="${INSTALLED_COMPRESSORS} ${KNOWN_COMPRESSORS_INSTALLED}" + fi + done + INSTALLED_COMPRESSORS=$(echo ${INSTALLED_COMPRESSORS} | sed 's/^ //') + for WANTED_COMPRESSOR in ${INSTALLED_COMPRESSORS}; do + if [ $(echo ${WANTED_COMPRESSOR} | grep -c pigz) -ne 0 ]; then + COMPRESSOR=${WANTED_COMPRESSOR} + break + fi + if [ $(echo ${WANTED_COMPRESSOR} | grep -c gzip) -ne 0 ]; then + COMPRESSOR=${WANTED_COMPRESSOR} + break + fi + if [ $(echo ${WANTED_COMPRESSOR} | grep -c xz) -ne 0 ]; then + COMPRESSOR=${WANTED_COMPRESSOR} + break + fi + done + DECOMPRESSOR="${COMPRESSOR} -d" fi + SSH="ssh" + if [ ! -z ${port} ]; then SSH="${SSH} -p ${port}" else @@ -146,16 +155,6 @@ echo "" >> $log rm ${pid} exit 1 - elif zpool status `echo $lfs | cut -f1 -d /` | grep "scan:" | egrep -qo "(scrub in progress|resilver in progress)"; then - echo "`date`: A Scrub or Resilver is currently in progress on source pool, aborting." >> $log - echo "" >> $log - rm ${pid} - exit 1 - elif su replicator -c "${SSH} $rh zpool status $rp | grep \"scan:\" | egrep -qo '(scrub in progress|resilver in progress)'"; then - echo "`date`: A Scrub or Resilver is currently in progress on remote pool, aborting." >> $log - echo "" >> $log - rm ${pid} - exit 1 fi if [ "$r" = "yes" ] @@ -177,7 +176,11 @@ echo "echo \"\`date\`: Initial snapshot(s) created\" >> $log" >> ${cmd} # Replicate data: - echo "sudo ${SEND} $lfs$r_old 2>> ${ERR} | ${SSH} $rh sudo zfs recv -du $rp/$lh 2>> ${ERR}" >> ${cmd} + if [ "${compress}" = "yes" ]; then + echo "sudo ${SEND} $lfs$r_old 2>> ${ERR} | ${COMPRESSOR} | ${SSH} $rh \"(${DECOMPRESSOR} | sudo zfs recv -du $rp/$lh)\" 2>> ${ERR}" >> ${cmd} + else + echo "sudo ${SEND} $lfs$r_old 2>> ${ERR} | ${SSH} $rh sudo zfs recv -du $rp/$lh 2>> ${ERR}" >> ${cmd} + fi echo errorcheck >> ${cmd} echo "echo \"\`date\`: Data replicated\" >> $log" >> ${cmd} @@ -220,18 +223,6 @@ ## Makes local incremental replication ## - if zpool status `echo $lfs | cut -f1 -d /` | grep "scan:" | egrep -qo "(scrub in progress|resilver in progress)"; then - echo "`date`: A Scrub or Resilver is currently in progress on source pool, aborting." >> $log - echo "" >> $log - rm ${pid} - exit 1 - elif zpool status $lp | grep "scan:" | egrep -qo "(scrub in progress|resilver in progress)"; then - echo "`date`: A Scrub or Resilver is currently in progress on destination pool, aborting." >> $log - echo "" >> $log - rm ${pid} - exit 1 - fi - tfs=`echo $lfs | cut -f 2-512 -d / | sed "s/^/$lp\//"` echo "`date`: Beginning local incremental replication sequence on \"$lfs\"" >> $log @@ -623,12 +614,33 @@ ## Makes remote incremental replication ## - SSH="ssh" - if [ "${compress}" = "yes" ]; then - SSH="ssh -C" + for KNOWN_COMPRESSORS in pigz gzip xz; do + KNOWN_COMPRESSORS_INSTALLED=$(whereis ${KNOWN_COMPRESSORS} | awk '{print $2}') + if [ $(echo ${KNOWN_COMPRESSORS_INSTALLED} | wc -w) -ne 0 ]; then + INSTALLED_COMPRESSORS="${INSTALLED_COMPRESSORS} ${KNOWN_COMPRESSORS_INSTALLED}" + fi + done + INSTALLED_COMPRESSORS=$(echo ${INSTALLED_COMPRESSORS} | sed 's/^ //') + for WANTED_COMPRESSOR in ${INSTALLED_COMPRESSORS}; do + if [ $(echo ${WANTED_COMPRESSOR} | grep -c pigz) -ne 0 ]; then + COMPRESSOR=${WANTED_COMPRESSOR} + break + fi + if [ $(echo ${WANTED_COMPRESSOR} | grep -c gzip) -ne 0 ]; then + COMPRESSOR=${WANTED_COMPRESSOR} + break + fi + if [ $(echo ${WANTED_COMPRESSOR} | grep -c xz) -ne 0 ]; then + COMPRESSOR=${WANTED_COMPRESSOR} + break + fi + done + DECOMPRESSOR="${COMPRESSOR} -d" fi + SSH="ssh" + if [ ! -z ${port} ]; then SSH="${SSH} -p ${port}" else @@ -641,17 +653,6 @@ rm ${pid} exit 1 fi - if zpool status `echo $lfs | cut -f1 -d /` | grep "scan:" | egrep -qo "(scrub in progress|resilver in progress)"; then - echo "`date`: A Scrub or Resilver is currently in progress on source pool, aborting." >> $log - echo "" >> $log - rm ${pid} - exit 1 - elif su replicator -c "${SSH} $rh zpool status $rp | grep \"scan:\" | egrep -qo '(scrub in progress|resilver in progress)'"; then - echo "`date`: A Scrub or Resilver is currently in progress on remote pool, aborting." >> $log - echo "" >> $log - rm ${pid} - exit 1 - fi tfs=`echo $lfs | cut -f 2-512 -d / | sed "s/^/$rp\/$lh\//"` @@ -725,7 +726,11 @@ echo "sudo ${SNAPSHOT} $lfs$r_old 2>> ${ERR}" >> ${cmd} echo errorcheck >> ${cmd} echo "echo \"\`date\`: Base snapshot(s) created\" >> $log" >> ${cmd} - echo "sudo ${SEND} $lfs$r_old 2>> ${ERR} | ${SSH} $rh sudo zfs recv -du $rp/$lh 2>> ${ERR}" >> ${cmd} + if [ "${compress}" = "yes" ]; then + echo "sudo ${SEND} $lfs$r_old 2>> ${ERR} | ${COMPRESSOR} | ${SSH} $rh \"(${DECOMPRESSOR} | sudo zfs recv -du $rp/$lh)\" 2>> ${ERR}" >> ${cmd} + else + echo "sudo ${SEND} $lfs$r_old 2>> ${ERR} | ${SSH} $rh sudo zfs recv -du $rp/$lh 2>> ${ERR}" >> ${cmd} + fi echo errorcheck >> ${cmd} echo "echo \"\`date\`: Data replicated\" >> $log" >> ${cmd} echo "echo \"\`date\`: Rebaseline complete\" >> $log" >> ${cmd} @@ -740,15 +745,31 @@ if [ "$r" = "yes" ]; then if [ "$none" = "yes" ]; then - DODRYRUN=$(zfs send -vRi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -oNoneEnabled=yes -oNoneSwitch=yes -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + if [ "${compress}" = "yes" ]; then + DODRYRUN=$(zfs send -vRi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${COMPRESSOR} | ${SSH} -oNoneEnabled=yes -oNoneSwitch=yes -i ${REPID} replicator@$rh "(${DECOMPRESSOR} | sudo zfs recv -vdn $rp/$lh)" 2>> ${DRYRUNOUT}) + else + DODRYRUN=$(zfs send -vRi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -oNoneEnabled=yes -oNoneSwitch=yes -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + fi else - DODRYRUN=$(zfs send -vRi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + if [ "${compress}" = "yes" ]; then + DODRYRUN=$(zfs send -vRi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${COMPRESSOR} | ${SSH} -i ${REPID} replicator@$rh "(${DECOMPRESSOR} | sudo zfs recv -vdn $rp/$lh)" 2>> ${DRYRUNOUT}) + else + DODRYRUN=$(zfs send -vRi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + fi fi else if [ "$none" = "yes" ]; then - DODRYRUN=$(zfs send -vi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -oNoneEnabled=yes -oNoneSwitch=yes -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + if [ "${compress}" = "yes" ]; then + DODRYRUN=$(zfs send -vi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${COMPRESSOR} | ${SSH} -oNoneEnabled=yes -oNoneSwitch=yes -i ${REPID} replicator@$rh "(${DECOMPRESSOR} | sudo zfs recv -vdn $rp/$lh)" 2>> ${DRYRUNOUT}) + else + DODRYRUN=$(zfs send -vi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -oNoneEnabled=yes -oNoneSwitch=yes -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + fi else - DODRYRUN=$(zfs send -vi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + if [ "${compress}" = "yes" ]; then + DODRYRUN=$(zfs send -vi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${COMPRESSOR} | ${SSH} -i ${REPID} replicator@$rh "(${DECOMPRESSOR} | sudo zfs recv -vdn $rp/$lh)" 2>> ${DRYRUNOUT}) + else + DODRYRUN=$(zfs send -vi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + fi fi fi @@ -1003,7 +1024,11 @@ fi # Replicate data: - echo "sudo ${SEND} -i $lfs$r_old $lfs$r_new 2>> ${ERR} | ${SSH} $rh sudo zfs recv -du $rp/$lh 2>> ${ERR}" >> ${cmd} + if [ "${compress}" = "yes" ]; then + echo "sudo ${SEND} -i $lfs$r_old $lfs$r_new 2>> ${ERR} | ${COMPRESSOR} | ${SSH} $rh \"(${DECOMPRESSOR} | sudo zfs recv -du $rp/$lh)\" 2>> ${ERR}" >> ${cmd} + else + echo "sudo ${SEND} -i $lfs$r_old $lfs$r_new 2>> ${ERR} | ${SSH} $rh sudo zfs recv -du $rp/$lh 2>> ${ERR}" >> ${cmd} + fi echo errorcheck >> ${cmd} echo "echo \"\`date\`: Data replicated\" >> $log" >> ${cmd} @@ -1052,18 +1077,6 @@ ## Cleans locally replicated file system ## - if zpool status `echo $lfs | cut -f1 -d /` | grep "scan:" | egrep -qo "(scrub in progress|resilver in progress)"; then - echo "`date`: A Scrub or Resilver is currently in progress on source pool, aborting." >> $log - echo "" >> $log - rm ${pid} - exit 1 - elif zpool status $lp | grep "scan:" | egrep -qo "(scrub in progress|resilver in progress)"; then - echo "`date`: A Scrub or Resilver is currently in progress on destination pool, aborting." >> $log - echo "" >> $log - rm ${pid} - exit 1 - fi - tfs=`echo $lfs | cut -f 2-512 -d / | sed "s/^/$lp\//"` if [ "$r" = "yes" ]; then @@ -1150,23 +1163,6 @@ SSH="${SSH} -p 22" fi - if [ `ping -c 1 -W 1000 $rh | grep -oe "[0-9] packets received" | awk '{print $1}'` -eq "0" ]; then - echo "`date`: Remote cleanup aborted on \"$lfs\"! No response from \"$rh\"." >> $log - echo "" >> $log - rm ${pid} - exit 1 - elif zpool status `echo $lfs | cut -f1 -d /` | grep "scan:" | egrep -qo "(scrub in progress|resilver in progress)"; then - echo "`date`: A Scrub or Resilver is currently in progress on source pool, aborting." >> $log - echo "" >> $log - rm ${pid} - exit 1 - elif su replicator -c "${SSH} $rh zpool status $rp | grep \"scan:\" | egrep -qo '(scrub in progress|resilver in progress)'"; then - echo "`date`: A Scrub or Resilver is currently in progress on destination pool, aborting." >> $log - echo "" >> $log - rm ${pid} - exit 1 - fi - tfs=`echo $lfs | cut -f 2-512 -d / | sed "s/^/$rp\/$lh\//"` if [ "$r" = "yes" ]; then @@ -1306,13 +1302,17 @@ if [ $(echo "${UNKNOWN_STATUS_PROCESSES}" | wc -w) -ne 0 ]; then for UNKNOWN_STATUS_PROCESS in ${UNKNOWN_STATUS_PROCESSES}; do if [ $(ps ax | grep "${UNKNOWN_STATUS_PROCESS}" | grep -v grep | egrep -c ' D.* ') -ne 0 ]; then - ERROR_MESSAGE="The previous replicate command set has been found with no running processes and seems to be stuck. A process from the last run has been found in \"D\" state which can not be killed, reboot required" - echo "${ERROR_MESSAGE}" >> ${mail} - echo "$(ps ax | grep "${UNKNOWN_STATUS_PROCESS}" | grep -v grep | egrep ' D.* ')" >> ${mail} - mail -s "$subject" $address < ${mail} - echo "$(date): ${ERROR_MESSAGE}" >> $log - echo "" >> $log - rm ${ERR}; rm ${cmd}; rm ${mail}; exit 1 + kill ${UNKNOWN_STATUS_PROCESS} + sleep 10 + if [ $(ps ax | grep "${UNKNOWN_STATUS_PROCESS}" | grep -v grep) -ne 0 ]; then + ERROR_MESSAGE="The previous replicate command set has been found with no running processes and seems to be stuck. A process from the last run has been found in \"D\" state which can not be killed, reboot required" + echo "${ERROR_MESSAGE}" >> ${mail} + echo "$(ps ax | grep "${UNKNOWN_STATUS_PROCESS}" | grep -v grep | egrep ' D.* ')" >> ${mail} + mail -s "$subject" $address < ${mail} + echo "$(date): ${ERROR_MESSAGE}" >> $log + echo "" >> $log + rm ${ERR}; rm ${cmd}; rm ${mail}; exit 1 + fi else kill ${UNKNOWN_STATUS_PROCESS} echo "$(date): No process from previous replicate run found running. Killed idle process \"${UNKNOWN_STATUS_PROCESS}\" as part of the cleanup before trying to start again." >> $log @@ -1390,8 +1390,12 @@ SSH="${SSH} -p 22" fi REPID="$(awk -F':' '{print $6}' /etc/passwd | grep replicator)/.ssh/id_rsa" - REMOTE_REPLICATE_COMMANDS=$(printf "${COMMANDS_IN_JOB}" | tr '|' '\n' | grep 'ssh' | sed -E -e "s/${SSH} ${rh}//" -e 's/.* sudo//' -e 's/^ //' | tr '\n' '|' | sed -E "s/\|$//") - REMOTE_REPLICATE_PROCESSES=$(${SSH} -i ${REPID} replicator@$rh ps -a -x -o pid,%cpu,command | egrep "zfs send -vRi|zfs send -vi|zfs recv -vdn|${REMOTE_REPLICATE_COMMANDS}" | egrep -v 'egrep| sudo ') + REMOTE_REPLICATE_COMMANDS=$(printf "${COMMANDS_IN_JOB}" | grep 'ssh' | tr '|' '\n' | sed -E -e "s/${SSH} ${rh}//" -e 's/.* sudo//' -e 's/^ //' -e 's/\"\(//g' -e 's/\)\"//g' | tr '\n' '|' | sed -E "s/\|$//") + if [ $(echo "${REMOTE_REPLICATE_COMMANDS}" | egrep -c '\|$') -ne 0 ]; then + REMOTE_REPLICATE_PROCESSES=$(${SSH} -i ${REPID} replicator@$rh "(NUM=1; while [ \${NUM} -le 30 ]; do ps -a -x -o pid,%cpu,command | egrep 'zfs send|zfs recv|zfs snapshot|zfs rename|zfs destroy|zfs list' | egrep -v 'egrep| sudo '; sleep 1; NUM=\$(expr \${NUM} + 1); done | sort -k 3 -u)") + else + REMOTE_REPLICATE_PROCESSES=$(${SSH} -i ${REPID} replicator@$rh "(NUM=1; while [ \${NUM} -le 30 ]; do ps -a -x -o pid,%cpu,command | egrep \"zfs send -vRi|zfs send -vi|zfs recv -vdn|${REMOTE_REPLICATE_COMMANDS}\" | egrep -v 'egrep| sudo '; sleep 1; NUM=\$(expr \${NUM} + 1); done | sort -k 3 -u)") + fi if [ "${DESTROYING}" = "-d" ]; then USED_CPU=$(printf "${REMOTE_REPLICATE_PROCESSES}" | egrep 'zfs destroy|zfs list' | awk '{sub(/\./,"");print$2}') else @@ -1440,7 +1444,26 @@ echo "" >> $log rm ${ERR}; rm ${cmd}; rm ${mail}; exit 1 else - replicate_cleanup + + # OK, a few sobs later, what if it is deleting snapshots? + # Let us see if the number of snapshots change. + + check_snapshot_number() { + ${SSH} -i ${REPID} replicator@$rh zfs list -r -t snapshot ${TARGET_FILESYSTEM} + } + SNAPSHOTS_BEFORE=$(check_snapshot_number) + sleep 120 + SNAPSHOTS_AFTER=$(check_snapshot_number) + if [ "${SNAPSHOTS_BEFORE}" -ne "${SNAPSHOTS_AFTER}" ]; then + + # If the snapshot number has changed, then it should still be working. + + echo "$(date): Started a new process, but the previous remote \"replicate\" is still running, aborting." >> $log + echo "" >> $log + rm ${ERR}; rm ${cmd}; rm ${mail}; exit 1 + else + replicate_cleanup + fi fi fi else @@ -1458,13 +1481,32 @@ SIZE_AFTER=$(check_target_size) if [ "${SIZE_BEFORE}" -ne "${SIZE_AFTER}" ]; then - # Finally, a hit! Then it should at least be doing something. + # Then it should at least be doing something. echo "$(date): Started a new process, but the previous local \"replicate\" is still running, aborting." >> $log echo "" >> $log rm ${ERR}; rm ${cmd}; rm ${mail}; exit 1 else - replicate_cleanup + + # OK, a few sobs later, what if it is deleting snapshots? + # Let us see if the number of snapshots change. + + check_snapshot_number() { + zfs list -r -t snapshot ${TARGET_FILESYSTEM} + } + SNAPSHOTS_BEFORE=$(check_snapshot_number) + sleep 120 + SNAPSHOTS_AFTER=$(check_snapshot_number) + if [ "${SNAPSHOTS_BEFORE}" -ne "${SNAPSHOTS_AFTER}" ]; then + + # If the snapshot number has changed, then it should still be working. + + echo "$(date): Started a new process, but the previous remote \"replicate\" is still running, aborting." >> $log + echo "" >> $log + rm ${ERR}; rm ${cmd}; rm ${mail}; exit 1 + else + replicate_cleanup + fi fi fi fi @@ -1551,14 +1593,14 @@ fi } if [ $(echo ${LAST_RUN_JOB} | wc -w) -ne 0 ]; then - REPLICATE_PROCESSES=$(ps -a -x -o pid,%cpu,command | egrep "zfs send -vRi|zfs send -vi|zfs recv -vdn|${LAST_RUN_JOB}|${COMMANDS_IN_JOB}" | grep -v egrep) + REPLICATE_PROCESSES=$(NUM=1; while [ ${NUM} -le 30 ]; do ps -a -x -o pid,%cpu,command | egrep "zfs send -vRi|zfs send -vi|zfs recv -vdn|${LAST_RUN_JOB}|${COMMANDS_IN_JOB}" | grep -v egrep; sleep 1; NUM=$(expr ${NUM} + 1); done | sort -k 3 -u) confirm_replicate_status else # No previous job found with valid commands. Retrying with broadened # search criteria: - REPLICATE_PROCESSES=$(ps -a -x -o pid,%cpu,command | egrep 'zfs send|zfs recv|zfs snapshot|zfs rename|zfs destroy|zfs list' | egrep -v 'egrep|zfSnap') + REPLICATE_PROCESSES=$(NUM=1; while [ ${NUM} -le 30 ]; do ps -a -x -o pid,%cpu,command | egrep 'zfs send|zfs recv|zfs snapshot|zfs rename|zfs destroy|zfs list' | egrep -v 'egrep|zfSnap'; sleep 1; NUM=$(expr ${NUM} + 1); done | sort -k 3 -u) confirm_replicate_status fi #fi