--- /usr/local/bin/replicate 2015-06-12 11:54:00.636078258 +0200 +++ /usr/local/bin/replicate 2015-06-12 12:04:41.308430670 +0200 @@ -131,16 +131,42 @@ SSH="ssh" - if [ "${compress}" = "yes" ]; then - SSH="ssh -C" - fi - if [ ! -z ${port} ]; then SSH="${SSH} -p ${port}" else SSH="${SSH} -p 22" fi + for COMPRESSOR_PROGRAM in gzip xz pigz; do + COMPRESSOR_AGENT=$(whereis ${COMPRESSOR_PROGRAM} | awk '{print $2}') + if [ $(echo ${COMPRESSOR_AGENT} | wc -w) -ne 0 ]; then + COMPRESSORS_FOUND="${COMPRESSORS_FOUND} ${COMPRESSOR_AGENT}" + fi + done + + COMPRESSORS_FOUND=$(echo ${COMPRESSORS_FOUND} | sed 's/^ //') + + for POTENTIAL_COMPRESSOR in ${COMPRESSORS_FOUND}; do + if echo "${POTENTIAL_COMPRESSOR}" | grep -q "pigz"; then + PIGZ_COMPRESSOR="${POTENTIAL_COMPRESSOR}" + fi + if echo "${POTENTIAL_COMPRESSOR}" | grep -q "gzip"; then + GZIP_COMPRESSOR="${POTENTIAL_COMPRESSOR}" + fi + if echo "${POTENTIAL_COMPRESSOR}" | grep -q "xz"; then + XZ_COMPRESSOR="${POTENTIAL_COMPRESSOR}" + fi + done + + if [ $(echo ${PIGZ_COMPRESSOR} | wc -w) -ne 0 ]; then + COMPRESSOR=${PIGZ_COMPRESSOR} + elif [ $(echo ${GZIP_COMPRESSOR} | wc -w) -ne 0 ]; then + COMPRESSOR=${GZIP_COMPRESSOR} + elif [ $(echo ${XZ_COMPRESSOR} | wc -w) -ne 0 ]; then + COMPRESSOR=${XZ_COMPRESSOR} + fi + DECOMPRESSOR="${COMPRESSOR} -d" + if [ `ping -c 1 -W 1000 $rh | grep -oe "[0-9] packets received" | awk '{print $1}'` -eq "0" ]; then echo "`date`: Remote baseline replication sequence aborted on \"$lfs\"! No response from \"$rh\"." >> $log echo "" >> $log @@ -172,13 +198,22 @@ echo "echo \"\`date\`: Beginning remote baseline replication sequence on \\\"$lfs\\\"\" >> ${log}" >> ${cmd} # Take initial snapshot(s): - echo "sudo ${SNAPSHOT} $lfs$r_old 2>> ${ERR}" >> ${cmd} - echo errorcheck >> ${cmd} + echo "sudo ${SNAPSHOT} $lfs$r_old 2>> ${ERR}" >> ${cmd} + echo errorcheck >> ${cmd} echo "echo \"\`date\`: Initial snapshot(s) created\" >> $log" >> ${cmd} # Replicate data: - echo "sudo ${SEND} $lfs$r_old 2>> ${ERR} | ${SSH} $rh sudo zfs recv -du $rp/$lh 2>> ${ERR}" >> ${cmd} - echo errorcheck >> ${cmd} + if [ "${compress}" = "yes" ]; then + if [ $(echo "${COMPRESSOR}" | wc -w) -ne 0 ]; then + echo "sudo ${SEND} $lfs$r_old 2>> ${ERR} | ${COMPRESSOR} | ${SSH} $rh \"${DECOMPRESSOR} | sudo zfs recv -du $rp/$lh\" 2>>{ERR}" >> ${cmd} + else + SSH="${SSH} -C" + echo "sudo ${SEND} $lfs$r_old 2>> ${ERR} | ${SSH} $rh sudo zfs recv -du $rp/$lh 2>> ${ERR}" >> ${cmd} + fi + else + echo "sudo ${SEND} $lfs$r_old 2>> ${ERR} | ${SSH} $rh sudo zfs recv -du $rp/$lh 2>> ${ERR}" >> ${cmd} + fi + echo errorcheck >> ${cmd} echo "echo \"\`date\`: Data replicated\" >> $log" >> ${cmd} # Clean unwanted, scheduled source snapshots from target: @@ -625,16 +660,42 @@ SSH="ssh" - if [ "${compress}" = "yes" ]; then - SSH="ssh -C" - fi - if [ ! -z ${port} ]; then SSH="${SSH} -p ${port}" else SSH="${SSH} -p 22" fi + for COMPRESSOR_PROGRAM in gzip xz pigz; do + COMPRESSOR_AGENT=$(whereis ${COMPRESSOR_PROGRAM} | awk '{print $2}') + if [ $(echo ${COMPRESSOR_AGENT} | wc -w) -ne 0 ]; then + COMPRESSORS_FOUND="${COMPRESSORS_FOUND} ${COMPRESSOR_AGENT}" + fi + done + + COMPRESSORS_FOUND=$(echo ${COMPRESSORS_FOUND} | sed 's/^ //') + + for POTENTIAL_COMPRESSOR in ${COMPRESSORS_FOUND}; do + if echo "${POTENTIAL_COMPRESSOR}" | grep -q "pigz"; then + PIGZ_COMPRESSOR="${POTENTIAL_COMPRESSOR}" + fi + if echo "${POTENTIAL_COMPRESSOR}" | grep -q "gzip"; then + GZIP_COMPRESSOR="${POTENTIAL_COMPRESSOR}" + fi + if echo "${POTENTIAL_COMPRESSOR}" | grep -q "xz"; then + XZ_COMPRESSOR="${POTENTIAL_COMPRESSOR}" + fi + done + + if [ $(echo ${PIGZ_COMPRESSOR} | wc -w) -ne 0 ]; then + COMPRESSOR=${PIGZ_COMPRESSOR} + elif [ $(echo ${GZIP_COMPRESSOR} | wc -w) -ne 0 ]; then + COMPRESSOR=${GZIP_COMPRESSOR} + elif [ $(echo ${XZ_COMPRESSOR} | wc -w) -ne 0 ]; then + COMPRESSOR=${XZ_COMPRESSOR} + fi + DECOMPRESSOR="${COMPRESSOR} -d" + if [ `ping -c 1 -W 1000 $rh | grep -oe "[0-9] packets received" | awk '{print $1}'` -eq "0" ]; then echo "`date`: Remote incremental replication sequence aborted on \"$lfs\"! No response from \"$rh\"." >> $log echo "" >> $log @@ -725,7 +786,16 @@ echo "sudo ${SNAPSHOT} $lfs$r_old 2>> ${ERR}" >> ${cmd} echo errorcheck >> ${cmd} echo "echo \"\`date\`: Base snapshot(s) created\" >> $log" >> ${cmd} - echo "sudo ${SEND} $lfs$r_old 2>> ${ERR} | ${SSH} $rh sudo zfs recv -du $rp/$lh 2>> ${ERR}" >> ${cmd} + if [ "${compress}" = "yes" ]; then + if [ $(echo "${COMPRESSOR}" | wc -w) -ne 0 ]; then + echo "sudo ${SEND} $lfs$r_old 2>> ${ERR} | ${COMPRESSOR} | ${SSH} $rh \"${DECOMPRESSOR} | sudo zfs recv -du $rp/$lh\" 2>>{ERR}" >> ${cmd} + else + SSH="${SSH} -C" + echo "sudo ${SEND} $lfs$r_old 2>> ${ERR} | ${SSH} $rh sudo zfs recv -du $rp/$lh 2>> ${ERR}" >> ${cmd} + fi + else + echo "sudo ${SEND} $lfs$r_old 2>> ${ERR} | ${SSH} $rh sudo zfs recv -du $rp/$lh 2>> ${ERR}" >> ${cmd} + fi echo errorcheck >> ${cmd} echo "echo \"\`date\`: Data replicated\" >> $log" >> ${cmd} echo "echo \"\`date\`: Rebaseline complete\" >> $log" >> ${cmd} @@ -734,27 +804,62 @@ echo "echo \"\`date\`: Delta snapshot(s) created\" >> $log" >> ${cmd} } - dryrun() - { + dryrun() { DRYRUNOUT=$(mktemp /tmp/replicate.dryrun.XXXXXX) if [ "$r" = "yes" ]; then if [ "$none" = "yes" ]; then - DODRYRUN=$(zfs send -vRi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -oNoneEnabled=yes -oNoneSwitch=yes -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + if [ "${compress}" = "yes" ]; then + if [ $(echo "${COMPRESSOR}" | wc -w) -ne 0 ]; then + DODRYRUN=$(zfs send -vRi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${COMPRESSOR} | ${SSH} -oNoneEnabled=yes -oNoneSwitch=yes -i ${REPID} replicator@$rh "${DECOMPRESSOR} | sudo zfs recv -vdn $rp/$lh" 2>> ${DRYRUNOUT}) + else + SSH="${SSH} -C" + DODRYRUN=$(zfs send -vRi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -oNoneEnabled=yes -oNoneSwitch=yes -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + fi + else + DODRYRUN=$(zfs send -vRi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -oNoneEnabled=yes -oNoneSwitch=yes -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + fi else - DODRYRUN=$(zfs send -vRi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + if [ "${compress}" = "yes" ]; then + if [ $(echo "${COMPRESSOR}" | wc -w) -ne 0 ]; then + DODRYRUN=$(zfs send -vRi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${COMPRESSOR} | ${SSH} -i ${REPID} replicator@$rh "${DECOMPRESSOR} | sudo zfs recv -vdn $rp/$lh" 2>> ${DRYRUNOUT} + else + SSH="${SSH} -C" + DODRYRUN=$(zfs send -vRi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + fi + else + DODRYRUN=$(zfs send -vRi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + fi fi else if [ "$none" = "yes" ]; then - DODRYRUN=$(zfs send -vi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -oNoneEnabled=yes -oNoneSwitch=yes -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + if [ "${compress}" = "yes" ]; then + if [ $(echo "${COMPRESSOR}" | wc -w) -ne 0 ]; then + DODRYRUN=$(zfs send -vi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${COMPRESSOR} | ${SSH} -oNoneEnabled=yes -oNoneSwitch=yes -i ${REPID} replicator@$rh "${DECOMPRESSOR} | sudo zfs recv -vdn $rp/$lh" 2>> ${DRYRUNOUT} + else + SSH="${SSH} -C" + DODRYRUN=$(zfs send -vi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -oNoneEnabled=yes -oNoneSwitch=yes -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + fi + else + DODRYRUN=$(zfs send -vi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -oNoneEnabled=yes -oNoneSwitch=yes -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + fi else - DODRYRUN=$(zfs send -vi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + if [ "${compress}" = "yes" ]; then + if [ $(echo "${COMPRESSOR}" | wc -w) -ne 0 ]; then + DODRYRUN=$(zfs send -vi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${COMPRESSOR} | ${SSH} -i ${REPID} replicator@$rh "${DECOMPRESSOR} | sudo zfs recv -vdn $rp/$lh" 2>> ${DRYRUNOUT} + else + SSH="${SSH} -C" + DODRYRUN=$(zfs send -vi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + fi + else + DODRYRUN=$(zfs send -vi $lfs$r_old $lfs$r_new 2> ${DRYRUNOUT} | ${SSH} -i ${REPID} replicator@$rh sudo zfs recv -vdn $rp/$lh 2>> ${DRYRUNOUT}) + fi fi fi DRYRUNOK=`egrep '(could not send|cannot receive)' ${DRYRUNOUT} | wc -l | sed 's/^[ \t]*//'` rm ${DRYRUNOUT}; echo ${DRYRUNOK} - } + } if [ ${SOURCEBASE} -gt "0" ] then @@ -1003,7 +1108,16 @@ fi # Replicate data: - echo "sudo ${SEND} -i $lfs$r_old $lfs$r_new 2>> ${ERR} | ${SSH} $rh sudo zfs recv -du $rp/$lh 2>> ${ERR}" >> ${cmd} + if [ "${compress}" = "yes" ]; then + if [ $(echo "${COMPRESSOR}" | wc -w) -ne 0 ]; then + echo "sudo ${SEND} -i $lfs$r_old $lfs$r_new 2>> ${ERR} | ${COMPRESSOR} | ${SSH} $rh \"${DECOMPRESSOR} | sudo zfs recv -du $rp/$lh\" 2>>{ERR}" >> ${cmd} + else + SSH="${SSH} -C" + echo "sudo ${SEND} -i $lfs$r_old $lfs$r_new 2>> ${ERR} | ${SSH} $rh sudo zfs recv -du $rp/$lh 2>> ${ERR}" >> ${cmd} + fi + else + echo "sudo ${SEND} -i $lfs$r_old $lfs$r_new 2>> ${ERR} | ${SSH} $rh sudo zfs recv -du $rp/$lh 2>> ${ERR}" >> ${cmd} + fi echo errorcheck >> ${cmd} echo "echo \"\`date\`: Data replicated\" >> $log" >> ${cmd}