--- /usr/local/bin/replicate.old 2014-10-27 15:15:43.340079059 +0100 +++ /usr/local/bin/replicate.new 2014-11-05 15:33:46.050101471 +0100 @@ -76,6 +76,8 @@ SEND="zfs send -p" fi + tfs=$(echo $lfs | cut -f 2-512 -d / | sed "s/^/$lp\//") + echo "echo \"\`date\`: Beginning local baseline replication sequence on \\\"$lfs\\\"\" >> ${log}" >> ${cmd} # Take initial snapshot(s): @@ -88,6 +90,33 @@ echo errorcheck >> ${cmd} echo "echo \"\`date\`: Data replicated\" >> $log" >> ${cmd} + # Delete unwanted, scheduled source snapshots from target: + if [ "$r" = "yes" ]; then + cat >> ${cmd} << EOF +if [ "\$(zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | egrep -v "$l_old|$l_new" | awk 'END{print NR}')" -gt "0" ]; then + for SNAPSHOT in \$(zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | egrep -v "$l_old|$l_new"); do + if [ "\$(zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep -c "\${SNAPSHOT}")" -gt "0" ]; then + sudo zfs destroy \${SNAPSHOT} 2>> ${ERR} + errorcheck + fi + done + echo "\$(date): Unwanted local target snapshot(s) destroyed" >> $log +fi +EOF + else + cat >> ${cmd} << EOF +if [ "\$(zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep "${tfs}@" | egrep -v "$l_old|$l_new" | awk 'END{print NR}')" -gt "0" ]; then + for SNAPSHOT in \$(zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep "${tfs}@" | egrep -v "$l_old|$l_new"); do + if [ "\$(zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep -c "\${SNAPSHOT}")" -gt "0" ]; then + sudo zfs destroy \${SNAPSHOT} 2>> ${ERR} + errorcheck + fi + done + echo "\$(date): Unwanted local target snapshot(s) destroyed" >> $log +fi +EOF + fi + echo "echo \"\`date\`: Local baseline replication sequence finished on \\\"$lfs\\\"\" >> ${log}" >> ${cmd} echo "echo \"\" >> $log" >> ${cmd} echo "exit 0" >> ${cmd} @@ -138,6 +167,8 @@ SEND="zfs send -p" fi + tfs=$(echo $lfs | cut -f 2-512 -d / | sed "s/^/$rp\/$lh\//") + echo "echo \"\`date\`: Beginning remote baseline replication sequence on \\\"$lfs\\\"\" >> ${log}" >> ${cmd} # Take initial snapshot(s): @@ -150,6 +181,33 @@ echo errorcheck >> ${cmd} echo "echo \"\`date\`: Data replicated\" >> $log" >> ${cmd} + # Clean unwanted, scheduled source snapshots from target: + if [ "$r" = "yes" ]; then + cat >> ${cmd} << EOF +if [ "\$(${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | egrep -v "$r_old|$r_new" | awk 'END{print NR}')" -gt "0" ]; then + for SNAPSHOT in \$(${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | egrep -v "$r_old|$r_new"); do + if [ "\$(${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep -c "\${SNAPSHOT}")" -gt "0" ]; then + ${SSH} $rh sudo zfs destroy \${SNAPSHOT} 2>> ${ERR} + errorcheck + fi + done + echo "\$(date): Unwanted remote target snapshot(s) destroyed" >> $log +fi +EOF + else + cat >> ${cmd} << EOF +if [ "\$(${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep "${tfs}@" | egrep -v "$r_old|$r_new" | awk 'END{print NR}')" -gt "0" ]; then + for SNAPSHOT in \$(${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep "${tfs}@" | egrep -v "$r_old|$r_new"); do + if [ "\$(${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep -c "\${SNAPSHOT}")" -gt "0" ]; then + ${SSH} $rh sudo zfs destroy \${SNAPSHOT} 2>> ${ERR} + errorcheck + fi + done + echo "\$(date): Unwanted remote target snapshot(s) destroyed" >> $log +fi +EOF + fi + echo "echo \"\`date\`: Remote baseline replication sequence finished on \\\"$lfs\\\"\" >> ${log}" >> ${cmd} echo "echo \"\" >> $log" >> ${cmd} echo "exit 0" >> ${cmd} @@ -180,21 +238,20 @@ # Take new source snapshot(s): - if [ "$r" = "yes" ] - then - SOURCEBASE=`zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep replicate.base | wc -l | sed 's/^[ \t]*//'` - SOURCEDELTA=`zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep replicate.delta | wc -l | sed 's/^[ \t]*//'` - DESTBASE=`zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep replicate.base | wc -l | sed 's/^[ \t]*//'` - DESTDELTA=`zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep replicate.delta | wc -l | sed 's/^[ \t]*//'` + if [ "$r" = "yes" ]; then + SOURCEBASE=$(zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep -c replicate.base) + SOURCEDELTA=$(zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep -c replicate.delta) + DESTBASE=$(zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep -c replicate.base) + DESTDELTA=$(zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep -c replicate.delta) SNAPSHOT="zfs snapshot -r" SEND="zfs send -R" DESTROY="zfs destroy -r" RENAME="zfs rename -r" - else - SOURCEBASE=`zfs list -H -t snapshot -o name $lfs$r_old 2>/dev/null | wc -l | sed 's/^[ \t]*//'` - SOURCEDELTA=`zfs list -H -t snapshot -o name $lfs$r_new 2>/dev/null | wc -l | sed 's/^[ \t]*//'` - DESTBASE=`zfs list -H -t snapshot -o name $tfs$r_old 2>/dev/null | wc -l | sed 's/^[ \t]*//'` - DESTDELTA=`zfs list -H -t snapshot -o name $tfs$r_new 2>/dev/null | wc -l | sed 's/^[ \t]*//'` + else + SOURCEBASE=$(zfs list -H -t snapshot -o name $lfs$r_old 2>/dev/null | awk 'END{print NR}') + SOURCEDELTA=$(zfs list -H -t snapshot -o name $lfs$r_new 2>/dev/null | awk 'END{print NR}') + DESTBASE=$(zfs list -H -t snapshot -o name $tfs$r_old 2>/dev/null | awk 'END{print NR}') + DESTDELTA=$(zfs list -H -t snapshot -o name $tfs$r_new 2>/dev/null | awk 'END{print NR}') SNAPSHOT="zfs snapshot" SEND="zfs send -p" DESTROY="zfs destroy" @@ -205,25 +262,42 @@ { echo "echo \"\`date\`: No appropriate correctional steps where found, have to rebaseline\" >> $log" >> ${cmd} if [ ${SOURCEBASE} -gt "0" ]; then - echo "sudo ${DESTROY} $lfs$l_old 2>> ${ERR}" >> ${cmd} - echo errorcheck >> ${cmd} - echo "echo \"\`date\`: Local base snapshot destroyed\" >> $log" >> ${cmd} - elif [ ${SOURCEDELTA} -gt "0" ]; then - echo "sudo ${DESTROY} $lfs$l_new 2>> ${ERR}" >> ${cmd} - echo errorcheck >> ${cmd} - echo "echo \"\`date\`: Local delta snapshot destroyed\" >> $log" >> ${cmd} - elif [ ${DESTBASE} -gt "0" ]; then - echo "sudo ${DESTROY} $tfs$l_old 2>> ${ERR}" >> ${cmd} - echo errorcheck >> ${cmd} - echo "echo \"\`date\`: Target base snapshot destroyed\" >> $log" >> ${cmd} - elif [ ${DESTDELTA} -gt "0" ]; then - echo "sudo ${DESTROY} $tfs$l_new 2>> ${ERR}" >> ${cmd} - echo errorcheck >> ${cmd} - echo "echo \"\`date\`: Target delta snapshot destroyed\" >> $log" >> ${cmd} - elif [ $(zfs list -H -o name -r $tfs 2>/dev/null | awk 'END{print NR}') -gt "0" ]; then - echo "sudo ${DESTROY} $tfs 2>> ${ERR}" >> ${cmd} - echo errorcheck >> ${cmd} - echo "echo \"\`date\`: Target filesystem destroyed\" >> $log" >> ${cmd} + echo "sudo ${DESTROY} $lfs$l_old 2>> ${ERR}" >> ${cmd} + echo errorcheck >> ${cmd} + echo "echo \"\`date\`: Local base snapshot destroyed\" >> $log" >> ${cmd} + fi + if [ ${SOURCEDELTA} -gt "0" ]; then + echo "sudo ${DESTROY} $lfs$l_new 2>> ${ERR}" >> ${cmd} + echo errorcheck >> ${cmd} + echo "echo \"\`date\`: Local delta snapshot destroyed\" >> $log" >> ${cmd} + fi + if [ ${DESTBASE} -gt "0" ]; then + echo "sudo ${DESTROY} $tfs$l_old 2>> ${ERR}" >> ${cmd} + echo errorcheck >> ${cmd} + echo "echo \"\`date\`: Target base snapshot destroyed\" >> $log" >> ${cmd} + fi + if [ ${DESTDELTA} -gt "0" ]; then + echo "sudo ${DESTROY} $tfs$l_new 2>> ${ERR}" >> ${cmd} + echo errorcheck >> ${cmd} + echo "echo \"\`date\`: Target delta snapshot destroyed\" >> $log" >> ${cmd} + fi + if [ "$r" != "yes" ]; then + cat >> ${cmd} << EOF +if [ "\$(zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep "${tfs}@" | egrep -v "$l_old|$l_new" | awk 'END{print NR}')" -gt "0" ]; then + for SNAPSHOT in \$(zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep "${tfs}@" | egrep -v "$l_old|$l_new"); do + if [ "\$(zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep -c "\${SNAPSHOT}")" -gt "0" ]; then + sudo zfs destroy \${SNAPSHOT} 2>> ${ERR} + errorcheck + fi + done + echo "\$(date): Unwanted local target snapshot(s) destroyed" >> $log +fi +EOF + fi + if [ $(zfs list -H -o name -r $tfs 2>/dev/null | awk 'END{print NR}') -gt "0" ]; then + echo "sudo ${DESTROY} $tfs 2>> ${ERR}" >> ${cmd} + echo errorcheck >> ${cmd} + echo "echo \"\`date\`: Target filesystem destroyed\" >> $log" >> ${cmd} fi echo "sudo ${SNAPSHOT} $lfs$l_old 2>> ${ERR}" >> ${cmd} echo errorcheck >> ${cmd} @@ -504,19 +578,22 @@ echo "echo \"\`date\`: Data replicated\" >> $log" >> ${cmd} # Destroy target .base snapshot(s): - if [ "$r" = "yes" ] - then - if [ `zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep replicate.base | wc -l | sed 's/^[ \t]*//'` -gt "0" ]; then - echo "sudo ${DESTROY} $tfs$l_old 2>> ${ERR}" >> ${cmd} - echo errorcheck >> ${cmd} - echo "echo \"\`date\`: Target .base snapshot(s) destroyed\" >> $log" >> ${cmd} - fi - else - if [ `zfs list -H -t snapshot -o name $tfs$l_old 2>/dev/null | wc -l | sed 's/^[ \t]*//'` -gt "0" ]; then - echo "sudo ${DESTROY} $tfs$l_old 2>> ${ERR}" >> ${cmd} - echo errorcheck >> ${cmd} - echo "echo \"\`date\`: Target .base snapshot(s) destroyed\" >> $log" >> ${cmd} - fi + if [ "$r" = "yes" ]; then + cat >> ${cmd} << EOF +if [ "\$(zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep -c replicate.base)" -gt "0" ]; then + sudo ${DESTROY} $tfs$l_old 2>> ${ERR} + errorcheck + echo "\$(date): Target .base snapshot(s) destroyed" >> $log +fi +EOF + else + cat >> ${cmd} << EOF +if [ "\$(zfs list -H -t snapshot -o name $tfs$l_old 2>/dev/null | awk 'END{print NR}')" -gt "0" ]; then + sudo ${DESTROY} $tfs$l_old 2>> ${ERR} + errorcheck + echo "\$(date): Target .base snapshot(s) destroyed" >> $log +fi +EOF fi # Rename target .delta snapshot(s) .old: @@ -584,21 +661,20 @@ REPID="`awk -F':' '{print $6}' /etc/passwd | grep replicator`/.ssh/id_rsa" - if [ "$r" = "yes" ] - then - SOURCEBASE=`zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep replicate.base | wc -l | sed 's/^[ \t]*//'` - SOURCEDELTA=`zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep replicate.delta | wc -l | sed 's/^[ \t]*//'` - DESTBASE=`su replicator -c "${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep replicate.base | wc -l | sed 's/^[ \t]*//'"` - DESTDELTA=`su replicator -c "${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep replicate.delta | wc -l | sed 's/^[ \t]*//'"` + if [ "$r" = "yes" ]; then + SOURCEBASE=$(zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep -c replicate.base) + SOURCEDELTA=$(zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep -c replicate.delta) + DESTBASE=$(su replicator -c "${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep -c replicate.base") + DESTDELTA=$(su replicator -c "${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep -c replicate.delta") SNAPSHOT="zfs snapshot -r" SEND="zfs send -R" DESTROY="zfs destroy -r" RENAME="zfs rename -r" - else - SOURCEBASE=`zfs list -H -t snapshot -o name $lfs$r_old 2>/dev/null | wc -l | sed 's/^[ \t]*//'` - SOURCEDELTA=`zfs list -H -t snapshot -o name $lfs$r_new 2>/dev/null | wc -l | sed 's/^[ \t]*//'` - DESTBASE=`su replicator -c "${SSH} $rh zfs list -H -t snapshot -o name $tfs$r_old 2>/dev/null | wc -l | sed 's/^[ \t]*//'"` - DESTDELTA=`su replicator -c "${SSH} $rh zfs list -H -t snapshot -o name $tfs$r_new 2>/dev/null | wc -l | sed 's/^[ \t]*//'"` + else + SOURCEBASE=$(zfs list -H -t snapshot -o name $lfs$r_old 2>/dev/null | awk 'END{print NR}') + SOURCEDELTA=$(zfs list -H -t snapshot -o name $lfs$r_new 2>/dev/null | awk 'END{print NR}') + DESTBASE=$(su replicator -c "${SSH} $rh zfs list -H -t snapshot -o name $tfs$r_old 2>/dev/null | awk 'END{print NR}'") + DESTDELTA=$(su replicator -c "${SSH} $rh zfs list -H -t snapshot -o name $tfs$r_new 2>/dev/null | awk 'END{print NR}'") SNAPSHOT="zfs snapshot" SEND="zfs send -p" DESTROY="zfs destroy" @@ -628,6 +704,19 @@ echo errorcheck >> ${cmd} echo "echo \"\`date\`: Target delta snapshot destroyed\" >> $log" >> ${cmd} fi + if [ "$r" != "yes" ]; then + cat >> ${cmd} << EOF +if [ "\$(${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep "${tfs}@" | egrep -v "$r_old|$r_new" | awk 'END{print NR}')" -gt "0" ]; then + for SNAPSHOT in \$(${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep "${tfs}@" | egrep -v "$r_old|$r_new"); do + if [ "\$(${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep -c "\${SNAPSHOT}")" -gt "0" ]; then + ${SSH} $rh sudo zfs destroy \${SNAPSHOT} 2>> ${ERR} + errorcheck + fi + done + echo "\$(date): Unwanted remote target snapshot(s) destroyed" >> $log +fi +EOF + fi if [ $(su replicator -c "${SSH} $rh zfs list -H -o name -r $tfs 2>/dev/null | awk 'END{print NR}'") -gt "0" ]; then echo "${SSH} $rh sudo ${DESTROY} $tfs 2>> ${ERR}" >> ${cmd} echo errorcheck >> ${cmd} @@ -912,10 +1001,22 @@ echo "echo \"\`date\`: Data replicated\" >> $log" >> ${cmd} # Destroy target .base snapshot(s): - if [ $(su replicator -c "${SSH} $rh zfs list -H -t snapshot -o name -r $tfs 2>/dev/null | grep replicate.base | awk 'END{print NR}'") -gt "0" ]; then - echo "${SSH} $rh sudo ${DESTROY} $tfs$r_old 2>> ${ERR}" >> ${cmd} - echo errorcheck >> ${cmd} - echo "echo \"\`date\`: Target .base snapshot(s) destroyed\" >> $log" >> ${cmd} + if [ "$r" = "yes" ]; then + cat >> ${cmd} << EOF +if [ "\$(${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep -c replicate.base)" -gt "0" ]; then + ${SSH} $rh sudo ${DESTROY} $tfs$r_old 2>> ${ERR} + errorcheck + echo "\$(date): Target .base snapshot(s) destroyed" >> $log +fi +EOF + else + cat >> ${cmd} << EOF +if [ "\$(${SSH} $rh zfs list -H -t snapshot -o name $tfs$r_old 2>/dev/null | awk 'END{print NR}')" -gt "0" ]; then + ${SSH} $rh sudo ${DESTROY} $tfs$r_old 2>> ${ERR} + errorcheck + echo "\$(date): Target .base snapshot(s) destroyed" >> $log +fi +EOF fi # Rename target .delta snapshot(s) to .base: @@ -958,26 +1059,25 @@ tfs=`echo $lfs | cut -f 2-512 -d / | sed "s/^/$lp\//"` - if [ "$r" = "yes" ] - then - SOURCEBASE=`zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep replicate.base | wc -l | sed 's/^[ \t]*//'` - SOURCEDELTA=`zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep replicate.delta | wc -l | sed 's/^[ \t]*//'` - DESTBASE=`zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep replicate.base | wc -l | sed 's/^[ \t]*//'` - DESTDELTA=`zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep replicate.delta | wc -l | sed 's/^[ \t]*//'` + if [ "$r" = "yes" ]; then + SOURCEBASE=$(zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep -c replicate.base) + SOURCEDELTA=$(zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep -c replicate.delta) + DESTBASE=$(zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep -c replicate.base) + DESTDELTA=$(zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep -c replicate.delta) SNAPSHOT="zfs snapshot -r" SEND="zfs send -R" DESTROY="zfs destroy -r" RENAME="zfs rename -r" - else - SOURCEBASE=`zfs list -H -t snapshot -o name $lfs$r_old 2>/dev/null | wc -l | sed 's/^[ \t]*//'` - SOURCEDELTA=`zfs list -H -t snapshot -o name $lfs$r_new 2>/dev/null | wc -l | sed 's/^[ \t]*//'` - DESTBASE=`zfs list -H -t snapshot -o name $tfs$r_old 2>/dev/null | wc -l | sed 's/^[ \t]*//'` - DESTDELTA=`zfs list -H -t snapshot -o name $tfs$r_new 2>/dev/null | wc -l | sed 's/^[ \t]*//'` + else + SOURCEBASE=$(zfs list -H -t snapshot -o name $lfs$r_old 2>/dev/null | awk 'END{print NR}') + SOURCEDELTA=$(zfs list -H -t snapshot -o name $lfs$r_new 2>/dev/null | awk 'END{print NR}') + DESTBASE=$(zfs list -H -t snapshot -o name $tfs$r_old 2>/dev/null | awk 'END{print NR}') + DESTDELTA=$(zfs list -H -t snapshot -o name $tfs$r_new 2>/dev/null | awk 'END{print NR}') SNAPSHOT="zfs snapshot" SEND="zfs send -p" DESTROY="zfs destroy" RENAME="zfs rename" - fi + fi echo "echo \"\`date\`: Beginning local cleaning process on \\\"$lfs\\\"\" >> ${log}" >> ${cmd} @@ -1001,6 +1101,19 @@ echo errorcheck >> ${cmd} echo "echo \"\`date\`: Target delta snapshot destroyed\" >> $log" >> ${cmd} fi + if [ "$r" != "yes" ]; then + cat >> ${cmd} << EOF +if [ "\$(zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep "${tfs}@" | egrep -v "$l_old|$l_new" | awk 'END{print NR}')" -gt "0" ]; then + for SNAPSHOT in \$(zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep "${tfs}@" | egrep -v "$l_old|$l_new"); do + if [ "\$(zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep -c "\${SNAPSHOT}")" -gt "0" ]; then + sudo zfs destroy \${SNAPSHOT} 2>> ${ERR} + errorcheck + fi + done + echo "\$(date): Unwanted local target snapshot(s) destroyed" >> $log +fi +EOF + fi if [ $(zfs list -H -o name -r $tfs 2>/dev/null | awk 'END{print NR}') -gt "0" ]; then echo "sudo ${DESTROY} $tfs 2>> ${ERR}" >> ${cmd} echo errorcheck >> ${cmd} @@ -1049,21 +1162,20 @@ tfs=`echo $lfs | cut -f 2-512 -d / | sed "s/^/$rp\/$lh\//"` - if [ "$r" = "yes" ] - then - SOURCEBASE=`zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep replicate.base | wc -l | sed 's/^[ \t]*//'` - SOURCEDELTA=`zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep replicate.delta | wc -l | sed 's/^[ \t]*//'` - DESTBASE=`su replicator -c "${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep replicate.base | wc -l | sed 's/^[ \t]*//'"` - DESTDELTA=`su replicator -c "${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep replicate.delta | wc -l | sed 's/^[ \t]*//'"` + if [ "$r" = "yes" ]; then + SOURCEBASE=$(zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep -c replicate.base) + SOURCEDELTA=$(zfs list -H -r -t snapshot -o name $lfs 2>/dev/null | grep -c replicate.delta) + DESTBASE=$(su replicator -c "${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep -c replicate.base") + DESTDELTA=$(su replicator -c "${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep -c replicate.delta") SNAPSHOT="zfs snapshot -r" SEND="zfs send -R" DESTROY="zfs destroy -r" RENAME="zfs rename -r" - else - SOURCEBASE=`zfs list -H -t snapshot -o name $lfs$r_old 2>/dev/null | wc -l | sed 's/^[ \t]*//'` - SOURCEDELTA=`zfs list -H -t snapshot -o name $lfs$r_new 2>/dev/null | wc -l | sed 's/^[ \t]*//'` - DESTBASE=`su replicator -c "${SSH} $rh zfs list -H -t snapshot -o name $tfs$r_old 2>/dev/null | wc -l | sed 's/^[ \t]*//'"` - DESTDELTA=`su replicator -c "${SSH} $rh zfs list -H -t snapshot -o name $tfs$r_new 2>/dev/null | wc -l | sed 's/^[ \t]*//'"` + else + SOURCEBASE=$(zfs list -H -t snapshot -o name $lfs$r_old 2>/dev/null | awk 'END{print NR}') + SOURCEDELTA=$(zfs list -H -t snapshot -o name $lfs$r_new 2>/dev/null | awk 'END{print NR}') + DESTBASE=$(su replicator -c "${SSH} $rh zfs list -H -t snapshot -o name $tfs$r_old 2>/dev/null | awk 'END{print NR}'") + DESTDELTA=$(su replicator -c "${SSH} $rh zfs list -H -t snapshot -o name $tfs$r_new 2>/dev/null | awk 'END{print NR}'") SNAPSHOT="zfs snapshot" SEND="zfs send -p" DESTROY="zfs destroy" @@ -1092,6 +1204,19 @@ echo errorcheck >> ${cmd} echo "echo \"\`date\`: Target delta snapshot destroyed\" >> $log" >> ${cmd} fi + if [ "$r" != "yes" ]; then + cat >> ${cmd} << EOF +if [ "\$(${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep "${tfs}@" | egrep -v "$r_old|$r_new" | awk 'END{print NR}')" -gt "0" ]; then + for SNAPSHOT in \$(${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep "${tfs}@" | egrep -v "$r_old|$r_new"); do + if [ "\$(${SSH} $rh zfs list -H -r -t snapshot -o name $tfs 2>/dev/null | grep -c "\${SNAPSHOT}")" -gt "0" ]; then + ${SSH} $rh sudo zfs destroy \${SNAPSHOT} 2>> ${ERR} + errorcheck + fi + done + echo "\$(date): Unwanted remote target snapshot(s) destroyed" >> $log +fi +EOF + fi if [ $(su replicator -c "${SSH} $rh zfs list -H -o name -r $tfs 2>/dev/null | awk 'END{print NR}'") -gt "0" ]; then echo "${SSH} $rh sudo ${DESTROY} $tfs 2>> ${ERR}" >> ${cmd} echo errorcheck >> ${cmd} @@ -1122,9 +1247,317 @@ sanity() { if [ -e $pid ]; then - echo "`date`: Started a new process, but the previous \"replicate\" is still running, aborting." >> $log - echo "" >> $log - rm ${ERR}; rm ${cmd}; rm ${mail}; exit 1 + CMD_BASE=$(echo ${cmd} | sed -E 's/\.[[:alnum:]]*$/\./') + + # Since replicate has fired again (that is why it is running now), it has + # created a new job for _this_ run. So when searching for latest job, we + # scroll back two steps, instead of just one, which we already know is going + # to be empty any way: + + LAST_RUN_JOB=$(ls -lU -D %y-%m-%d_%H-%M-%S ${CMD_BASE}* | egrep -o "([0-9]|\-|\_){17}.*$" | sort -n -t '_' -k 1,8 | tail -2 | head -1 | cut -d ' ' -f 2) + COMMANDS_IN_JOB=$(sed -E -e 's/\$\{.*\}//' -e 's/^\ {4}if \[ "\$\(//' -e 's#2>/dev/null.*$##' ${LAST_RUN_JOB} | egrep -v '^(echo|errorcheck|exit|if|fi|for|done|^#|^$)|^\ (\{|\})|^\ {2}(echo|errorcheck|if|fi|for|done)|^\ {4}(tail|mail|cat|exit|errorcheck|if|fi)|^\ {6}errorcheck' | tr '|' '\n' | sed -E -e 's/^\ *//' -e 's/ 2>>.*$//') + if [ $(echo "${COMMANDS_IN_JOB}" | wc -w) -le 0 ]; then + for JOB in $(ls -lU -D %y-%m-%d_%H-%M-%S ${CMD_BASE}* | egrep -o "([0-9]|\-|\_){17}.*$" | sort -r -n -t '_' -k 1,8 | cut -d ' ' -f 2); do + COMMANDS_IN_JOB=$(sed -E -e 's/\$\{.*\}//' -e 's/^\ {4}if \[ "\$\(//' -e 's#2>/dev/null.*$##' ${JOB} | egrep -v '^(echo|errorcheck|exit|if|fi|for|done|^#|^$)|^\ (\{|\})|^\ {2}(echo|errorcheck|if|fi|for|done)|^\ {4}(tail|mail|cat|exit|errorcheck|if|fi)|^\ {6}errorcheck' | tr '|' '\n' | sed -E -e 's/^\ *//' -e 's/ 2>>.*$//') + if [ $(echo "${COMMANDS_IN_JOB}" | wc -w) -ne 0 ]; then + LAST_RUN_JOB=${JOB} + break + else + LAST_RUN_JOB="" + fi + done + fi + replicate_cleanup() { + REPLICATE_PID_LOOP=$(printf "${REPLICATE_PROCESSES}" | awk '{print$1}') + for REPLICATE_PID in ${REPLICATE_PID_LOOP}; do + USED_CPU=$(ps -a -x -o pid,%cpu,command | grep "${REPLICATE_PID}" | grep -v grep | awk '{sub(/\./,"");print$2}') + if [ $(echo "${USED_CPU}" | wc -w) -ne 0 ]; then + + # OK, so there is at least a process there: + + if [ $(printf "${USED_CPU}" | wc -l) -gt 1 ]; then + USED_CPU=$(printf "${USED_CPU}" | awk '{if($1>a)a=$1};{if(a=="")a="00"};END{print a}') + fi + else + USED_CPU="00" + fi + if [ "${USED_CPU}" = "00" ]; then + USED_CPU="0" + else + USED_CPU=$(echo "${USED_CPU}" | sed 's/0//g') + fi + if [ ${USED_CPU} -ne 0 ]; then + RUNNING_PROCESSES="${RUNNING_PROCESSES} ${REPLICATE_PID}" + else + UNKNOWN_STATUS_PROCESSES="${UNKNOWN_STATUS_PROCESSES} ${REPLICATE_PID}" + fi + done + if [ $(echo "${RUNNING_PROCESSES}" | wc -w) -ne 0 ]; then + echo "$(date): Started a new process, but previous \"replicate\" processes are still running, aborting." >> $log + echo "" >> $log + rm ${ERR}; rm ${cmd}; rm ${mail}; exit 1 + else + if [ $(echo "${UNKNOWN_STATUS_PROCESSES}" | wc -w) -ne 0 ]; then + for UNKNOWN_STATUS_PROCESS in ${UNKNOWN_STATUS_PROCESSES}; do + if [ $(ps ax | grep "${UNKNOWN_STATUS_PROCESS}" | grep -v grep | egrep -c ' D.* ') -ne 0 ]; then + ERROR_MESSAGE="The previous replicate command set has been found with no running processes and seems to be stuck. A process from the last run has been found in \"D\" state which can not be killed, reboot required" + echo "${ERROR_MESSAGE}" >> ${mail} + echo "$(ps ax | grep "${UNKNOWN_STATUS_PROCESS}" | grep -v grep | egrep ' D.* ')" >> ${mail} + mail -s "$subject" $address < ${mail} + echo "$(date): ${ERROR_MESSAGE}" >> $log + echo "" >> $log + rm ${ERR}; rm ${cmd}; rm ${mail}; exit 1 + else + kill ${UNKNOWN_STATUS_PROCESS} + echo "$(date): No process from previous replicate run found running. Killed idle process \"${UNKNOWN_STATUS_PROCESS}\" as part of the cleanup before trying to start again." >> $log + echo "" >> $log + fi + done + else + echo "$(date): No idle or running processes from previous replicate run found running." >> $log + echo "" >> $log + fi + fi + } + confirm_activity() { + + while [ "$#" -gt "0" ]; do + case "$1" in + "-r") + MODE="$1" + ;; + "-d") + DESTROYING="$1" + ;; + *) + echo "Unknown argument '$1'" >&2 + ;; + esac + shift + done + + if [ "${MODE}" = "-r" ]; then + if [ "${DESTROYING}" = "-d" ]; then + USED_CPU=$(printf "${REPLICATE_PROCESSES}" | grep 'ssh' | egrep 'zfs destroy|zfs list' | awk '{sub(/\./,"");print$2}') + else + USED_CPU=$(printf "${REPLICATE_PROCESSES}" | grep 'ssh' | grep 'zfs recv' | awk '{sub(/\./,"");print$2}') + fi + else + if [ "${DESTROYING}" = "-d" ]; then + USED_CPU=$(printf "${REPLICATE_PROCESSES}" | egrep 'zfs destroy|zfs list' | awk '{sub(/\./,"");print$2}') + else + USED_CPU=$(printf "${REPLICATE_PROCESSES}" | grep 'zfs recv' | awk '{sub(/\./,"");print$2}') + fi + fi + if [ $(printf "${USED_CPU}" | wc -l) -gt 1 ]; then + + # After a very thourough screening we still found more than one active + # process that match but we can only have one to decide with, so letting + # 'awk' choose the process with the highest CPU usage: + + USED_CPU=$(printf "${USED_CPU}" | awk '{if($1>a)a=$1};{if(a=="")a="00"};END{print a}') + fi + if [ ${USED_CPU} = "00" ]; then + USED_CPU="0" + else + USED_CPU=$(echo "${USED_CPU}" | sed 's/0//g') + fi + if [ ${USED_CPU} -ne 0 ]; then + + # Found running process: + + echo "$(date): Started a new process, but the previous remote \"replicate\" is still running, aborting." >> $log + echo "" >> $log + rm ${ERR}; rm ${cmd}; rm ${mail}; exit 1 + else + if [ "${MODE}" = "-r" ]; then + + # Found 'zfs recv' process but considered idle. Need to check if + # still active on receiving end. + + SSH="ssh" + if [ ! -z ${port} ]; then + SSH="${SSH} -p ${port}" + else + SSH="${SSH} -p 22" + fi + REPID="$(awk -F':' '{print $6}' /etc/passwd | grep replicator)/.ssh/id_rsa" + REMOTE_REPLICATE_COMMANDS=$(printf "${REPLICATE_PROCESSES}" | egrep ' ssh ' | tr '|' '\n' | sed -E -e "s/^[0-9]+\ +[0-9]+\.[0-9]+\ //" -e 's/.* sudo //' | tr '\n' '|' | sed -E "s/\|$//") + REMOTE_REPLICATE_PROCESSES=$(${SSH} -i ${REPID} replicator@$rh ps -a -x -o pid,%cpu,command | egrep "${REMOTE_REPLICATE_COMMANDS}" | egrep -v 'egrep| sudo ') + if [ "${DESTROYING}" = "-d" ]; then + USED_CPU=$(printf "${REMOTE_REPLICATE_PROCESSES}" | egrep 'zfs destroy|zfs list' | awk '{sub(/\./,"");print$2}') + else + USED_CPU=$(printf "${REMOTE_REPLICATE_PROCESSES}" | grep 'zfs recv' | awk '{sub(/\./,"");print$2}') + fi + if [ $(printf "${USED_CPU}" | wc -l) -gt 1 ]; then + USED_CPU=$(printf "${USED_CPU}" | awk '{if($1>a)a=$1};{if(a=="")a="00"};END{print a}') + fi + if [ ${USED_CPU} = "00" ]; then + USED_CPU="0" + else + USED_CPU=$(echo "${USED_CPU}" | sed 's/0//g') + fi + if [ ${USED_CPU} -ne 0 ]; then + + # Found running process: + + echo "$(date): Started a new process, but the previous remote \"replicate\" is still running, aborting." >> $log + echo "" >> $log + rm ${ERR}; rm ${cmd}; rm ${mail}; exit 1 + else + + # But for crying out loud, it isn't using any CPU time either? + # OK, what is left to check then, is the size changing? + + LAST_RUN_FILESYSTEM=$(printf "${COMMANDS_IN_JOB}" | grep 'zfs send' | awk '{print $NF}' | cut -d '@' -f 1) + TARGET_FILESYSTEM=$(echo ${LAST_RUN_FILESYSTEM} | cut -f 2-512 -d / | sed "s/^/$rp\/$lh\//") + check_target_size() { + ${SSH} -i ${REPID} replicator@$rh zfs get -H -p -o value used ${TARGET_FILESYSTEM} + } + SIZE_BEFORE=$(check_target_size) + sleep 120 + SIZE_AFTER=$(check_target_size) + if [ "${SIZE_BEFORE}" -ne "${SIZE_AFTER}" ]; then + + # Finally, a hit! Then it should at least be doing something. + + echo "$(date): Started a new process, but the previous remote \"replicate\" is still running, aborting." >> $log + echo "" >> $log + rm ${ERR}; rm ${cmd}; rm ${mail}; exit 1 + else + replicate_cleanup + fi + fi + else + + # But for crying out loud, it isn't using any CPU time either? + # OK, what is left to check then, is the size changing? + + LAST_RUN_FILESYSTEM=$(printf "${COMMANDS_IN_JOB}" | grep 'zfs send' | awk '{print $NF}' | cut -d '@' -f 1) + TARGET_FILESYSTEM=$(echo ${LAST_RUN_FILESYSTEM} | cut -f 2-512 -d / | sed "s/^/$lp\//") + check_target_size() { + zfs get -H -p -o value used ${TARGET_FILESYSTEM} + } + SIZE_BEFORE=$(check_target_size) + sleep 120 + SIZE_AFTER=$(check_target_size) + if [ "${SIZE_BEFORE}" -ne "${SIZE_AFTER}" ]; then + + # Finally, a hit! Then it should at least be doing something. + + echo "$(date): Started a new process, but the previous local \"replicate\" is still running, aborting." >> $log + echo "" >> $log + rm ${ERR}; rm ${cmd}; rm ${mail}; exit 1 + else + replicate_cleanup + fi + fi + fi + } + confirm_active_recv() { + if [ ! -z ${rh} ]; then + + # Means it is a remote replication job: + + if [ $(printf "${REPLICATE_PROCESSES}" | grep -c 'ssh') -ne 0 ]; then + + # Found active 'ssh' transfer: + + if [ $(printf "${REPLICATE_PROCESSES}" | grep 'ssh' | grep -c 'zfs recv') -ne 0 ]; then + + # Found active 'zfs recv': + + confirm_activity -r + else + + # No active 'zfs recv' found running. Is it destroying? + + if [ $(printf "${REPLICATE_PROCESSES}" | grep 'ssh' | egrep -c 'zfs destroy|zfs list') -ne 0 ]; then + + # It's supposed to be destroying snapshots at the moment. Let's make + # sure: + + confirm_activity -r -d + else + replicate_cleanup + fi + fi + else + + # No active 'ssh' transfer found running when set as remote: + + replicate_cleanup + fi + else + + # If not remote, then it is a local replication job: + + if [ $(printf "${REPLICATE_PROCESSES}" | grep -c 'zfs recv') -ne 0 ]; then + + # Found active 'zfs recv': + + confirm_activity + else + + # No active 'zfs recv' found running. Is it destroying? + + if [ $(printf "${REPLICATE_PROCESSES}" | egrep -c 'zfs destroy|zfs list') -ne 0 ]; then + + # It's supposed to be destroying snapshots at the moment. Let's make + # sure: + + confirm_activity -d + else + replicate_cleanup + fi + fi + fi + } + confirm_replicate_status() { + if [ $(printf "${REPLICATE_PROCESSES}" | wc -w) -ne 0 ]; then + if [ $(printf "${REPLICATE_PROCESSES}" | egrep -c 'zfs send|zfs recv|zfs destroy|zfs list') -ne 0 ]; then + if [ $(printf "${REPLICATE_PROCESSES}" | grep -c 'zfs send') -ne 0 ]; then + + # Found active 'zfs send': + + confirm_active_recv + else + + # No active 'zfs send' found running: + + confirm_active_recv + fi + else + + # No active 'zfs send' or 'zfs recv' found running: + + replicate_cleanup + fi + else + + # No processes left running. Starting new job. + + : + fi + } + if [ $(echo ${LAST_RUN_JOB} | wc -w) -ne 0 ]; then + if [ $(ps ax | egrep "${LAST_RUN_JOB}" | grep -v egrep -c) -ne 0 ]; then + REPLICATE_PROCESSES=$(ps -a -x -o pid,%cpu,command | egrep "${LAST_RUN_JOB}|${COMMANDS_IN_JOB}" | grep -v egrep) + confirm_replicate_status + else + + # The last known job to have commands is not running any more: + + replicate_cleanup + fi + else + + # No previous job found with valid commands. Retrying with broadened + # search criteria: + + REPLICATE_PROCESSES=$(ps -a -x -o pid,%cpu,command | egrep 'zfs send|zfs recv|zfs snapshot|zfs rename|zfs destroy|zfs list' | egrep -v 'egrep|zfSnap') + confirm_replicate_status + fi fi chown replicator ${mail}; chmod 600 ${mail} chown replicator ${ERR}; chmod 600 ${ERR} @@ -1193,8 +1626,8 @@ for job in $jobs do - sanity . $job + sanity touch $pid if [ -z "$rh" ] then @@ -1244,8 +1677,8 @@ if [ "$2" = "" ]; then for job in `find $path -type f ! -iname sample_* ! -iname noauto_* | sort -rd` do - sanity . $job + sanity touch $pid if [ -z "$rh" ] then @@ -1267,8 +1700,8 @@ ## if [ -e ${path}$2 ]; then - sanity . $path$2 + sanity touch $pid if [ -z "$rh" ] then @@ -1332,8 +1765,8 @@ if [ -e ${path}$1 ] then - sanity . $path$1 + sanity touch $pid if [ -z "$rh" ] then