patrick/zfs_autobackup
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added resume support via receive_resume_token. moved stuff to main function to make it clear that only args should be used globally

Browse Source
This commit is contained in:
Edwin Eefting
2017-07-26 02:20:14 +02:00
parent 5474a0a156
commit b4507bae27
1 changed files with 272 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files

162
zfs_autobackup
View File

@ -79,7 +79,6 @@ def run(cmd, input=None, ssh_to="local", tab_split=False, valid_exitcodes=[ 0 ],
return(ret)
"""determine filesystems that should be backupped by looking at the special autobackup-property"""
def zfs_get_selected_filesystems(ssh_to, backup_name):
#get all source filesystems that have the backup property
@ -113,6 +112,23 @@ def zfs_get_selected_filesystems(ssh_to, backup_name):
return(selected_filesystems)
"""determine filesystems that can be resumed via receive_resume_token"""
def zfs_get_resumable_filesystems(ssh_to, filesystems):
cmd=[ "zfs", "get", "-t", "volume,filesystem", "-o", "name,value", "-H", "receive_resume_token" ]
cmd.extend(filesystems)
#TODO: get rid of ugly errors for non-existing target filesystems
resumable_filesystems=run(ssh_to=ssh_to, tab_split=True, cmd=cmd, valid_exitcodes= [ 0,1 ] )
ret={}
for (resumable_filesystem,token) in resumable_filesystems:
if token!='-':
ret[resumable_filesystem]=token
return(ret)
"""deferred destroy list of snapshots (in @format). """
def zfs_destroy_snapshots(ssh_to, snapshots):
@ -167,6 +183,7 @@ def zfs_get_snapshots(ssh_to, filesystems, backup_name):
ret={}
if filesystems:
#TODO: get rid of ugly errors for non-existing target filesystems
snapshots=run(ssh_to=ssh_to, input="\0".join(filesystems), valid_exitcodes=[ 0,1 ], cmd=
[ "xargs", "-0", "-n", "1", "zfs", "list", "-d", "1", "-r", "-t" ,"snapshot", "-H", "-o", "name" ]
)
@ -193,7 +210,10 @@ def zfs_get_snapshots(ssh_to, filesystems, backup_name):
"""transfer a zfs snapshot from source to target. both can be either local or via ssh.
specify buffer_size to use mbuffer (or alike) to apply buffering where neccesary
TODO:
buffering: specify buffer_size to use mbuffer (or alike) to apply buffering where neccesary
local to local:
local send -> local buffer -> local receive
@ -204,9 +224,14 @@ remote send -> remote buffer -> ssh -> local buffer -> local receive
remote to remote:
remote send -> remote buffer -> ssh -> local buffer -> ssh -> remote buffer -> remote receive
TODO: can we string together all the zfs sends and recvs, so that we only need to use 1 ssh connection? should be faster if there are many small snaphots
"""
def zfs_transfer(ssh_source, source_filesystem, first_snapshot, second_snapshot,
ssh_target, target_filesystem, buffer_size=None):
ssh_target, target_filesystem, resume_token=None, buffer_size=None):
#### build source command
source_cmd=[]
@ -218,27 +243,35 @@ def zfs_transfer(ssh_source, source_filesystem, first_snapshot, second_snapshot,
if args.compress:
source_cmd.append("-C")
source_cmd.extend(["zfs", "send", "-p" ])
source_cmd.extend(["zfs", "send", ])
#only verbose in debug mode, lots of output
if args.debug:
source_cmd.append("-v")
if not first_snapshot:
verbose("Tranferring "+source_filesystem+" initial backup snapshot "+second_snapshot)
txt="Initial transfer of "+source_filesystem+" snapshot "+second_snapshot
else:
verbose("Tranferring "+source_filesystem+" incremental backup between snapshots "+first_snapshot+"..."+second_snapshot)
txt="Incremental transfer of "+source_filesystem+" between snapshots "+first_snapshot+"..."+second_snapshot
if resume_token:
source_cmd.extend([ "-t", resume_token ])
verbose("RESUMING "+txt)
else:
source_cmd.append("-p")
if first_snapshot:
source_cmd.extend([ "-i", first_snapshot ])
# FIXME needs attention
if ssh_source != "local":
source_cmd.append(source_filesystem.replace(' ', '\ ') + "@" + second_snapshot)
else:
source_cmd.append(source_filesystem + "@" + second_snapshot)
# if ssh_source != "local":
# #add buffer
# source_cmd.append("|dd")
verbose(txt)
#### build target command
target_cmd=[]
@ -255,12 +288,19 @@ def zfs_transfer(ssh_source, source_filesystem, first_snapshot, second_snapshot,
#also verbose in --verbose mode so we can see the transfer speed when its completed
if args.verbose or args.debug:
target_cmd.append("-v")
if args.resume:
target_cmd.append("-s")
# FIXME needs attention
if ssh_target != "local":
target_cmd.append(target_filesystem.replace(' ', '\ '))
else:
target_cmd.append(target_filesystem)
#### make sure parent on target exists
parent_filesystem= "/".join(target_filesystem.split("/")[:-1])
run(ssh_to=ssh_target, test=args.test, input=parent_filesystem + "\0", cmd=
@ -349,38 +389,7 @@ def lstrip_path(path, count):
################################################################## ENTRY POINT
############## parse arguments
import argparse
parser = argparse.ArgumentParser(description='ZFS autobackup v2.0')
parser.add_argument('--ssh-source', default="local", help='Source host to get backup from. (user@hostname) Default %(default)s.')
parser.add_argument('--ssh-target', default="local", help='Target host to push backup to. (user@hostname) Default %(default)s.')
parser.add_argument('--ssh-cipher', default=None, help='SSH cipher to use (default %(default)s)')
parser.add_argument('--keep-source', type=int, default=30, help='Number of days to keep old snapshots on source. Default %(default)s.')
parser.add_argument('--keep-target', type=int, default=30, help='Number of days to keep old snapshots on target. Default %(default)s.')
parser.add_argument('backup_name', help='Name of the backup (you should set the zfs property "autobackup:backup-name" to true on filesystems you want to backup')
parser.add_argument('target_fs', help='Target filesystem')
parser.add_argument('--no-snapshot', action='store_true', help='dont create new snapshot (usefull for finishing uncompleted backups, or cleanups)')
parser.add_argument('--no-send', action='store_true', help='dont send snapshots (usefull to only do a cleanup)')
parser.add_argument('--strip-path', default=0, type=int, help='number of directory to strip from path (use 1 when cloning zones between 2 SmartOS machines)')
parser.add_argument('--destroy-stale', action='store_true', help='Destroy stale backups that have no more snapshots. Be sure to verify the output before using this! ')
parser.add_argument('--clear-refreservation', action='store_true', help='Set refreservation property to none for new filesystems. Usefull when backupping SmartOS volumes. (recommended)')
parser.add_argument('--clear-mountpoint', action='store_true', help='Sets canmount=noauto property, to prevent the received filesystem from mounting over existing filesystems. (recommended)')
parser.add_argument('--rollback', action='store_true', help='Rollback changes on the target before starting a backup. (normally you can prevent changes by setting the readonly property on the target_fs to on)')
parser.add_argument('--compress', action='store_true', help='use compression during zfs send/recv')
parser.add_argument('--test', action='store_true', help='dont change anything, just show what would be done (still does all read-only operations)')
parser.add_argument('--verbose', action='store_true', help='verbose output')
parser.add_argument('--debug', action='store_true', help='debug output (shows commands that are executed)')
args = parser.parse_args()
def zfs_autobackup():
@ -390,6 +399,9 @@ if args.test:
args.verbose=True
verbose("RUNNING IN TEST-MODE, NOT MAKING ACTUAL BACKUP!")
### getting and determinging source/target filesystems
# get selected filesystem on backup source
verbose("Getting selected source filesystems for backup {0} on {1}".format(args.backup_name,args.ssh_source))
source_filesystems=zfs_get_selected_filesystems(args.ssh_source, args.backup_name)
@ -406,13 +418,27 @@ for source_filesystem in source_filesystems:
#append args.target_fs prefix and strip args.strip_path paths from source_filesystem
target_filesystems.append(args.target_fs + "/" + lstrip_path(source_filesystem, args.strip_path))
### creating snapshots
# this is one of the first things we do, so that in case of failures we still have snapshots.
#create new snapshot?
if not args.no_snapshot:
new_snapshot_name=args.backup_name+"-"+time.strftime("%Y%m%d%H%M%S")
verbose("Creating source snapshot {0} on {1} ".format(new_snapshot_name, args.ssh_source))
zfs_create_snapshot(args.ssh_source, source_filesystems, new_snapshot_name)
#get all snapshots of all selected filesystems on both source and target
### get resumable transfers
resumable_target_filesystems={}
if args.resume:
verbose("Checking for aborted transfers that can be resumed")
resumable_target_filesystems=zfs_get_resumable_filesystems(args.ssh_target, target_filesystems)
debug("Resumable filesystems: "+str(pprint.pformat(resumable_target_filesystems)))
### get all snapshots of all selected filesystems on both source and target
verbose("Getting source snapshot-list from {0}".format(args.ssh_source))
source_snapshots=zfs_get_snapshots(args.ssh_source, source_filesystems, args.backup_name)
debug("Source snapshots: " + str(pprint.pformat(source_snapshots)))
@ -491,12 +517,21 @@ for source_filesystem in source_filesystems:
for send_snapshot in send_snapshots:
#resumable?
if target_filesystem in resumable_target_filesystems:
resume_token=resumable_target_filesystems.pop(target_filesystem)
else:
resume_token=None
zfs_transfer(
ssh_source=args.ssh_source, source_filesystem=source_filesystem,
first_snapshot=latest_target_snapshot, second_snapshot=send_snapshot,
ssh_target=args.ssh_target, target_filesystem=target_filesystem
ssh_target=args.ssh_target, target_filesystem=target_filesystem,
resume_token=resume_token
)
#now that we succesfully transferred this snapshot, the previous snapshot is obsolete:
if latest_target_snapshot:
target_obsolete_snapshots[target_filesystem].append(latest_target_snapshot)
@ -564,3 +599,42 @@ if target_destroys:
verbose("All done")
################################################################## ENTRY POINT
# parse arguments
import argparse
parser = argparse.ArgumentParser(description='ZFS autobackup v2.0')
parser.add_argument('--ssh-source', default="local", help='Source host to get backup from. (user@hostname) Default %(default)s.')
parser.add_argument('--ssh-target', default="local", help='Target host to push backup to. (user@hostname) Default %(default)s.')
parser.add_argument('--ssh-cipher', default=None, help='SSH cipher to use (default %(default)s)')
parser.add_argument('--keep-source', type=int, default=30, help='Number of days to keep old snapshots on source. Default %(default)s.')
parser.add_argument('--keep-target', type=int, default=30, help='Number of days to keep old snapshots on target. Default %(default)s.')
parser.add_argument('backup_name', help='Name of the backup (you should set the zfs property "autobackup:backup-name" to true on filesystems you want to backup')
parser.add_argument('target_fs', help='Target filesystem')
parser.add_argument('--no-snapshot', action='store_true', help='dont create new snapshot (usefull for finishing uncompleted backups, or cleanups)')
parser.add_argument('--no-send', action='store_true', help='dont send snapshots (usefull to only do a cleanup)')
parser.add_argument('--resume', action='store_true', help='support resuming of interrupted transfers by using the zfs extensible_dataset feature (both zpools should have it enabled)')
parser.add_argument('--strip-path', default=0, type=int, help='number of directory to strip from path (use 1 when cloning zones between 2 SmartOS machines)')
parser.add_argument('--destroy-stale', action='store_true', help='Destroy stale backups that have no more snapshots. Be sure to verify the output before using this! ')
parser.add_argument('--clear-refreservation', action='store_true', help='Set refreservation property to none for new filesystems. Usefull when backupping SmartOS volumes. (recommended)')
parser.add_argument('--clear-mountpoint', action='store_true', help='Sets canmount=noauto property, to prevent the received filesystem from mounting over existing filesystems. (recommended)')
parser.add_argument('--rollback', action='store_true', help='Rollback changes on the target before starting a backup. (normally you can prevent changes by setting the readonly property on the target_fs to on)')
parser.add_argument('--compress', action='store_true', help='use compression during zfs send/recv')
parser.add_argument('--test', action='store_true', help='dont change anything, just show what would be done (still does all read-only operations)')
parser.add_argument('--verbose', action='store_true', help='verbose output')
parser.add_argument('--debug', action='store_true', help='debug output (shows commands that are executed)')
#note args is the only global variable we use, since its a global readonly setting anyway
args = parser.parse_args()
zfs_autobackup()