added resume support via receive_resume_token. moved stuff to main function to make it clear that only args should be used globally

zfs_autobackup

@@ -79,7 +79,6 @@ def run(cmd, input=None, ssh_to="local", tab_split=False, valid_exitcodes=[ 0 ],
         return(ret)
 
 
-
 """determine filesystems that should be backupped by looking at the special autobackup-property"""
 def zfs_get_selected_filesystems(ssh_to, backup_name):
     #get all source filesystems that have the backup property
@@ -113,6 +112,23 @@ def zfs_get_selected_filesystems(ssh_to, backup_name):
     return(selected_filesystems)
 
 
+"""determine filesystems that can be resumed via receive_resume_token"""
+def zfs_get_resumable_filesystems(ssh_to, filesystems):
+
+    cmd=[ "zfs", "get", "-t",  "volume,filesystem", "-o", "name,value", "-H", "receive_resume_token" ]
+    cmd.extend(filesystems)
+
+    #TODO: get rid of ugly errors for non-existing target filesystems
+    resumable_filesystems=run(ssh_to=ssh_to, tab_split=True, cmd=cmd, valid_exitcodes= [ 0,1 ] )
+
+    ret={}
+
+    for (resumable_filesystem,token) in resumable_filesystems:
+        if token!='-':
+            ret[resumable_filesystem]=token
+
+    return(ret)
+
 
 """deferred destroy list of snapshots (in @format). """
 def zfs_destroy_snapshots(ssh_to, snapshots):
@@ -167,6 +183,7 @@ def zfs_get_snapshots(ssh_to, filesystems, backup_name):
     ret={}
 
     if filesystems:
+        #TODO: get rid of ugly errors for non-existing target filesystems
         snapshots=run(ssh_to=ssh_to, input="\0".join(filesystems), valid_exitcodes=[ 0,1 ], cmd=
             [ "xargs", "-0", "-n", "1", "zfs", "list", "-d", "1", "-r", "-t" ,"snapshot", "-H", "-o", "name" ]
         )
@@ -193,7 +210,10 @@ def zfs_get_snapshots(ssh_to, filesystems, backup_name):
 
 """transfer a zfs snapshot from source to target. both can be either local or via ssh.
 
-specify buffer_size to use mbuffer (or alike) to apply buffering where neccesary
+
+TODO:
+
+buffering: specify buffer_size to use mbuffer (or alike) to apply buffering where neccesary
 
 local to local:
 local send -> local buffer -> local receive
@@ -204,9 +224,14 @@ remote send -> remote buffer -> ssh -> local buffer -> local receive
 
 remote to remote:
 remote send -> remote buffer -> ssh -> local buffer -> ssh -> remote buffer -> remote receive
+
+TODO: can we string together all the zfs sends and recvs, so that we only need to use 1 ssh connection? should be faster if there are many small snaphots
+
+
+
 """
 def zfs_transfer(ssh_source, source_filesystem, first_snapshot, second_snapshot,
-                 ssh_target, target_filesystem, buffer_size=None):
+                 ssh_target, target_filesystem, resume_token=None, buffer_size=None):
 
     #### build source command
     source_cmd=[]
@@ -218,27 +243,35 @@ def zfs_transfer(ssh_source, source_filesystem, first_snapshot, second_snapshot,
         if args.compress:
             source_cmd.append("-C")
 
-    source_cmd.extend(["zfs", "send", "-p" ])
+    source_cmd.extend(["zfs", "send",  ])
 
     #only verbose in debug mode, lots of output
     if args.debug:
         source_cmd.append("-v")
 
+
     if not first_snapshot:
-        verbose("Tranferring "+source_filesystem+" initial backup snapshot "+second_snapshot)
+        txt="Initial transfer of "+source_filesystem+" snapshot "+second_snapshot
     else:
-        verbose("Tranferring "+source_filesystem+" incremental backup between snapshots "+first_snapshot+"..."+second_snapshot)
+        txt="Incremental transfer of "+source_filesystem+" between snapshots "+first_snapshot+"..."+second_snapshot
+
+    if resume_token:
+        source_cmd.extend([ "-t", resume_token ])
+        verbose("RESUMING "+txt)
+
+    else:
+        source_cmd.append("-p")
+
+        if first_snapshot:
             source_cmd.extend([ "-i", first_snapshot ])
+
         # FIXME needs attention
         if ssh_source != "local":
             source_cmd.append(source_filesystem.replace(' ', '\ ') + "@" + second_snapshot)
         else:
             source_cmd.append(source_filesystem + "@" + second_snapshot)
 
-    # if ssh_source != "local":
+        verbose(txt)
-    #     #add buffer
-    #     source_cmd.append("|dd")
-
 
     #### build target command
     target_cmd=[]
@@ -255,12 +288,19 @@ def zfs_transfer(ssh_source, source_filesystem, first_snapshot, second_snapshot,
     #also verbose in --verbose mode so we can see the transfer speed when its completed
     if args.verbose or args.debug:
         target_cmd.append("-v")
+
+    if args.resume:
+        target_cmd.append("-s")
+
+
     # FIXME needs attention
     if ssh_target != "local":
         target_cmd.append(target_filesystem.replace(' ', '\ '))
     else:
         target_cmd.append(target_filesystem)
 
+
+
     #### make sure parent on target exists
     parent_filesystem= "/".join(target_filesystem.split("/")[:-1])
     run(ssh_to=ssh_target, test=args.test, input=parent_filesystem + "\0", cmd=
@@ -349,38 +389,7 @@ def lstrip_path(path, count):
 
 
 
-################################################################## ENTRY POINT
+def zfs_autobackup():
-
-
-
-############## parse arguments
-import argparse
-parser = argparse.ArgumentParser(description='ZFS autobackup v2.0')
-parser.add_argument('--ssh-source', default="local", help='Source host to get backup from. (user@hostname) Default %(default)s.')
-parser.add_argument('--ssh-target', default="local", help='Target host to push backup to. (user@hostname) Default  %(default)s.')
-parser.add_argument('--ssh-cipher', default=None, help='SSH cipher to use (default  %(default)s)')
-parser.add_argument('--keep-source', type=int, default=30, help='Number of days to keep old snapshots on source. Default %(default)s.')
-parser.add_argument('--keep-target', type=int, default=30, help='Number of days to keep old snapshots on target. Default %(default)s.')
-parser.add_argument('backup_name',    help='Name of the backup (you should set the zfs property "autobackup:backup-name" to true on filesystems you want to backup')
-parser.add_argument('target_fs',    help='Target filesystem')
-
-parser.add_argument('--no-snapshot', action='store_true', help='dont create new snapshot (usefull for finishing uncompleted backups, or cleanups)')
-parser.add_argument('--no-send', action='store_true', help='dont send snapshots (usefull to only do a cleanup)')
-
-parser.add_argument('--strip-path', default=0, type=int, help='number of directory to strip from path (use 1 when cloning zones between 2 SmartOS machines)')
-
-
-parser.add_argument('--destroy-stale', action='store_true', help='Destroy stale backups that have no more snapshots. Be sure to verify the output before using this! ')
-parser.add_argument('--clear-refreservation', action='store_true', help='Set refreservation property to none for new filesystems. Usefull when backupping SmartOS volumes. (recommended)')
-parser.add_argument('--clear-mountpoint', action='store_true', help='Sets canmount=noauto property, to prevent the received filesystem from mounting over existing filesystems. (recommended)')
-parser.add_argument('--rollback', action='store_true', help='Rollback changes on the target before starting a backup. (normally you can prevent changes by setting the readonly property on the target_fs to on)')
-
-
-parser.add_argument('--compress', action='store_true', help='use compression during zfs send/recv')
-parser.add_argument('--test', action='store_true', help='dont change anything, just show what would be done (still does all read-only operations)')
-parser.add_argument('--verbose', action='store_true', help='verbose output')
-parser.add_argument('--debug', action='store_true', help='debug output (shows commands that are executed)')
-args = parser.parse_args()
 
 
 
@@ -390,6 +399,9 @@ if args.test:
         args.verbose=True
         verbose("RUNNING IN TEST-MODE, NOT MAKING ACTUAL BACKUP!")
 
+
+    ### getting and determinging source/target filesystems
+
     # get selected filesystem on backup source
     verbose("Getting selected source filesystems for backup {0} on {1}".format(args.backup_name,args.ssh_source))
     source_filesystems=zfs_get_selected_filesystems(args.ssh_source, args.backup_name)
@@ -406,13 +418,27 @@ for source_filesystem in source_filesystems:
         #append args.target_fs prefix and strip args.strip_path paths from source_filesystem
         target_filesystems.append(args.target_fs + "/" + lstrip_path(source_filesystem, args.strip_path))
 
+
+    ### creating snapshots
+    # this is one of the first things we do, so that in case of failures we still have snapshots.
+
     #create new snapshot?
     if not args.no_snapshot:
         new_snapshot_name=args.backup_name+"-"+time.strftime("%Y%m%d%H%M%S")
         verbose("Creating source snapshot {0} on {1} ".format(new_snapshot_name, args.ssh_source))
         zfs_create_snapshot(args.ssh_source, source_filesystems, new_snapshot_name)
 
-#get all snapshots of all selected filesystems on both source and target
+
+    ### get resumable transfers
+    resumable_target_filesystems={}
+    if args.resume:
+        verbose("Checking for aborted transfers that can be resumed")
+        resumable_target_filesystems=zfs_get_resumable_filesystems(args.ssh_target, target_filesystems)
+        debug("Resumable filesystems: "+str(pprint.pformat(resumable_target_filesystems)))
+
+
+    ### get all snapshots of all selected filesystems on both source and target
+
     verbose("Getting source snapshot-list from {0}".format(args.ssh_source))
     source_snapshots=zfs_get_snapshots(args.ssh_source, source_filesystems, args.backup_name)
     debug("Source snapshots: " + str(pprint.pformat(source_snapshots)))
@@ -491,12 +517,21 @@ for source_filesystem in source_filesystems:
 
                 for send_snapshot in send_snapshots:
 
+                    #resumable?
+                    if target_filesystem in resumable_target_filesystems:
+                        resume_token=resumable_target_filesystems.pop(target_filesystem)
+                    else:
+                        resume_token=None
+
                     zfs_transfer(
                         ssh_source=args.ssh_source, source_filesystem=source_filesystem,
                         first_snapshot=latest_target_snapshot, second_snapshot=send_snapshot,
-                    ssh_target=args.ssh_target, target_filesystem=target_filesystem
+                        ssh_target=args.ssh_target, target_filesystem=target_filesystem,
+                        resume_token=resume_token
                     )
 
+
+
                     #now that we succesfully transferred this snapshot, the previous snapshot is obsolete:
                     if latest_target_snapshot:
                         target_obsolete_snapshots[target_filesystem].append(latest_target_snapshot)
@@ -564,3 +599,42 @@ if target_destroys:
 
 
     verbose("All done")
+
+
+
+################################################################## ENTRY POINT
+
+# parse arguments
+import argparse
+parser = argparse.ArgumentParser(description='ZFS autobackup v2.0')
+parser.add_argument('--ssh-source', default="local", help='Source host to get backup from. (user@hostname) Default %(default)s.')
+parser.add_argument('--ssh-target', default="local", help='Target host to push backup to. (user@hostname) Default  %(default)s.')
+parser.add_argument('--ssh-cipher', default=None, help='SSH cipher to use (default  %(default)s)')
+parser.add_argument('--keep-source', type=int, default=30, help='Number of days to keep old snapshots on source. Default %(default)s.')
+parser.add_argument('--keep-target', type=int, default=30, help='Number of days to keep old snapshots on target. Default %(default)s.')
+parser.add_argument('backup_name',    help='Name of the backup (you should set the zfs property "autobackup:backup-name" to true on filesystems you want to backup')
+parser.add_argument('target_fs',    help='Target filesystem')
+
+parser.add_argument('--no-snapshot', action='store_true', help='dont create new snapshot (usefull for finishing uncompleted backups, or cleanups)')
+parser.add_argument('--no-send', action='store_true', help='dont send snapshots (usefull to only do a cleanup)')
+parser.add_argument('--resume', action='store_true', help='support resuming of interrupted transfers by using the zfs extensible_dataset feature (both zpools should have it enabled)')
+
+parser.add_argument('--strip-path', default=0, type=int, help='number of directory to strip from path (use 1 when cloning zones between 2 SmartOS machines)')
+
+
+parser.add_argument('--destroy-stale', action='store_true', help='Destroy stale backups that have no more snapshots. Be sure to verify the output before using this! ')
+parser.add_argument('--clear-refreservation', action='store_true', help='Set refreservation property to none for new filesystems. Usefull when backupping SmartOS volumes. (recommended)')
+parser.add_argument('--clear-mountpoint', action='store_true', help='Sets canmount=noauto property, to prevent the received filesystem from mounting over existing filesystems. (recommended)')
+parser.add_argument('--rollback', action='store_true', help='Rollback changes on the target before starting a backup. (normally you can prevent changes by setting the readonly property on the target_fs to on)')
+
+
+parser.add_argument('--compress', action='store_true', help='use compression during zfs send/recv')
+parser.add_argument('--test', action='store_true', help='dont change anything, just show what would be done (still does all read-only operations)')
+parser.add_argument('--verbose', action='store_true', help='verbose output')
+parser.add_argument('--debug', action='store_true', help='debug output (shows commands that are executed)')
+
+#note args is the only global variable we use, since its a global readonly setting anyway
+args = parser.parse_args()
+
+
+zfs_autobackup()