#!/usr/bin/env python2 # -*- coding: utf8 -*- #(C)edwin@datux.nl -- Edwin Eefting #Release under GPL. from __future__ import print_function import os import sys import re import traceback import subprocess import pprint import time def error(txt): print(txt, file=sys.stderr) def verbose(txt): if args.verbose: print(txt) def debug(txt): if args.debug: print(txt) #fatal abort execution, exit code 255 def abort(txt): error(txt) sys.exit(255) """run a command. specifiy ssh user@host to run remotely""" def run(cmd, input=None, ssh_to="local", tab_split=False, valid_exitcodes=[ 0 ], test=False): encoded_cmd=[] #use ssh? if ssh_to != "local": encoded_cmd.extend(["ssh", ssh_to]) #make sure the command gets all the data in utf8 format: #(this is neccesary if LC_ALL=en_US.utf8 is not set in the environment) for arg in cmd: #add single quotes for remote commands to support spaces and other wierd stuff (remote commands are executed in a shell) encoded_cmd.append( ("'"+arg+"'").encode('utf-8')) else: for arg in cmd: encoded_cmd.append(arg.encode('utf-8')) #the accurate way of displaying it whould be: print encoded_cmd #However, we use the more human-readable way, but this is not always properly escaped! #(most of the time it should be copypastable however.) debug_txt="# "+" ".join(encoded_cmd) if test: debug("[TEST] "+debug_txt) else: debug(debug_txt) if input: debug("INPUT:\n"+input.rstrip()) stdin=subprocess.PIPE else: stdin=None if test: return p=subprocess.Popen(encoded_cmd, env=os.environ, stdout=subprocess.PIPE, stdin=stdin) output=p.communicate(input=input)[0] if p.returncode not in valid_exitcodes: raise(subprocess.CalledProcessError(p.returncode, encoded_cmd)) lines=output.splitlines() if not tab_split: return(lines) else: ret=[] for line in lines: ret.append(line.split("\t")) return(ret) """determine filesystems that should be backupped by looking at the special autobackup-property""" def zfs_get_selected_filesystems(ssh_to, backup_name): #get all source filesystems that have the backup property source_filesystems=run(ssh_to=ssh_to, tab_split=True, cmd=[ "zfs", "get", "-t", "volume,filesystem", "-o", "name,value,source", "-s", "local,inherited", "-H", "autobackup:"+backup_name ]) #determine filesystems that should be actually backupped selected_filesystems=[] direct_filesystems=[] for source_filesystem in source_filesystems: (name,value,source)=source_filesystem if value=="false": verbose("* Ignored : {0} (disabled)".format(name)) else: if source=="local" and ( value=="true" or value=="child"): direct_filesystems.append(name) if source=="local" and value=="true": selected_filesystems.append(name) verbose("* Selected: {0} (direct selection)".format(name)) elif source.find("inherited from ")==0 and (value=="true" or value=="child"): inherited_from=re.sub("^inherited from ", "", source) if inherited_from in direct_filesystems: selected_filesystems.append(name) verbose("* Selected: {0} (inherited selection)".format(name)) else: verbose("* Ignored : {0} (already a backup)".format(name)) else: verbose("* Ignored : {0} (only childs)".format(name)) return(selected_filesystems) """determine filesystems that can be resumed via receive_resume_token""" def zfs_get_resumable_filesystems(ssh_to, filesystems): cmd=[ "zfs", "get", "-t", "volume,filesystem", "-o", "name,value", "-H", "receive_resume_token" ] cmd.extend(filesystems) resumable_filesystems=run(ssh_to=ssh_to, tab_split=True, cmd=cmd, valid_exitcodes= [ 0,1 ] ) ret={} for (resumable_filesystem,token) in resumable_filesystems: if token!='-': ret[resumable_filesystem]=token return(ret) """deferred destroy list of snapshots (in @format). """ def zfs_destroy_snapshots(ssh_to, snapshots): #zfs can only destroy one filesystem at once so we use xargs and stdin run(ssh_to=ssh_to, test=args.test, input="\0".join(snapshots), cmd= [ "xargs", "-0", "-n", "1", "zfs", "destroy", "-d" ] ) """destroy list of filesystems """ def zfs_destroy(ssh_to, filesystems, recursive=False): cmd=[ "xargs", "-0", "-n", "1", "zfs", "destroy" ] if recursive: cmd.append("-r") #zfs can only destroy one filesystem at once so we use xargs and stdin run(ssh_to=ssh_to, test=args.test, input="\0".join(filesystems), cmd=cmd) #simulate snapshots for --test option test_snapshots={} """create snapshot on multiple filesystems at once (atomicly per pool)""" def zfs_create_snapshot(ssh_to, filesystems, snapshot): #collect per pool, zfs can only take atomic snapshots per pool pools={} for filesystem in filesystems: pool=filesystem.split('/')[0] if pool not in pools: pools[pool]=[] pools[pool].append(filesystem) for pool in pools: cmd=[ "zfs", "snapshot" ] for filesystem in pools[pool]: cmd.append(filesystem+"@"+snapshot) #in testmode we dont actually make changes, so keep them in a list to simulate # if args.test: # if not ssh_to in test_snapshots: # test_snapshots[ssh_to]={} # if not filesystem in test_snapshots[ssh_to]: # test_snapshots[ssh_to][filesystem]=[] # test_snapshots[ssh_to][filesystem].append(snapshot) run(ssh_to=ssh_to, tab_split=False, cmd=cmd, test=args.test) """get names of all snapshots for specified filesystems belonging to backup_name return[filesystem_name]=[ "snashot1", "snapshot2", ... ] """ def zfs_get_snapshots(ssh_to, filesystems, backup_name): ret={} if filesystems: cmd=[ "zfs", "list", "-d", "1", "-r", "-t" ,"snapshot", "-H", "-o", "name" ] cmd.extend(filesystems) snapshots=run(ssh_to=ssh_to, tab_split=False, cmd=cmd, valid_exitcodes=[ 0 ]) for snapshot in snapshots: (filesystem, snapshot_name)=snapshot.split("@") if re.match("^"+backup_name+"-[0-9]*$", snapshot_name): if not filesystem in ret: ret[filesystem]=[] ret[filesystem].append(snapshot_name) #also add any test-snapshots that where created with --test mode # if args.test: # if ssh_to in test_snapshots: # for filesystem in filesystems: # if filesystem in test_snapshots[ssh_to]: # if not filesystem in ret: # ret[filesystem]=[] # ret[filesystem].extend(test_snapshots[ssh_to][filesystem]) return(ret) def default_tag(): return("zfs_autobackup:"+args.backup_name) """hold a snapshot so it cant be destroyed accidently by admin or other processes""" def zfs_hold_snapshot(ssh_to, snapshot, tag=None): cmd=[ "zfs", "hold", tag or default_tag(), snapshot ] run(ssh_to=ssh_to, test=args.test, tab_split=False, cmd=cmd, valid_exitcodes=[ 0, 1 ]) """release a snapshot""" def zfs_release_snapshot(ssh_to, snapshot, tag=None): cmd=[ "zfs", "release", tag or default_tag(), snapshot ] run(ssh_to=ssh_to, test=args.test, tab_split=False, cmd=cmd, valid_exitcodes=[ 0, 1 ]) """transfer a zfs snapshot from source to target. both can be either local or via ssh. TODO: (parially implemented, local buffer is a bit more annoying to do) buffering: specify buffer_size to use mbuffer (or alike) to apply buffering where neccesary local to local: local send -> local buffer -> local receive local to remote and remote to local: local send -> local buffer -> ssh -> remote buffer -> remote receive remote send -> remote buffer -> ssh -> local buffer -> local receive remote to remote: remote send -> remote buffer -> ssh -> local buffer -> ssh -> remote buffer -> remote receive """ def zfs_transfer(ssh_source, source_filesystem, first_snapshot, second_snapshot, ssh_target, target_filesystem, resume_token=None, buffer_size=None): #### build source command source_cmd=[] if ssh_source != "local": source_cmd.extend([ "ssh", ssh_source ]) source_cmd.extend(["zfs", "send", ]) #all kind of performance options: source_cmd.append("-L") # large block support source_cmd.append("-e") # WRITE_EMBEDDED, more compact stream source_cmd.append("-c") # use compressed WRITE records if not args.resume: source_cmd.append("-D") # dedupped stream, sends less duplicate data #only verbose in debug mode, lots of output if args.debug : source_cmd.append("-v") if not first_snapshot: txt=">>> Transfer: "+source_filesystem+"@"+second_snapshot else: txt=">>> Transfer: "+source_filesystem+"@"+first_snapshot+"...@"+second_snapshot if resume_token: source_cmd.extend([ "-t", resume_token ]) txt=txt+" [RESUMED]" else: source_cmd.append("-p") if first_snapshot: source_cmd.extend([ "-i", first_snapshot ]) if ssh_source != "local": source_cmd.append("'" + source_filesystem + "@" + second_snapshot + "'") else: source_cmd.append(source_filesystem + "@" + second_snapshot) verbose(txt) if args.buffer and args.ssh_source!="local": source_cmd.append("|mbuffer -m {}".format(args.buffer)) #### build target command target_cmd=[] if ssh_target != "local": target_cmd.extend([ "ssh", ssh_target ]) target_cmd.extend(["zfs", "recv", "-u" ]) # filter certain properties on receive (usefull for linux->freebsd in some cases) if args.filter_properties: for filter_property in args.filter_properties: target_cmd.extend([ "-x" , filter_property ]) if args.debug: target_cmd.append("-v") if args.resume: target_cmd.append("-s") if ssh_target!="local": target_cmd.append("'" + target_filesystem + "'") else: target_cmd.append(target_filesystem) if args.buffer and args.ssh_target!="local": target_cmd.append("|mbuffer -m {}".format(args.buffer)) #### make sure parent on target exists parent_filesystem= "/".join(target_filesystem.split("/")[:-1]) run(ssh_to=ssh_target, cmd=[ "zfs", "create" ,"-p", parent_filesystem ], test=args.test) ### execute pipe debug_txt="# "+source_cmd[0]+" '"+("' '".join(source_cmd[1:]))+"'" + " | " + target_cmd[0]+" '"+("' '".join(target_cmd[1:]))+"'" if args.test: debug("[TEST] "+debug_txt) return else: debug(debug_txt) source_proc=subprocess.Popen(source_cmd, env=os.environ, stdout=subprocess.PIPE) target_proc=subprocess.Popen(target_cmd, env=os.environ, stdin=source_proc.stdout) source_proc.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits. target_proc.communicate() if not args.ignore_transfer_errors: if source_proc.returncode: raise(subprocess.CalledProcessError(source_proc.returncode, source_cmd)) #zfs recv sometimes gives an exitcode 1 while the transfer was succesfull, therefore we ignore exit 1's and do an extra check to see if the snapshot is there. if target_proc.returncode and target_proc.returncode!=1: raise(subprocess.CalledProcessError(target_proc.returncode, target_cmd)) debug("Verifying if snapshot exists on target") run(ssh_to=ssh_target, cmd=["zfs", "list", target_filesystem+"@"+second_snapshot ]) #NOTE: unreliable when using with autobackup:bla=child # """get filesystems that where already backupped to a target. """ # def zfs_get_backupped_filesystems(ssh_to, backup_name, target_path): # #get all target filesystems that have received or inherited the backup propert, under the target_path tree # ret=run(ssh_to=ssh_to, tab_split=False, valid_exitcodes=[ 0,1 ], cmd=[ # "zfs", "get", "-r", "-t", "volume,filesystem", "-o", "name", "-s", "received,inherited", "-H", "autobackup:"+backup_name, target_path # ]) # # return(ret) """get existing filesystems """ def zfs_get_existing_filesystems(ssh_to, target_path): #get all target filesystems that have received or inherited the backup propert, under the target_path tree ret=run(ssh_to=ssh_to, tab_split=False, valid_exitcodes=[ 0,1 ], cmd=[ "zfs", "list", "-r", "-t", "volume,filesystem", "-o", "name", "-H", target_path ]) return(ret) """get filesystems that where once backupped to target but are no longer selected on source these are filesystems that are not in the list in target_filesystems. this happens when filesystems are destroyed or unselected on the source. """ def get_stale_backupped_filesystems(backup_name, target_path, target_filesystems, existing_target_filesystems): #determine backupped filesystems that are not in target_filesystems anymore stale_backupped_filesystems=[] for existing_target_filesystem in existing_target_filesystems: if existing_target_filesystem not in target_filesystems: stale_backupped_filesystems.append(existing_target_filesystem) return(stale_backupped_filesystems) now=time.time() """determine list of snapshot (in @format) to destroy, according to age""" def determine_destroy_list(snapshots, days): ret=[] for filesystem in snapshots: for snapshot in snapshots[filesystem]: time_str=re.findall("^.*-([0-9]*)$", snapshot)[0] if len(time_str)==14: #new format: time_secs=time.mktime(time.strptime(time_str,"%Y%m%d%H%M%S")) else: time_secs=int(time_str) # verbose("time_secs"+time_str) if (now-time_secs) > (24 * 3600 * days): ret.append(filesystem+"@"+snapshot) return(ret) def lstrip_path(path, count): return("/".join(path.split("/")[count:])) """get list of filesystems that are changed, compared to specified latest snapshot. """ def zfs_get_unchanged_snapshots(ssh_to, snapshots): ret=[] for ( filesystem, snapshot_list ) in snapshots.items(): latest_snapshot=snapshot_list[-1] cmd=[ "zfs", "get","-H" ,"-ovalue", "written@"+latest_snapshot, filesystem ] output=run(ssh_to=ssh_to, tab_split=False, cmd=cmd, valid_exitcodes=[ 0 ]) if output[0]=="0B" or output[0]=="0": ret.append(filesystem) return(ret) """get filesytems that are have changed since any snapshot.""" def zfs_get_unchanged_filesystems(ssh_to, filesystems): ret=[] cmd=[ "zfs", "get","-H" ,"-oname,value", "written" ] cmd.extend(filesystems) output=run(ssh_to=ssh_to, tab_split=True, cmd=cmd, valid_exitcodes=[ 0 ]) for ( filesystem , written ) in output: if written=="0B" or written=="0": ret.append(filesystem) return(ret) #fugly.. failures=0 #something failed, but we try to continue with the rest def failed(txt): global failures failures=failures+1 error("FAILURE: "+txt+"\n") def zfs_autobackup(): ############## data gathering section if args.test: args.verbose=True verbose("RUNNING IN TEST-MODE, NOT MAKING ACTUAL BACKUP!") ### getting and determinging source/target filesystems # get selected filesystems on backup source verbose("Getting selected source filesystems for backup {0} on {1}".format(args.backup_name,args.ssh_source)) source_filesystems=zfs_get_selected_filesystems(args.ssh_source, args.backup_name) #nothing todo if not source_filesystems: abort("No source filesystems selected, please do a 'zfs set autobackup:{0}=true' on {1}".format(args.backup_name,args.ssh_source)) if args.ignore_replicated: replicated_filesystems=zfs_get_unchanged_filesystems(args.ssh_source, source_filesystems) for replicated_filesystem in replicated_filesystems: if replicated_filesystem in source_filesystems: source_filesystems.remove(replicated_filesystem) verbose("* Already replicated: {}".format(replicated_filesystem)) if not source_filesystems: verbose("Nothing to do, all filesystems are already replicated.") sys.exit(0) # determine target filesystems target_filesystems=[] for source_filesystem in source_filesystems: #append args.target_path prefix and strip args.strip_path paths from source_filesystem target_filesystems.append(args.target_path + "/" + lstrip_path(source_filesystem, args.strip_path)) debug("Wanted target filesystems:\n"+str(pprint.pformat(target_filesystems))) # get actual existing target filesystems. (including ones that might not be in the backupset anymore) verbose("Getting existing target filesystems") existing_target_filesystems=zfs_get_existing_filesystems(ssh_to=args.ssh_target, target_path=args.target_path) debug("Existing target filesystems:\n"+str(pprint.pformat(existing_target_filesystems))) common_target_filesystems=list(set(target_filesystems) & set(existing_target_filesystems)) debug("Common target filesystems (target filesystems that also exist on source):\n"+str(pprint.pformat(common_target_filesystems))) ### get resumable transfers from target resumable_target_filesystems={} if args.resume: verbose("Checking for aborted transfers that can be resumed") #Important: use target_filesystem, not existing_target_filesystems (during initial transfer its resumable but doesnt exsit yet) resumable_target_filesystems=zfs_get_resumable_filesystems(args.ssh_target, target_filesystems) debug("Resumable filesystems:\n"+str(pprint.pformat(resumable_target_filesystems))) ### get existing target snapshots target_snapshots={} if common_target_filesystems: verbose("Getting target snapshot-list from {0}".format(args.ssh_target)) target_snapshots=zfs_get_snapshots(args.ssh_target, common_target_filesystems, args.backup_name) # except subprocess.CalledProcessError: # verbose("(ignoring errors, probably initial backup for this filesystem)") # pass debug("Target snapshots:\n" + str(pprint.pformat(target_snapshots))) ### get eixsting source snapshots verbose("Getting source snapshot-list from {0}".format(args.ssh_source)) source_snapshots=zfs_get_snapshots(args.ssh_source, source_filesystems, args.backup_name) debug("Source snapshots:\n" + str(pprint.pformat(source_snapshots))) ### create new snapshots on source if not args.no_snapshot: #determine which filesystems changed since last snapshot if not args.allow_empty and not args.ignore_replicated: #determine which filesystemn are unchanged since OUR snapshots. (not since ANY snapshot) unchanged_filesystems=zfs_get_unchanged_snapshots(args.ssh_source, source_snapshots) else: unchanged_filesystems=[] snapshot_filesystems=[] for source_filesystem in source_filesystems: if source_filesystem not in unchanged_filesystems: snapshot_filesystems.append(source_filesystem) else: verbose("* Not snapshotting {}, no changes found.".format(source_filesystem)) #create snapshots if snapshot_filesystems: new_snapshot_name=args.backup_name+"-"+time.strftime("%Y%m%d%H%M%S") verbose("Creating source snapshots {0} on {1} ".format(new_snapshot_name, args.ssh_source)) zfs_create_snapshot(args.ssh_source, snapshot_filesystems, new_snapshot_name) else: verbose("No changes at all, not creating snapshot.") #add it to the list of source filesystems for snapshot_filesystem in snapshot_filesystems: source_snapshots.setdefault(snapshot_filesystem,[]).append(new_snapshot_name) #obsolete snapshots that may be removed source_obsolete_snapshots={} target_obsolete_snapshots={} ############## backup section #determine which snapshots to send for each filesystem for source_filesystem in source_filesystems: try: target_filesystem=args.target_path + "/" + lstrip_path(source_filesystem, args.strip_path) if source_filesystem not in source_snapshots: #this happens if you use --no-snapshot and there are new filesystems without snapshots verbose("* Skipping source filesystem {0}, no snapshots found".format(source_filesystem)) else: #incremental or initial send? if target_filesystem in target_snapshots and target_snapshots[target_filesystem]: #incremental mode, determine what to send and what is obsolete #latest succesfully send snapshot, should be common on both source and target latest_target_snapshot=target_snapshots[target_filesystem][-1] if latest_target_snapshot not in source_snapshots[source_filesystem]: #cant find latest target anymore. find first common snapshot and inform user error_msg="Cant find latest target snapshot on source for '{}', did you destroy/rename it?".format(source_filesystem) error_msg=error_msg+"\nLatest on target : "+target_filesystem+"@"+latest_target_snapshot error_msg=error_msg+"\nMissing on source: "+source_filesystem+"@"+latest_target_snapshot found=False for latest_target_snapshot in reversed(target_snapshots[target_filesystem]): if latest_target_snapshot in source_snapshots[source_filesystem]: error_msg=error_msg+"\nYou could solve this by rolling back to this common snapshot on target: "+target_filesystem+"@"+latest_target_snapshot found=True break if not found: error_msg=error_msg+"\nAlso could not find an earlier common snapshot to rollback to." else: if args.ignore_new: verbose("* Skipping source filesystem '{0}', target already has newer snapshots.".format(source_filesystem)) continue raise(Exception(error_msg)) #send all new source snapshots that come AFTER the last target snapshot latest_source_index=source_snapshots[source_filesystem].index(latest_target_snapshot) send_snapshots=source_snapshots[source_filesystem][latest_source_index+1:] #source snapshots that come BEFORE last target snapshot are obsolete source_obsolete_snapshots[source_filesystem]=source_snapshots[source_filesystem][0:latest_source_index] #target snapshots that come BEFORE last target snapshot are obsolete latest_target_index=target_snapshots[target_filesystem].index(latest_target_snapshot) target_obsolete_snapshots[target_filesystem]=target_snapshots[target_filesystem][0:latest_target_index] else: #initial mode, send all snapshots, nothing is obsolete: latest_target_snapshot=None send_snapshots=source_snapshots[source_filesystem] target_obsolete_snapshots[target_filesystem]=[] source_obsolete_snapshots[source_filesystem]=[] #now actually send the snapshots if not args.no_send: if send_snapshots and args.rollback and latest_target_snapshot: #roll back any changes on target debug("Rolling back target to latest snapshot.") run(ssh_to=args.ssh_target, test=args.test, cmd=["zfs", "rollback", target_filesystem+"@"+latest_target_snapshot ]) for send_snapshot in send_snapshots: #resumable? if target_filesystem in resumable_target_filesystems: resume_token=resumable_target_filesystems.pop(target_filesystem) else: resume_token=None #hold the snapshot we're sending on the source if not args.no_holds: zfs_hold_snapshot(ssh_to=args.ssh_source, snapshot=source_filesystem+"@"+send_snapshot) zfs_transfer( ssh_source=args.ssh_source, source_filesystem=source_filesystem, first_snapshot=latest_target_snapshot, second_snapshot=send_snapshot, ssh_target=args.ssh_target, target_filesystem=target_filesystem, resume_token=resume_token ) #hold the snapshot we just send to the target zfs_hold_snapshot(ssh_to=args.ssh_target, snapshot=target_filesystem+"@"+send_snapshot) #now that we succesfully transferred this snapshot, the previous snapshot is obsolete: if latest_target_snapshot: zfs_release_snapshot(ssh_to=args.ssh_target, snapshot=target_filesystem+"@"+latest_target_snapshot) target_obsolete_snapshots[target_filesystem].append(latest_target_snapshot) if not args.no_holds: zfs_release_snapshot(ssh_to=args.ssh_source, snapshot=source_filesystem+"@"+latest_target_snapshot) source_obsolete_snapshots[source_filesystem].append(latest_target_snapshot) #we just received a new filesytem? else: if args.clear_refreservation: debug("Clearing refreservation to save space.") run(ssh_to=args.ssh_target, test=args.test, cmd=["zfs", "set", "refreservation=none", target_filesystem ]) if args.clear_mountpoint: debug("Setting canmount=noauto to prevent auto-mounting in the wrong place. (ignoring errors)") run(ssh_to=args.ssh_target, test=args.test, cmd=["zfs", "set", "canmount=noauto", target_filesystem ], valid_exitcodes= [0, 1] ) latest_target_snapshot=send_snapshot # failed, skip this source_filesystem except Exception as e: failed(str(e)) ############## cleanup section #we only do cleanups after everything is complete, to keep everything consistent (same snapshots everywhere) if not args.ignore_replicated: #find stale backups on target that have become obsolete stale_target_filesystems=get_stale_backupped_filesystems(backup_name=args.backup_name, target_path=args.target_path, target_filesystems=target_filesystems, existing_target_filesystems=existing_target_filesystems) debug("Stale target filesystems: {0}".format("\n".join(stale_target_filesystems))) stale_target_snapshots=zfs_get_snapshots(args.ssh_target, stale_target_filesystems, args.backup_name) debug("Stale target snapshots: " + str(pprint.pformat(stale_target_snapshots))) target_obsolete_snapshots.update(stale_target_snapshots) #determine stale filesystems that have no snapshots left (the can be destroyed) stale_target_destroys=[] for stale_target_filesystem in stale_target_filesystems: if stale_target_filesystem not in stale_target_snapshots: stale_target_destroys.append(stale_target_filesystem) if stale_target_destroys: #NOTE: dont destroy automaticly..not safe enough. # if args.destroy_stale: # verbose("Destroying stale filesystems on target {0}:\n{1}".format(args.ssh_target, "\n".join(stale_target_destroys))) # zfs_destroy(ssh_to=args.ssh_target, filesystems=stale_target_destroys, recursive=True) # else: verbose("Stale filesystems on {0}:\n{1}".format(args.ssh_target, "\n".join(stale_target_destroys))) else: verbose("NOTE: Cant determine stale target filesystems while using ignore_replicated.") #now actually destroy the old snapshots source_destroys=determine_destroy_list(source_obsolete_snapshots, args.keep_source) if source_destroys: verbose("Destroying old snapshots on source {0}:\n{1}".format(args.ssh_source, "\n".join(source_destroys))) try: zfs_destroy_snapshots(ssh_to=args.ssh_source, snapshots=source_destroys) except Exception as e: failed(str(e)) target_destroys=determine_destroy_list(target_obsolete_snapshots, args.keep_target) if target_destroys: verbose("Destroying old snapshots on target {0}:\n{1}".format(args.ssh_target, "\n".join(target_destroys))) try: zfs_destroy_snapshots(ssh_to=args.ssh_target, snapshots=target_destroys) except Exception as e: failed(str(e)) ################################################################## ENTRY POINT # parse arguments import argparse parser = argparse.ArgumentParser( description='ZFS autobackup v2.4', epilog='When a filesystem fails, zfs_backup will continue and report the number of failures at that end. Also the exit code will indicate the number of failures.') parser.add_argument('--ssh-source', default="local", help='Source host to get backup from. (user@hostname) Default %(default)s.') parser.add_argument('--ssh-target', default="local", help='Target host to push backup to. (user@hostname) Default %(default)s.') parser.add_argument('--keep-source', type=int, default=30, help='Number of days to keep old snapshots on source. Default %(default)s.') parser.add_argument('--keep-target', type=int, default=30, help='Number of days to keep old snapshots on target. Default %(default)s.') parser.add_argument('backup_name', help='Name of the backup (you should set the zfs property "autobackup:backup-name" to true on filesystems you want to backup') parser.add_argument('target_path', help='Target path') parser.add_argument('--no-snapshot', action='store_true', help='dont create new snapshot (usefull for finishing uncompleted backups, or cleanups)') parser.add_argument('--no-send', action='store_true', help='dont send snapshots (usefull to only do a cleanup)') parser.add_argument('--allow-empty', action='store_true', help='if nothing has changed, still create empty snapshots.') parser.add_argument('--ignore-replicated', action='store_true', help='Ignore datasets that seem to be replicated some other way. (No changes since lastest snapshot. Usefull for proxmox HA replication)') parser.add_argument('--no-holds', action='store_true', help='Dont lock snapshots on the source. (Usefull to allow proxmox HA replication to switches nodes)') parser.add_argument('--ignore-new', action='store_true', help='Ignore filesystem if there are already newer snapshots for it on the target (use with caution)') parser.add_argument('--resume', action='store_true', help='support resuming of interrupted transfers by using the zfs extensible_dataset feature (both zpools should have it enabled) Disadvantage is that you need to use zfs recv -A if another snapshot is created on the target during a receive. Otherwise it will keep failing.') parser.add_argument('--strip-path', default=0, type=int, help='number of directory to strip from path (use 1 when cloning zones between 2 SmartOS machines)') parser.add_argument('--buffer', default="", help='Use mbuffer with specified size to speedup zfs transfer. (e.g. --buffer 1G) Will also show nice progress output.') # parser.add_argument('--destroy-stale', action='store_true', help='Destroy stale backups that have no more snapshots. Be sure to verify the output before using this! ') parser.add_argument('--clear-refreservation', action='store_true', help='Set refreservation property to none for new filesystems. Usefull when backupping SmartOS volumes. (recommended)') parser.add_argument('--clear-mountpoint', action='store_true', help='Sets canmount=noauto property, to prevent the received filesystem from mounting over existing filesystems. (recommended)') parser.add_argument('--filter-properties', action='append', help='Filter properties when receiving filesystems. Can be specified multiple times. (Example: If you send data from Linux to FreeNAS, you should filter xattr)') parser.add_argument('--rollback', action='store_true', help='Rollback changes on the target before starting a backup. (normally you can prevent changes by setting the readonly property on the target_path to on)') parser.add_argument('--ignore-transfer-errors', action='store_true', help='Ignore transfer errors (still checks if received filesystem exists. usefull for acltype errors)') parser.add_argument('--test', action='store_true', help='dont change anything, just show what would be done (still does all read-only operations)') parser.add_argument('--verbose', action='store_true', help='verbose output') parser.add_argument('--debug', action='store_true', help='debug output (shows commands that are executed)') #note args is the only global variable we use, since its a global readonly setting anyway args = parser.parse_args() if args.ignore_replicated and args.allow_empty: abort("Cannot use allow_empty with ignore_replicated.") try: zfs_autobackup() if not failures: verbose("All operations completed succesfully.") sys.exit(0) else: verbose("{} OPERATION(S) FAILED!".format(failures)) #exit with the number of failures. sys.exit(min(255,failures)) except Exception as e: if args.debug: raise else: print(str(e)) abort("FATAL ERROR")