Changed order of operations: target stuff is now done as last step. (in case ssh-target is unreachable, snapshots are still being made). Also allow operating as a snapshot tool when not specifying target_path. Implements #46

bin/zfs-autobackup

@@ -26,8 +26,8 @@ if sys.stdout.isatty():
     except ImportError:
         pass
 
-VERSION="3.0-rc11"
+VERSION="3.0-rc12"
-HEADER="zfs-autobackup v{} - Copyright 2020 E.H.Eefting (edwin@datux.nl)\n".format(VERSION)
+HEADER="zfs-autobackup v{} - Copyright 2020 E.H.Eefting (edwin@datux.nl)".format(VERSION)
 
 class Log:
     def __init__(self, show_debug=False, show_verbose=False):
@@ -1093,13 +1093,16 @@ class ZfsDataset():
         return(self.zfs_node.thinner.thin(snapshots, keep_objects=keeps))
 
 
-    def thin(self):
+    def thin(self, skip_holds=False):
         """destroys snapshots according to thin_list, except last snapshot"""
 
         (keeps, obsoletes)=self.thin_list(keeps=self.our_snapshots[-1:])
         for obsolete in obsoletes:
-            obsolete.destroy()
+            if skip_holds and obsolete.is_hold():
-            self.snapshots.remove(obsolete)
+                obsolete.verbose("Keeping (common snapshot)")
+            else:
+                obsolete.destroy()
+                self.snapshots.remove(obsolete)
 
 
     def find_common_snapshot(self, target_dataset):
@@ -1552,7 +1555,7 @@ class ZfsAutobackup:
         parser.add_argument('--keep-target', type=str, default="10,1d1w,1w1m,1m1y", help='Thinning schedule for old target snapshots. Default: %(default)s')
 
         parser.add_argument('backup_name',    help='Name of the backup (you should set the zfs property "autobackup:backup-name" to true on filesystems you want to backup')
-        parser.add_argument('target_path', default=None, nargs='?',  help='Target ZFS filesystem')
+        parser.add_argument('target_path', default=None, nargs='?',  help='Target ZFS filesystem (optional: if not specified, zfs-autobackup will only operate as snapshot-tool on source)')
 
         parser.add_argument('--other-snapshots', action='store_true', help='Send over other snapshots as well, not just the ones created by this tool.')
         parser.add_argument('--no-snapshot', action='store_true', help='Don\'t create new snapshots (useful for finishing uncompleted backups, or cleanups)')
@@ -1625,6 +1628,92 @@ class ZfsAutobackup:
         self.log.verbose("")
         self.log.verbose("#### "+title)
 
+    # sync datasets, or thin-only on both sides
+    # target is needed for this.
+    def sync_datasets(self, source_node, source_datasets):
+
+        description="[Target]"
+
+        self.set_title("Target settings")
+
+        target_thinner=Thinner(self.args.keep_target)
+        target_node=ZfsNode(self.args.backup_name, self, ssh_config=self.args.ssh_config, ssh_to=self.args.ssh_target, readonly=self.args.test, debug_output=self.args.debug_output, description=description, thinner=target_thinner)
+        target_node.verbose("Receive datasets under: {}".format(self.args.target_path))
+
+        if self.args.no_send:        
+            self.set_title("Thinning source and target")
+        else:
+            self.set_title("Sending and thinning")
+
+        #check if exists, to prevent vague errors
+        target_dataset=ZfsDataset(target_node, self.args.target_path)
+        if not target_dataset.exists:
+            self.error("Target path '{}' does not exist. Please create this dataset first.".format(target_dataset))
+            return(255)
+
+
+        if self.args.filter_properties:
+            filter_properties=self.args.filter_properties.split(",")
+        else:
+            filter_properties=[]
+
+        if self.args.set_properties:
+            set_properties=self.args.set_properties.split(",")
+        else:
+            set_properties=[]
+
+        if self.args.clear_refreservation:
+            filter_properties.append("refreservation")
+
+        if self.args.clear_mountpoint:
+            set_properties.append("canmount=noauto")
+
+        #sync datasets
+        fail_count=0
+        target_datasets=[]
+        for source_dataset in source_datasets:
+
+            try:
+                #determine corresponding target_dataset
+                target_name=self.args.target_path + "/" + source_dataset.lstrip_path(self.args.strip_path)
+                target_dataset=ZfsDataset(target_node, target_name)
+                target_datasets.append(target_dataset)
+
+                #ensure parents exists
+                #TODO: this isnt perfect yet, in some cases it can create parents when it shouldn't.
+                if not self.args.no_send and not target_dataset.parent in target_datasets and not target_dataset.parent.exists:
+                    target_dataset.parent.create_filesystem(parents=True)
+
+                #determine common zpool features
+                source_features=source_node.get_zfs_pool(source_dataset.split_path()[0]).features
+                target_features=target_node.get_zfs_pool(target_dataset.split_path()[0]).features
+                common_features=source_features and target_features
+                # source_dataset.debug("Common features: {}".format(common_features))
+
+                source_dataset.sync_snapshots(target_dataset, show_progress=self.args.progress, features=common_features, filter_properties=filter_properties, set_properties=set_properties, ignore_recv_exit_code=self.args.ignore_transfer_errors, source_holds= not self.args.no_holds, rollback=self.args.rollback, raw=self.args.raw, other_snapshots=self.args.other_snapshots, no_send=self.args.no_send, destroy_incompatible=self.args.destroy_incompatible)
+            except Exception as e:
+                fail_count=fail_count+1
+                source_dataset.error("FAILED: "+str(e))
+                if self.args.debug:
+                    raise
+
+        #also thin target_datasets that are not on the source any more
+        self.debug("Thinning obsolete datasets")
+        for dataset in ZfsDataset(target_node, self.args.target_path).recursive_datasets:
+            if dataset not in target_datasets:
+                dataset.debug("Missing on source")
+                dataset.thin()
+
+            return(fail_count)
+
+
+    def thin_source(self, source_datasets):
+        self.set_title("Thinning source")
+
+        for source_dataset in source_datasets:
+            source_dataset.thin(skip_holds=True)
+
+
     def run(self):
 
         try:
@@ -1633,26 +1722,13 @@ class ZfsAutobackup:
             if self.args.test:
                 self.verbose("TEST MODE - SIMULATING WITHOUT MAKING ANY CHANGES")
 
-            self.set_title("Settings summary")
+            self.set_title("Source settings")
 
             description="[Source]"
             source_thinner=Thinner(self.args.keep_source)
             source_node=ZfsNode(self.args.backup_name, self, ssh_config=self.args.ssh_config, ssh_to=self.args.ssh_source, readonly=self.args.test, debug_output=self.args.debug_output, description=description, thinner=source_thinner)
             source_node.verbose("Send all datasets that have 'autobackup:{}=true' or 'autobackup:{}=child'".format(self.args.backup_name, self.args.backup_name))
 
-            self.verbose("")
-
-            description="[Target]"
-            target_thinner=Thinner(self.args.keep_target)
-            target_node=ZfsNode(self.args.backup_name, self, ssh_config=self.args.ssh_config, ssh_to=self.args.ssh_target, readonly=self.args.test, debug_output=self.args.debug_output, description=description, thinner=target_thinner)
-            target_node.verbose("Receive datasets under: {}".format(self.args.target_path))
-
-            #check if exists, to prevent vague errors
-            target_dataset=ZfsDataset(target_node, self.args.target_path)
-            if not target_dataset.exists:
-                self.error("Target path '{}' does not exist. Please create this dataset first.".format(target_dataset))
-                return(255)
-
             self.set_title("Selecting")
             selected_source_datasets=source_node.selected_datasets
             if not selected_source_datasets:
@@ -1661,7 +1737,6 @@ class ZfsAutobackup:
 
             source_datasets=[]
 
-
             #filter out already replicated stuff?
             if not self.args.ignore_replicated:
                 source_datasets=selected_source_datasets
@@ -1673,80 +1748,34 @@ class ZfsAutobackup:
                     else:
                         selected_source_dataset.verbose("Ignoring, already replicated")
 
-
             if not self.args.no_snapshot:
                 self.set_title("Snapshotting")
                 source_node.consistent_snapshot(source_datasets, source_node.new_snapshotname(), min_changed_bytes=self.args.min_change)
 
-
+            #if target is specified, we sync the datasets, otherwise we just thin the source.
-            if self.args.no_send:        
+            if self.args.target_path:
-                self.set_title("Thinning")
+                fail_count=self.sync_datasets(source_node, source_datasets)
             else:
-                self.set_title("Sending and thinning")
+                self.thin_source(source_datasets)
-
+                fail_count=0
-            if self.args.filter_properties:
-                filter_properties=self.args.filter_properties.split(",")
-            else:
-                filter_properties=[]
-
-            if self.args.set_properties:
-                set_properties=self.args.set_properties.split(",")
-            else:
-                set_properties=[]
-
-            if self.args.clear_refreservation:
-                filter_properties.append("refreservation")
-
-            if self.args.clear_mountpoint:
-                set_properties.append("canmount=noauto")
-
-            #sync datasets
-            fail_count=0
-            target_datasets=[]
-            for source_dataset in source_datasets:
-
-                try:
-                    #determine corresponding target_dataset
-                    target_name=self.args.target_path + "/" + source_dataset.lstrip_path(self.args.strip_path)
-                    target_dataset=ZfsDataset(target_node, target_name)
-                    target_datasets.append(target_dataset)
-
-                    #ensure parents exists
-                    #TODO: this isnt perfect yet, in some cases it can create parents when it shouldn't.
-                    if not self.args.no_send and not target_dataset.parent in target_datasets and not target_dataset.parent.exists:
-                        target_dataset.parent.create_filesystem(parents=True)
-
-                    #determine common zpool features
-                    source_features=source_node.get_zfs_pool(source_dataset.split_path()[0]).features
-                    target_features=target_node.get_zfs_pool(target_dataset.split_path()[0]).features
-                    common_features=source_features and target_features
-                    # source_dataset.debug("Common features: {}".format(common_features))
-
-                    source_dataset.sync_snapshots(target_dataset, show_progress=self.args.progress, features=common_features, filter_properties=filter_properties, set_properties=set_properties, ignore_recv_exit_code=self.args.ignore_transfer_errors, source_holds= not self.args.no_holds, rollback=self.args.rollback, raw=self.args.raw, other_snapshots=self.args.other_snapshots, no_send=self.args.no_send, destroy_incompatible=self.args.destroy_incompatible)
-                except Exception as e:
-                    fail_count=fail_count+1
-                    source_dataset.error("FAILED: "+str(e))
-                    if self.args.debug:
-                        raise
-
-            #also thin target_datasets that are not on the source any more
-            self.debug("Thinning obsolete datasets")
-            for dataset in ZfsDataset(target_node, self.args.target_path).recursive_datasets:
-                if dataset not in target_datasets:
-                    dataset.debug("Missing on source")
-                    dataset.thin()
 
 
             if not fail_count:
                 if self.args.test:
                     self.set_title("All tests successfull.")
                 else:
-                    self.set_title("All backups completed successfully")
+                    self.set_title("All operations completed successfully")
+                    if not self.args.target_path:
+                        self.verbose("(No target_path specified, only operated as snapshot tool.)")
+
             else:
-                self.error("{} datasets failed!".format(fail_count))
+                if fail_count!=255:
+                    self.error("{} failures!".format(fail_count))
+
 
             if self.args.test:
-                self.verbose("TEST MODE - DID NOT MAKE ANY BACKUPS!")
+                self.verbose("")
+                self.verbose("TEST MODE - DID NOT MAKE ANY CHANGES!")
 
             return(fail_count)
 

test_zfsautobackup.py

@@ -13,6 +13,52 @@ class TestZfsAutobackup(unittest2.TestCase):
 
         self.assertEqual(ZfsAutobackup("test test_target1 --keep-source -1".split(" ")).run(), 255)
 
+    def  test_snapshotmode(self):
+
+        with patch('time.strftime', return_value="20101111000000"):
+            self.assertFalse(ZfsAutobackup("test test_target1 --verbose".split(" ")).run())
+
+        with patch('time.strftime', return_value="20101111000001"):
+            self.assertFalse(ZfsAutobackup("test test_target1 --allow-empty --verbose".split(" ")).run())
+
+        with patch('time.strftime', return_value="20101111000002"):
+            self.assertFalse(ZfsAutobackup("test --verbose --allow-empty --keep-source 0".split(" ")).run())
+
+        #on source: only has 1 and 2
+        #on target: has 0 and 1  
+
+        r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS)
+        self.assertMultiLineEqual(r,"""
+test_source1
+test_source1/fs1
+test_source1/fs1@test-20101111000001
+test_source1/fs1@test-20101111000002
+test_source1/fs1/sub
+test_source1/fs1/sub@test-20101111000001
+test_source1/fs1/sub@test-20101111000002
+test_source2
+test_source2/fs2
+test_source2/fs2/sub
+test_source2/fs2/sub@test-20101111000001
+test_source2/fs2/sub@test-20101111000002
+test_source2/fs3
+test_source2/fs3/sub
+test_target1
+test_target1/test_source1
+test_target1/test_source1/fs1
+test_target1/test_source1/fs1@test-20101111000000
+test_target1/test_source1/fs1@test-20101111000001
+test_target1/test_source1/fs1/sub
+test_target1/test_source1/fs1/sub@test-20101111000000
+test_target1/test_source1/fs1/sub@test-20101111000001
+test_target1/test_source2
+test_target1/test_source2/fs2
+test_target1/test_source2/fs2/sub
+test_target1/test_source2/fs2/sub@test-20101111000000
+test_target1/test_source2/fs2/sub@test-20101111000001
+""")
+
+
 
     def  test_defaults(self):