update doc

README.md

@@ -510,27 +510,25 @@ Look in man ssh_config for many more options.
 
 ## Usage
 
-(NOTE: Quite a lot has changed since the current stable version 3.0. The page your are viewing is for upcoming version 3.1 which is still in beta.)
-
 ```console
 usage: zfs-autobackup [-h] [--ssh-config CONFIG-FILE] [--ssh-source USER@HOST]
-                      [--ssh-target USER@HOST] [--keep-source SCHEDULE]
-                      [--keep-target SCHEDULE] [--pre-snapshot-cmd COMMAND]
-                      [--post-snapshot-cmd COMMAND] [--other-snapshots]
-                      [--no-snapshot] [--no-send] [--no-thinning] [--no-holds]
-                      [--min-change BYTES] [--allow-empty]
-                      [--ignore-replicated] [--strip-path N]
-                      [--clear-refreservation] [--clear-mountpoint]
-                      [--filter-properties PROPERY,...]
-                      [--set-properties PROPERTY=VALUE,...] [--rollback]
-                      [--destroy-incompatible] [--destroy-missing SCHEDULE]
-                      [--ignore-transfer-errors] [--decrypt] [--encrypt]
-                      [--test] [--verbose] [--debug] [--debug-output]
-                      [--progress] [--send-pipe COMMAND] [--recv-pipe COMMAND]
-                      [--compress TYPE] [--rate DATARATE] [--buffer SIZE]
-                      backup-name [target-path]
+                   [--ssh-target USER@HOST] [--keep-source SCHEDULE]
+                   [--keep-target SCHEDULE] [--pre-snapshot-cmd COMMAND]
+                   [--post-snapshot-cmd COMMAND] [--other-snapshots]
+                   [--no-snapshot] [--no-send] [--no-thinning] [--no-holds]
+                   [--min-change BYTES] [--allow-empty] [--ignore-replicated]
+                   [--strip-path N] [--clear-refreservation]
+                   [--clear-mountpoint] [--filter-properties PROPERTY,...]
+                   [--set-properties PROPERTY=VALUE,...] [--rollback]
+                   [--destroy-incompatible] [--destroy-missing SCHEDULE]
+                   [--ignore-transfer-errors] [--decrypt] [--encrypt]
+                   [--zfs-compressed] [--test] [--verbose] [--debug]
+                   [--debug-output] [--progress] [--send-pipe COMMAND]
+                   [--recv-pipe COMMAND] [--compress TYPE] [--rate DATARATE]
+                   [--buffer SIZE]
+                   backup-name [target-path]
 
-zfs-autobackup v3.1-beta6 - (c)2021 E.H.Eefting (edwin@datux.nl)
+zfs-autobackup v3.1 - (c)2021 E.H.Eefting (edwin@datux.nl)
 
 positional arguments:
   backup-name           Name of the backup (you should set the zfs property
@@ -576,7 +574,6 @@ optional arguments:
   --ignore-replicated   Ignore datasets that seem to be replicated some other
                         way. (No changes since lastest snapshot. Useful for
                         proxmox HA replication)
-
   --strip-path N        Number of directories to strip from target path (use 1
                         when cloning zones between 2 SmartOS machines)
   --clear-refreservation
@@ -585,7 +582,7 @@ optional arguments:
   --clear-mountpoint    Set property canmount=noauto for new datasets.
                         (recommended, prevents mount conflicts. same as --set-
                         properties canmount=noauto)
-  --filter-properties PROPERY,...
+  --filter-properties PROPERTY,...
                         List of properties to "filter" when receiving
                         filesystems. (you can still restore them with zfs
                         inherit -S)
@@ -608,6 +605,8 @@ optional arguments:
                         filesystem exists. useful for acltype errors)
   --decrypt             Decrypt data before sending it over.
   --encrypt             Encrypt data after receiving it.
+  --zfs-compressed      Transfer blocks that already have zfs-compression as-
+                        is.
   --test                dont change anything, just show what would be done
                         (still does all read-only operations)
   --verbose             verbose output
@@ -621,14 +620,15 @@ optional arguments:
   --recv-pipe COMMAND   pipe zfs recv input through COMMAND (can be used
                         multiple times)
   --compress TYPE       Use compression during transfer, zstd-fast
-                        recommended. (zstd-slow, xz, pigz-fast, lz4, pigz-
-                        slow, zstd-fast, gzip, lzo)
+                        recommended. (xz, pigz-slow, zstd-slow, zstd-fast,
+                        lzo, gzip, pigz-fast, lz4)
   --rate DATARATE       Limit data transfer rate (e.g. 128K. requires
                         mbuffer.)
   --buffer SIZE         Add zfs send and recv buffers to smooth out IO bursts.
                         (e.g. 128M. requires mbuffer)
 
 Full manual at: https://github.com/psy0rz/zfs_autobackup
+
 ```
 
 ## Troubleshooting
@@ -754,7 +754,7 @@ for HOST in $HOSTS; do
   ssh $HOST "zfs set autobackup:data_$NAME=child rpool/data"
 
   #backup data filesystems to a common directory
-  zfs-autobackup --keep-source=1d1w,1w1m --ssh-source $HOST data_$NAME $TARGET/data --clear-mountpoint --clear-refreservation --ignore-transfer-errors --strip-path 2 --verbose  --ignore-replicated --min-change 200000 --no-holds   $@
+  zfs-autobackup --keep-source=1d1w,1w1m --ssh-source $HOST data_$NAME $TARGET/data --clear-mountpoint --clear-refreservation --ignore-transfer-errors --strip-path 2 --verbose  --ignore-replicated --min-change 300000 --no-holds   $@
 
   zabbix-job-status backup_$HOST""_data_$NAME daily $? >/dev/null 2>/dev/null
 

tests/test_regressions.py

@@ -8,12 +8,51 @@ class TestZfsNode(unittest2.TestCase):
         prepare_zpools()
         self.longMessage=True
 
-    # #resume initial backup
-    # def test_keepsource0(self):
+    def test_keepsource0target10queuedsend(self):
+        """Test if thinner doesnt destroy too much early on if there are no common snapshots YET. Issue #84"""
 
-    #     #somehow only specifying --allow-empty --keep-source 0 failed:
-    #     with patch('time.strftime', return_value="20101111000000"):
-    #         self.assertFalse(ZfsAutobackup("test test_target1 --verbose --allow-empty --keep-source 0".split(" ")).run())
+        with patch('time.strftime', return_value="20101111000000"):
+            self.assertFalse(ZfsAutobackup(
+                "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty --no-send".split(
+                    " ")).run())
 
-    #     with patch('time.strftime', return_value="20101111000001"):
-    #         self.assertFalse(ZfsAutobackup("test test_target1 --verbose --allow-empty --keep-source 0".split(" ")).run())
+        with patch('time.strftime', return_value="20101111000001"):
+            self.assertFalse(ZfsAutobackup(
+                "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty --no-send".split(
+                    " ")).run())
+
+        with patch('time.strftime', return_value="20101111000002"):
+            self.assertFalse(ZfsAutobackup(
+                "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty".split(
+                    " ")).run())
+
+        r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS)
+        self.assertMultiLineEqual(r, """
+test_source1
+test_source1/fs1
+test_source1/fs1@test-20101111000002
+test_source1/fs1/sub
+test_source1/fs1/sub@test-20101111000002
+test_source2
+test_source2/fs2
+test_source2/fs2/sub
+test_source2/fs2/sub@test-20101111000002
+test_source2/fs3
+test_source2/fs3/sub
+test_target1
+test_target1/test_source1
+test_target1/test_source1/fs1
+test_target1/test_source1/fs1@test-20101111000000
+test_target1/test_source1/fs1@test-20101111000001
+test_target1/test_source1/fs1@test-20101111000002
+test_target1/test_source1/fs1/sub
+test_target1/test_source1/fs1/sub@test-20101111000000
+test_target1/test_source1/fs1/sub@test-20101111000001
+test_target1/test_source1/fs1/sub@test-20101111000002
+test_target1/test_source2
+test_target1/test_source2/fs2
+test_target1/test_source2/fs2/sub
+test_target1/test_source2/fs2/sub@test-20101111000000
+test_target1/test_source2/fs2/sub@test-20101111000001
+test_target1/test_source2/fs2/sub@test-20101111000002
+""")

tests/test_zfsautobackup.py

@@ -312,8 +312,6 @@ test_target1/test_source2/fs2
             self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-send --no-progress".split(" ")).run())
 
             r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS)
-            #(only parents are created )
-            #TODO: it probably shouldn't create these
             self.assertMultiLineEqual(r,"""
 test_source1
 test_source1/fs1
@@ -337,8 +335,6 @@ test_target1
             self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --ignore-replicated".split(" ")).run())
 
             r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS)
-            #(only parents are created )
-            #TODO: it probably shouldn't create these
             self.assertMultiLineEqual(r,"""
 test_source1
 test_source1/fs1
@@ -851,7 +847,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003
 """)
 
 
-        # make snapshot 4, since we used no-holds, it will delete 3 on the source, breaking the backup
+        # run with snapshot-only for 4, since we used no-holds, it will delete 3 on the source, breaking the backup
         with patch('time.strftime', return_value="20101111000004"):
             self.assertFalse(ZfsAutobackup("test --no-progress --verbose --keep-source=0 --keep-target=0 --allow-empty".split(" ")).run())
 

zfs_autobackup/ZfsAutobackup.py

@@ -16,7 +16,7 @@ from zfs_autobackup.ThinnerRule import ThinnerRule
 class ZfsAutobackup:
     """main class"""
 
-    VERSION = "3.1-rc4"
+    VERSION = "3.1"
     HEADER = "zfs-autobackup v{} - (c)2021 E.H.Eefting (edwin@datux.nl)".format(VERSION)
 
     def __init__(self, argv, print_arguments=True):

zfs_autobackup/ZfsDataset.py

@@ -874,9 +874,13 @@ class ZfsDataset:
             :type target_keeps: list of ZfsDataset
         """
 
-        # on source: destroy all obsoletes before common.
+        # on source: destroy all obsoletes before common. (since we cant send them anyways)
         # But after common, only delete snapshots that target also doesn't want
-        before_common = True
+        if common_snapshot:
+            before_common = True
+        else:
+            before_common = False
+
         for source_snapshot in self.snapshots:
             if common_snapshot and source_snapshot.snapshot_name == common_snapshot.snapshot_name:
                 before_common = False
@@ -888,8 +892,8 @@ class ZfsDataset:
 
         # on target: destroy everything thats obsolete, except common_snapshot
         for target_snapshot in target_dataset.snapshots:
-            if (target_snapshot in target_obsoletes) and (
-                    not common_snapshot or target_snapshot.snapshot_name != common_snapshot.snapshot_name):
+            if (target_snapshot in target_obsoletes) \
+                    and ( not common_snapshot or (target_snapshot.snapshot_name != common_snapshot.snapshot_name)):
                 if target_snapshot.exists:
                     target_snapshot.destroy()