allow to only select parent dataset, non-recursively. (usefull for /var/lib/docker )

tests/test_zfsautobackup.py

@@ -892,7 +892,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003
         r = shelltest("zfs snapshot test_source1@test")
 
         l=LogConsole(show_verbose=True, show_debug=False, color=False)
-        n=ZfsNode(snapshot_time_format="bla", hold_name="bla", logger=l)
+        n=ZfsNode(utc=False, snapshot_time_format="bla", hold_name="bla", logger=l)
         d=ZfsDataset(n,"test_source1@test")
 
         sp=d.send_pipe([], prev_snapshot=None, resume_token=None, show_progress=True, raw=False, send_pipes=[], send_properties=True, write_embedded=True, zfs_compressed=True)

tests/test_zfsnode.py

@@ -12,7 +12,7 @@ class TestZfsNode(unittest2.TestCase):
     def test_consistent_snapshot(self):
         logger = LogStub()
         description = "[Source]"
-        node = ZfsNode(snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description)
+        node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description)
 
         with self.subTest("first snapshot"):
             node.consistent_snapshot(node.selected_datasets(property_name="autobackup:test",exclude_paths=[], exclude_received=False, exclude_unchanged=False, min_change=200000), "test-20101111000001", 100000)
@@ -74,7 +74,7 @@ test_target1
     def test_consistent_snapshot_prepostcmds(self):
         logger = LogStub()
         description = "[Source]"
-        node = ZfsNode(snapshot_time_format="test", hold_name="test", logger=logger, description=description, debug_output=True)
+        node = ZfsNode(utc=False, snapshot_time_format="test", hold_name="test", logger=logger, description=description, debug_output=True)
 
         with self.subTest("Test if all cmds are executed correctly (no failures)"):
             with OutputIO() as buf:
@@ -124,6 +124,21 @@ test_target1
                 self.assertIn("STDOUT > post1", buf.getvalue())
                 self.assertIn("STDOUT > post2", buf.getvalue())
 
+    def test_timestamps(self):
+        # Assert that timestamps keep relative order both for utc and for localtime
+        logger = LogStub()
+        description = "[Source]"
+        node_local = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description)
+        node_utc = ZfsNode(utc=True, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description)
+
+        for node in [node_local, node_utc]:
+            with self.subTest("timestamp ordering " + ("utc" if node == node_utc else "localtime")):
+                dataset_a = ZfsDataset(node,"test_source1@test-20101111000001")
+                dataset_b = ZfsDataset(node,"test_source1@test-20101111000002")
+                dataset_c = ZfsDataset(node,"test_source1@test-20240101020202")
+                self.assertGreater(dataset_b.timestamp, dataset_a.timestamp)
+                self.assertGreater(dataset_c.timestamp, dataset_b.timestamp)
+
 
     def test_getselected(self):
 
@@ -131,18 +146,24 @@ test_target1
         shelltest("zfs create test_source1/fs1/subexcluded")
         shelltest("zfs set autobackup:test=false test_source1/fs1/subexcluded")
 
+        # only select parent
+        shelltest("zfs create test_source1/fs1/onlyparent")
+        shelltest("zfs create test_source1/fs1/onlyparent/child")
+        shelltest("zfs set autobackup:test=parent test_source1/fs1/onlyparent")
+
         # should be excluded by being unchanged
         shelltest("zfs create test_source1/fs1/unchanged")
         shelltest("zfs snapshot test_source1/fs1/unchanged@somesnapshot")
 
         logger = LogStub()
         description = "[Source]"
-        node = ZfsNode(snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description)
+        node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description)
         s = pformat(node.selected_datasets(property_name="autobackup:test", exclude_paths=[], exclude_received=False, exclude_unchanged=True, min_change=1))
         print(s)
 
         # basics
         self.assertEqual(s, """[(local): test_source1/fs1,
+ (local): test_source1/fs1/onlyparent,
  (local): test_source1/fs1/sub,
  (local): test_source2/fs2/sub]""")
 
@@ -150,7 +171,7 @@ test_target1
     def test_validcommand(self):
         logger = LogStub()
         description = "[Source]"
-        node = ZfsNode(snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description)
+        node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description)
 
         with self.subTest("test invalid option"):
             self.assertFalse(node.valid_command(["zfs", "send", "--invalid-option", "nonexisting"]))
@@ -160,7 +181,7 @@ test_target1
     def test_supportedsendoptions(self):
         logger = LogStub()
         description = "[Source]"
-        node = ZfsNode(snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description)
+        node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description)
         # -D propably always supported
         self.assertGreater(len(node.supported_send_options), 0)
 
@@ -168,7 +189,7 @@ test_target1
         logger = LogStub()
         description = "[Source]"
         # NOTE: this could hang via ssh if we dont close filehandles properly. (which was a previous bug)
-        node = ZfsNode(snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description, ssh_to='localhost')
+        node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description, ssh_to='localhost')
         self.assertIsInstance(node.supported_recv_options, list)
 
 

zfs_autobackup/CliBase.py

@@ -10,7 +10,7 @@ class CliBase(object):
     Overridden in subclasses that add stuff for the specific programs."""
 
     # also used by setup.py
-    VERSION = "3.2-alpha2"
+    VERSION = "3.2-beta1"
     HEADER = "{} v{} - (c)2022 E.H.Eefting (edwin@datux.nl)".format(os.path.basename(sys.argv[0]), VERSION)
 
     def __init__(self, argv, print_arguments=True):
@@ -80,6 +80,8 @@ class CliBase(object):
                             help='show zfs progress output. Enabled automaticly on ttys. (use --no-progress to disable)')
         group.add_argument('--no-progress', action='store_true',
                             help=argparse.SUPPRESS)  # needed to workaround a zfs recv -v bug
+        group.add_argument('--utc', action='store_true',
+                            help='Use UTC instead of local time when dealing with timestamps for both formatting and parsing. To snapshot in an ISO 8601 compliant time format you may for example specify --snapshot-format "{}-%%Y-%%m-%%dT%%H:%%M:%%SZ". Changing this parameter after-the-fact (existing snapshots) will cause their timestamps to be interpreted as a different time than before.')
         group.add_argument('--version', action='store_true',
                             help='Show version.')
 
@@ -106,4 +108,4 @@ class CliBase(object):
 
     def set_title(self, title):
         self.log.verbose("")
-        self.log.verbose("#### " + title)
+        self.log.verbose("#### " + title)

zfs_autobackup/ZfsAuto.py

@@ -61,6 +61,7 @@ class ZfsAuto(CliBase):
         self.verbose("")
         self.verbose("Selecting dataset property : {}".format(self.property_name))
         self.verbose("Snapshot format            : {}".format(self.snapshot_time_format))
+        self.verbose("Timezone                   : {}".format("UTC" if args.utc else "Local"))
 
         return args
 
@@ -93,8 +94,7 @@ class ZfsAuto(CliBase):
         group.add_argument('--hold-format', metavar='FORMAT', default="zfs_autobackup:{}",
                             help='ZFS hold string format. Default: %(default)s')
         group.add_argument('--strip-path', metavar='N', default=0, type=int,
-                           help='Number of directories to strip from target path (use 1 when cloning zones between 2 '
-                                'SmartOS machines)')
+                           help='Number of directories to strip from target path.')
 
         group=parser.add_argument_group("Selection options")
         group.add_argument('--ignore-replicated', action='store_true', help=argparse.SUPPRESS)

zfs_autobackup/ZfsAutobackup.py

@@ -1,6 +1,7 @@
 import time
 
 import argparse
+from datetime import datetime
 from signal import signal, SIGPIPE
 from .util import output_redir, sigpipe_handler
 
@@ -12,7 +13,6 @@ from .Thinner import Thinner
 from .ZfsDataset import ZfsDataset
 from .ZfsNode import ZfsNode
 from .ThinnerRule import ThinnerRule
-import os.path
 
 class ZfsAutobackup(ZfsAuto):
     """The main zfs-autobackup class. Start here, at run() :)"""
@@ -70,6 +70,9 @@ class ZfsAutobackup(ZfsAuto):
                            help='If nothing has changed, still create empty snapshots. (Faster. Same as --min-change=0)')
         group.add_argument('--other-snapshots', action='store_true',
                            help='Send over other snapshots as well, not just the ones created by this tool.')
+        group.add_argument('--set-snapshot-properties', metavar='PROPERTY=VALUE,...', type=str,
+                           help='List of properties to set on the snapshot.')
+
 
         group = parser.add_argument_group("Transfer options")
         group.add_argument('--no-send', action='store_true',
@@ -411,6 +414,15 @@ class ZfsAutobackup(ZfsAuto):
 
         return set_properties
 
+    def set_snapshot_properties_list(self):
+
+        if self.args.set_snapshot_properties:
+            set_snapshot_properties = self.args.set_snapshot_properties.split(",")
+        else:
+            set_snapshot_properties = []
+
+        return set_snapshot_properties
+
     def run(self):
 
         try:
@@ -423,7 +435,8 @@ class ZfsAutobackup(ZfsAuto):
                 source_thinner = None
             else:
                 source_thinner = Thinner(self.args.keep_source)
-            source_node = ZfsNode(snapshot_time_format=self.snapshot_time_format, hold_name=self.hold_name, logger=self,
+            source_node = ZfsNode(utc=self.args.utc,
+                                  snapshot_time_format=self.snapshot_time_format, hold_name=self.hold_name, logger=self,
                                   ssh_config=self.args.ssh_config,
                                   ssh_to=self.args.ssh_source, readonly=self.args.test,
                                   debug_output=self.args.debug_output, description=description, thinner=source_thinner)
@@ -442,11 +455,13 @@ class ZfsAutobackup(ZfsAuto):
             ################# snapshotting
             if not self.args.no_snapshot:
                 self.set_title("Snapshotting")
-                snapshot_name = time.strftime(self.snapshot_time_format)
+                dt = datetime.utcnow() if self.args.utc else datetime.now()
+                snapshot_name = dt.strftime(self.snapshot_time_format)
                 source_node.consistent_snapshot(source_datasets, snapshot_name,
                                                 min_changed_bytes=self.args.min_change,
                                                 pre_snapshot_cmds=self.args.pre_snapshot_cmd,
-                                                post_snapshot_cmds=self.args.post_snapshot_cmd)
+                                                post_snapshot_cmds=self.args.post_snapshot_cmd,
+                                                set_snapshot_properties=self.set_snapshot_properties_list())
 
             ################# sync
             # if target is specified, we sync the datasets, otherwise we just thin the source. (e.g. snapshot mode)
@@ -458,7 +473,8 @@ class ZfsAutobackup(ZfsAuto):
                     target_thinner = None
                 else:
                     target_thinner = Thinner(self.args.keep_target)
-                target_node = ZfsNode(snapshot_time_format=self.snapshot_time_format, hold_name=self.hold_name,
+                target_node = ZfsNode(utc=self.args.utc,
+                                      snapshot_time_format=self.snapshot_time_format, hold_name=self.hold_name,
                                       logger=self, ssh_config=self.args.ssh_config,
                                       ssh_to=self.args.ssh_target,
                                       readonly=self.args.test, debug_output=self.args.debug_output,
@@ -524,7 +540,8 @@ def cli():
 
     signal(SIGPIPE, sigpipe_handler)
 
-    sys.exit(ZfsAutobackup(sys.argv[1:], False).run())
+    failed_datasets=ZfsAutobackup(sys.argv[1:], False).run()
+    sys.exit(min(failed_datasets, 255))
 
 
 if __name__ == "__main__":

zfs_autobackup/ZfsAutoverify.py

@@ -231,7 +231,8 @@ class ZfsAutoverify(ZfsAuto):
             self.set_title("Source settings")
 
             description = "[Source]"
-            source_node = ZfsNode(snapshot_time_format=self.snapshot_time_format, hold_name=self.hold_name, logger=self,
+            source_node = ZfsNode(utc=self.args.utc,
+                                  snapshot_time_format=self.snapshot_time_format, hold_name=self.hold_name, logger=self,
                                   ssh_config=self.args.ssh_config,
                                   ssh_to=self.args.ssh_source, readonly=self.args.test,
                                   debug_output=self.args.debug_output, description=description)
@@ -249,7 +250,8 @@ class ZfsAutoverify(ZfsAuto):
 
             # create target_node
             self.set_title("Target settings")
-            target_node = ZfsNode(snapshot_time_format=self.snapshot_time_format, hold_name=self.hold_name,
+            target_node = ZfsNode(utc=self.args.utc,
+                                  snapshot_time_format=self.snapshot_time_format, hold_name=self.hold_name,
                                   logger=self, ssh_config=self.args.ssh_config,
                                   ssh_to=self.args.ssh_target,
                                   readonly=self.args.test, debug_output=self.args.debug_output,
@@ -306,8 +308,8 @@ def cli():
     import sys
 
     signal(SIGPIPE, sigpipe_handler)
-
-    sys.exit(ZfsAutoverify(sys.argv[1:], False).run())
+    failed = ZfsAutoverify(sys.argv[1:], False).run()
+    sys.exit(min(failed,255))
 
 
 if __name__ == "__main__":

zfs_autobackup/ZfsCheck.py

@@ -18,7 +18,7 @@ class ZfsCheck(CliBase):
         # NOTE: common options argument parsing are in CliBase
         super(ZfsCheck, self).__init__(argv, print_arguments)
 
-        self.node = ZfsNode(self.log, readonly=self.args.test, debug_output=self.args.debug_output)
+        self.node = ZfsNode(self.log, utc=self.args.utc, readonly=self.args.test, debug_output=self.args.debug_output)
 
         self.block_hasher = BlockHasher(count=self.args.count, bs=self.args.block_size, skip=self.args.skip)
 
@@ -302,8 +302,8 @@ class ZfsCheck(CliBase):
 def cli():
     import sys
     signal(SIGPIPE, sigpipe_handler)
-
-    sys.exit(ZfsCheck(sys.argv[1:], False).run())
+    failed=ZfsCheck(sys.argv[1:], False).run()
+    sys.exit(min(failed,255))
 
 
 if __name__ == "__main__":

zfs_autobackup/ZfsDataset.py

@@ -1,4 +1,6 @@
 import re
+from datetime import datetime
+import sys
 import time
 
 from .CachedProperty import CachedProperty
@@ -129,7 +131,7 @@ class ZfsDataset:
             :type exclude_unchanged: bool
             :type min_change: bool
 
-            :param value: Value of the zfs property ("false"/"true"/"child"/"-")
+            :param value: Value of the zfs property ("false"/"true"/"child"/parent/"-")
             :param source: Source of the zfs property ("local"/"received", "-")
             :param inherited: True of the value/source was inherited from a higher dataset.
         """
@@ -140,7 +142,7 @@ class ZfsDataset:
             raise (Exception(
                 "{} autobackup-property has illegal source: '{}' (possible BUG)".format(self.name, source)))
 
-        if value not in ["false", "true", "child", "-"]:
+        if value not in ["false", "true", "child", "parent", "-"]:
             # user error
             raise (Exception(
                 "{} autobackup-property has illegal value: '{}'".format(self.name, value)))
@@ -153,6 +155,10 @@ class ZfsDataset:
         if value == "child" and not inherited:
             return False
 
+        # only select parent, no childs, ignore
+        if value == "parent" and inherited:
+            return False
+
         # manually excluded by property
         if value == "false":
             self.verbose("Excluded")
@@ -375,9 +381,22 @@ class ZfsDataset:
         """get timestamp from snapshot name. Only works for our own snapshots
         with the correct format.
         """
-
-        time_secs = time.mktime(time.strptime(self.snapshot_name, self.zfs_node.snapshot_time_format))
-        return time_secs
+        dt = datetime.strptime(self.snapshot_name, self.zfs_node.snapshot_time_format)
+        if sys.version_info[0] >= 3:
+            from datetime import timezone
+            if self.zfs_node.utc:
+                dt = dt.replace(tzinfo=timezone.utc)
+            seconds = dt.timestamp()
+        else:
+            # python2 has no good functions to deal with UTC. Yet the unix timestamp
+            # must be in UTC to allow comparison against `time.time()` in on other parts
+            # of this project (e.g. Thinner.py). If we are handling UTC timestamps,
+            # we must adjust for that here.
+            if self.zfs_node.utc:
+                seconds = (dt - datetime(1970, 1, 1)).total_seconds()
+            else:
+                seconds = time.mktime(dt.timetuple())
+        return seconds
 
     def from_names(self, names):
         """convert a list of names to a list ZfsDatasets for this zfs_node
@@ -1137,7 +1156,7 @@ class ZfsDataset:
         self.debug("Unmounting")
 
         cmd = [
-            "umount", "-l", self.name
+            "umount", self.name
         ]
 
 

zfs_autobackup/ZfsNode.py

@@ -17,10 +17,11 @@ from .ExecuteNode import ExecuteError
 class ZfsNode(ExecuteNode):
     """a node that contains zfs datasets. implements global (systemwide/pool wide) zfs commands"""
 
-    def __init__(self, logger, snapshot_time_format="", hold_name="", ssh_config=None, ssh_to=None, readonly=False,
+    def __init__(self, logger, utc=False, snapshot_time_format="", hold_name="", ssh_config=None, ssh_to=None, readonly=False,
                  description="",
                  debug_output=False, thinner=None):
 
+        self.utc = utc
         self.snapshot_time_format = snapshot_time_format
         self.hold_name = hold_name
 
@@ -180,7 +181,7 @@ class ZfsNode(ExecuteNode):
         self.logger.debug("{} {}".format(self.description, txt))
 
     def consistent_snapshot(self, datasets, snapshot_name, min_changed_bytes, pre_snapshot_cmds=[],
-                            post_snapshot_cmds=[]):
+                            post_snapshot_cmds=[], set_snapshot_properties=[]):
         """create a consistent (atomic) snapshot of specified datasets, per pool.
         """
 
@@ -218,6 +219,8 @@ class ZfsNode(ExecuteNode):
             # create consistent snapshot per pool
             for (pool_name, snapshots) in pools.items():
                 cmd = ["zfs", "snapshot"]
+                for snapshot_property in set_snapshot_properties:
+                    cmd += ['-o', snapshot_property]
 
                 cmd.extend(map(lambda snapshot_: str(snapshot_), snapshots))