added tar-mode. moved static methods. more compatible /dev checking without udevadm

tests/test_verify.py

@@ -67,7 +67,7 @@ class TestZfsEncryption(unittest2.TestCase):
             self.assertFalse(ZfsAutoverify("test test_target1 --verbose --test".split(" ")).run())
 
         with self.subTest("rsync, remote source and target. (not supported, all 6 fail)"):
-            self.assertEqual(6, ZfsAutoverify("test test_target1 --ssh-source=localhost --ssh-target=localhost --verbose --exclude-received".split(" ")).run())
+            self.assertEqual(6, ZfsAutoverify("test test_target1 --ssh-source=localhost --ssh-target=localhost --verbose --exclude-received --fs-compare=rsync".split(" ")).run())
 
         def runchecked(testname, command):
             with self.subTest(testname):
@@ -81,10 +81,17 @@ class TestZfsEncryption(unittest2.TestCase):
                     self.assertRegex(buf.getvalue(), "bad_filesystem: FAILED:")
                     self.assertRegex(buf.getvalue(), "bad_zvol: FAILED:")
 
-        runchecked("rsync, remote source", "test test_target1 --ssh-source=localhost --verbose --exclude-received")
+        runchecked("rsync, remote source", "test test_target1 --ssh-source=localhost --verbose --exclude-received --fs-compare=rsync")
-        runchecked("rsync, remote target", "test test_target1 --ssh-target=localhost --verbose --exclude-received")
+        runchecked("rsync, remote target", "test test_target1 --ssh-target=localhost --verbose --exclude-received --fs-compare=rsync")
-        runchecked("rsync, local", "test test_target1 --verbose --exclude-received")
+        runchecked("rsync, local", "test test_target1 --verbose --exclude-received --fs-compare=rsync")
 
+        runchecked("tar, remote source and remote target",
+                   "test test_target1 --ssh-source=localhost --ssh-target=localhost --verbose --exclude-received --fs-compare=tar")
+        runchecked("tar, remote source",
+                   "test test_target1 --ssh-source=localhost --verbose --exclude-received --fs-compare=tar")
+        runchecked("tar, remote target",
+                   "test test_target1 --ssh-target=localhost --verbose --exclude-received --fs-compare=tar")
+        runchecked("tar, local", "test test_target1 --verbose --exclude-received --fs-compare=tar")
 
         with self.subTest("no common snapshot"):
             #destroy common snapshot, now 3 should fail

zfs_autobackup/ExecuteNode.py

@@ -78,10 +78,11 @@ class ExecuteNode(LogStub):
         return self.ssh_to is None
 
     def run(self, cmd, inp=None, tab_split=False, valid_exitcodes=None, readonly=False, hide_errors=False,
-            return_stderr=False, pipe=False):
+            return_stderr=False, pipe=False, return_all=False):
         """run a command on the node , checks output and parses/handle output and returns it
 
-        Either uses a local shell (sh -c) or remote shell (ssh) to execute the command. Therefore the command can have stuff like actual pipes in it, if you dont want to use pipe=True to pipe stuff.
+        Either uses a local shell (sh -c) or remote shell (ssh) to execute the command.
+        Therefore the command can have stuff like actual pipes in it, if you dont want to use pipe=True to pipe stuff.
 
         :param cmd: the actual command, should be a list, where the first item is the command
                     and the rest are parameters. use ExecuteNode.PIPE to add an unescaped |
@@ -94,6 +95,7 @@ class ExecuteNode(LogStub):
         :param readonly: make this True if the command doesn't make any changes and is safe to execute in testmode
         :param hide_errors: don't show stderr output as error, instead show it as debugging output (use to hide expected errors)
         :param return_stderr: return both stdout and stderr as a tuple. (normally only returns stdout)
+        :param return_all: return both stdout and stderr and exit_code as a tuple. (normally only returns stdout)
 
         """
 
@@ -106,6 +108,7 @@ class ExecuteNode(LogStub):
 
         # stderr parser
         error_lines = []
+        returned_exit_code=None
 
         def stderr_handler(line):
             if tab_split:
@@ -155,7 +158,9 @@ class ExecuteNode(LogStub):
         if not cmd_pipe.execute(stdout_handler=stdout_handler):
             raise(ExecuteError("Last command returned error"))
 
-        if return_stderr:
+        if return_all:
+            return output_lines, error_lines, cmd_item.process and cmd_item.process.returncode
+        elif return_stderr:
             return output_lines, error_lines
         else:
             return output_lines

zfs_autobackup/ZfsAutoverify.py

@@ -1,50 +1,44 @@
 import os
+import time
 
+from .ExecuteNode import ExecuteNode
 from .ZfsAuto import ZfsAuto
 from .ZfsDataset import ZfsDataset
 from .ZfsNode import ZfsNode
 import sys
 import platform
 
+def hash_tree_tar(node, path):
+    """calculate md5sum of a directory tree, using tar"""
 
-class ZfsAutoverify(ZfsAuto):
+    node.debug("Hashing filesystem {} ".format(path))
-    """The zfs-autoverify class, default agruments and stuff come from ZfsAuto"""
 
-    def __init__(self, argv, print_arguments=True):
+    cmd=[ "tar", "-cf", "-", "-C", path, ".",
+          ExecuteNode.PIPE, "md5sum"]
 
-        # NOTE: common options and parameters are in ZfsAuto
+    stdout = node.run(cmd)
-        super(ZfsAutoverify, self).__init__(argv, print_arguments)
 
-    def parse_args(self, argv):
+    if node.readonly:
-        """do extra checks on common args"""
+        hashed=None
+    else:
+        hashed = stdout[0].split(" ")[0]
 
-        args=super(ZfsAutoverify, self).parse_args(argv)
+    node.debug("Hash of {} filesytem is {}".format(path, hashed))
 
-        if args.target_path == None:
+    return hashed
-            self.log.error("Please specify TARGET-PATH")
-            sys.exit(255)
 
-        return args
 
-    def get_parser(self):
+def compare_trees_tar(source_node, source_path, target_node, target_path):
-        """extend common parser with  extra stuff needed for zfs-autobackup"""
-
-        parser=super(ZfsAutoverify, self).get_parser()
-
-        group=parser.add_argument_group("Verify options")
-        group.add_argument('--fs-compare', metavar='METHOD', default="tar", choices=["tar", "rsync"],
-                            help='Compare method to use for filesystems. (tar, rsync) Default: %(default)s ')
-
-        return parser
-
-    def compare_trees_tar(self , source_node, source_path, target_node, target_path):
     """compare two trees using tar. compatible and simple"""
 
-        self.error("XXX implement")
+    source_hash= hash_tree_tar(source_node, source_path)
-        pass
+    target_hash= hash_tree_tar(target_node, target_path)
+
+    if source_hash != target_hash:
+        raise Exception("md5hash difference: {} != {}".format(source_hash, target_hash))
 
 
-    def compare_trees_rsync(self , source_node, source_path, target_node, target_path):
+def compare_trees_rsync(source_node, source_path, target_node, target_path):
     """use rsync to compare two trees.
      Advantage is that we can see which individual files differ.
      But requires rsync and cant do remote to remote."""
@@ -78,7 +72,8 @@ class ZfsAutoverify(ZfsAuto):
     if stderr:
         raise Exception("Dataset verify failed, see above list for differences")
 
-    def verify_filesystem(self, source_snapshot, source_mnt, target_snapshot, target_mnt):
+
+def verify_filesystem(source_snapshot, source_mnt, target_snapshot, target_mnt, method):
     """Compare the contents of two zfs filesystem snapshots """
 
     try:
@@ -87,16 +82,22 @@ class ZfsAutoverify(ZfsAuto):
         source_snapshot.mount(source_mnt)
         target_snapshot.mount(target_mnt)
 
-            self.compare_trees_rsync(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt)
+        if method=='rsync':
+            compare_trees_rsync(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt)
+        elif method == 'tar':
+            compare_trees_tar(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt)
+        else:
+            raise(Exception("program errror, unknown method"))
 
     finally:
         source_snapshot.unmount()
         target_snapshot.unmount()
 
-    def hash_dev(self, node, dev):
+
+def hash_dev(node, dev):
     """calculate md5sum of a device on a node"""
 
-        node.debug("Hashing {} ".format(dev))
+    node.debug("Hashing volume {} ".format(dev))
 
     cmd = [ "md5sum", dev ]
 
@@ -107,35 +108,109 @@ class ZfsAutoverify(ZfsAuto):
     else:
         hashed = stdout[0].split(" ")[0]
 
-        node.debug("Hash of {} is {}".format(dev, hashed))
+    node.debug("Hash of volume {} is {}".format(dev, hashed))
 
     return hashed
 
-    def verify_volume(self, source_dataset, source_snapshot, target_dataset, target_snapshot):
+def activate_volume_snapshot(dataset, snapshot):
+    """enables snapdev, waits and tries to findout /dev path to the volume, in a compatible way. (linux/freebsd/smartos)"""
+
+    dataset.set("snapdev", "visible")
+
+    #NOTE: add smartos location to this list as well
+    locations=[
+        "/dev/zvol/" + snapshot.name
+    ]
+
+    dataset.debug("Waiting for /dev entry to appear...")
+    time.sleep(0.1)
+
+    start_time=time.time()
+    while time.time()-start_time<10:
+        for location in locations:
+            stdout, stderr, exit_code=dataset.zfs_node.run(["test", "-e", location], return_all=True, valid_exitcodes=[0,1])
+
+            #fake it in testmode
+            if dataset.zfs_node.readonly:
+                return location
+
+            if exit_code==0:
+                return location
+        time.sleep(1)
+
+    raise(Exception("Timeout while waiting for {} entry to appear.".format(locations)))
+
+def deacitvate_volume_snapshot(dataset):
+    dataset.inherit("snapdev")
+
+
+def verify_volume(source_dataset, source_snapshot, target_dataset, target_snapshot):
     """compare the contents of two zfs volume snapshots"""
 
     try:
-            #make sure the volume snapshot is visible in /dev
+        source_dev= activate_volume_snapshot(source_dataset, source_snapshot)
-            source_dataset.set("snapdev", "visible")
+        target_dev= activate_volume_snapshot(target_dataset, target_snapshot)
-            target_dataset.set("snapdev", "visible")
 
-            # fixme: not compatible with freebsd and others.
+        source_hash= hash_dev(source_snapshot.zfs_node, source_dev)
-            source_dev="/dev/zvol/"+source_snapshot.name
+        target_hash= hash_dev(target_snapshot.zfs_node, target_dev)
-            target_dev="/dev/zvol/"+target_snapshot.name
-            source_dataset.zfs_node.run(["udevadm", "trigger", source_dev])
-            target_dataset.zfs_node.run(["udevadm", "trigger", target_dev])
-
-            source_hash=self.hash_dev(source_snapshot.zfs_node, source_dev)
-            target_hash=self.hash_dev(target_snapshot.zfs_node, target_dev)
 
         if source_hash!=target_hash:
             raise Exception("md5hash difference: {} != {}".format(source_hash, target_hash))
 
     finally:
-            source_dataset.inherit("snapdev")
+        deacitvate_volume_snapshot(source_dataset)
-            target_dataset.inherit("snapdev")
+        deacitvate_volume_snapshot(target_dataset)
 
 
+def create_mountpoints(source_node, target_node):
+
+    # prepare mount points
+    source_node.debug("Create temporary mount point")
+    source_mnt = "/tmp/zfs-autoverify_source_{}_{}".format(platform.node(), os.getpid())
+    source_node.run(["mkdir", source_mnt])
+
+    target_node.debug("Create temporary mount point")
+    target_mnt = "/tmp/zfs-autoverify_target_{}_{}".format(platform.node(), os.getpid())
+    target_node.run(["mkdir", target_mnt])
+
+    return source_mnt, target_mnt
+
+
+def cleanup_mountpoint(node, mnt):
+    node.debug("Cleaning up temporary mount point")
+    node.run([ "rmdir", mnt ], hide_errors=True, valid_exitcodes=[] )
+
+
+class ZfsAutoverify(ZfsAuto):
+    """The zfs-autoverify class, default agruments and stuff come from ZfsAuto"""
+
+    def __init__(self, argv, print_arguments=True):
+
+        # NOTE: common options and parameters are in ZfsAuto
+        super(ZfsAutoverify, self).__init__(argv, print_arguments)
+
+    def parse_args(self, argv):
+        """do extra checks on common args"""
+
+        args=super(ZfsAutoverify, self).parse_args(argv)
+
+        if args.target_path == None:
+            self.log.error("Please specify TARGET-PATH")
+            sys.exit(255)
+
+        return args
+
+    def get_parser(self):
+        """extend common parser with  extra stuff needed for zfs-autobackup"""
+
+        parser=super(ZfsAutoverify, self).get_parser()
+
+        group=parser.add_argument_group("Verify options")
+        group.add_argument('--fs-compare', metavar='METHOD', default="tar", choices=["tar", "rsync"],
+                            help='Compare method to use for filesystems. (tar, rsync) Default: %(default)s ')
+
+        return parser
+
     def verify_datasets(self, source_mnt, source_datasets, target_node, target_mnt):
 
         fail_count=0
@@ -162,9 +237,9 @@ class ZfsAutoverify(ZfsAuto):
                 target_snapshot.verbose("Verifying...")
 
                 if source_dataset.properties['type']=="filesystem":
-                    self.verify_filesystem(source_snapshot, source_mnt, target_snapshot, target_mnt)
+                    verify_filesystem(source_snapshot, source_mnt, target_snapshot, target_mnt, self.args.fs_compare)
                 elif source_dataset.properties['type']=="volume":
-                    self.verify_volume(source_dataset, source_snapshot, target_dataset, target_snapshot)
+                    verify_volume(source_dataset, source_snapshot, target_dataset, target_snapshot)
                 else:
                     raise(Exception("{} has unknown type {}".format(source_dataset, source_dataset.properties['type'])))
 
@@ -178,24 +253,6 @@ class ZfsAutoverify(ZfsAuto):
 
         return fail_count
 
-    def create_mountpoints(self, source_node, target_node):
-
-        # prepare mount points
-        source_node.debug("Create temporary mount point")
-        source_mnt = "/tmp/zfs-autoverify_source_{}_{}".format(platform.node(), os.getpid())
-        source_node.run(["mkdir", source_mnt])
-
-        target_node.debug("Create temporary mount point")
-        target_mnt = "/tmp/zfs-autoverify_target_{}_{}".format(platform.node(), os.getpid())
-        target_node.run(["mkdir", target_mnt])
-
-        return source_mnt, target_mnt
-
-    def cleanup_mountpoint(self, node, mnt):
-        node.debug("Cleaning up temporary mount point")
-        node.run([ "rmdir", mnt ], hide_errors=True, valid_exitcodes=[] )
-
-
     def run(self):
 
         source_node=None
@@ -237,7 +294,7 @@ class ZfsAutoverify(ZfsAuto):
 
             self.set_title("Verifying")
 
-            source_mnt, target_mnt=self.create_mountpoints(source_node, target_node)
+            source_mnt, target_mnt= create_mountpoints(source_node, target_node)
 
             fail_count = self.verify_datasets(
                 source_mnt=source_mnt,
@@ -273,10 +330,10 @@ class ZfsAutoverify(ZfsAuto):
 
             # cleanup
             if source_mnt is not None:
-                self.cleanup_mountpoint(source_node, source_mnt)
+                cleanup_mountpoint(source_node, source_mnt)
 
             if target_mnt is not None:
-                self.cleanup_mountpoint(target_node, target_mnt)
+                cleanup_mountpoint(target_node, target_mnt)