now yields errors and mismatches

tests/test_blockhasher.py

@@ -1,6 +1,7 @@
 from basetest import *
 from zfs_autobackup.BlockHasher import BlockHasher
 
+
 # make VERY sure this works correctly under all circumstances.
 
 # sha1 sums of files, (bs=4096)
@@ -11,7 +12,7 @@ from zfs_autobackup.BlockHasher import BlockHasher
 # 959e6b58078f0cfd2fb3d37e978fda51820473ff  whole_whole2
 # 309ffffba2e1977d12f3b7469971f30d28b94bd8  whole_whole2_partial
 
-class TestBlockHahser(unittest2.TestCase):
+class TestBlockHasher(unittest2.TestCase):
 
     def setUp(self):
         pass
@@ -77,24 +78,23 @@ class TestBlockHahser(unittest2.TestCase):
             ])
 
     def test_blockhash_compare(self):
-
+        block_hasher = BlockHasher(count=1)
+        generator = block_hasher.generate("tests/data/whole_whole2_partial")
+        self.assertEqual([], list(block_hasher.compare("tests/data/whole_whole2_partial", generator)))
 
         block_hasher = BlockHasher(count=1)
         generator = block_hasher.generate("tests/data/whole_whole2_partial")
-        self.assertEqual(3,block_hasher.compare("tests/data/whole_whole2_partial", generator))
+        self.assertEqual(
-
+            [(1, '2e863f1fcccd6642e4e28453eba10d2d3f74d798', 'EOF'),
-        block_hasher=BlockHasher(count=1)
+             (2, '642027d63bb0afd7e0ba197f2c66ad03e3d70de1', 'EOF')],
-        with self.assertRaisesRegexp(Exception, "^Block 1 mismatched!"):
+            list(block_hasher.compare("tests/data/whole", generator)))
-            generator=block_hasher.generate("tests/data/whole_whole2_partial")
-            self.assertEqual(3,block_hasher.compare("tests/data/whole", generator))
 
         block_hasher = BlockHasher(count=10)
         generator = block_hasher.generate("tests/data/whole_whole2_partial")
-        self.assertEqual(1,block_hasher.compare("tests/data/whole_whole2_partial", generator))
+        self.assertEqual([], list(block_hasher.compare("tests/data/whole_whole2_partial", generator)))
 
         # different order to make sure seek functions
         block_hasher = BlockHasher(count=1)
         checksums = list(block_hasher.generate("tests/data/whole_whole2_partial"))
         checksums.reverse()
-        self.assertEqual(3,block_hasher.compare("tests/data/whole_whole2_partial", checksums))
+        self.assertEqual([], list(block_hasher.compare("tests/data/whole_whole2_partial", checksums)))
-

tests/test_treehasher.py

@@ -1,6 +1,7 @@
 from basetest import *
 from zfs_autobackup.BlockHasher import BlockHasher
 
+
 # sha1 sums of files, (bs=4096)
 # da39a3ee5e6b4b0d3255bfef95601890afd80709  empty
 # 642027d63bb0afd7e0ba197f2c66ad03e3d70de1  partial
@@ -12,10 +13,7 @@ from zfs_autobackup.BlockHasher import BlockHasher
 
 class TestTreeHasher(unittest2.TestCase):
 
-
-
     def test_treehasher(self):
-
         shelltest("rm -rf /tmp/treehashertest; mkdir /tmp/treehashertest")
         shelltest("cp tests/data/whole /tmp/treehashertest")
         shelltest("mkdir /tmp/treehashertest/emptydir")
@@ -40,19 +38,22 @@ class TestTreeHasher(unittest2.TestCase):
 
         with self.subTest("Test compare"):
             generator = tree_hasher.generate("/tmp/treehashertest")
-            count=tree_hasher.compare("/tmp/treehashertest", generator)
+            errors = list(tree_hasher.compare("/tmp/treehashertest", generator))
-            self.assertEqual(count,2)
+            self.assertEqual(errors, [])
 
         with self.subTest("Test mismatch"):
             generator = list(tree_hasher.generate("/tmp/treehashertest"))
             shelltest("cp tests/data/whole2 /tmp/treehashertest/whole")
 
-            with self.assertRaisesRegex(Exception,"mismatch"):
+            self.assertEqual(list(tree_hasher.compare("/tmp/treehashertest", generator)),
-                tree_hasher.compare("/tmp/treehashertest", generator)
+                             [('whole',
+                               0,
+                               '3c0bf91170d873b8e327d3bafb6bc074580d11b7',
+                               '2e863f1fcccd6642e4e28453eba10d2d3f74d798')])
 
-        with self.subTest("Test missig file compare"):
+        with self.subTest("Test missing file compare"):
             generator = list(tree_hasher.generate("/tmp/treehashertest"))
             shelltest("rm /tmp/treehashertest/whole")
 
-            with self.assertRaises(Exception):
+            self.assertEqual(list(tree_hasher.compare("/tmp/treehashertest", generator)),
-                tree_hasher.compare("/tmp/treehashertest", generator)
+                             [('whole', '-', '-', "ERROR: [Errno 2] No such file or directory: 'whole'")])

zfs_autobackup/BlockHasher.py

@@ -9,12 +9,12 @@ class BlockHasher():
 
     Its also possible to only read a certain percentage of blocks to just check a sample.
     """
+
     def __init__(self, count=10000, bs=4096, hash_class=hashlib.sha1):
         self.count = count
         self.bs = bs
         self.hash_class = hash_class
 
-
     def generate(self, fname):
         """Generates checksums
 
@@ -39,12 +39,13 @@ class BlockHasher():
                 yield (chunk_nr, hash.hexdigest())
 
     def compare(self, fname, generator):
-        """reads from generator and compares blocks, raises exception on error"""
+        """reads from generator and compares blocks, yields mismatches"""
 
+        try:
             checked = 0
             with open(fname, "rb") as f:
                 for (chunk_nr, hexdigest) in generator:
-                # print ("comparing {} {} {}".format(fname, chunk_nr, hexdigest))
+                    try:
 
                         checked = checked + 1
                         hash = self.hash_class()
@@ -56,8 +57,14 @@ class BlockHasher():
                             if block_nr == self.count:
                                 break
 
-                if (hash.hexdigest()!=hexdigest):
+                        if block_nr == 0:
-                    raise Exception("Block {} mismatched! Hash is {}, but should be {}".format(chunk_nr, hash.hexdigest(), hexdigest))
+                            yield (chunk_nr, hexdigest, 'EOF')
 
-        return checked
+                        elif (hash.hexdigest() != hexdigest):
+                            yield (chunk_nr, hexdigest, hash.hexdigest())
 
+                    except Exception as e:
+                        yield ( chunk_nr , hexdigest, 'ERROR: '+str(e))
+
+        except Exception as e:
+            yield ( '-', '-', 'ERROR: '+ str(e))

zfs_autobackup/TreeHasher.py

@@ -53,11 +53,11 @@ class TreeHasher():
             for file_name, group_generator in itertools.groupby(generator, lambda x: x[0]):
                 count=count+1
                 block_generator=itertools.starmap(filter_file_name, group_generator)
-                self.block_hasher.compare(file_name, block_generator)
+                for ( chunk_nr, compare_hexdigest, actual_hexdigest) in self.block_hasher.compare(file_name, block_generator):
+                    yield ( file_name, chunk_nr, compare_hexdigest, actual_hexdigest )
         finally:
             os.chdir(cwd)
 
-        return count