patrick/zfs_autobackup
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

central timehandling and better mocking during test

Browse Source
This commit is contained in:
Edwin Eefting
2023-10-02 16:29:46 +02:00
parent 477e980ba2
commit 5cca819916
8 changed files with 75 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
tests/basetest.py
View File

@ -4,6 +4,8 @@
import sys
import zfs_autobackup.util
#dirty hack for this error:
#AttributeError: module 'collections' has no attribute 'MutableMapping'
@ -28,6 +30,9 @@ import contextlib
import sys
import io
import datetime
TEST_POOLS="test_source1 test_source2 test_target1"
ZFS_USERSPACE= subprocess.check_output("dpkg-query -W zfsutils-linux |cut -f2", shell=True).decode('utf-8').rstrip()
ZFS_KERNEL= subprocess.check_output("modinfo zfs|grep ^version |sed 's/.* //'", shell=True).decode('utf-8').rstrip()
@ -105,3 +110,18 @@ def prepare_zpools():
subprocess.check_call("zfs set autobackup:test=child test_source2/fs2", shell=True)
print("Prepare done")
@contextlib.contextmanager
def mocktime(time_str, format="%Y%m%d%H%M%S"):
def fake_datetime_now():
return datetime.datetime.strptime(time_str, format)
with patch.object(zfs_autobackup.util,'datetime_now_mock', fake_datetime_now()):
yield

48
tests/test_zfsautobackup32.py
View File

@ -1,5 +1,4 @@
from basetest import *
import time
class TestZfsAutobackup32(unittest2.TestCase):
"""various new 3.2 features"""
@ -137,18 +136,39 @@ test_target1/test_source2/fs2/sub@test-20101111000001
""")
# def test_stuff(self):
#
#
# shelltest("zfs set autobackup:test=true test_source2")
# # shelltest("zfs set readonly=on test_target1")
#
# with patch('time.strftime', return_value="test-20101111000000"):
# self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --clear-mountpoint".split(" ")).run())
#
# # shelltest("zfs mount test_target1/test_source2/fs2/sub" )
#
# with patch('time.strftime', return_value="test-20101111000001"):
# self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --rollback".split(" ")).run())
#XXX: VERBERTERING VAN ADD VIRTUALSNAPSHOTS IN GIT STASH!
def test_thinning(self):
# time_str = "20111112000000" # month in the "future"
# future_timestamp = time_secs = time.mktime(time.strptime(time_str, "%Y%m%d%H%M%S"))
# with patch('time.time', return_value=future_timestamp):
with mocktime("20001001000000"):
print(datetime_now(False))
self.assertFalse(ZfsAutobackup("test --allow-empty --clear-mountpoint --verbose".split(" ")).run())
# with patch('time.strftime', return_value="test-20001101000000"):
# self.assertFalse(ZfsAutobackup("test --allow-empty --clear-mountpoint test_target1 --no-progress --allow-empty --clear-mountpoint".split(" ")).run())
#
# with patch('time.strftime', return_value="test-20001201000000"):
# self.assertFalse(ZfsAutobackup("test --allow-empty --clear-mountpoint".split(" ")).run())
#
# with patch('time.strftime', return_value="test-20001202000000"):
# self.assertFalse(ZfsAutobackup("test --allow-empty --clear-mountpoint".split(" ")).run())
#
# time_str="test-20001203000000"
# with patch('time.time', return_value=time.mktime(time.strptime(time_str, "test-%Y%m%d%H%M%S"))):
# with patch('time.strftime', return_value=time_str):
# self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --allow-empty --clear-mountpoint --keep-source=1d2d".split(" ")).run())
#
#
#
# r=shelltest("zfs list -H -o name -r -t snapshot test_source1 test_target1")
# self.assertMultiLineEqual(r,"""
# /test_target1
# /test_target1/test_source1/fs1
# /test_target1/test_source1/fs1/sub
# /test_target1/test_source2/fs2/sub
# """)

8
zfs_autobackup/Thinner.py
View File

@ -1,4 +1,3 @@
import time
from .ThinnerRule import ThinnerRule
@ -37,7 +36,7 @@ class Thinner:
return ret
def thin(self, objects, keep_objects=None, now=None):
def thin(self, objects, keep_objects, now):
"""thin list of objects with current schedule rules. objects: list of
objects to thin. every object should have timestamp attribute.
@ -49,8 +48,6 @@ class Thinner:
now: if specified, use this time as current time
"""
if not keep_objects:
keep_objects = []
# always keep a number of the last objets?
if self.always_keep:
@ -68,9 +65,6 @@ class Thinner:
for rule in self.rules:
time_blocks[rule.period] = {}
if not now:
now = int(time.time())
keeps = []
removes = []

4
zfs_autobackup/ZfsAuto.py
View File

@ -1,8 +1,8 @@
import argparse
import sys
from datetime import time, datetime
from .CliBase import CliBase
from .util import datetime_now
class ZfsAuto(CliBase):
@ -59,7 +59,7 @@ class ZfsAuto(CliBase):
self.snapshot_time_format = args.snapshot_format.format(args.backup_name)
self.hold_name = args.hold_format.format(args.backup_name)
dt = datetime.utcnow() if args.utc else datetime.now()
dt = datetime_now(args.utc)
self.verbose("")
self.verbose("Current time {} : {}".format(args.utc and "UTC" or " ", dt.strftime("%Y-%m-%d %H:%M:%S")))

9
zfs_autobackup/ZfsAutobackup.py
View File

@ -1,9 +1,7 @@
import time
import argparse
from datetime import datetime
from signal import signal, SIGPIPE
from .util import output_redir, sigpipe_handler
from .util import output_redir, sigpipe_handler, datetime_now
from .ZfsAuto import ZfsAuto
@ -203,7 +201,7 @@ class ZfsAutobackup(ZfsAuto):
else:
# past the deadline?
deadline_ttl = ThinnerRule("0s" + self.args.destroy_missing).ttl
now = int(time.time())
now = datetime_now(self.args.utc).timestamp()
if dataset.our_snapshots[-1].timestamp + deadline_ttl > now:
dataset.verbose("Destroy missing: Waiting for deadline.")
else:
@ -461,8 +459,7 @@ class ZfsAutobackup(ZfsAuto):
################# snapshotting
if not self.args.no_snapshot:
self.set_title("Snapshotting")
dt = datetime.utcnow() if self.args.utc else datetime.now()
snapshot_name = dt.strftime(self.snapshot_time_format)
snapshot_name = datetime_now(self.args.utc).strftime(self.snapshot_time_format)
source_node.consistent_snapshot(source_datasets, snapshot_name,
min_changed_bytes=self.args.min_change,
pre_snapshot_cmds=self.args.pre_snapshot_cmd,

4
zfs_autobackup/ZfsNode.py
View File

@ -12,6 +12,7 @@ from .CachedProperty import CachedProperty
from .ZfsPool import ZfsPool
from .ZfsDataset import ZfsDataset
from .ExecuteNode import ExecuteError
from .util import datetime_now
class ZfsNode(ExecuteNode):
@ -59,7 +60,8 @@ class ZfsNode(ExecuteNode):
def thin(self, objects, keep_objects):
# NOTE: if thinning is disabled with --no-thinning, self.__thinner will be none.
if self.__thinner is not None:
return self.__thinner.thin(objects, keep_objects)
return self.__thinner.thin(objects, keep_objects, datetime_now(self.utc))
else:
return (keep_objects, [])

129
zfs_autobackup/test.py
View File

@ -1,129 +0,0 @@
import os.path
import os
import subprocess
import sys
import time
from signal import signal, SIGPIPE
import util
signal(SIGPIPE, util.sigpipe_handler)
try:
print ("voor eerste")
raise Exception("eerstre")
except Exception as e:
print ("voor tweede")
raise Exception("tweede")
finally:
print ("JO")
def generator():
try:
util.deb('in generator')
print ("TRIGGER SIGPIPE")
sys.stdout.flush()
util.deb('after trigger')
# if False:
yield ("bla")
# yield ("bla")
except GeneratorExit as e:
util.deb('GENEXIT '+str(e))
raise
except Exception as e:
util.deb('EXCEPT '+str(e))
finally:
util.deb('FINALLY')
print("nog iets")
sys.stdout.flush()
util.deb('after print in finally WOOP!')
util.deb('START')
g=generator()
util.deb('after generator')
for bla in g:
# print ("heb wat ontvangen")
util.deb('ontvangen van gen')
break
# raise Exception("moi")
pass
raise Exception("moi")
util.deb('after for')
while True:
pass
#
# with open('test.py', 'rb') as fh:
#
# # fsize = fh.seek(10000, os.SEEK_END)
# # print(fsize)
#
# start=time.time()
# for i in range(0,1000000):
# # fh.seek(0, 0)
# fsize=fh.seek(0, os.SEEK_END)
# # fsize=fh.tell()
# # os.path.getsize('test.py')
# print(time.time()-start)
#
#
# print(fh.tell())
#
# sys.exit(0)
#
#
#
# checked=1
# skipped=1
# coverage=0.1
#
# max_skip=0
#
#
# skipinarow=0
# while True:
# total=checked+skipped
#
# skip=coverage<random()
# if skip:
# skipped = skipped + 1
# print("S {:.2f}%".format(checked * 100 / total))
#
# skipinarow = skipinarow+1
# if skipinarow>max_skip:
# max_skip=skipinarow
# else:
# skipinarow=0
# checked=checked+1
# print("C {:.2f}%".format(checked * 100 / total))
#
# print(max_skip)
#
# skip=0
# while True:
#
# total=checked+skipped
# if skip>0:
# skip=skip-1
# skipped = skipped + 1
# print("S {:.2f}%".format(checked * 100 / total))
# else:
# checked=checked+1
# print("C {:.2f}%".format(checked * 100 / total))
#
# #calc new skip
# skip=skip+((1/coverage)-1)*(random()*2)
# # print(skip)
# if skip> max_skip:
# max_skip=skip
#
# print(max_skip)

26
zfs_autobackup/util.py
View File

@ -1,21 +1,9 @@
# root@psyt14s:/home/psy/zfs_autobackup# ls -lh /home/psy/Downloads/carimage.zip
# -rw-rw-r-- 1 psy psy 990M Nov 26 2020 /home/psy/Downloads/carimage.zip
# root@psyt14s:/home/psy/zfs_autobackup# time sha1sum /home/psy/Downloads/carimage.zip
# a682e1a36e16fe0d0c2f011104f4a99004f19105 /home/psy/Downloads/carimage.zip
#
# real 0m2.558s
# user 0m2.105s
# sys 0m0.448s
# root@psyt14s:/home/psy/zfs_autobackup# time python3 -m zfs_autobackup.ZfsCheck
#
# real 0m1.459s
# user 0m0.993s
# sys 0m0.462s
# NOTE: surprisingly sha1 in via python3 is faster than the native sha1sum utility, even in the way we use below!
import os
import platform
import sys
from datetime import datetime
def tmp_name(suffix=""):
@ -48,7 +36,7 @@ def output_redir():
def sigpipe_handler(sig, stack):
#redir output so we dont get more SIGPIPES during cleanup. (which my try to write to stdout)
output_redir()
deb('redir')
#deb('redir')
# def check_output():
# """make sure stdout still functions. if its broken, this will trigger a SIGPIPE which will be handled by the sigpipe_handler."""
@ -63,3 +51,13 @@ def sigpipe_handler(sig, stack):
# fh.write("DEB: "+txt+"\n")
# This should be the only source of trueth for the current datetime.
# This function will be mocked during unit testing.
datetime_now_mock=None
def datetime_now(utc):
if datetime_now_mock is None:
return( datetime.utcnow() if utc else datetime.now())
else:
return datetime_now_mock