patrick/zfs_autobackup
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement chunk size argument and refactor mbuffer command generation

Browse Source 
Fixes #203
This commit is contained in:
Pierre-Elliott Bécue
2023-07-18 16:46:32 +02:00
committed by DatuX
parent 7586cacb49
commit 7feae675a6
1 changed files with 32 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
zfs_autobackup/ZfsAutobackup.py
View File

@ -119,6 +119,8 @@ class ZfsAutobackup(ZfsAuto):
help='Limit data transfer rate in Bytes/sec (e.g. 128K. requires mbuffer.)') help='Limit data transfer rate in Bytes/sec (e.g. 128K. requires mbuffer.)')
group.add_argument('--buffer', metavar='SIZE', default=None, group.add_argument('--buffer', metavar='SIZE', default=None,
help='Add zfs send and recv buffers to smooth out IO bursts. (e.g. 128M. requires mbuffer)') help='Add zfs send and recv buffers to smooth out IO bursts. (e.g. 128M. requires mbuffer)')
parser.add_argument('--chunk-size', metavar="CHUNKSIZE", default=None,
help='Tune chunk size when mbuffer is used. (requires mbuffer.)')
group.add_argument('--send-pipe', metavar="COMMAND", default=[], action='append', group.add_argument('--send-pipe', metavar="COMMAND", default=[], action='append',
help='pipe zfs send output through COMMAND (can be used multiple times)') help='pipe zfs send output through COMMAND (can be used multiple times)')
group.add_argument('--recv-pipe', metavar="COMMAND", default=[], action='append', group.add_argument('--recv-pipe', metavar="COMMAND", default=[], action='append',
@ -242,11 +244,22 @@ class ZfsAutobackup(ZfsAuto):
"""determine the zfs send pipe""" """determine the zfs send pipe"""
ret = [] ret = []
_mbuffer = False
_buffer = "16M"
_cs = "128k"
_rate = False
# IO buffer # IO buffer
if self.args.buffer: if self.args.buffer:
logger("zfs send buffer : {}".format(self.args.buffer)) logger("zfs send buffer : {}".format(self.args.buffer))
ret.extend([ExecuteNode.PIPE, "mbuffer", "-q", "-s128k", "-m" + self.args.buffer]) _mbuffer = True
_buffer = self.args.buffer
# IO chunk size
if self.args.chunk_size:
logger("zfs send chunk size : {}".format(self.args.chunk_size))
_mbuffer = True
_cs = self.args.chunk_size
# custom pipes # custom pipes
for send_pipe in self.args.send_pipe: for send_pipe in self.args.send_pipe:
@ -264,7 +277,14 @@ class ZfsAutobackup(ZfsAuto):
# transfer rate # transfer rate
if self.args.rate: if self.args.rate:
logger("zfs send transfer rate : {}".format(self.args.rate)) logger("zfs send transfer rate : {}".format(self.args.rate))
ret.extend([ExecuteNode.PIPE, "mbuffer", "-q", "-s128k", "-m16M", "-R" + self.args.rate]) _mbuffer = True
_rate = self.args.rate
if _mbuffer:
cmd = [ExecuteNode.PIPE, "mbuffer", "-q", "-s{}".format(_cs), "-m{}".format(_buffer)]
if _rate:
cmd.append("-R{}".format(self.args.rate))
ret.extend(cmd)
return ret return ret
@ -286,11 +306,19 @@ class ZfsAutobackup(ZfsAuto):
logger("zfs recv custom pipe : {}".format(recv_pipe)) logger("zfs recv custom pipe : {}".format(recv_pipe))
# IO buffer # IO buffer
if self.args.buffer: if self.args.buffer or self.args.chunk_size:
_cs = "128k"
_buffer = "16M"
# only add second buffer if its usefull. (e.g. non local transfer or other pipes active) # only add second buffer if its usefull. (e.g. non local transfer or other pipes active)
if self.args.ssh_source != None or self.args.ssh_target != None or self.args.recv_pipe or self.args.send_pipe or self.args.compress != None: if self.args.ssh_source != None or self.args.ssh_target != None or self.args.recv_pipe or self.args.send_pipe or self.args.compress != None:
logger("zfs recv buffer : {}".format(self.args.buffer)) logger("zfs recv buffer : {}".format(self.args.buffer))
ret.extend(["mbuffer", "-q", "-s128k", "-m" + self.args.buffer, ExecuteNode.PIPE])
if self.args.chunk_size:
_cs = self.args.chunk_size
if self.args.buffer:
_buffer = self.args.buffer
ret.extend(["mbuffer", "-q", "-s{}".format(_cs), "-m{}".format(_buffer), ExecuteNode.PIPE])
return ret return ret