refactoring for oop and a better diff-engine

This commit is contained in:
Edwin Eefting
2017-07-28 01:46:13 +02:00
parent 8c121fab16
commit 441a323fb2
2 changed files with 134 additions and 135 deletions

View File

@ -23,6 +23,10 @@ It has the following features:
* Easy installation: * Easy installation:
* Only one host needs the zfs_autobackup script. The other host just needs ssh and the zfs command. * Only one host needs the zfs_autobackup script. The other host just needs ssh and the zfs command.
* Written in python and uses zfs-commands, no 3rd party dependencys or libraries. * Written in python and uses zfs-commands, no 3rd party dependencys or libraries.
* Tested on:
* SmartOS
* FreeNAS
* (others should work as well)
Usage Usage
==== ====

View File

@ -1,9 +1,5 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf8 -*- # -*- coding: utf8 -*-
from __future__ import print_function from __future__ import print_function
import os import os
import sys import sys
@ -18,34 +14,35 @@ import time
def error(txt): def error(txt):
print(txt, file=sys.stderr) print(txt, file=sys.stderr)
def verbose(txt): def verbose(txt):
if args.verbose: if args.verbose:
print(txt) print(txt)
def debug(txt): def debug(txt):
if args.debug: if args.debug:
print(txt) print(txt)
"""run a command. specifiy ssh user@host to run remotely""" class Node(Object):
def run(cmd, input=None, ssh_to="local", tab_split=False, valid_exitcodes=[ 0 ], test=False): """an endpoint that contains zfs filesystems. can be local or remote"""
def __init__(self, ssh_to='local'):
self.backup_name=backup_name
self.ssh_to=ssh_to
def run(cmd, input=None, tab_split=False, valid_exitcodes=[ 0 ], test=False):
"""run a command on the node"""
encoded_cmd=[] encoded_cmd=[]
#use ssh? #use ssh?
if ssh_to != "local": if self.ssh_to != "local":
encoded_cmd.extend(["ssh", ssh_to]) encoded_cmd.extend(["ssh", self.ssh_to])
if args.ssh_cipher: if args.ssh_cipher:
encoded_cmd.extend(["-c", args.ssh_cipher]) encoded_cmd.extend(["-c", args.ssh_cipher])
if args.compress: if args.compress:
encoded_cmd.append("-C") encoded_cmd.append("-C")
#make sure the command gets all the data in utf8 format: #make sure the command gets all the data in utf8 format:
#(this is neccesary if LC_ALL=en_US.utf8 is not set in the environment) #(this is neccesary if LC_ALL=en_US.utf8 is not set in the environment)
for arg in cmd: for arg in cmd:
@ -56,14 +53,11 @@ def run(cmd, input=None, ssh_to="local", tab_split=False, valid_exitcodes=[ 0 ],
for arg in cmd: for arg in cmd:
encoded_cmd.append(arg.encode('utf-8')) encoded_cmd.append(arg.encode('utf-8'))
#debug and test stuff
#the accurate way of displaying it whould be: print encoded_cmd
#However, we use the more human-readable way, but this is not always properly escaped!
#(most of the time it should be copypastable however.)
debug_txt="# "+" ".join(encoded_cmd) debug_txt="# "+" ".join(encoded_cmd)
if test: if test:
debug("[TEST] "+debug_txt) debug("[SKIPPING] "+debug_txt)
else: else:
debug(debug_txt) debug(debug_txt)
@ -76,6 +70,7 @@ def run(cmd, input=None, ssh_to="local", tab_split=False, valid_exitcodes=[ 0 ],
if test: if test:
return return
#execute and parse/return results
p=subprocess.Popen(encoded_cmd, env=os.environ, stdout=subprocess.PIPE, stdin=stdin) p=subprocess.Popen(encoded_cmd, env=os.environ, stdout=subprocess.PIPE, stdin=stdin)
output=p.communicate(input=input)[0] output=p.communicate(input=input)[0]
if p.returncode not in valid_exitcodes: if p.returncode not in valid_exitcodes:
@ -91,11 +86,12 @@ def run(cmd, input=None, ssh_to="local", tab_split=False, valid_exitcodes=[ 0 ],
return(ret) return(ret)
def zfs_get_selected_filesystems():
"""determine filesystems that should be backupped by looking at the special autobackup-property""" """determine filesystems that should be backupped by looking at the special autobackup-property"""
def zfs_get_selected_filesystems(ssh_to, backup_name):
#get all source filesystems that have the backup property #get all source filesystems that have the backup property
source_filesystems=run(ssh_to=ssh_to, tab_split=True, cmd=[ source_filesystems=self.run(tab_split=True, cmd=[
"zfs", "get", "-t", "volume,filesystem", "-o", "name,value,source", "-s", "local,inherited", "-H", "autobackup:"+backup_name "zfs", "get", "-t", "volume,filesystem", "-o", "name,value,source", "-s", "local,inherited", "-H", "autobackup:"+args.backup_name
]) ])
#determine filesystems that should be actually backupped #determine filesystems that should be actually backupped
@ -124,14 +120,13 @@ def zfs_get_selected_filesystems(ssh_to, backup_name):
return(selected_filesystems) return(selected_filesystems)
"""determine filesystems that can be resumed via receive_resume_token""" def zfs_get_resumable_filesystems(filesystems):
def zfs_get_resumable_filesystems(ssh_to, filesystems): """determine filesystems that can be resumed via receive_resume_token (should be executed on target)"""
cmd=[ "zfs", "get", "-t", "volume,filesystem", "-o", "name,value", "-H", "receive_resume_token" ] cmd=[ "zfs", "get", "-t", "volume,filesystem", "-o", "name,value", "-H", "receive_resume_token" ]
cmd.extend(filesystems) cmd.extend(filesystems)
#TODO: get rid of ugly errors for non-existing target filesystems resumable_filesystems=self.run(tab_split=True, cmd=cmd)
resumable_filesystems=run(ssh_to=ssh_to, tab_split=True, cmd=cmd, valid_exitcodes= [ 0,1 ] )
ret={} ret={}
@ -142,16 +137,17 @@ def zfs_get_resumable_filesystems(ssh_to, filesystems):
return(ret) return(ret)
def zfs_destroy_snapshots(snapshots):
"""deferred destroy list of snapshots (in @format). """ """deferred destroy list of snapshots (in @format). """
def zfs_destroy_snapshots(ssh_to, snapshots):
#zfs can only destroy one filesystem at once so we use xargs and stdin #zfs can only destroy one filesystem at once so we use xargs and stdin
run(ssh_to=ssh_to, test=args.test, input="\0".join(snapshots), cmd= self.run(test=args.test, input="\0".join(snapshots), cmd=
[ "xargs", "-0", "-n", "1", "zfs", "destroy", "-d" ] [ "xargs", "-0", "-n", "1", "zfs", "destroy", "-d" ]
) )
def zfs_destroy(filesystems, recursive=False):
"""destroy list of filesystems """ """destroy list of filesystems """
def zfs_destroy(ssh_to, filesystems, recursive=False):
cmd=[ "xargs", "-0", "-n", "1", "zfs", "destroy" ] cmd=[ "xargs", "-0", "-n", "1", "zfs", "destroy" ]
@ -159,15 +155,14 @@ def zfs_destroy(ssh_to, filesystems, recursive=False):
cmd.append("-r") cmd.append("-r")
#zfs can only destroy one filesystem at once so we use xargs and stdin #zfs can only destroy one filesystem at once so we use xargs and stdin
run(ssh_to=ssh_to, test=args.test, input="\0".join(filesystems), cmd=cmd) self.run(test=args.test, input="\0".join(filesystems), cmd=cmd)
#simulate snapshots for --test option #simulate snapshots for --test option
#FIXME
test_snapshots={} test_snapshots={}
def zfs_create_snapshot(filesystems, snapshot):
"""create snapshot on multiple filesystems at once (atomicly)""" """create snapshot on multiple filesystems at once (atomicly)"""
def zfs_create_snapshot(ssh_to, filesystems, snapshot):
cmd=[ "zfs", "snapshot" ] cmd=[ "zfs", "snapshot" ]