Update python-publish.yml

debug in zfs-autobackup

.github/workflows/python-publish.yml

@@ -0,0 +1,32 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+    - name: Build and publish
+      env:
+        TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+        TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+      run: |
+        python setup.py sdist bdist_wheel
+        python3 -m twine check dist/*
+        twine upload dist/*

.github/workflows/regression.yml

@@ -0,0 +1,59 @@
+name: Regression tests
+
+
+on: ["push", "pull_request"]
+  
+
+
+jobs:
+     
+  ubuntu20:
+    runs-on: ubuntu-20.04
+    
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2.3.4
+        
+        
+       
+      - name: Prepare
+        run: lsmod && sudo apt update && sudo apt install zfsutils-linux && sudo -H pip3 install coverage unittest2 mock==3.0.5 coveralls
+        
+        
+      - name: Regression test
+        run: sudo -E ./run_tests
+       
+        
+      - name: Coveralls
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: coveralls --service=github
+        
+  ubuntu18:
+    runs-on: ubuntu-18.04
+    
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2.3.4
+        
+        
+       
+      - name: Prepare
+        run: lsmod && sudo apt update && sudo apt install zfsutils-linux python3-setuptools && sudo -H pip3 install coverage unittest2 mock==3.0.5 coveralls
+        
+        
+      - name: Regression test
+        run: sudo -E ./run_tests
+       
+        
+      - name: Coveralls
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: coveralls --service=github
+        
+      
+       
+       
+           
+       
+      

.travis.yml

@@ -1,31 +1,32 @@
+#MOVING TO GITHUB ACTIONS
 
-jobs:
-  include:
-    - os: linux
-      dist: xenial
-      language: python
-      python: 2.7
-    - os: linux
-      dist: xenial
-      language: python
-      python: 3.6
-    - os: linux
-      dist: bionic
-      language: python
-      python: 2.7
-    - os: linux
-      dist: bionic
-      language: python
-      python: 3.6
+# jobs:
+#   include:
+#     - os: linux
+#       dist: xenial
+#       language: python
+#       python: 2.7
+#     - os: linux
+#       dist: xenial
+#       language: python
+#       python: 3.6
+#     - os: linux
+#       dist: bionic
+#       language: python
+#       python: 2.7
+#     - os: linux
+#       dist: bionic
+#       language: python
+#       python: 3.6
 
 
 
 
-before_install:
-  - sudo apt-get update
-  - sudo apt-get install zfsutils-linux
+# before_install:
+#   - sudo apt-get update
+#   - sudo apt-get install zfsutils-linux
 
-script:
-  # - sudo -E ./ngrok.sh
-  - sudo -E ./run_tests
-  # - sudo -E pip --version
+# script:
+#   # - sudo -E ./ngrok.sh
+#   - sudo -E ./run_tests
+#   # - sudo -E pip --version

README.md

@@ -1,6 +1,7 @@
+
 # ZFS autobackup
 
-[![Coverage Status](https://coveralls.io/repos/github/psy0rz/zfs_autobackup/badge.svg)](https://coveralls.io/github/psy0rz/zfs_autobackup) [![Build Status](https://travis-ci.org/psy0rz/zfs_autobackup.svg?branch=master)](https://travis-ci.org/psy0rz/zfs_autobackup)
+[![Coverage Status](https://coveralls.io/repos/github/psy0rz/zfs_autobackup/badge.svg)](https://coveralls.io/github/psy0rz/zfs_autobackup) [![Regression tests](https://github.com/psy0rz/zfs_autobackup/workflows/Regression%20tests/badge.svg)](https://github.com/psy0rz/zfs_autobackup/actions?query=workflow%3A%22Regression+tests%22)
 
 ## New in v3
 
@@ -91,7 +92,7 @@ On older servers you might have to use easy_install
 
 ### Direct download
 
-Its also possible to just download <https://raw.githubusercontent.com/psy0rz/zfs_autobackup/master/bin/zfs-autobackup> and run it directly.
+Its also possible to just download <https://raw.githubusercontent.com/psy0rz/zfs_autobackup/v3.0/bin/zfs-autobackup> and run it directly.
 
 The only requirement that is sometimes missing is the `argparse` python module. Optionally you can install `colorama` for colors.
 
@@ -260,7 +261,7 @@ You can use zfs-autobackup to only make snapshots.
 Just dont specify the target-path:
 ```console
 root@ws1:~# zfs-autobackup test --verbose 
-  zfs-autobackup v3.0-rc12 - Copyright 2020 E.H.Eefting (edwin@datux.nl)
+  zfs-autobackup v3.0 - Copyright 2020 E.H.Eefting (edwin@datux.nl)
   
   #### Source settings
   [Source] Datasets are local
@@ -569,54 +570,88 @@ zabbix-job-status backup_smartos01_fs1 daily $?
 
 This will update the zabbix server with the exit code and will also alert you if the job didn't run for more than 2 days.
 
-## Backuping up a proxmox cluster with HA replication
+## Backup a proxmox cluster with HA replication
 
 Due to the nature of proxmox we had to make a few enhancements to zfs-autobackup. This will probably also benefit other systems that use their own replication in combination with zfs-autobackup.
 
 All data under rpool/data can be on multiple nodes of the cluster. The naming of those filesystem is unique over the whole cluster. Because of this we should backup rpool/data of all nodes to the same destination. This way we wont have duplicate backups of the filesystems that are replicated. Because of various options, you can even migrate hosts and zfs-autobackup will be fine. (and it will get the next backup from the new node automatically)
 
-In the example below we have 3 nodes, named h4, h5 and h6.
-
-The backup will go to a machine named smartos03.
+In the example below we have 3 nodes, named pve1, pve2 and pve3.
 
 ### Preparing the proxmox nodes
 
-On each node select the filesystems as following:
+No preparation is needed, the script will take care of everything. You only need to setup the ssh keys, so that the backup server can access the proxmox server.
 
-```console
-root@h4:~# zfs set autobackup:h4_smartos03=true rpool
-root@h4:~# zfs set autobackup:h4_smartos03=false rpool/data
-root@h4:~# zfs set autobackup:data_smartos03=child rpool/data
+TIP: make sure your backup server is firewalled and cannot be reached from any production machine.
+
+### SSH config on backup server
+
+I use ~/.ssh/config to specify how to reach the various hosts.
+
+In this example we are making an offsite copy and use portforwarding to reach the proxmox machines:
+```
+Host *
+    ControlPath ~/.ssh/control-master-%r@%h:%p
+    ControlMaster auto
+    ControlPersist 3600
+    Compression yes
+
+Host pve1
+    Hostname some.host.com
+    Port 10001
+
+Host pve2
+    Hostname some.host.com
+    Port 10002
+
+Host pve3
+    Hostname some.host.com
+    Port 10003
 ```
 
-* rpool will be backuped the usual way, and is named h4_smartos03. (each node will have a unique name)
-* rpool/data will be excluded from the usual backup
-* The CHILDREN of rpool/data be selected for a cluster wide backup named data_smartos03. (each node uses the same backup name)
+### Backup script 
 
-### Preparing the backup server
+I use the following backup script on the backup server.
 
-Extra options needed for proxmox with HA:
-
-* --no-holds: To allow proxmox to destroy our snapshots if a VM migrates to another node.
-* --ignore-replicated: To ignore the replicated filesystems of proxmox on the receiving proxmox nodes. (e.g: only backup from the node where the VM is active)
-* --min-change 200000: Ignore replicated works by checking if there are no changes since the last snapshot. However for some reason proxmox always has some small changes. (Probably house-keeping data are something? This always was fine and suddenly changed with an update)
-
-I use the following backup script on the backup server:
+Adjust the variables HOSTS TARGET and NAME to your needs.
 
 ```shell
-for H in h4 h5 h6; do
-  echo "################################### DATA $H"
-  #backup data filesystems to a common place
-  ./zfs-autobackup --ssh-source root@$H data_smartos03 zones/backup/zfsbackups/pxe1_data --clear-refreservation --clear-mountpoint  --ignore-transfer-errors --strip-path 2 --verbose --ignore-replicated --min-change 200000 --no-holds $@
-  zabbix-job-status backup_$H""_data_smartos03 daily $? >/dev/null 2>/dev/null
+#!/bin/bash
+
+HOSTS="pve1 pve2 pve3"
+TARGET=rpool/pvebackups
+NAME=prox
+
+zfs create -p $TARGET/data &>/dev/null
+for HOST in $HOSTS; do
+
+  echo "################################### RPOOL $HOST"
+
+  # enable backup
+  ssh $HOST "zfs set autobackup:rpool_$NAME=child rpool/ROOT"
+
+  #backup rpool to specific directory per host
+  zfs create -p $TARGET/rpools/$HOST &>/dev/null
+  zfs-autobackup --keep-source=1d1w,1w1m --ssh-source $HOST rpool_$NAME $TARGET/rpools/$HOST --clear-mountpoint --clear-refreservation --ignore-transfer-errors --strip-path 2 --verbose   --no-holds   $@
+
+  zabbix-job-status backup_$HOST""_rpool_$NAME daily $? >/dev/null 2>/dev/null
+
+
+  echo "################################### DATA $HOST"
+
+  # enable backup
+  ssh $HOST "zfs set autobackup:data_$NAME=child rpool/data"
+
+  #backup data filesystems to a common directory
+  zfs-autobackup --keep-source=1d1w,1w1m --ssh-source $HOST data_$NAME $TARGET/data --clear-mountpoint --clear-refreservation --ignore-transfer-errors --strip-path 2 --verbose  --ignore-replicated --min-change 200000 --no-holds   $@
+
+  zabbix-job-status backup_$HOST""_data_$NAME daily $? >/dev/null 2>/dev/null
 
-  echo "################################### RPOOL $H"
-  #backup rpool to own place
-  ./zfs-autobackup --ssh-source root@$H $H""_smartos03 zones/backup/zfsbackups/$H --verbose --clear-refreservation --clear-mountpoint --ignore-transfer-errors $@
-  zabbix-job-status backup_$H""_smartos03 daily $? >/dev/null 2>/dev/null
 done
 ```
 
+This script will also send the backup status to Zabbix. (if you've installed my zabbix-job-status script)
+
 # Sponsor list
 
 This project was sponsorred by:

run_tests

@@ -16,16 +16,16 @@ if ! [ -e /root/.ssh/id_rsa ]; then
     ssh -oStrictHostKeyChecking=no localhost true || exit 1
 fi
 
-coverage run --source bin.zfs_autobackup -m unittest discover -vv
+coverage run --source bin.zfs_autobackup -m unittest discover -vv $@
 EXIT=$?
 
 echo 
 coverage report
 
 #this does automatic travis CI/https://coveralls.io/ intergration:
-if which coveralls > /dev/null; then
-        echo "Submitting to coveralls.io:"
-        coveralls 
-fi
+# if which coveralls > /dev/null; then
+#        echo "Submitting to coveralls.io:"
+#        coveralls 
+# fi
 
 exit $EXIT

test_executenode.py

@@ -45,10 +45,10 @@ class TestExecuteNode(unittest2.TestCase):
 
         #input a string and check it via cat
         with self.subTest("stdin input string"):
-            self.assertEqual(node.run(["cat"], input="test"), ["test"])
+            self.assertEqual(node.run(["cat"], inp="test"), ["test"])
 
         #command that wants input, while we dont have input, shouldnt hang forever.
-        with self.subTest("stdin process with input=None (shouldn't hang)"):
+        with self.subTest("stdin process with inp=None (shouldn't hang)"):
             self.assertEqual(node.run(["cat"]), [])
 
     def test_basics_local(self):
@@ -74,36 +74,36 @@ class TestExecuteNode(unittest2.TestCase):
 
         with self.subTest("pipe data"):
             output=nodea.run(["dd", "if=/dev/zero", "count=1000"], pipe=True)
-            self.assertEqual(nodeb.run(["md5sum"], input=output), ["816df6f64deba63b029ca19d880ee10a  -"])
+            self.assertEqual(nodeb.run(["md5sum"], inp=output), ["816df6f64deba63b029ca19d880ee10a  -"])
 
         with self.subTest("exit code both ends of pipe ok"):
             output=nodea.run(["true"], pipe=True)
-            nodeb.run(["true"], input=output)
+            nodeb.run(["true"], inp=output)
     
         with self.subTest("error on pipe input side"):
             with self.assertRaises(subprocess.CalledProcessError):
                 output=nodea.run(["false"], pipe=True)
-                nodeb.run(["true"], input=output)
+                nodeb.run(["true"], inp=output)
 
         with self.subTest("error on pipe output side "):
             with self.assertRaises(subprocess.CalledProcessError):
                 output=nodea.run(["true"], pipe=True)
-                nodeb.run(["false"], input=output)
+                nodeb.run(["false"], inp=output)
 
         with self.subTest("error on both sides of pipe"):
             with self.assertRaises(subprocess.CalledProcessError):
                 output=nodea.run(["false"], pipe=True)
-                nodeb.run(["false"], input=output)
+                nodeb.run(["false"], inp=output)
 
         with self.subTest("check stderr on pipe output side"):
             output=nodea.run(["true"], pipe=True)
-            (stdout, stderr)=nodeb.run(["ls", "nonexistingfile"], input=output, return_stderr=True, valid_exitcodes=[0,2])
+            (stdout, stderr)=nodeb.run(["ls", "nonexistingfile"], inp=output, return_stderr=True, valid_exitcodes=[0,2])
             self.assertEqual(stdout,[])
             self.assertRegex(stderr[0], "nonexistingfile" )
 
         with self.subTest("check stderr on pipe input side (should be only printed)"):
             output=nodea.run(["ls", "nonexistingfile"], pipe=True)
-            (stdout, stderr)=nodeb.run(["true"], input=output, return_stderr=True, valid_exitcodes=[0,2])
+            (stdout, stderr)=nodeb.run(["true"], inp=output, return_stderr=True, valid_exitcodes=[0,2])
             self.assertEqual(stdout,[])
             self.assertEqual(stderr,[] )
             

test_thinner.py

@@ -1,4 +1,5 @@
 from basetest import *
+import pprint
 
 #randint is different in python 2 vs 3
 randint_compat = lambda lo, hi: lo + int(random.random() * (hi + 1 - lo))

test_zfsautobackup.py

@@ -481,7 +481,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000  refreservation  -
 
 
         with patch('time.strftime', return_value="20101111000000"):
-            self.assertFalse(ZfsAutobackup("test test_target1 --verbose --clear-mountpoint".split(" ")).run())
+            self.assertFalse(ZfsAutobackup("test test_target1 --verbose --clear-mountpoint --debug".split(" ")).run())
 
             r=shelltest("zfs get canmount -r test_source1 test_source2 test_target1")
             self.assertMultiLineEqual(r,"""