[IMP][ADD]RsyncVolumes : allow options and remote sync

* Do not use envsubst anymore but python os.environ ;
* Options for config : PLAN, SOURCE and DEST, EXCLUDE_WORDS ;
* Add synchronization with rsync+SSH with REMOTE env variables.

backups/rsyncvolumes/Dockerfile

@@ -5,7 +5,7 @@ ENV SOURCE /var/lib/docker/volumes
 ENV DEST /var/backups/
 
 RUN apt-get update && apt-get install --no-install-recommends -y \
-      cron gettext-base rsync python-minimal libpython-stdlib
+      cron gettext-base rsync ssh sshpass python-minimal libpython-stdlib
 
 # Avoid overlayfs bug with tail
 VOLUME /var/log
@@ -13,7 +13,8 @@ VOLUME /var/log
 WORKDIR /opt/backup
 COPY crontab ./crontab
 COPY rsync.template .
-COPY plan.tpl.py .
+COPY plan.py .
+COPY remote.py .
 COPY launch.sh .
 
 CMD bash launch.sh

backups/rsyncvolumes/Dockerfile.c7

@@ -4,7 +4,7 @@ MAINTAINER Yaltik - Fabien Bourgeois <fabien@yaltik.com>
 ENV SOURCE /var/lib/docker/volumes
 ENV DEST /var/backups/
 
-RUN yum -y install gettext rsync cronie
+RUN yum -y install gettext rsync ssh sshpass cronie
 
 # Avoid overlayfs bug with tail
 VOLUME /var/log
@@ -12,7 +12,8 @@ VOLUME /var/log
 WORKDIR /opt/backup
 COPY crontab ./crontab
 COPY rsync.template .
-COPY plan.tpl.py .
+COPY plan.py .
+COPY remote.py .
 COPY launch.c7.sh launch.sh
 
 CMD bash launch.sh

backups/rsyncvolumes/launch.c7.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
 
 envsubst < /opt/backup/rsync.template > /opt/backup/rsync.sh
-envsubst < /opt/backup/plan.tpl.py > /opt/backup/plan.py
 crontab ./crontab && touch /var/log/cron.log && crond && tail -f /var/log/cron.log

backups/rsyncvolumes/launch.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
 
 envsubst < /opt/backup/rsync.template > /opt/backup/rsync.sh
-envsubst < /opt/backup/plan.tpl.py > /opt/backup/plan.py
 crontab ./crontab && touch /var/log/cron.log && cron && tail -f /var/log/cron.log

backups/rsyncvolumes/plan.py

@@ -2,10 +2,11 @@
 
 from zipfile import ZipFile, ZIP_DEFLATED
 from datetime import datetime
-from os import walk, unlink
+from os import walk, unlink, environ
 from os.path import join as osjoin
 from os.path import isfile
 from glob import glob
+from remote import sync
 
 # Backup plan for Docker volumes
 # ::::::::::::::::::::::::::::::
@@ -14,33 +15,40 @@ from glob import glob
 #
 # - all saves the last week
 # - one save per week the last month
-# - one save per month the last year
+# - one save per month during 6 months
 
 # Volumes Backup
 # ==============
 
+DEST = environ.get('DEST')
 dt = datetime.now()
 zipname = '{0}.zip'.format(dt.strftime('%Y-%m-%d_%Hh%Mm%Ss'))
-zippath = '${DEST}' + zipname
+zippath = DEST + zipname
 
 # Zip files
 
+EXCLUDE_WORDS = environ.get('EXCLUDE_WORDS', '')
+EXCLUDE_WORDS = EXCLUDE_WORDS.split(',')
 
 def zipdir(path, ziph):
     # ziph is zipfile handle
     for root, dirs, files in walk(path):
         for file in files:
             pfile = osjoin(root, file)
-            if isfile(pfile):
+            if isfile(pfile) and all(map(lambda w: w not in pfile, EXCLUDE_WORDS)):
                 ziph.write(pfile)
 
+print('Compression started')
 f = ZipFile(zippath, 'w', ZIP_DEFLATED, allowZip64=True)
-zipdir('${DEST}/volumes', f)
+zipdir('%s/volumes' % DEST, f)
 f.close()
+print('Compression ended')
 
 # Filter all obsolete save files
 # ==============================
 
+PLAN = environ.get('PLAN', u'180,15,2')
+PLAN = map(int, PLAN.split(u','))
 
 def notkeep(fname):
     fname = fname.split('/')[-1]
@@ -50,12 +58,12 @@ def notkeep(fname):
     ds = ds[0]
     d = datetime.strptime(ds, '%Y-%m-%d')
     delta = dt - d
-    if delta.days > 180:
+    if delta.days > PLAN[0]:
         return True
-    elif delta.days > 15:
+    elif delta.days > PLAN[1]:
-        if (d.day != 1):
+        if d.day != 1:
             return True
-    elif delta.days > 2:
+    elif delta.days > PLAN[2]:
         if (d.weekday() != 0) and (d.day != 1):
             return True
     else:
@@ -64,7 +72,7 @@ def notkeep(fname):
 # Filters zip files and keeps only legitimate ones
 # ================================================
 
-files = glob('${DEST}/*.zip')
+files = glob('%s/*.zip' % DEST)
 backups_to_remove = filter(notkeep, files)
 
 if len(backups_to_remove) > 0:
@@ -73,3 +81,5 @@ if len(backups_to_remove) > 0:
 
 for b in backups_to_remove:
     unlink(b)
+
+sync()

backups/rsyncvolumes/remote.py

@@ -0,0 +1,18 @@
+from os import environ
+from subprocess import Popen
+from shlex import split as shplit
+
+# Remote syncing, when needed
+
+def sync():
+    """ Remote syncing """
+    DEST = environ.get('DEST')
+    REMOTE_HOST = environ.get('REMOTE_HOST', False)
+    if REMOTE_HOST:
+        REMOTE_USER = environ.get('REMOTE_USER')
+        REMOTE_PASS = environ.get('REMOTE_PASS')
+        print('Remote host %s configured : will sync with it' % REMOTE_HOST)
+        command = ('sshpass -p %s rsync -av --delete -e '
+                   '"ssh -p 23 -o StrictHostKeyChecking=no" '
+                   '%s %s@%s:./backups') % (REMOTE_PASS, DEST, REMOTE_USER, REMOTE_HOST)
+        Popen(shplit(command))