[IMP][ADD]RsyncVolumes : allow options and remote sync

* Do not use envsubst anymore but python os.environ ;
* Options for config : PLAN, SOURCE and DEST, EXCLUDE_WORDS ;
* Add synchronization with rsync+SSH with REMOTE env variables.
This commit is contained in:
Fabien BOURGEOIS 2021-03-12 19:48:43 +01:00
parent edff1295de
commit cf48101dfb
6 changed files with 44 additions and 16 deletions

View File

@ -5,7 +5,7 @@ ENV SOURCE /var/lib/docker/volumes
ENV DEST /var/backups/
RUN apt-get update && apt-get install --no-install-recommends -y \
cron gettext-base rsync python-minimal libpython-stdlib
cron gettext-base rsync ssh sshpass python-minimal libpython-stdlib
# Avoid overlayfs bug with tail
VOLUME /var/log
@ -13,7 +13,8 @@ VOLUME /var/log
WORKDIR /opt/backup
COPY crontab ./crontab
COPY rsync.template .
COPY plan.tpl.py .
COPY plan.py .
COPY remote.py .
COPY launch.sh .
CMD bash launch.sh

View File

@ -4,7 +4,7 @@ MAINTAINER Yaltik - Fabien Bourgeois <fabien@yaltik.com>
ENV SOURCE /var/lib/docker/volumes
ENV DEST /var/backups/
RUN yum -y install gettext rsync cronie
RUN yum -y install gettext rsync ssh sshpass cronie
# Avoid overlayfs bug with tail
VOLUME /var/log
@ -12,7 +12,8 @@ VOLUME /var/log
WORKDIR /opt/backup
COPY crontab ./crontab
COPY rsync.template .
COPY plan.tpl.py .
COPY plan.py .
COPY remote.py .
COPY launch.c7.sh launch.sh
CMD bash launch.sh

View File

@ -1,5 +1,4 @@
#!/bin/bash
envsubst < /opt/backup/rsync.template > /opt/backup/rsync.sh
envsubst < /opt/backup/plan.tpl.py > /opt/backup/plan.py
crontab ./crontab && touch /var/log/cron.log && crond && tail -f /var/log/cron.log

View File

@ -1,5 +1,4 @@
#!/bin/bash
envsubst < /opt/backup/rsync.template > /opt/backup/rsync.sh
envsubst < /opt/backup/plan.tpl.py > /opt/backup/plan.py
crontab ./crontab && touch /var/log/cron.log && cron && tail -f /var/log/cron.log

View File

@ -2,10 +2,11 @@
from zipfile import ZipFile, ZIP_DEFLATED
from datetime import datetime
from os import walk, unlink
from os import walk, unlink, environ
from os.path import join as osjoin
from os.path import isfile
from glob import glob
from remote import sync
# Backup plan for Docker volumes
# ::::::::::::::::::::::::::::::
@ -14,33 +15,40 @@ from glob import glob
#
# - all saves the last week
# - one save per week the last month
# - one save per month the last year
# - one save per month during 6 months
# Volumes Backup
# ==============
DEST = environ.get('DEST')
dt = datetime.now()
zipname = '{0}.zip'.format(dt.strftime('%Y-%m-%d_%Hh%Mm%Ss'))
zippath = '${DEST}' + zipname
zippath = DEST + zipname
# Zip files
EXCLUDE_WORDS = environ.get('EXCLUDE_WORDS', '')
EXCLUDE_WORDS = EXCLUDE_WORDS.split(',')
def zipdir(path, ziph):
# ziph is zipfile handle
for root, dirs, files in walk(path):
for file in files:
pfile = osjoin(root, file)
if isfile(pfile):
if isfile(pfile) and all(map(lambda w: w not in pfile, EXCLUDE_WORDS)):
ziph.write(pfile)
print('Compression started')
f = ZipFile(zippath, 'w', ZIP_DEFLATED, allowZip64=True)
zipdir('${DEST}/volumes', f)
zipdir('%s/volumes' % DEST, f)
f.close()
print('Compression ended')
# Filter all obsolete save files
# ==============================
PLAN = environ.get('PLAN', u'180,15,2')
PLAN = map(int, PLAN.split(u','))
def notkeep(fname):
fname = fname.split('/')[-1]
@ -50,12 +58,12 @@ def notkeep(fname):
ds = ds[0]
d = datetime.strptime(ds, '%Y-%m-%d')
delta = dt - d
if delta.days > 180:
if delta.days > PLAN[0]:
return True
elif delta.days > 15:
if (d.day != 1):
elif delta.days > PLAN[1]:
if d.day != 1:
return True
elif delta.days > 2:
elif delta.days > PLAN[2]:
if (d.weekday() != 0) and (d.day != 1):
return True
else:
@ -64,7 +72,7 @@ def notkeep(fname):
# Filters zip files and keeps only legitimate ones
# ================================================
files = glob('${DEST}/*.zip')
files = glob('%s/*.zip' % DEST)
backups_to_remove = filter(notkeep, files)
if len(backups_to_remove) > 0:
@ -73,3 +81,5 @@ if len(backups_to_remove) > 0:
for b in backups_to_remove:
unlink(b)
sync()

View File

@ -0,0 +1,18 @@
from os import environ
from subprocess import Popen
from shlex import split as shplit
# Remote syncing, when needed
def sync():
""" Remote syncing """
DEST = environ.get('DEST')
REMOTE_HOST = environ.get('REMOTE_HOST', False)
if REMOTE_HOST:
REMOTE_USER = environ.get('REMOTE_USER')
REMOTE_PASS = environ.get('REMOTE_PASS')
print('Remote host %s configured : will sync with it' % REMOTE_HOST)
command = ('sshpass -p %s rsync -av --delete -e '
'"ssh -p 23 -o StrictHostKeyChecking=no" '
'%s %s@%s:./backups') % (REMOTE_PASS, DEST, REMOTE_USER, REMOTE_HOST)
Popen(shplit(command))