Select Git revision
JavaCCChunks.jrag
secondary.py 7.57 KiB
#!/usr/bin/python3
import atexit
import command
import hashtoc
import loghandler
import os
import socket
import subprocess
import time
def cond_unlink(path, log):
try:
os.unlink(path)
log.DEBUG('removed %s' % path)
except FileNotFoundError:
pass
def cond_kill(p):
try:
p.kill()
except:
pass
class Status:
def __init__(self, log):
self.checked = 0
self.added = 0
self.deleted = 0
self.replaced = 0
self.unchanged = 0
self.metadata = 0
self.extract_OK = -1
def report():
log.MESSAGE('STATUS %d = +%d -%d =%d ?%d (%d)' % (
self.checked, self.added, self.deleted,
self.replaced, self.metadata,
self.extract_OK))
atexit.register(report)
class Backup:
def __init__(self, primary_star, mount, path, status, log):
self.primary_star = primary_star
self.primary_in = primary_star.makefile('wb')
self.primary_out = primary_star.makefile('rb')
self.mount = mount
self.path = path
self.status = status
self.log = log
self.dst_root = os.path.join(mount, path).encode('utf-8')
self.trash_root = os.path.join(mount, 'TRASH').encode('utf-8')
self.trash = os.path.join(self.trash_root,
str(int(time.time())).encode('utf-8'))
extract_cmd = [ '/bin/star', '-x', '-nowarn', '-no-statistics' ]
self.extract = subprocess.Popen(extract_cmd,
cwd=os.path.join(mount, path),
stdin=self.primary_out)
atexit.register(cond_kill, self.extract)
# Make sure that the generated star archive is not empty
self.primary_in.write(b'.\0')
def close(self):
self.primary_in.flush()
self.primary_star.shutdown(socket.SHUT_WR)
self.status.extract_OK = self.extract.wait()
def check(self, src, dst):
if src.name != dst.name:
raise Exception('Names differ: %s, %s' % (src, dst))
dst_path = os.path.join(self.dst_root, dst.name)
if src.kind != dst.kind or src.sum != dst.sum or src.size != dst.size:
self.log.DEBUG('Replace...', src.name, dst.name,
src.sum, dst.sum, src.size, dst.size)
self.status.replaced += 1
self.delete(dst)
self.add(src)
elif os.path.lexists(dst_path):
changed = False
if src.kind in [ b'F', b'D'] and src.mode != dst.mode:
self.log.DEBUG('MODE', dst.name, src.mode, dst.mode)
os.chmod(dst_path, int(src.mode, 8))
changed = True
if (src.kind in [ b'F', b'D', b'L', b'S'] and
(src.uid != dst.uid or src.gid != dst.gid)):
self.log.DEBUG('UID/GID', dst.name, src.uid, src.gid,
dst.uid, dst.gid)
os.lchown(dst_path, int(src.uid), int(src.gid))
changed = True
if src.kind == b'F' and src.mtime != dst.mtime:
self.log.DEBUG('MTIME', src.name, src.mtime, dst.mtime)
atime = os.stat(dst_path).st_atime
os.utime(dst_path, (int(atime), int(src.mtime)))
changed = True
if changed:
self.status.metadata += 1
else:
self.status.unchanged += 1
def make_room(self, size):
for p in sorted(os.listdir(self.trash_root)):
stat = os.statvfs(self.dst_root)
free = stat.f_frsize * stat.f_bavail
need = size + stat.f_frsize
if free > need:
break
self.log.MESSAGE("Need to free:",
need - free, (need, free), self.trash_root)
d = os.path.join(self.trash_root, p)
if os.path.isdir(d):
self.log.MESSAGE('Removing dir', d)
shutil.rmtree(d)
pass
else:
self.log.MESSAGE('Removing file', d)
os.unlink(d)
def add(self, src):
self.log.DEBUG('Add:', src.name)
if len(src.size) == 0:
size = 0
else:
size = int(src.size)
self.make_room(size)
parent = os.path.dirname(src.name)
while len(parent) != 0:
# Make sure directories get the correct modes
self.primary_in.write(parent + b'\0')
parent = os.path.dirname(parent)
self.primary_in.write(src.name + b'\0')
def delete(self, dst):
self.log.DEBUG('Delete:', dst.name)
dst_path = os.path.join(self.dst_root, dst.name)
if os.path.exists(dst_path):
trash_path = os.path.join(self.trash, dst.name)
trash_dir = os.path.dirname(trash_path)
if not os.path.exists(trash_dir):
os.makedirs(trash_dir, mode=0o700)
os.rename(dst_path, trash_path)
def do_backup(hash_name, options, socket_path, mount, path):
if options.debug:
log = loghandler.LOG(loghandler.LOG_DEBUG)
else:
log = loghandler.LOG(loghandler.LOG_WARNING)
atexit.register(cond_unlink, socket_path, log)
status = Status(log)
config_path = '%s/TOTALBACKUP.config' % (mount)
if not os.path.exists(config_path):
raise Exception('"%s" does not exists' % (config_path))
# Connect to server config/hashtoc socket
config_hash = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
config_hash.connect(socket_path)
# Send secondary config to primary
config_hash.makefile('w').write(open(config_path).read())
config_hash.shutdown(socket.SHUT_WR)
# Make ready to read primary TOC (src)
src = hashtoc.HashTOC(config_hash.makefile('rb'), rename={hash_name:'sum'})
# Create secondary hashtoc (dst)
cmd = ( command.Command('/usr/bin/hashtoc')
.flag('--%s' % hash_name)
.flag('--zero-terminated')
.flag('--xattr', options.xattr)
.option('--max-age', options.max_age)
.option('--jobs', options.jobs)
.option('--lookahead', options.lookahead)
.arg('.') )
p = subprocess.Popen(cmd,
cwd=os.path.join(mount, path),
stdout=subprocess.PIPE)
atexit.register(cond_kill, p)
dst = hashtoc.HashTOC(p.stdout, rename={hash_name:'sum'})
# Connect to server star socket
primary_star = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
primary_star.connect(socket_path)
backup = Backup(primary_star=primary_star,
mount=mount, path=path, status=status, log=log)
while True:
if src.name == None and dst.name == None:
# All done
break
status.checked += 1
if src.name == None:
status.deleted += 1
backup.delete(dst)
dst.next()
elif dst.name == None:
status.added += 1
backup.add(src)
src.next()
elif src.name == dst.name:
backup.check(src=src, dst=dst)
src.next()
dst.next()
elif src.name < dst.name:
status.added += 1
backup.add(src)
src.next()
elif src.name > dst.name:
status.deleted += 1
backup.delete(dst)
dst.next()
else:
raise Exception()
backup.close()
log.DEBUG('hashtoc result', p.wait())
config_hash.shutdown(socket.SHUT_RD)
config_hash.close()