From 45fc6f108659b9eebfdff0979f08c47afa3288cc Mon Sep 17 00:00:00 2001
From: Anders Blomdell <anders.blomdell@control.lth.se>
Date: Wed, 20 Jan 2021 18:05:58 +0100
Subject: [PATCH] Use tar_stream/tarfile instead of star in secondary.py
---
secondary.py | 53 +++++++++++++++++++++++++++-------------------------
1 file changed, 28 insertions(+), 25 deletions(-)
diff --git a/secondary.py b/secondary.py
index 3b1bf39..c7494bd 100644
--- a/secondary.py
+++ b/secondary.py
@@ -7,9 +7,11 @@ import loghandler
import os
import socket
import subprocess
+import threading
import time
import shutil
import sys
+import tar_stream
def cond_unlink(path, log):
try:
@@ -43,10 +45,10 @@ class Status:
class Backup:
- def __init__(self, primary_star, mount, path, status, log):
- self.primary_star = primary_star
- self.primary_in = primary_star.makefile('wb')
- self.primary_out = primary_star.makefile('rb')
+ def __init__(self, primary_tar, mount, path, status, log):
+ self.primary_tar = primary_tar
+ self.primary_in = primary_tar.makefile('wb')
+ self.primary_out = primary_tar.makefile('rb')
self.mount = mount
self.path = path
self.status = status
@@ -55,20 +57,25 @@ class Backup:
self.trash_root = os.path.join(mount, 'TRASH').encode('utf-8')
self.trash = os.path.join(self.trash_root,
str(int(time.time())).encode('utf-8'))
- extract_cmd = [ '/bin/star', '-x', '-no-fifo',
- '-nowarn', '-no-statistics' ]
- self.extract = subprocess.Popen(extract_cmd,
- cwd=os.path.join(mount, path),
- stdin=self.primary_out)
- atexit.register(cond_kill, self.extract)
-
- # Make sure that the generated star archive is not empty
+ self.extractor = threading.Thread(daemon=True, target=self.run)
+ self.extractor.start()
+ # Make sure that the generated tar archive is not empty
self.primary_in.write(b'.\0')
+ pass
+
+ def run(self):
+ cwd = os.path.join(self.mount, self.path)
+ reader = tar_stream.TarReader(self.primary_out)
+ for e in reader:
+ self.make_room(e.size)
+ e.tarfile.extract(e, path=cwd)
+ pass
def close(self):
self.primary_in.flush()
- self.primary_star.shutdown(socket.SHUT_WR)
- self.status.extract_OK = self.extract.wait()
+ self.primary_tar.shutdown(socket.SHUT_WR)
+ self.extractor.join()
+ pass
def check(self, src, dst):
if src.name != dst.name:
@@ -104,7 +111,7 @@ class Backup:
self.status.unchanged += 1
def make_room(self, size):
- for p in sorted(os.listdir(self.trash_root)):
+ while True:
stat = os.statvfs(self.dst_root)
free = stat.f_frsize * stat.f_bavail
need = size + stat.f_frsize
@@ -112,7 +119,8 @@ class Backup:
break
self.log.MESSAGE("Need to free:",
need - free, (need, free), self.trash_root)
- d = os.path.join(self.trash_root, p)
+ oldest = sorted(os.listdir(self.trash_root))[0]
+ d = os.path.join(self.trash_root, oldest)
if os.path.isdir(d):
self.log.MESSAGE('Removing dir', d)
shutil.rmtree(d)
@@ -123,11 +131,6 @@ class Backup:
def add(self, src):
self.log.DEBUG('Add:', src.name)
- if len(src.size) == 0:
- size = 0
- else:
- size = int(src.size)
- self.make_room(size)
parent = os.path.dirname(src.name)
while len(parent) != 0:
# Make sure directories get the correct modes
@@ -184,11 +187,11 @@ def do_backup(hash_name, options, socket_path, mount, path):
atexit.register(cond_kill, p)
dst = hashtoc.HashTOC(p.stdout, rename={hash_name:'sum'})
- # Connect to server star socket
- primary_star = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
- primary_star.connect(socket_path)
+ # Connect to server tar socket
+ primary_tar = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ primary_tar.connect(socket_path)
- backup = Backup(primary_star=primary_star,
+ backup = Backup(primary_tar=primary_tar,
mount=mount, path=path, status=status, log=log)
while True:
if src.name == None and dst.name == None:
--
GitLab