Commit 821d3622 authored by Anders Blomdell's avatar Anders Blomdell
Browse files

Add tar_stream wraps tarfile with memory optimizations

parent 6e592645
#!/usr/bin/python3
import tarfile
import os
import threading
class TarWriter:
def __init__(self, fileobj):
self.tarfile = tarfile.open(fileobj=fileobj,
mode='w|',
format=tarfile.PAX_FORMAT)
def add(self, path):
self.tarfile.add(path, recursive=False)
# Keep memory consumption down
self.tarfile.members = []
self.tarfile.inodes = {}
pass
def close(self):
self.tarfile.close()
self.tarfile = None
class TarReader:
def __init__(self, fileobj):
self.fileobj = fileobj
self.tarfile = None
pass
def __iter__(self):
if self.tarfile == None:
self.tarfile = tarfile.open(fileobj=self.fileobj,
mode='r|')
pass
for e in self.tarfile:
yield e
# Keep memory consumption down
self.tarfile.members = []
pass
pass
def close(self):
self.tarfile.close()
self.tarfile = None
#!/usr/bin/python3
from tar_stream import TarReader, TarWriter
import time
import os
import threading
def walk_files(root):
for p,d,f in os.walk(root):
for n in sorted(f):
yield os.path.join(p,n)
pass
for n in sorted(d):
yield os.path.join(p,n)
pass
pass
pass
def write_tar(t, files):
for p in files:
t.add(p)
pass
pass
def reader(tar_reader):
print("READER", tar_reader)
for e in tar_reader:
print('R', e.size, e.name)
if e.isfile():
a = tar_reader.tarfile.extractfile(e).read()
b = open(e.name, 'rb').read()
if a != b:
raise Exception('DIFFERS...', a, b)
pass
elif e.issym():
print(e.linkname)
print(e.linkpath)
pass
pass
pass
if __name__ == '__main__':
(r,w) = os.pipe()
tw = TarWriter(os.fdopen(w, 'wb'))
tr = TarReader(os.fdopen(r, 'rb'))
t = threading.Thread(target=reader, args=(tr,))
t.daemon=True
t.start()
write_tar(tw, walk_files('.'))
tw.close()
t.join()
tr.close()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment