Add tar_stream wraps tarfile with memory optimizations

tar_stream.py

+#!/usr/bin/python3
+
+import tarfile
+import os
+import threading
+
+class TarWriter:
+
+    def __init__(self, fileobj):
+        self.tarfile = tarfile.open(fileobj=fileobj,
+                                    mode='w|',
+                                    format=tarfile.PAX_FORMAT)
+
+    def add(self, path):
+        self.tarfile.add(path, recursive=False)
+        # Keep memory consumption down
+        self.tarfile.members = []
+        self.tarfile.inodes = {}
+        pass
+
+    def close(self):
+        self.tarfile.close()
+        self.tarfile = None
+        
+class TarReader:
+
+    def __init__(self, fileobj):
+        self.fileobj = fileobj
+        self.tarfile = None
+        pass
+
+    def __iter__(self):
+        if self.tarfile == None:
+            self.tarfile = tarfile.open(fileobj=self.fileobj,
+                                        mode='r|')
+            pass
+        for e in self.tarfile:
+            yield e
+            # Keep memory consumption down
+            self.tarfile.members = []
+            pass
+        pass
+        
+    def close(self):
+        self.tarfile.close()
+        self.tarfile = None
+        

test/test_tar_stream.py

+#!/usr/bin/python3
+
+from tar_stream import TarReader, TarWriter
+import time
+import os
+import threading
+
+
+def walk_files(root):
+    for p,d,f in os.walk(root):
+        for n in sorted(f):
+            yield os.path.join(p,n)
+            pass
+        for n in sorted(d):
+            yield os.path.join(p,n)
+            pass
+        pass
+    pass
+    
+def write_tar(t, files):
+    for p in files:
+        t.add(p)
+        pass
+    pass
+
+
+def reader(tar_reader):
+    print("READER", tar_reader)
+    for e in tar_reader:
+        print('R', e.size, e.name)
+        if e.isfile():
+            a = tar_reader.tarfile.extractfile(e).read()
+            b = open(e.name, 'rb').read()
+            if a != b:
+                raise Exception('DIFFERS...', a, b)
+            pass
+        elif e.issym():
+            print(e.linkname)
+            print(e.linkpath)
+            pass
+            
+        pass
+    pass
+
+if __name__ == '__main__':
+    (r,w) = os.pipe()
+    tw = TarWriter(os.fdopen(w, 'wb'))
+    tr = TarReader(os.fdopen(r, 'rb'))
+    t = threading.Thread(target=reader, args=(tr,))
+    t.daemon=True
+    t.start()
+
+    write_tar(tw, walk_files('.'))
+    tw.close()
+    t.join()
+    tr.close()