secondary.py 7.34 KB
Newer Older
Anders Blomdell's avatar
Anders Blomdell committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
#!/usr/bin/python3

import atexit
import md5toc
import os
import socket
import subprocess
import loghandler
import time

def cond_unlink(path, log):
    try:
        os.unlink(path)
        log.DEBUG('removed %s' % path)
    except FileNotFoundError:
        pass

def cond_kill(p):
    try:
        p.kill()
    except:
        pass

class Status:

    def __init__(self, log):
        self.checked = 0
        self.added = 0
        self.deleted = 0
        self.replaced = 0
        self.unchanged = 0
        self.metadata = 0
        self.extract_OK = -1
        def report():
            log.MESSAGE('STATUS %d = +%d -%d =%d ?%d (%d)' % (
                self.checked, self.added, self.deleted,
                self.replaced, self.metadata,
                self.extract_OK))
        atexit.register(report)
        
class Backup:

43
44
45
46
    def __init__(self, primary_star, mount, path, status, log):
        self.primary_star = primary_star
        self.primary_in = primary_star.makefile('wb')
        self.primary_out = primary_star.makefile('rb')
Anders Blomdell's avatar
Anders Blomdell committed
47
48
49
50
51
52
53
54
        self.mount = mount
        self.path = path
        self.status = status
        self.log = log
        self.dst_root = os.path.join(mount, path).encode('utf-8')
        self.trash_root = os.path.join(mount, 'TRASH').encode('utf-8')
        self.trash = os.path.join(self.trash_root,
                                  str(int(time.time())).encode('utf-8'))
55
        extract_cmd = [ '/bin/star', '-x', '-nowarn', '-no-statistics' ]
Anders Blomdell's avatar
Anders Blomdell committed
56
57
58
59
        self.extract = subprocess.Popen(extract_cmd,
                                        cwd=os.path.join(mount, path),
                                        stdin=self.primary_out)
        atexit.register(cond_kill, self.extract)
60
61
        # Make sure that the generated star archive is not empty
        self.primary_in.write(b'.\n')
Anders Blomdell's avatar
Anders Blomdell committed
62
63
64

    def close(self):
        self.primary_in.flush()
65
        self.primary_star.shutdown(socket.SHUT_WR)
Anders Blomdell's avatar
Anders Blomdell committed
66
67
68
69
70
        self.status.extract_OK = self.extract.wait()

    def check(self, src, dst):
        if src.name != dst.name:
            raise Exception('Names differ: %s, %s' % (src, dst))
71
        dst_path = os.path.join(self.dst_root, dst.name)
Anders Blomdell's avatar
Anders Blomdell committed
72
73
74
75
76
77
        if src.kind != dst.kind or src.md5 != dst.md5 or src.size != dst.size:
            self.log.DEBUG('Replace...', src.name, dst.name,
                           src.md5, dst.md5, src.size, dst.size)
            self.status.replaced += 1
            self.delete(dst)
            self.add(src)
78
        elif os.path.exists(dst_path):
Anders Blomdell's avatar
Anders Blomdell committed
79
            changed = False
80
            if src.kind in [ b'F', b'D'] and src.mode != dst.mode:
Anders Blomdell's avatar
Anders Blomdell committed
81
                self.log.DEBUG('MODE', dst.name, src.mode, dst.mode)
82
                os.chmod(dst_path, int(src.mode, 8))
Anders Blomdell's avatar
Anders Blomdell committed
83
                changed = True
84
85
            if (src.kind in [ b'F', b'D'] and
                (src.uid != dst.uid or src.gid != dst.gid)):
Anders Blomdell's avatar
Anders Blomdell committed
86
87
                self.log.DEBUG('UID/GID', dst.name, src.uid, src.gid,
                               dst.uid, dst.gid)
88
                os.lchown(dst_path, int(src.uid), int(src.gid))
Anders Blomdell's avatar
Anders Blomdell committed
89
                changed = True
90
            if src.kind == b'F' and src.mtime != dst.mtime:
Anders Blomdell's avatar
Anders Blomdell committed
91
                self.log.DEBUG('MTIME', src.name, src.mtime, dst.mtime)
92
93
                atime = os.stat(dst_path).st_atime
                os.utime(dst_path, (int(atime), int(src.mtime)))
Anders Blomdell's avatar
Anders Blomdell committed
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
                changed = True
            if changed:
                self.status.metadata += 1
            else:
                self.status.unchanged += 1
                
    def make_room(self, size):
        for p in sorted(os.listdir(self.trash_root)):
            stat = os.statvfs(self.dst_root)
            free = stat.f_frsize * stat.f_bavail
            need = size + stat.f_frsize
            if free > need:
                break
            self.log.MESSAGE("Need to free:",
                             need - free, (need, free), self.trash_root)
            d = os.path.join(self.trash_root, p)
            if os.path.isdir(d):
                self.log.MESSAGE('Removing dir', d)
                shutil.rmtree(d)
                pass
            else:
                self.log.MESSAGE('Removing file', d)
                os.unlink(d)

    def add(self, src):
        self.log.DEBUG('Add:', src.name)
        if len(src.size) == 0:
            size = 0
        else:
            size = int(src.size)
        self.make_room(size)
        parent = os.path.dirname(src.name)
        while len(parent) != 0:
            # Make sure directories get the correct modes
            self.primary_in.write(parent + b'\n')
            parent = os.path.dirname(parent)
        self.primary_in.write(src.name + b'\n')

    def delete(self, dst):
        self.log.DEBUG('Delete:', dst.name)
        dst_path = os.path.join(self.dst_root, dst.name)
        if os.path.exists(dst_path):
            trash_path = os.path.join(self.trash, dst.name)
            trash_dir = os.path.dirname(trash_path)
            if not os.path.exists(trash_dir):
                os.makedirs(trash_dir, mode=0o700)
            os.rename(dst_path, trash_path)


def do_backup(options, socket_path, mount, path):
    if options.debug:
        log = loghandler.LOG(loghandler.LOG_DEBUG)
    else:
        log = loghandler.LOG(loghandler.LOG_WARNING)
    atexit.register(cond_unlink, socket_path, log)
    status = Status(log)
    
    config_path = '%s/TOTALBACKUP.config' % (mount)
        
    if not os.path.exists(config_path):
        raise Exception('"%s" does not exists' % (config_path))

    # Connect to server config/md5toc socket
    config_md5 = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    config_md5.connect(socket_path)
    # Send secondary config to primary
    config_md5.makefile('w').write(open(config_path).read())
    config_md5.shutdown(socket.SHUT_WR)
    # Make ready to read primary TOC (src)
    src = md5toc.MD5TOC(config_md5.makefile('rb'))

    # Create secondary md5toc (dst)
    cmd = (
        [ '/usr/bin/md5toc' ] +
        ( options.xattr and [ '--xattr' ] or []) +
        ( options.xattr and options.max_age and [ '--max-age', options.max_age ]
          or []) +
        [ '.' ]
    )
    p = subprocess.Popen(cmd,
                         cwd=os.path.join(mount, path),
                         stdout=subprocess.PIPE)
    atexit.register(cond_kill, p)
    dst = md5toc.MD5TOC(p.stdout)
            
179
180
181
    # Connect to server star socket
    primary_star = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    primary_star.connect(socket_path)
Anders Blomdell's avatar
Anders Blomdell committed
182
    
183
    backup = Backup(primary_star=primary_star,
Anders Blomdell's avatar
Anders Blomdell committed
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
                    mount=mount, path=path, status=status, log=log)
    while True:
        if src.name == None and dst.name == None:
            # All done
            break
        status.checked += 1
        if src.name == None:
            status.deleted += 1
            backup.delete(dst)
            dst.next()
        elif dst.name == None:
            status.added += 1
            backup.add(src)
            src.next()
        elif src.name == dst.name:
            backup.check(src=src, dst=dst)
            src.next()
            dst.next()
        elif src.name < dst.name:
            status.added += 1
            backup.add(src)
            src.next()
        elif src.name > dst.name:
            status.deleted += 1
            backup.delete(dst)
            dst.next()
        else:
            raise Exception()

    backup.close()
        
    log.DEBUG('md5toc result', p.wait())
    config_md5.shutdown(socket.SHUT_RD)
    config_md5.close()