secondary.py 7.57 KB
Newer Older
Anders Blomdell's avatar
Anders Blomdell committed
1
2
3
#!/usr/bin/python3

import atexit
4
5
6
import command
import hashtoc
import loghandler
Anders Blomdell's avatar
Anders Blomdell committed
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
import os
import socket
import subprocess
import time

def cond_unlink(path, log):
    try:
        os.unlink(path)
        log.DEBUG('removed %s' % path)
    except FileNotFoundError:
        pass

def cond_kill(p):
    try:
        p.kill()
    except:
        pass

class Status:

    def __init__(self, log):
        self.checked = 0
        self.added = 0
        self.deleted = 0
        self.replaced = 0
        self.unchanged = 0
        self.metadata = 0
        self.extract_OK = -1
        def report():
            log.MESSAGE('STATUS %d = +%d -%d =%d ?%d (%d)' % (
                self.checked, self.added, self.deleted,
                self.replaced, self.metadata,
                self.extract_OK))
        atexit.register(report)
        
class Backup:

44
45
46
47
    def __init__(self, primary_star, mount, path, status, log):
        self.primary_star = primary_star
        self.primary_in = primary_star.makefile('wb')
        self.primary_out = primary_star.makefile('rb')
Anders Blomdell's avatar
Anders Blomdell committed
48
49
50
51
52
53
54
55
        self.mount = mount
        self.path = path
        self.status = status
        self.log = log
        self.dst_root = os.path.join(mount, path).encode('utf-8')
        self.trash_root = os.path.join(mount, 'TRASH').encode('utf-8')
        self.trash = os.path.join(self.trash_root,
                                  str(int(time.time())).encode('utf-8'))
56
        extract_cmd = [ '/bin/star', '-x', '-nowarn', '-no-statistics' ]
Anders Blomdell's avatar
Anders Blomdell committed
57
58
59
60
        self.extract = subprocess.Popen(extract_cmd,
                                        cwd=os.path.join(mount, path),
                                        stdin=self.primary_out)
        atexit.register(cond_kill, self.extract)
61
        # Make sure that the generated star archive is not empty
62
        self.primary_in.write(b'.\0')
Anders Blomdell's avatar
Anders Blomdell committed
63
64
65

    def close(self):
        self.primary_in.flush()
66
        self.primary_star.shutdown(socket.SHUT_WR)
Anders Blomdell's avatar
Anders Blomdell committed
67
68
69
70
71
        self.status.extract_OK = self.extract.wait()

    def check(self, src, dst):
        if src.name != dst.name:
            raise Exception('Names differ: %s, %s' % (src, dst))
72
        dst_path = os.path.join(self.dst_root, dst.name)
73
        if src.kind != dst.kind or src.sum != dst.sum or src.size != dst.size:
Anders Blomdell's avatar
Anders Blomdell committed
74
            self.log.DEBUG('Replace...', src.name, dst.name,
75
                           src.sum, dst.sum, src.size, dst.size)
Anders Blomdell's avatar
Anders Blomdell committed
76
77
78
            self.status.replaced += 1
            self.delete(dst)
            self.add(src)
79
        elif os.path.lexists(dst_path):
Anders Blomdell's avatar
Anders Blomdell committed
80
            changed = False
81
            if src.kind in [ b'F', b'D'] and src.mode != dst.mode:
Anders Blomdell's avatar
Anders Blomdell committed
82
                self.log.DEBUG('MODE', dst.name, src.mode, dst.mode)
83
                os.chmod(dst_path, int(src.mode, 8))
Anders Blomdell's avatar
Anders Blomdell committed
84
                changed = True
85
            if (src.kind in [ b'F', b'D', b'L', b'S'] and
86
                (src.uid != dst.uid or src.gid != dst.gid)):
Anders Blomdell's avatar
Anders Blomdell committed
87
88
                self.log.DEBUG('UID/GID', dst.name, src.uid, src.gid,
                               dst.uid, dst.gid)
89
                os.lchown(dst_path, int(src.uid), int(src.gid))
Anders Blomdell's avatar
Anders Blomdell committed
90
                changed = True
91
            if src.kind == b'F' and src.mtime != dst.mtime:
Anders Blomdell's avatar
Anders Blomdell committed
92
                self.log.DEBUG('MTIME', src.name, src.mtime, dst.mtime)
93
94
                atime = os.stat(dst_path).st_atime
                os.utime(dst_path, (int(atime), int(src.mtime)))
Anders Blomdell's avatar
Anders Blomdell committed
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
                changed = True
            if changed:
                self.status.metadata += 1
            else:
                self.status.unchanged += 1
                
    def make_room(self, size):
        for p in sorted(os.listdir(self.trash_root)):
            stat = os.statvfs(self.dst_root)
            free = stat.f_frsize * stat.f_bavail
            need = size + stat.f_frsize
            if free > need:
                break
            self.log.MESSAGE("Need to free:",
                             need - free, (need, free), self.trash_root)
            d = os.path.join(self.trash_root, p)
            if os.path.isdir(d):
                self.log.MESSAGE('Removing dir', d)
                shutil.rmtree(d)
                pass
            else:
                self.log.MESSAGE('Removing file', d)
                os.unlink(d)

    def add(self, src):
        self.log.DEBUG('Add:', src.name)
        if len(src.size) == 0:
            size = 0
        else:
            size = int(src.size)
        self.make_room(size)
        parent = os.path.dirname(src.name)
        while len(parent) != 0:
            # Make sure directories get the correct modes
129
            self.primary_in.write(parent + b'\0')
Anders Blomdell's avatar
Anders Blomdell committed
130
            parent = os.path.dirname(parent)
131
        self.primary_in.write(src.name + b'\0')
Anders Blomdell's avatar
Anders Blomdell committed
132
133
134
135

    def delete(self, dst):
        self.log.DEBUG('Delete:', dst.name)
        dst_path = os.path.join(self.dst_root, dst.name)
Anders Blomdell's avatar
Anders Blomdell committed
136
        if os.path.lexists(dst_path):
Anders Blomdell's avatar
Anders Blomdell committed
137
138
139
140
141
142
143
            trash_path = os.path.join(self.trash, dst.name)
            trash_dir = os.path.dirname(trash_path)
            if not os.path.exists(trash_dir):
                os.makedirs(trash_dir, mode=0o700)
            os.rename(dst_path, trash_path)


144
def do_backup(hash_name, options, socket_path, mount, path):
Anders Blomdell's avatar
Anders Blomdell committed
145
146
147
148
149
150
151
152
153
154
155
156
    if options.debug:
        log = loghandler.LOG(loghandler.LOG_DEBUG)
    else:
        log = loghandler.LOG(loghandler.LOG_WARNING)
    atexit.register(cond_unlink, socket_path, log)
    status = Status(log)
    
    config_path = '%s/TOTALBACKUP.config' % (mount)
        
    if not os.path.exists(config_path):
        raise Exception('"%s" does not exists' % (config_path))

157
158
159
    # Connect to server config/hashtoc socket
    config_hash = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    config_hash.connect(socket_path)
Anders Blomdell's avatar
Anders Blomdell committed
160
    # Send secondary config to primary
161
162
    config_hash.makefile('w').write(open(config_path).read())
    config_hash.shutdown(socket.SHUT_WR)
Anders Blomdell's avatar
Anders Blomdell committed
163
    # Make ready to read primary TOC (src)
164
    src = hashtoc.HashTOC(config_hash.makefile('rb'), rename={hash_name:'sum'})
Anders Blomdell's avatar
Anders Blomdell committed
165

166
167
168
169
    # Create secondary hashtoc (dst)
    cmd = ( command.Command('/usr/bin/hashtoc')
            .flag('--%s' % hash_name)
            .flag('--zero-terminated')
Anders Blomdell's avatar
Anders Blomdell committed
170
            .flag('--xattr', options.xattr)
171
172
173
174
            .option('--max-age', options.max_age)
            .option('--jobs', options.jobs)
            .option('--lookahead', options.lookahead)
            .arg('.') )
Anders Blomdell's avatar
Anders Blomdell committed
175
176
177
178
    p = subprocess.Popen(cmd,
                         cwd=os.path.join(mount, path),
                         stdout=subprocess.PIPE)
    atexit.register(cond_kill, p)
179
    dst = hashtoc.HashTOC(p.stdout,  rename={hash_name:'sum'})
Anders Blomdell's avatar
Anders Blomdell committed
180
            
181
182
183
    # Connect to server star socket
    primary_star = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    primary_star.connect(socket_path)
Anders Blomdell's avatar
Anders Blomdell committed
184
    
185
    backup = Backup(primary_star=primary_star,
Anders Blomdell's avatar
Anders Blomdell committed
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
                    mount=mount, path=path, status=status, log=log)
    while True:
        if src.name == None and dst.name == None:
            # All done
            break
        status.checked += 1
        if src.name == None:
            status.deleted += 1
            backup.delete(dst)
            dst.next()
        elif dst.name == None:
            status.added += 1
            backup.add(src)
            src.next()
        elif src.name == dst.name:
            backup.check(src=src, dst=dst)
            src.next()
            dst.next()
        elif src.name < dst.name:
            status.added += 1
            backup.add(src)
            src.next()
        elif src.name > dst.name:
            status.deleted += 1
            backup.delete(dst)
            dst.next()
        else:
            raise Exception()

    backup.close()
        
217
218
219
    log.DEBUG('hashtoc result', p.wait())
    config_hash.shutdown(socket.SHUT_RD)
    config_hash.close()