diff --git a/README.md b/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..f1508d5ae0e40436888d7edd323e183820665c50
--- /dev/null
+++ b/README.md
@@ -0,0 +1,25 @@
+dist-sync
+=========
+
+Simple python program to keep (rpm-based) distribution packages/files
+locally up to date. Requires dnf-plugins-core > 3.0.1 (or backported
+--downloadcomps functionality)
+
+
+```
+usage: dist-sync [-h] [-v] [--cleanup] [--commit] [--reposync] [--rsync]
+ [--nocreaterepo]
+ PATH [PATH ...]
+
+positional arguments:
+ PATH RPM distribution directory
+
+optional arguments:
+ -h, --help show this help message and exit
+ -v, --verbose print progress information
+ --cleanup remove files that has disappeared from remote repository
+ --commit move updated files to active location
+ --reposync Run rsync actions as well as reposync
+ --rsync Run rsync actions as well as reposync
+ --nocreaterepo Skip createrepo
+```
\ No newline at end of file
diff --git a/dist-sync b/dist-sync
index 8951c19bf90c3a6e8cc5cfd7cf4e2765d2b40fff..0e4c25ad43db52ccb8bef990db6261d4aa928afc 100755
--- a/dist-sync
+++ b/dist-sync
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
import argparse
import contextlib
@@ -8,7 +8,6 @@ import subprocess
import sys
import tempfile
import threading
-import lockfile
import fcntl
import errno
import time
@@ -51,7 +50,7 @@ def file_lock(lock_file, progress=None):
try:
fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
break
- except IOError, e:
+ except IOError as e:
if e.errno != errno.EAGAIN:
raise
pass
@@ -105,8 +104,8 @@ def read_files(path):
return result
def read_rpms(path):
- return dict(filter(lambda (k,v): k.endswith('.rpm'),
- read_files(path).iteritems()))
+ return dict([ (k,v) for k,v in read_files(path).items()
+ if k.endswith('.rpm')])
class Repo:
@@ -123,7 +122,7 @@ class Repo:
def add(self, target, source):
if target == 'ARCH':
self.arch = source
- elif target in ['baseurl', 'mirrorlist']:
+ elif target in ['baseurl', 'mirrorlist', 'metalink' ]:
if self.mirrored:
raise Exception('Multiply mirrored %s, %s' %
(self.mirrored, (target, source)))
@@ -280,16 +279,13 @@ class Repo:
# so let us put everything in a subdirectory manually
packages = '%s/Packages' % (destdir)
cmd = []
- cmd.extend(['/usr/local/bin/reposync',
- '--arch', self.arch,
- '--releasever', self.release,
+ cmd.extend(['dnf', 'reposync',
'--config', conf_name,
- '--download_path', packages,
- '--norepopath',
- '--download-metadata',
- '--delete' ])
+ '--download-path', os.path.dirname(destdir),
+ '--downloadcomps'
+ ] + [ '--verbose' for i in range(args.verbose) ])
if args.verbose:
- print ' '.join(cmd)
+ print(' '.join(cmd))
p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
def progress(pipe):
while True:
@@ -298,7 +294,7 @@ class Repo:
break
if not args.verbose:
continue
- m = re.match('.*[^0-9](\d+)/(\d+).*', l)
+ m = re.match(b'.*[^0-9](\d+)/(\d+).*', l)
if m:
n = int(m.group(1))
m = int(m.group(2))
@@ -346,20 +342,18 @@ class Repo:
'-exec', 'rm', '{}', ';'])
comps = filter(lambda p: re.match('^.*comps.*\.xml$', p),
os.listdir('.'))
- comps = sorted(comps,
- cmp=lambda a,b: cmp(os.stat(a).st_mtime,
- os.stat(b).st_mtime))
+ comps = sorted(comps, key=lambda p: os.stat(p).st_mtime)
if len(comps) == 0:
comps = []
pass
else:
comps = [ '-g', comps[-1] ]
pass
- cmd = list([ '/usr/bin/createrepo', '-c', 'cache' ] +
+ cmd = list([ '/usr/bin/createrepo_c', '-c', 'cache' ] +
comps +
['-v', '--update', '--pretty', '.'])
if args.verbose:
- print ' '.join(cmd)
+ print(' '.join(cmd))
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
@@ -370,7 +364,7 @@ class Repo:
break
if not args.verbose:
continue
- m = re.match('^Worker\s+[^:]:\s+reading\s+(.*)', l)
+ m = re.match(b'^Worker\s+[^:]:\s+reading\s+(.*)', l)
if m:
if args.verbose == 1:
sys.stderr.write('\r%-60.60s\r' % m.group(1))
@@ -378,7 +372,7 @@ class Repo:
sys.stderr.write('%s\n' % m.group(1))
sys.stderr.flush()
continue
- m = re.match('^Using data .* for (\S+)$', l)
+ m = re.match(b'^Using data .* for (\S+)$', l)
if m:
if args.verbose == 1:
sys.stderr.write('\r%-60.60s\r' % m.group(1))
@@ -407,7 +401,7 @@ class Repo:
pass
for cachedir,_,files in os.walk('.'):
for f in files:
- os.chmod('%s/%s' % (cachedir, f), 0644)
+ os.chmod('%s/%s' % (cachedir, f), 0o644)
pass
pass
pass
@@ -475,12 +469,12 @@ def parse_config(config):
return result
if __name__ == '__main__':
- parser = argparse.ArgumentParser(usage='%(prog)s [options] DISTRIBUTION*')
- parser.add_argument('distribution',
- metavar='DISTRIBUTION',
+ parser = argparse.ArgumentParser()
+ parser.add_argument('path',
+ metavar='PATH',
type=str,
nargs='+',
- help='a rpm distribution directory ')
+ help='RPM distribution directory ')
parser.add_argument('-v', '--verbose',
default=0,
action='count',
@@ -491,6 +485,9 @@ if __name__ == '__main__':
parser.add_argument('--commit',
action='store_true',
help='move updated files to active location')
+ parser.add_argument('--reposync',
+ action='store_true',
+ help='Run rsync actions as well as reposync')
parser.add_argument('--rsync',
action='store_true',
help='Run rsync actions as well as reposync')
@@ -498,30 +495,45 @@ if __name__ == '__main__':
action='store_true',
help='Skip createrepo')
args = parser.parse_args()
- for d in args.distribution:
+
+ todo = []
+ for p in args.path:
+ if os.path.isfile(p) and os.path.basename(p) == 'REPOS':
+ todo.append((os.path.dirname(p), p))
+ continue
+ if os.path.isdir(p) and os.path.isfile(os.path.join(p, 'REPOS')):
+ todo.append((p, os.path.join(p, 'REPOS')))
+ continue
+ raise Exception("No 'REPOS' file in '%s'" % p)
+
+ for d,r in todo:
lockfile = open('%s/LOCK' % d, 'a')
def progress(f):
log('Waiting for %s to be unlocked' % f.name)
pass
log('%s:' % os.path.realpath(d))
with file_lock(lockfile, progress=progress):
- repofile = open('%s/REPOS' % d)
+ repofile = open(r)
repos = parse_config(repofile)
- if args.commit:
- log('Commiting changes...')
+ if args.rsync:
+ log("Running 'rsync'")
for repo in repos:
- repo.commit(args)
+ repo.rsync(args)
pass
pass
- else:
+ if args.reposync:
+ log("Running 'dnf reposync'...")
for repo in repos:
- if args.rsync:
- repo.rsync(args)
- pass
repo.get_updates(args)
repo.update_db(args)
pass
pass
+ if args.commit:
+ log('Commiting changes...')
+ for repo in repos:
+ repo.commit(args)
+ pass
+ pass
pass
pass
pass