diff --git a/src/mio.py b/src/mio.py
index 465cf50c465c807fd073780065ed9e103cefdfd6..d284830d23b7346f7a3de2919596c57cd7c37081 100755
--- a/src/mio.py
+++ b/src/mio.py
@@ -1,12 +1,13 @@
#!/usr/bin/python
+import mio.filecache
import mio.installer
import mio.log
import mio.transform
import mio.parser
import mio.repository
import mio.node
-import optparse
+import argparse
import os
import os.path
import re
@@ -19,13 +20,9 @@ if not hasattr(__builtins__, "True"):
__builtins__["True"] = 1
__builtins__["False"] = 0
-class VerboseOptionParser(optparse.OptionParser):
- def error(self, msg):
- self.print_help()
- optparse.OptionParser.error(self, msg)
-
-def parse(url, repository):
- tree = mio.parser.parse(url)
+def parse(path, filename, repository):
+ f = mio.filecache.loadfile(path, filename)
+ tree = mio.parser.parse(f.filename, url=f.key)
if tree._tag == "comps":
# Transform comps to canonical form
@@ -42,21 +39,23 @@ def parse(url, repository):
def find_and_parse(target, builder, url=None):
path = list(options.path)
if url:
+ print "CHECK THIS", path, target, url
path.insert(0, os.path.dirname(url))
- for dir in path:
+ for p in path:
if target.startswith("/"):
m = re.match("^/([^/]*)", target)
- file = "%s/%s.mio" % (dir, m.group(1))
+ filename = "%s.mio" % (m.group(1))
pass
elif target.startswith("@"):
- file = "%s/base/comps.xml" % dir
+ filename = "base/comps.xml"
try:
# Try to locate group file via repomd.xml
- tree = mio.parser.parse("%s/repodata/repomd.xml" % dir)
+ f = mio.filecache.loadfile(p, '/repodata/repomd.xml')
+ tree = mio.parser.parse(f.filename)
for l in tree._data_._location_:
if l.type[1] == 'group':
- file = "%s/%s" % (dir, l.href[0])
+ filename = l.href[0]
break
pass
pass
@@ -64,10 +63,10 @@ def find_and_parse(target, builder, url=None):
continue
pass
else:
- file = "%s/hostinfo.xml" % dir
+ filename = "hostinfo.xml"
pass
try:
- parse(file, builder)
+ parse(p, filename, builder)
return
except IOError, e:
pass
@@ -90,67 +89,69 @@ def find_and_parse(target, builder, url=None):
print "Failed to locate: %s" % target
if __name__ == '__main__':
- optParser = VerboseOptionParser(usage="%prog [options] targets*")
- optParser.add_option("--backup",
- action="store",
- metavar="DIR",
- help="saved a backup of changed files in DIR")
- optParser.add_option("--backup-suffix",
- action="store",
- metavar="SUFFIX",
- help="append SUFFIX to backup files")
- optParser.add_option("--dump",
- action="store_true", default=False,
- help="when listing, dump XML tree")
- optParser.add_option("--exclude-rpm",
- action="append", default=[],
- metavar="RPM",
- help="exclude RPM")
- optParser.add_option("--install",
- action="append", default=[],
- metavar="TARGET",
- help="install TARGET")
- optParser.add_option("-l","--list",
- action="append", default=[],
- metavar="XML",
- help="list available targets in XML")
- optParser.add_option("--list-rpms",
- action="append", default=[],
- metavar="TARGET",
- help="list rpms needed for TARGET")
- optParser.add_option("--noyum",
- action="store_true",
- help="don't run yum (rpms will not be installed)")
- optParser.add_option("-p", "--path",
- action="append", default=[],
- metavar="DIR",
- help="search path for files")
- optParser.add_option("--prefix",
- action="store",
- metavar="DIR",
- help="prefix for installed files")
- optParser.add_option("-q","--quiet",
- action="count", default=0,
- dest="quiet",
- help="decrease verbosity")
- optParser.add_option("--reachable-files",
- action="append", default=[],
- metavar="XML",
- help="list files reachable from XML")
- optParser.add_option("--reachable-rpms",
- action="append", default=[],
- metavar="XML",
- help="list RPMs reachable from XML")
- optParser.add_option("--test",
- action="append", default=[],
- metavar="TARGET",
- help="test TARGET")
- optParser.add_option("-v","--verbose",
- action="count",
- dest="verbose", default=0,
- help="increase verbosity")
+ optParser = argparse.ArgumentParser(usage="%(prog)s [options]")
+ optParser.add_argument("--backup",
+ action="store",
+ metavar="DIR",
+ help="saved a backup of changed files in DIR")
+ optParser.add_argument("--backup-suffix",
+ action="store",
+ metavar="SUFFIX",
+ help="append SUFFIX to backup files")
+ optParser.add_argument("--dump",
+ action="store_true", default=False,
+ help="when listing, dump XML tree")
+ optParser.add_argument("--exclude-rpm",
+ action="append", default=[],
+ metavar="RPM",
+ help="exclude RPM")
+ optParser.add_argument("--install",
+ action="append", default=[],
+ metavar="TARGET",
+ help="install TARGET")
+ optParser.add_argument("-l","--list",
+ action="append", default=[],
+ metavar="XML",
+ help="list available targets in XML")
+ optParser.add_argument("--list-rpms",
+ action="append", default=[],
+ metavar="TARGET",
+ help="list rpms needed for TARGET")
+ optParser.add_argument("--noyum",
+ action="store_true",
+ help="don't run yum (rpms will not be installed)")
+ optParser.add_argument("-p", "--path",
+ nargs='+',
+ action="append", default=[],
+ metavar="DIR",
+ help="search path for files")
+ optParser.add_argument("--prefix",
+ action="store",
+ metavar="DIR",
+ help="prefix for installed files")
+ optParser.add_argument("-q","--quiet",
+ action="count", default=0,
+ dest="quiet",
+ help="decrease verbosity")
+ optParser.add_argument("--reachable-files",
+ action="append", default=[],
+ metavar="XML",
+ help="list files reachable from XML")
+ optParser.add_argument("--reachable-rpms",
+ action="append", default=[],
+ metavar="XML",
+ help="list RPMs reachable from XML")
+ optParser.add_argument("--test",
+ action="append", default=[],
+ metavar="TARGET",
+ help="test TARGET")
+ optParser.add_argument("-v","--verbose",
+ action="count",
+ dest="verbose", default=0,
+ help="increase verbosity")
- (options, args) = optParser.parse_args(sys.argv[1:])
+ options = optParser.parse_args(sys.argv[1:])
+ options.path = map(tuple, options.path)
mio.log.set_verbosity(NORMAL + options.verbose - options.quiet)
diff --git a/src/mio/filecache.py b/src/mio/filecache.py
index 5fec2ed70eae6a696a6e49bf63b81560eecb05de..923be5a7e2d135c8ed802805ee3079e9900ec22a 100755
--- a/src/mio/filecache.py
+++ b/src/mio/filecache.py
@@ -6,44 +6,127 @@ import re
import sys
import tempfile
import time
-import urllib2
+import urlgrabber
+import urlgrabber.mirror
import urlparse
+import urllib
import mio.exception
+import socket
+import pycurl
+
+temp = []
+cache = {}
+mirror = {}
-def join(elements):
- """join and normalize elements to a proper (url) path"""
- join = ''
- for e in elements:
- if e != None:
- if len(join) > 0:
- join += '/'
- join += e
- (scheme, host, path, param, query, frag) = urlparse.urlparse(join)
- if scheme:
- path = re.sub('/+', '/', path)
- result = urlparse.urlunparse((scheme, host, path, param, query, frag))
- else:
- result = re.sub('/+', '/', join)
- return result
+def subpath(path, *tail):
+ result = []
+ for p in path:
+ u = urlparse.urlsplit(p)
+ result.append(urlparse.urlunsplit([
+ u.scheme,
+ u.hostname,
+ '/'.join([u.path] + list(tail)),
+ u.query,
+ u.fragment]))
+ pass
+ return tuple(result)
+
+
+def Key(path, name):
+ class Key(tuple, object):
+ @property
+ def path(self):
+ return self[0]
+ @property
+ def name(self):
+ return self[1]
+ def __repr__(self):
+ return 'Key(path=%s, name=%s)' % (self[0], self[1])
+ return Key([path, name])
+
+class CacheEntry:
+
+ def __init__(self, filename, key):
+ self.filename = filename
+ self.key = key
+
+ def __repr__(self):
+ return 'CacheEntry(filename=%s, key=%s)' % (self.filename, self.key)
-def geturl(p):
- (scheme, host, path, param, query, frag) = urlparse.urlparse(p)
- if not scheme:
- scheme = 'file'
- host =''
- path = os.path.abspath(p)
- param = ''
- query = ''
- frag = ''
- return urlparse.urlunparse((scheme, host, path, param, query, frag))
-
-def getfile(p):
- (scheme, host, path, param, query, frag) = urlparse.urlparse(geturl(p))
- if scheme == 'file':
- return path
- else:
- return None
-
+class Unalias:
+
+ def __init__(self):
+ self.by_name = {}
+
+ def unalias_host(self, host, port):
+ if not host in self.by_name:
+ byaddr = set()
+ for _,_,_,_,sa in socket.getaddrinfo(host, port,
+ socket.AF_UNSPEC,
+ socket.SOCK_STREAM):
+ byaddr.add(sa[0])
+ pass
+ self.by_name[host] = set()
+ for name,_,_ in map(socket.gethostbyaddr, byaddr):
+ self.by_name[host].add(str(name))
+ pass
+ pass
+ return sorted(self.by_name[host])
+
+ def __call__(self, url):
+ result = []
+ u = urlparse.urlsplit(str(url))
+ if u.hostname:
+ for hostname in self.unalias_host(u.hostname, u.port or 80):
+ result.append(urlparse.urlunsplit((
+ u.scheme,
+ u.netloc.replace(u.hostname, hostname),
+ u.path,
+ u.query,
+ u.fragment)))
+ pass
+ pass
+ elif u.scheme in [ '', 'file' ]:
+ result.append(urlparse.urlunsplit(('file', '', u.path, '', '')))
+ pass
+ else:
+ print u
+ pass
+ return result
+
+unalias = Unalias()
+
+class Mirror:
+
+ def __init__(self, path):
+ self.path = path
+ self.urls = []
+ for m in map(unalias, path):
+ self.urls.extend(m)
+ pass
+ grabber = urlgrabber.grabber.URLGrabber()
+ self.mirror = urlgrabber.mirror.MirrorGroup(grabber, self.urls)
+ pass
+
+ def urlopen(self, path):
+ return self.mirror.urlopen(str(path))
+
+def expand_mirror(path):
+ class Info:
+ def __init__(self, id, path, urls):
+ self.id = 'repo_%d' % id
+ self.path = path
+ self.urls = urls
+ pass
+ def __repr__(self):
+ return 'Repo(id=%s, path=%s, urls=%s)' % (self.id,
+ self.path,
+ self.urls)
+ pass
+ result = []
+ for p,i in zip(path, range(len(path))):
+ result.append(Info(i, p, mirror[p].urls))
+ return result
def cleanup():
for f in temp:
@@ -65,56 +148,36 @@ def createdir():
return d
-def loadfile(path):
- name = join(path)
- if not cache.has_key(name):
- file = getfile(name)
- if file:
- cache[name] = file
+def loadfile(path, *name):
+ key = Key(path, os.path.normpath('/'.join(name)))
+ if not key in cache:
+ if not path in mirror:
+ mirror[path] = Mirror(key.path)
+ pass
+ u = mirror[path].urlopen(key.name)
+ if u.scheme == 'file':
+ p = urllib.unquote(urlparse.urlsplit(u.geturl()).path)
+ cache[key] = CacheEntry(p, key)
pass
else:
- try:
- u = urllib2.urlopen(name)
- pass
- except urllib2.HTTPError, e:
- raise mio.exception.HTTPError(name, e)
+ mtime = u.curl_obj.getinfo(pycurl.INFO_FILETIME)
f = createfile()
f.write(u.read())
u.close()
f.close()
- try:
- modified = u.info()["Last-Modified"]
- mtime = calendar.timegm(
- time.strptime(modified, "%a, %d %b %Y %H:%M:%S %Z"))
- os.utime(f.name, (mtime, mtime))
- except:
- pass
- cache[name] = f.name
- return cache[name]
+ os.utime(f.name, (mtime, mtime))
+ cache[key] = CacheEntry(f.name, key)
+ pass
+ pass
+ return cache[key]
-def loadscript(path):
- result = loadfile(path)
- if result in temp:
- os.chmod(result, 0700)
+def loadscript(path, *name):
+ result = loadfile(path, *name)
+ if result.filename in temp:
+ os.chmod(result.filename, 0700)
return result
-def locatefile(path):
- name = join(path)
- if not cache.has_key(name):
- file = getfile(name)
- if file:
- cache[name] = file
- else:
- raise mio.exception.NonLocalPath(join(path))
- return cache[name]
-
-def locatedir(path):
- dir = getfile(join(path))
- if not dir:
- raise mio.exception.NonLocalPath(join(path))
- return dir
-
+def localpath(prefix, *path):
+ return os.path.normpath('/'.join(filter(None, [prefix] + list(path))))
-temp = []
-cache = {}
atexit.register(cleanup)
diff --git a/src/mio/installer.py b/src/mio/installer.py
index a945b0f01e0141fd7d7d03b2e83662214d4917c7..ea472d4125bb2d1c91d3d05f5aba5261ed6268dc 100755
--- a/src/mio/installer.py
+++ b/src/mio/installer.py
@@ -131,14 +131,14 @@ class Installer:
def add_rpm(self, rpm):
- try:
- dir = mio.filecache.locatedir([ os.path.dirname(rpm.decl._url),
- rpm.decl.rpms[0:] ])
- except mio.exception.NonLocalPath, e:
- dir = e.path
-
- if not dir in self.rpmdir:
- self.rpmdir.append(dir)
+ if rpm.decl.rpms[0:]:
+ path = mio.filecache.subpath(rpm.decl._url.path,
+ rpm.decl.rpms[0:])
+ if not path in self.rpmdir:
+ self.rpmdir.append(path)
+ pass
+ pass
+
if rpm in self.rpm:
# All rpm's with the same name are considered identical
self.rpm[rpm].append(rpm)
diff --git a/src/mio/node.py b/src/mio/node.py
index b6704b8293c5021cbc896f31fa97e5d7392e122f..56dddd468ea427d37a561148a4ed7f956ad5dd81 100755
--- a/src/mio/node.py
+++ b/src/mio/node.py
@@ -186,7 +186,7 @@ class dir_node(target_node):
def do_trig(self, prefix, backup, do_pre):
result = False
- self.target = mio.filecache.locatedir([prefix, self.decl.name[0]])
+ self.target = mio.filecache.localpath(prefix, self.decl.name[0])
self.uid = mio.util.uid(self.owner)
self.gid = mio.util.uid(self.group)
@@ -269,8 +269,7 @@ class file_node(target_node):
def do_trig(self, prefix, backup, do_pre):
result = False
- dir = os.path.dirname(self.decl._url)
- self.target = mio.filecache.locatefile([prefix, self.decl.name[0]])
+ self.target = mio.filecache.localpath(prefix, self.decl.name[0])
self.uid = mio.util.uid(self.owner)
self.gid = mio.util.uid(self.group)
@@ -285,9 +284,9 @@ class file_node(target_node):
else:
if self.decl.source[0] != "":
file = self.decl.source[0] or self.decl.name[0]
- self.source = mio.filecache.loadfile([dir,
- self.decl.files[0:],
- file])
+ self.source = mio.filecache.loadfile(self.decl._url.path,
+ self.decl.files[0:],
+ file).filename
source_sum = self.digest(self.source)
try:
target_sum = self.digest(self.target)
@@ -421,7 +420,7 @@ class symlink_node(target_node):
def do_trig(self, prefix, backup, do_pre):
result = False
- self.target = mio.filecache.locatedir([prefix, self.decl.name[0]])
+ self.target = mio.filecache.localpath(prefix, self.decl.name[0])
self.uid = mio.util.uid(self.owner)
self.gid = mio.util.uid(self.group)
@@ -511,12 +510,11 @@ class script_node:
self.decl = decl
def run(self, when):
- dir = os.path.dirname(self.decl._url)
scripts = self.decl.scripts[0:]
name = self.decl.script[0]
- script = mio.filecache.loadscript([dir, scripts, name])
+ script = mio.filecache.loadscript(self.decl._url.path, scripts, name)
log(NORMAL, declaration(self), "running '%s'" % name)
- mio.daemon_cage.system("%s %s %s" % (script, name, when))
+ mio.daemon_cage.system("%s %s %s" % (script.filename, name, when))
def test(self, when):
dir = os.path.dirname(self.decl._url)
diff --git a/src/mio/parser.py b/src/mio/parser.py
index 2c4e4a760a16e3f5525b2e27965ffe4bfa645946..2fbd4fdd48c68ec21ab63f41e473521310e8a221 100755
--- a/src/mio/parser.py
+++ b/src/mio/parser.py
@@ -521,7 +521,7 @@ class Parser:
parent = self.current[-1]
parent._add(Comment(data, self.parser.CurrentLineNumber))
-def parse(source, include_comments=False):
+def parse(source, include_comments=False, url=None):
"""Parse the given 'source' returning an easily traversable tree"""
(scheme, host, path, param, query, frag) = urlparse.urlparse(source)
if not scheme:
@@ -540,7 +540,7 @@ def parse(source, include_comments=False):
pass
pass
pass
- return Parser(data, url=source, mtime=mtime,
+ return Parser(data, url=url or source, mtime=mtime,
include_comments=include_comments).tree
def parse_string(source, include_comments=False):
diff --git a/src/mio/yum.py b/src/mio/yum.py
index 8ad2ed76b7384d4f04d46034768cad95c9ada707..6ce341ef048b8171473e8e15fc4aca056b4fafba 100755
--- a/src/mio/yum.py
+++ b/src/mio/yum.py
@@ -19,31 +19,29 @@ def newest_rpm(name):
def yumdir(path):
result = []
- for d in path:
+ for p in path:
repo_date = 0
try:
- file = mio.filecache.loadfile([ d, 'repodata/repomd.xml' ])
- if os.path.exists(file):
- stat = os.stat(file)
- repo_date = max(repo_date, stat.st_mtime)
- if not d in result:
- result.append(d)
+ f = mio.filecache.loadfile(p, 'repodata/repomd.xml')
+ stat = os.stat(f.filename)
+ repo_date = max(repo_date, stat.st_mtime)
+ if not p in result:
+ result.append(p)
except IOError:
pass
header_date = 0
try:
- file = mio.filecache.loadfile([ d, 'headers/header.info' ])
- if os.path.exists(file):
- stat = os.stat(file)
- header_date = max(header_date, stat.st_mtime)
- if not d in result:
- result.append(d)
+ f = mio.filecache.loadfile(p, 'headers/header.info')
+ stat = os.stat(f.filename)
+ header_date = max(header_date, stat.st_mtime)
+ if not p in result:
+ result.append(p)
except IOError:
pass
if header_date or repo_date:
- rpm_date = newest_rpm([ d ])
+ rpm_date = newest_rpm(p)
if repo_date and repo_date < rpm_date:
raise Exception("repodata/repomd.xml is too old %s" % d)
if header_date and header_date < rpm_date:
@@ -64,10 +62,10 @@ def conf(path):
"obsoletes=1" ]
dir = yumdir(path)
- for d in dir:
- result.append("[%s_]" % d.replace("/", "_"))
- result.append("name=RPM's from %s" % d)
- result.append("baseurl=%s" % mio.filecache.geturl(d))
+ for m in mio.filecache.expand_mirror(dir):
+ result.append("[%s]" % m.id)
+ result.append("name=RPM's from %s" % ','.join(m.path))
+ result.append("baseurl=%s" % '\n '.join(m.urls))
return "\n".join(result)