mio.py

#!/usr/bin/python

from __future__ import print_function
import mio.filecache
import mio.installer
import mio.log
import mio.transform
import mio.parser
import mio.repository
import mio.node
import argparse
import os
import os.path
import re
import sys
from mio.exception import *
from mio.log import log, MUTE, SILENT, NORMAL, VERBOSE
import locale

def parse_with_path(path, filename, repository=None):
    f = mio.filecache.loadfile(path, filename)
    tree = mio.parser.parse(f.filename, url=f.key)
    
    join = None
    if tree._tag == "comps":
        # Transform comps to canonical form
        tree = mio.transform.from_fedora(tree)
        join = mio.transform.join
        pass
    elif tree._tag == "hostinfo":
        # Transform hostinfo to canonical form
        tree = mio.transform.from_hostinfo(tree)
        pass
    elif tree._tag == "mio":
        # Transform mio tree to canonical form
        tree = mio.transform.from_mio(tree)
        pass
    if repository != None:
        repository.add(tree._group_, join=join)
        pass
    return tree

def parse(filename, repository=None):
    for p in options.path:
        try:
            tree = parse_with_path(p, filename, repository)
            return tree
        except IOError as e:
            print(e)
            raise
            pass
    raise Exception("Could not parse %s" % filename)

class FindAndParse:

    def __init__(self):
        self.tried = set()

    def parse_with_path(self, path, filename, builder, target):
        if not (path,filename) in self.tried:
            try:
                parse_with_path(path, filename, builder) 
                self.tried.add((path,filename))
                return True
            except IOError as e:
                self.tried.add((path,filename))
                pass
            except ValueError as e:
                pass
            except DuplicateGroupException as e:
                print(e)
                traceback.print_exc()
                print("Failed to locate: %s" % target)
                sys.exit(1)
                pass
            pass
        return False
            

    def __call__(self, target, builder, url=None):
        path = list(options.path)
        if url:
            for p in url.path:
                path.insert(0, os.path.dirname(p))
                
        OK = False
        for p in path:
            if target.startswith("/"):
                m = re.match("^/([^/]*)", target)
                OK |= self.parse_with_path(p, "%s.mio" % (m.group(1)), builder, target)
                pass
            elif target.startswith("@"):
                filename = "base/comps.xml"
                try:
                    # Try to locate group file via repomd.xml
                    f = mio.filecache.loadfile(p, '/repodata/repomd.xml')
                    tree = mio.parser.parse(f.filename)
                    for l in tree._data_._location_:
                        if l.type[1] == 'group':
                            OK |= self.parse_with_path(p, l.href[0], builder, target)
                            break
                        pass
                    pass
                except IOError as e:
                    continue
                pass
            else:
                OK |= self.parse_with_path(p, "hostinfo.xml", builder, target)
                pass
        
        if OK:
            return
        if options.noyum and target.startswith('@'):
            # Synthetic target
            tmp1 = mio.parser.Node("mio", 0)
            tmp2 =  mio.parser.Node("group", 0, { "name": target })
            tmp1._add(tmp2)
            builder.add(tmp1._group_)
            pass
        else:
            print("Failed to locate: %s" % target)

if __name__ == '__main__':
    locale.setlocale(locale.LC_TIME, 'C')
    optParser = argparse.ArgumentParser(usage="%(prog)s [options]")
    optParser.add_argument("--backup",
                           action="store",
                           metavar="DIR",
                           help="saved a backup of changed files in DIR")
    optParser.add_argument("--backup-suffix",
                           action="store",
                           metavar="SUFFIX",
                           help="append SUFFIX to backup files")
    optParser.add_argument("--dump",
                           action="store_true", default=False, 
                           help="when listing, dump XML tree")
    optParser.add_argument("--exclude-rpm",
                           action="append", default=[],
                           metavar="RPM",
                           help="exclude RPM")
    optParser.add_argument("--install",
                           action="append", default=[],
                           metavar="TARGET",
                           help="install TARGET")
    optParser.add_argument("-l","--list",
                           action="append", default=[],
                           metavar="XML",
                           help="list available targets in XML")
    optParser.add_argument("--list-rpms",
                           action="append", default=[],
                           metavar="TARGET",
                           help="list rpms needed for TARGET")
    optParser.add_argument("--noyum",
                           action="store_true",
                           help="don't run yum (rpms will not be installed)")
    optParser.add_argument("-p", "--path",
                           nargs='+',
                           action="append", default=[],
                           metavar="DIR",
                           help="search path for files")
    optParser.add_argument("--prefix",
                           action="store",
                           metavar="DIR",
                           help="prefix for installed files")
    optParser.add_argument("-q","--quiet",
                           action="count", default=0,
                           dest="quiet", 
                           help="decrease verbosity")
    optParser.add_argument("--reachable-files",
                           action="append", default=[],
                           metavar="XML",
                           help="list files reachable from XML")
    optParser.add_argument("--reachable-rpms",
                           action="append", default=[],
                           metavar="XML",
                           help="list RPMs reachable from XML")
    optParser.add_argument("--test",
                           action="append", default=[],
                           metavar="TARGET",
                           help="test TARGET")
    optParser.add_argument("--dependency-tree",
                           action="append", nargs='*', default=None,
                           metavar="TARGET",
                           help="print dependency tree for TARGET")
    optParser.add_argument("-v","--verbose",
                           action="count",
                           dest="verbose", default=0,
                           help="increase verbosity")
    
    options = optParser.parse_args(sys.argv[1:])
    options.path = [ tuple(p,) for p in options.path ]
    mio.log.set_verbosity(NORMAL + options.verbose - options.quiet)

    find_and_parse = FindAndParse()
    rules = mio.repository.Repository(find_and_parse)

    rpm_result = None

    if options.prefix:
        os.environ["MIO_PREFIX"] = options.prefix
    
    if options.list:
        # List all targets reachable from the given source files
        for url in options.list:
            parse(url, rules)

        error = {}
        print("Reachable targets:")
        for g in rules.get_all(error):
            if not options.dump:
                print("  ",g.name[0])
            else:
                print(g._xml(indent=1))

        for e in error:
            print("Unresolved group: '%s'" % e)
            for r in error[e]:
                print("  from: '%s' in %s:%s" % (r.name[0], r._url, r._line))

    if options.list_rpms:
        installer = mio.installer.Installer(rules)
        for g in options.list_rpms:
            installer.add_group(g)
            pass

        log(NORMAL, "# List rpms for %s" % options.list_rpms)
        for r in installer.list_rpms():
            print(r)
            pass
        pass
    
    if options.reachable_files:
        # List all files reachable from the given source files
        reachable_rules = mio.repository.Repository(find_and_parse,
                                                    lambda x:x.startswith('@'))
        for url in options.reachable_files:
            parse(url, reachable_rules)
            pass
        
        error = {}
        log(VERBOSE, "Reachable targets:")
        files = {}
        for g in reachable_rules.get_all(error):
            installer = mio.installer.Installer(reachable_rules)
            log(VERBOSE, "Scanning: %s" % g.name[0])
            installer.add_group(g.name[0])
            for f in installer.file:
                if f.decl.source[0] == "":
                    # Exists in target system, used to change protection
                    continue
                if mio.node.parse_boolean(f.decl.delete[0]):
                    # Might exists in target system, but should be deleted
                    continue
                source = f.decl.source[0] or f.decl.name[0]
                path = "%s%s" % (f.decl.files[0:],source)
                try:
                    files[path].append(f.name)
                except KeyError:
                    files[path] = [ f.name ]
                pass
            pass
        
        if not error:
            for k in sorted(files.keys()):
                print(k)
            pass
        else:
            for e in error:
                print("Unresolved group: '%s'" % e)
                for r in error[e]:
                    print("  from: '%s' in %s:%s" % (r.name[0],
                                                     r._url, r._line))
                    pass
                pass
            pass
        pass

    if options.reachable_rpms:
        # List all RPMs reachable from the given source files
        reachable_rules = mio.repository.Repository(find_and_parse,
                                                    lambda x:x.startswith('@'))
        for url in options.reachable_rpms:
            parse(url, reachable_rules)
            pass
        
        error = {}
        log(VERBOSE, "Reachable targets:")
        rpms = {}
        for g in reachable_rules.get_all(error):
            installer = mio.installer.Installer(reachable_rules)
            log(VERBOSE, "Scanning: %s" % g.name[0])
            installer.add_group(g.name[0])
            for r in installer.rpm:
                rpms[r.name] = r.name
                pass
            pass
        
        if not error:
            for k in sorted(rpms.keys()):
                print(k)
            pass
        else:
            for e in error:
                print("Unresolved group: '%s'" % e)
                for r in error[e]:
                    print("  from: '%s' in %s:%s" % (r.name[0],
                                                     r._url, r._line))
                    pass
                pass
            pass
        pass

    if options.dependency_tree != None:
        def dump_tree(name, indent=0):
            def command_or_script(cos, prefix=''):
                if cos.command[0] != None:
                    print("%s  [%s command] %s" % (
                        ('  '*indent, prefix, cos.command[0])))
                    pass
                pass
                if cos.script[0] != None:
                    print("%s  [%s script] %s" % (
                        ('  '*indent, prefix, cos.script[0])))
                    pass
                pass
            print('%s%s' % ('  '*indent, name))
            tree = rules.get(name)
            for p in tree._pre_:
                command_or_script(p, 'pre')
            if not options.noyum:
                for r in tree._rpm_:
                    print("%s[r] %s" % ('  '*(indent+1), r.name[0]))
                    pass
                pass
            for s in tree._symlink_:
                print('%s[s] %s -> %s' % ('  '*(indent+1),
                                          s.name[0],
                                          s.value[0]))
                pass
            for f in tree._file_:
                if f.delete[0] == 'yes' or f.source[0] == '':
                    continue
                src = os.path.normpath('%s/%s' % (f.files[0:],
                                                  f.source[0] or f.name[0]))
                print('%s[f] %s = %s' % ('  '*(indent+1), f.name[0], src))
                pass
            for p in tree._post_:
                command_or_script(p, 'post')
            for d in tree._dependency_:
                dump_tree(d.name[0], indent+1)
                pass
            pass
        todo = [i for l in options.dependency_tree for i in l] # Flatten list
        if len(todo) == 0:
            tree = parse('hostinfo.xml')
            todo = [ h.name[0] for h in tree._group_]
        for target in todo:
            dump_tree(target)
            pass
        pass

    if options.test:
        installer = mio.installer.Installer(rules)
        for g in options.test:
            installer.add_group(g)

        if not options.noyum:
            log(NORMAL, "Testing rpms")
            installer.test_rpms(options.path,
                                options.exclude_rpm)
 
        log(NORMAL, "Testing files")
        installer.test_groups(options.prefix,
                              (options.backup, options.backup_suffix))
        
    if options.install:
        installer = mio.installer.Installer(rules)
        for g in options.install:
            installer.add_group(g)

        if not options.noyum:
            log(NORMAL, "Installing rpms")
            try:
                (rpm_result, failed) = installer.install_rpms(
                    options.path, options.exclude_rpm)
                pass
            except Exception as e:
                log(MUTE, "Failed to install rpm's, rerun with '--noyum'")
                raise()
                exit(1)
                pass
        log(NORMAL, "Installing files")
        installer.install_groups(options.prefix,
                                 (options.backup, options.backup_suffix))

    if rpm_result:
        print("Yum failed with %d" % rpm_result)
        for r in failed:
            print("Failed to install %s" % r)
        exit(1)