1#@!#!123s

: / usr / lib64 / python2.7 / site-packages / reportclient /
Filename : debuginfo.py
back
"""
    This module provides classes and functions used to download and manage
    debuginfos.
"""

import sys
import os
import time
import errno
import shutil
from subprocess import Popen

from yum import _, YumBase
from yum.callbacks import DownloadBaseCallback
from yum.Errors import YumBaseError

from reportclient import (_, log1, log2, RETURN_OK, RETURN_FAILURE,
                          RETURN_CANCEL_BY_USER, verbose, ask_yes_no,
                          error_msg)

TMPDIR=""

def ensure_abrt_uid(fn):
    """
    Ensures that the function is called using abrt's uid and gid

    Returns:
        Either an unchanged function object or a wrapper function object for
        the function.
    """

    import pwd
    current_uid = os.getuid()
    current_gid = os.getgid()
    abrt = pwd.getpwnam("abrt")

    # if we're are already running as abrt, don't do anything
    if abrt.pw_uid == current_uid and abrt.pw_gid == current_gid:
        return fn

    def wrapped(*args, **kwargs):
        """
        Wrapper function around the called function.

        Sets up uid and gid to match abrt's and after the function finishes
        rolls its uid and gid back.

        Returns:
            Return value of the wrapped function.
        """

        # switch to abrt
        os.setegid(abrt.pw_gid)
        os.seteuid(abrt.pw_uid)
        # extract the files as abrt:abrt
        retval = fn(*args, **kwargs)
        # switch back to whatever we were
        os.seteuid(current_uid)
        os.setegid(current_gid)
        return retval

    return wrapped

# TODO: unpack just required debuginfo and not entire rpm?
# ..that can lead to: foo.c No such file and directory
# files is not used...
@ensure_abrt_uid
def unpack_rpm(package_file_name, files, tmp_dir, destdir, keeprpm, exact_files=False):
    """
    Unpacks a single rpm located in tmp_dir into destdir.

    Arguments:
        package_file_name - name of the rpm file
        files - files to extract from the rpm
        tmp_dir - temporary directory where the rpm file is located
        destdir - destination directory for the rpm package extraction
        keeprpm - check if the user wants to delete rpms from the tmp directory
        exact_files - extract only specified files

    Returns:
        RETURN_FAILURE in case of a serious problem
    """

    package_full_path = tmp_dir + "/" + package_file_name
    log1("Extracting %s to %s", package_full_path, destdir)
    log2("%s", files)
    print _("Extracting cpio from {0}").format(package_full_path)
    unpacked_cpio_path = tmp_dir + "/unpacked.cpio"
    try:
        unpacked_cpio = open(unpacked_cpio_path, 'wb')
    except IOError, ex:
        print _("Can't write to '{0}': {1}").format(unpacked_cpio_path, ex)
        return RETURN_FAILURE

    rpm2cpio = Popen(["rpm2cpio", package_full_path],
                       stdout = unpacked_cpio, bufsize = -1)
    retcode = rpm2cpio.wait()

    if retcode == 0:
        log1("cpio written OK")
        if not keeprpm:
            log1("keeprpms = False, removing %s", package_full_path)
            #print _("Removing temporary rpm file")
            os.unlink(package_full_path)
    else:
        unpacked_cpio.close()
        print _("Can't extract package '{0}'").format(package_full_path)
        return RETURN_FAILURE

    # close the file
    unpacked_cpio.close()
    # and open it for reading
    unpacked_cpio = open(unpacked_cpio_path, 'rb')

    print _("Caching files from {0} made from {1}").format("unpacked.cpio", package_file_name)

    file_patterns = ""
    cpio_args = ["cpio", "-idu"]
    if exact_files:
        for filename in files:
            file_patterns += "." + filename + " "
        cpio_args = ["cpio", "-idu", file_patterns.strip()]

    with open("/dev/null", "w") as null:
        cpio = Popen(cpio_args, cwd=destdir, bufsize=-1,
                     stdin=unpacked_cpio, stdout=null, stderr=null)
        retcode = cpio.wait()

    if retcode == 0:
        log1("files extracted OK")
        #print _("Removing temporary cpio file")
        os.unlink(unpacked_cpio_path)
    else:
        print _("Can't extract files from '{0}'").format(unpacked_cpio_path)
        return RETURN_FAILURE

def clean_up():
    """
    Removes the temporary directory.
    """

    if TMPDIR:
        try:
            shutil.rmtree(TMPDIR)
        except OSError, ex:
            if ex.errno != errno.ENOENT:
                error_msg(_("Can't remove '{0}': {1}").format(TMPDIR, ex))

class MyDownloadCallback(DownloadBaseCallback):
    """
    This class serves as a download progress handler for yum's progress bar.
    """

    def __init__(self, total_pkgs):
        """
        Sets up instance variables

        Arguments:
            total_pkgs - number of packages to download
        """

        self.total_pkgs = total_pkgs
        self.downloaded_pkgs = 0
        self.last_pct = 0
        self.last_time = 0
        DownloadBaseCallback.__init__(self)

    def updateProgress(self, name, frac, fread, ftime):
        """
        A method used to update the progress

        Arguments:
            name - filename
            frac - progress fracment (0 -> 1)
            fread - formated string containing BytesRead
            ftime - formated string containing remaining or elapsed time
        """

        pct = int(frac * 100)
        if pct == self.last_pct:
            log2("percentage is the same, not updating progress")
            return

        self.last_pct = pct
        # if run from terminal we can have fancy output
        if sys.stdout.isatty():
            sys.stdout.write("\033[sDownloading (%i of %i) %s: %3u%%\033[u"
                    % (self.downloaded_pkgs + 1, self.total_pkgs, name, pct)
            )
            if pct == 100:
                #print (_("Downloading (%i of %i) %s: %3u%%")
                #        % (self.downloaded_pkgs + 1, self.total_pkgs, name, pct)
                #)
                print (_("Downloading ({0} of {1}) {2}: {3:3}%").format(
                        self.downloaded_pkgs + 1, self.total_pkgs, name, pct
                        )
                )
        # but we want machine friendly output when spawned from abrt-server
        else:
            t = time.time()
            if self.last_time == 0:
                self.last_time = t
            # update only every 5 seconds
            if pct == 100 or self.last_time > t or t - self.last_time >= 5:
                print (_("Downloading ({0} of {1}) {2}: {3:3}%").format(
                        self.downloaded_pkgs + 1, self.total_pkgs, name, pct
                        )
                )
                self.last_time = t
                if pct == 100:
                    self.last_time = 0

        sys.stdout.flush()

def downloadErrorCallback(callBackObj):
    """
    A callback function for mirror errors.
    """

    print _("Problem '{0!s}' occured while downloading from mirror: '{1!s}'. Trying next one").format(
        callBackObj.exception, callBackObj.mirror)
    # explanation of the return value can be found here:
    # /usr/lib/python2.7/site-packages/urlgrabber/mirror.py
    return {'fail':0}

class DebugInfoDownload(YumBase):
    """
    This class is used to manage download of debuginfos.
    """

    def __init__(self, cache, tmp, repo_pattern="*debug*", keep_rpms=False,
                 noninteractive=True):
        self.old_stdout = -1
        self.cachedir = cache
        self.tmpdir = tmp
        global TMPDIR
        TMPDIR = tmp
        self.keeprpms = keep_rpms
        self.noninteractive = noninteractive
        self.repo_pattern=repo_pattern
        YumBase.__init__(self)
        self.mute_stdout()
        #self.conf.cache = os.geteuid() != 0
        # Setup yum (Ts, RPM db, Repo & Sack)
        # doConfigSetup() takes some time, let user know what we are doing
        print _("Initializing yum")
        try:
            # Saw this exception here:
            # cannot open Packages index using db3 - Permission denied (13)
            # yum.Errors.YumBaseError: Error: rpmdb open failed
            self.doConfigSetup()
        except YumBaseError, ex:
            self.unmute_stdout()
            print _("Error initializing yum (YumBase.doConfigSetup): '{0!s}'").format(ex)
            #return 1 - can't do this in constructor
            exit(1)
        self.unmute_stdout()

    def mute_stdout(self):
        """
        Links sys.stdout with /dev/null and saves the old stdout
        """

        if verbose < 2:
            self.old_stdout = sys.stdout
            sys.stdout = open("/dev/null", "w")

    def unmute_stdout(self):
        """
        Replaces sys.stdout by stdout saved using mute
        """

        if verbose < 2:
            if self.old_stdout != -1:
                sys.stdout = self.old_stdout
            else:
                print "ERR: unmute called without mute?"

    @ensure_abrt_uid
    def setup_tmp_dirs(self):
        if not os.path.exists(self.tmpdir):
            try:
                os.makedirs(self.tmpdir)
            except OSError, ex:
                print "Can't create tmpdir: %s" % ex
                return RETURN_FAILURE
        if not os.path.exists(self.cachedir):
            try:
                os.makedirs(self.cachedir)
            except OSError, ex:
                print "Can't create cachedir: %s" % ex
                return RETURN_FAILURE

        return RETURN_OK

    # return value will be used as exitcode. So 0 = ok, !0 - error
    def download(self, files, download_exact_files=False):
        """
        Downloads rpms into a temporary directory

        Arguments:
            package_files_dict - a dict containing {pkg: file list} entries
            total_pkgs - total number of packages to download
            download_exact_files - extract only specified files

        Returns:
            RETURN_OK if all goes well.
            RETURN_FAILURE in case it cannot set up either of the directories.
        """

        installed_size = 0
        total_pkgs = 0
        todownload_size = 0
        downloaded_pkgs = 0
        # nothing to download?
        if not files:
            return RETURN_FAILURE

        #if verbose == 0:
        #    # this suppress yum messages about setting up repositories
        #    mute_stdout()

        # make yumdownloader work as non root user
        if not self.setCacheDir():
            print _("Error: can't make cachedir, exiting")
            return RETURN_FAILURE

        # disable all not needed
        for repo in self.repos.listEnabled():
            try:
                repo.close()
                self.repos.disableRepo(repo.id)
            except YumBaseError, ex:
                print _("Can't disable repository '{0!s}': {1!s}").format(repo.id, str(ex))

        # This takes some time, let user know what we are doing
        print _("Setting up yum repositories")
        # setting-up repos one-by-one, so we can skip the broken ones...
        # this helps when users are using 3rd party repos like rpmfusion
        # in rawhide it results in: Can't find valid base url...
        for r in self.repos.findRepos(pattern=self.repo_pattern):
            try:
                rid = self.repos.enableRepo(r.id)
                self.repos.doSetup(thisrepo=str(r.id))
                log1("enabled repo %s", rid)
                setattr(r, "skip_if_unavailable", True)
                # yes, we want async download, otherwise our progressCallback
                # is not called and the internal yum's one  is used,
                # which causes artifacts on output
                try:
                    setattr(r, "_async", False)
                except (NameError, AttributeError), ex:
                    print ex
                    print _("Can't disable async download, the output might contain artifacts!")
            except YumBaseError, ex:
                print _("Can't setup {0}: {1}, disabling").format(r.id, ex)
                self.repos.disableRepo(r.id)

        # This is somewhat "magic", it unpacks the metadata making it usable.
        # Looks like this is the moment when yum talks to remote servers,
        # which takes time (sometimes minutes), let user know why
        # we have "paused":
        print _("Looking for needed packages in repositories")
        try:
            self.repos.populateSack(mdtype='metadata', cacheonly=1)
        except YumBaseError, ex:
            print _("Error retrieving metadata: '{0!s}'").format(ex)
            #we don't want to die here, some metadata might be already retrieved
            # so there is a chance we already have what we need
            #return 1

        try:
            # Saw this exception here:
            # raise Errors.NoMoreMirrorsRepoError, errstr
            # NoMoreMirrorsRepoError: failure:
            # repodata/7e6632b82c91a2e88a66ad848e231f14c48259cbf3a1c3e992a77b1fc0e9d2f6-filelists.sqlite.bz2
            # from fedora-debuginfo: [Errno 256] No more mirrors to try.
            self.repos.populateSack(mdtype='filelists', cacheonly=1)
        except YumBaseError, ex:
            print _("Error retrieving filelists: '{0!s}'").format(ex)
            # we don't want to die here, some repos might be already processed
            # so there is a chance we already have what we need
            #return 1

        #if verbose == 0:
        #    # re-enable the output to stdout
        #    unmute_stdout()

        not_found = []
        package_files_dict = {}
        for debuginfo_path in files:
            log2("yum whatprovides %s", debuginfo_path)
            pkg = self.pkgSack.searchFiles(debuginfo_path)
            # sometimes one file is provided by more rpms, we can use either of
            # them, so let's use the first match
            if pkg:
                if pkg[0] in package_files_dict.keys():
                    package_files_dict[pkg[0]].append(debuginfo_path)
                else:
                    package_files_dict[pkg[0]] = [debuginfo_path]
                    todownload_size += float(pkg[0].size)
                    installed_size += float(pkg[0].installedsize)
                    total_pkgs += 1

                log2("found pkg for %s: %s", debuginfo_path, pkg[0])
            else:
                log2("not found pkg for %s", debuginfo_path)
                not_found.append(debuginfo_path)

        # connect our progress update callback
        dnlcb = MyDownloadCallback(total_pkgs)
        self.repos.setProgressBar(dnlcb)
        self.repos.setMirrorFailureCallback(downloadErrorCallback)

        if verbose != 0 or len(not_found) != 0:
            print _("Can't find packages for {0} debuginfo files").format(len(not_found))
        if verbose != 0 or total_pkgs != 0:
            print _("Packages to download: {0}").format(total_pkgs)
            question = _("Downloading {0:.2f}Mb, installed size: {1:.2f}Mb. Continue?").format(
                         todownload_size / (1024*1024),
                         installed_size / (1024*1024)
                        )
            if self.noninteractive == False and not ask_yes_no(question):
                print _("Download cancelled by user")
                return RETURN_CANCEL_BY_USER
            # set up tmp and cache dirs so that we can check free space in both
            retval = self.setup_tmp_dirs()
            if retval != RETURN_OK:
                return retval
            # check if there is enough free space in both tmp and cache
            res = os.statvfs(self.tmpdir)
            tmp_space = float(res.f_bsize * res.f_bavail) / (1024*1024)
            if (todownload_size / (1024*1024)) > tmp_space:
                question = _("Warning: Not enough free space in tmp dir '{0}'"
                             " ({1:.2f}Mb left). Continue?").format(
                    self.tmpdir, tmp_space)
                if not self.noninteractive and not ask_yes_no(question):
                    print _("Download cancelled by user")
                    return RETURN_CANCEL_BY_USER
            res = os.statvfs(self.cachedir)
            cache_space = float(res.f_bsize * res.f_bavail) / (1024*1024)
            if (installed_size / (1024*1024)) > cache_space:
                question = _("Warning: Not enough free space in cache dir "
                             "'{0}' ({1:.2f}Mb left). Continue?").format(
                    self.cachedir, cache_space)
                if not self.noninteractive and not ask_yes_no(question):
                    print _("Download cancelled by user")
                    return RETURN_CANCEL_BY_USER

        for pkg, files in package_files_dict.iteritems():
            dnlcb.downloaded_pkgs = downloaded_pkgs
            repo.cache = 0
            remote = pkg.returnSimple('relativepath')
            local = os.path.basename(remote)
            retval = self.setup_tmp_dirs()
            # continue only if the tmp dirs are ok
            if retval != RETURN_OK:
                return retval

            remote_path = pkg.returnSimple('remote_url')
            # check if the pkg is in a local repo and copy it if it is
            err = None
            if remote_path.startswith('file:///'):
                pkg_path = remote_path[7:]
                log2("copying from local repo: %s", remote)
                try:
                    shutil.copy(pkg_path, self.tmpdir)
                except OSError, ex:
                    print _("Cannot copy file '{0}': {1}").format(pkg_path, ex)
                    continue
            else:
                # pkg is in a remote repo, we need to download it to tmpdir
                local = os.path.join(self.tmpdir, local)
                pkg.localpath = local # Hack: to set the localpath we want
                err = self.downloadPkgs(pkglist=[pkg])
            # normalize the name
            # just str(pkg) doesn't work because it can have epoch
            pkg_nvra = pkg.name + "-" + pkg.version + "-" + pkg.release + "." + pkg.arch
            package_file_name = pkg_nvra + ".rpm"
            if err:
                # I observed a zero-length file left on error,
                # which prevents cleanup later. Fix it:
                try:
                    os.unlink(self.tmpdir + "/" + package_file_name)
                except OSError:
                    pass
                print (_("Downloading package {0} failed").format(pkg))
            else:
                unpack_result = unpack_rpm(package_file_name, files, self.tmpdir,
                                           self.cachedir, self.keeprpms,
                                           exact_files=download_exact_files)
                if unpack_result == RETURN_FAILURE:
                    # recursively delete the temp dir on failure
                    print _("Unpacking failed, aborting download...")
                    clean_up()
                    return RETURN_FAILURE

            downloaded_pkgs += 1

        if not self.keeprpms and os.path.exists(self.tmpdir):
            # Was: "All downloaded packages have been extracted, removing..."
            # but it was appearing even if no packages were in fact extracted
            # (say, when there was one package, and it has download error).
            print (_("Removing {0}").format(self.tmpdir))
            try:
                os.rmdir(self.tmpdir)
            except OSError:
                error_msg(_("Can't remove %s, probably contains an error log").format(self.tmpdir))

        return RETURN_OK

def build_ids_to_path(pfx, build_ids):
    """
    Transforms build ids into a path.

    build_id1=${build_id:0:2}
    build_id2=${build_id:2}
    file="usr/lib/debug/.build-id/$build_id1/$build_id2.debug"
    """

    return ["%s/usr/lib/debug/.build-id/%s/%s.debug" % (pfx, b_id[:2], b_id[2:]) for b_id in build_ids]

# beware this finds only missing libraries, but not the executable itself ..

def filter_installed_debuginfos(build_ids, cache_dirs):
    """
    Checks for installed debuginfos.

    Arguments:
        build_ids - string containing build ids
        cache_dirs - list of cache directories

    Returns:
        List of missing debuginfo files.
    """

    files = build_ids_to_path("", build_ids)
    missing = []

    # 1st pass -> search in /usr/lib
    for debuginfo_path in files:
        log2("looking: %s", debuginfo_path)
        if os.path.exists(debuginfo_path):
            log2("found: %s", debuginfo_path)
            continue
        log2("not found: %s", debuginfo_path)
        missing.append(debuginfo_path)

    if missing:
        files = missing
        missing = []
    else: # nothing is missing, we can stop looking
        return missing

    for cache_dir in cache_dirs:
        log2("looking in %s" % cache_dir)
        for debuginfo_path in files:
            cache_debuginfo_path = cache_dir + debuginfo_path
            log2("looking: %s", cache_debuginfo_path)
            if os.path.exists(cache_debuginfo_path):
                log2("found: %s", cache_debuginfo_path)
                continue
            log2("not found: %s", debuginfo_path)
            missing.append(debuginfo_path)
        # in next iteration look only for files missing
        # from previous iterations
        if missing:
            files = missing
            missing = []
        else: # nothing is missing, we can stop looking
            return missing

    return files