"""
This module provides classes and functions used to download and manage
debuginfos.
"""
import sys
import os
import time
import errno
import shutil
from subprocess import Popen
from yum import _, YumBase
from yum.callbacks import DownloadBaseCallback
from yum.Errors import YumBaseError
from reportclient import (_, log1, log2, RETURN_OK, RETURN_FAILURE,
RETURN_CANCEL_BY_USER, verbose, ask_yes_no,
error_msg)
TMPDIR=""
def ensure_abrt_uid(fn):
"""
Ensures that the function is called using abrt's uid and gid
Returns:
Either an unchanged function object or a wrapper function object for
the function.
"""
import pwd
current_uid = os.getuid()
current_gid = os.getgid()
abrt = pwd.getpwnam("abrt")
# if we're are already running as abrt, don't do anything
if abrt.pw_uid == current_uid and abrt.pw_gid == current_gid:
return fn
def wrapped(*args, **kwargs):
"""
Wrapper function around the called function.
Sets up uid and gid to match abrt's and after the function finishes
rolls its uid and gid back.
Returns:
Return value of the wrapped function.
"""
# switch to abrt
os.setegid(abrt.pw_gid)
os.seteuid(abrt.pw_uid)
# extract the files as abrt:abrt
retval = fn(*args, **kwargs)
# switch back to whatever we were
os.seteuid(current_uid)
os.setegid(current_gid)
return retval
return wrapped
# TODO: unpack just required debuginfo and not entire rpm?
# ..that can lead to: foo.c No such file and directory
# files is not used...
@ensure_abrt_uid
def unpack_rpm(package_file_name, files, tmp_dir, destdir, keeprpm, exact_files=False):
"""
Unpacks a single rpm located in tmp_dir into destdir.
Arguments:
package_file_name - name of the rpm file
files - files to extract from the rpm
tmp_dir - temporary directory where the rpm file is located
destdir - destination directory for the rpm package extraction
keeprpm - check if the user wants to delete rpms from the tmp directory
exact_files - extract only specified files
Returns:
RETURN_FAILURE in case of a serious problem
"""
package_full_path = tmp_dir + "/" + package_file_name
log1("Extracting %s to %s", package_full_path, destdir)
log2("%s", files)
print _("Extracting cpio from {0}").format(package_full_path)
unpacked_cpio_path = tmp_dir + "/unpacked.cpio"
try:
unpacked_cpio = open(unpacked_cpio_path, 'wb')
except IOError, ex:
print _("Can't write to '{0}': {1}").format(unpacked_cpio_path, ex)
return RETURN_FAILURE
rpm2cpio = Popen(["rpm2cpio", package_full_path],
stdout = unpacked_cpio, bufsize = -1)
retcode = rpm2cpio.wait()
if retcode == 0:
log1("cpio written OK")
if not keeprpm:
log1("keeprpms = False, removing %s", package_full_path)
#print _("Removing temporary rpm file")
os.unlink(package_full_path)
else:
unpacked_cpio.close()
print _("Can't extract package '{0}'").format(package_full_path)
return RETURN_FAILURE
# close the file
unpacked_cpio.close()
# and open it for reading
unpacked_cpio = open(unpacked_cpio_path, 'rb')
print _("Caching files from {0} made from {1}").format("unpacked.cpio", package_file_name)
file_patterns = ""
cpio_args = ["cpio", "-idu"]
if exact_files:
for filename in files:
file_patterns += "." + filename + " "
cpio_args = ["cpio", "-idu", file_patterns.strip()]
with open("/dev/null", "w") as null:
cpio = Popen(cpio_args, cwd=destdir, bufsize=-1,
stdin=unpacked_cpio, stdout=null, stderr=null)
retcode = cpio.wait()
if retcode == 0:
log1("files extracted OK")
#print _("Removing temporary cpio file")
os.unlink(unpacked_cpio_path)
else:
print _("Can't extract files from '{0}'").format(unpacked_cpio_path)
return RETURN_FAILURE
def clean_up():
"""
Removes the temporary directory.
"""
if TMPDIR:
try:
shutil.rmtree(TMPDIR)
except OSError, ex:
if ex.errno != errno.ENOENT:
error_msg(_("Can't remove '{0}': {1}").format(TMPDIR, ex))
class MyDownloadCallback(DownloadBaseCallback):
"""
This class serves as a download progress handler for yum's progress bar.
"""
def __init__(self, total_pkgs):
"""
Sets up instance variables
Arguments:
total_pkgs - number of packages to download
"""
self.total_pkgs = total_pkgs
self.downloaded_pkgs = 0
self.last_pct = 0
self.last_time = 0
DownloadBaseCallback.__init__(self)
def updateProgress(self, name, frac, fread, ftime):
"""
A method used to update the progress
Arguments:
name - filename
frac - progress fracment (0 -> 1)
fread - formated string containing BytesRead
ftime - formated string containing remaining or elapsed time
"""
pct = int(frac * 100)
if pct == self.last_pct:
log2("percentage is the same, not updating progress")
return
self.last_pct = pct
# if run from terminal we can have fancy output
if sys.stdout.isatty():
sys.stdout.write("\033[sDownloading (%i of %i) %s: %3u%%\033[u"
% (self.downloaded_pkgs + 1, self.total_pkgs, name, pct)
)
if pct == 100:
#print (_("Downloading (%i of %i) %s: %3u%%")
# % (self.downloaded_pkgs + 1, self.total_pkgs, name, pct)
#)
print (_("Downloading ({0} of {1}) {2}: {3:3}%").format(
self.downloaded_pkgs + 1, self.total_pkgs, name, pct
)
)
# but we want machine friendly output when spawned from abrt-server
else:
t = time.time()
if self.last_time == 0:
self.last_time = t
# update only every 5 seconds
if pct == 100 or self.last_time > t or t - self.last_time >= 5:
print (_("Downloading ({0} of {1}) {2}: {3:3}%").format(
self.downloaded_pkgs + 1, self.total_pkgs, name, pct
)
)
self.last_time = t
if pct == 100:
self.last_time = 0
sys.stdout.flush()
def downloadErrorCallback(callBackObj):
"""
A callback function for mirror errors.
"""
print _("Problem '{0!s}' occured while downloading from mirror: '{1!s}'. Trying next one").format(
callBackObj.exception, callBackObj.mirror)
# explanation of the return value can be found here:
# /usr/lib/python2.7/site-packages/urlgrabber/mirror.py
return {'fail':0}
class DebugInfoDownload(YumBase):
"""
This class is used to manage download of debuginfos.
"""
def __init__(self, cache, tmp, repo_pattern="*debug*", keep_rpms=False,
noninteractive=True):
self.old_stdout = -1
self.cachedir = cache
self.tmpdir = tmp
global TMPDIR
TMPDIR = tmp
self.keeprpms = keep_rpms
self.noninteractive = noninteractive
self.repo_pattern=repo_pattern
YumBase.__init__(self)
self.mute_stdout()
#self.conf.cache = os.geteuid() != 0
# Setup yum (Ts, RPM db, Repo & Sack)
# doConfigSetup() takes some time, let user know what we are doing
print _("Initializing yum")
try:
# Saw this exception here:
# cannot open Packages index using db3 - Permission denied (13)
# yum.Errors.YumBaseError: Error: rpmdb open failed
self.doConfigSetup()
except YumBaseError, ex:
self.unmute_stdout()
print _("Error initializing yum (YumBase.doConfigSetup): '{0!s}'").format(ex)
#return 1 - can't do this in constructor
exit(1)
self.unmute_stdout()
def mute_stdout(self):
"""
Links sys.stdout with /dev/null and saves the old stdout
"""
if verbose < 2:
self.old_stdout = sys.stdout
sys.stdout = open("/dev/null", "w")
def unmute_stdout(self):
"""
Replaces sys.stdout by stdout saved using mute
"""
if verbose < 2:
if self.old_stdout != -1:
sys.stdout = self.old_stdout
else:
print "ERR: unmute called without mute?"
@ensure_abrt_uid
def setup_tmp_dirs(self):
if not os.path.exists(self.tmpdir):
try:
os.makedirs(self.tmpdir)
except OSError, ex:
print "Can't create tmpdir: %s" % ex
return RETURN_FAILURE
if not os.path.exists(self.cachedir):
try:
os.makedirs(self.cachedir)
except OSError, ex:
print "Can't create cachedir: %s" % ex
return RETURN_FAILURE
return RETURN_OK
# return value will be used as exitcode. So 0 = ok, !0 - error
def download(self, files, download_exact_files=False):
"""
Downloads rpms into a temporary directory
Arguments:
package_files_dict - a dict containing {pkg: file list} entries
total_pkgs - total number of packages to download
download_exact_files - extract only specified files
Returns:
RETURN_OK if all goes well.
RETURN_FAILURE in case it cannot set up either of the directories.
"""
installed_size = 0
total_pkgs = 0
todownload_size = 0
downloaded_pkgs = 0
# nothing to download?
if not files:
return RETURN_FAILURE
#if verbose == 0:
# # this suppress yum messages about setting up repositories
# mute_stdout()
# make yumdownloader work as non root user
if not self.setCacheDir():
print _("Error: can't make cachedir, exiting")
return RETURN_FAILURE
# disable all not needed
for repo in self.repos.listEnabled():
try:
repo.close()
self.repos.disableRepo(repo.id)
except YumBaseError, ex:
print _("Can't disable repository '{0!s}': {1!s}").format(repo.id, str(ex))
# This takes some time, let user know what we are doing
print _("Setting up yum repositories")
# setting-up repos one-by-one, so we can skip the broken ones...
# this helps when users are using 3rd party repos like rpmfusion
# in rawhide it results in: Can't find valid base url...
for r in self.repos.findRepos(pattern=self.repo_pattern):
try:
rid = self.repos.enableRepo(r.id)
self.repos.doSetup(thisrepo=str(r.id))
log1("enabled repo %s", rid)
setattr(r, "skip_if_unavailable", True)
# yes, we want async download, otherwise our progressCallback
# is not called and the internal yum's one is used,
# which causes artifacts on output
try:
setattr(r, "_async", False)
except (NameError, AttributeError), ex:
print ex
print _("Can't disable async download, the output might contain artifacts!")
except YumBaseError, ex:
print _("Can't setup {0}: {1}, disabling").format(r.id, ex)
self.repos.disableRepo(r.id)
# This is somewhat "magic", it unpacks the metadata making it usable.
# Looks like this is the moment when yum talks to remote servers,
# which takes time (sometimes minutes), let user know why
# we have "paused":
print _("Looking for needed packages in repositories")
try:
self.repos.populateSack(mdtype='metadata', cacheonly=1)
except YumBaseError, ex:
print _("Error retrieving metadata: '{0!s}'").format(ex)
#we don't want to die here, some metadata might be already retrieved
# so there is a chance we already have what we need
#return 1
try:
# Saw this exception here:
# raise Errors.NoMoreMirrorsRepoError, errstr
# NoMoreMirrorsRepoError: failure:
# repodata/7e6632b82c91a2e88a66ad848e231f14c48259cbf3a1c3e992a77b1fc0e9d2f6-filelists.sqlite.bz2
# from fedora-debuginfo: [Errno 256] No more mirrors to try.
self.repos.populateSack(mdtype='filelists', cacheonly=1)
except YumBaseError, ex:
print _("Error retrieving filelists: '{0!s}'").format(ex)
# we don't want to die here, some repos might be already processed
# so there is a chance we already have what we need
#return 1
#if verbose == 0:
# # re-enable the output to stdout
# unmute_stdout()
not_found = []
package_files_dict = {}
for debuginfo_path in files:
log2("yum whatprovides %s", debuginfo_path)
pkg = self.pkgSack.searchFiles(debuginfo_path)
# sometimes one file is provided by more rpms, we can use either of
# them, so let's use the first match
if pkg:
if pkg[0] in package_files_dict.keys():
package_files_dict[pkg[0]].append(debuginfo_path)
else:
package_files_dict[pkg[0]] = [debuginfo_path]
todownload_size += float(pkg[0].size)
installed_size += float(pkg[0].installedsize)
total_pkgs += 1
log2("found pkg for %s: %s", debuginfo_path, pkg[0])
else:
log2("not found pkg for %s", debuginfo_path)
not_found.append(debuginfo_path)
# connect our progress update callback
dnlcb = MyDownloadCallback(total_pkgs)
self.repos.setProgressBar(dnlcb)
self.repos.setMirrorFailureCallback(downloadErrorCallback)
if verbose != 0 or len(not_found) != 0:
print _("Can't find packages for {0} debuginfo files").format(len(not_found))
if verbose != 0 or total_pkgs != 0:
print _("Packages to download: {0}").format(total_pkgs)
question = _("Downloading {0:.2f}Mb, installed size: {1:.2f}Mb. Continue?").format(
todownload_size / (1024*1024),
installed_size / (1024*1024)
)
if self.noninteractive == False and not ask_yes_no(question):
print _("Download cancelled by user")
return RETURN_CANCEL_BY_USER
# set up tmp and cache dirs so that we can check free space in both
retval = self.setup_tmp_dirs()
if retval != RETURN_OK:
return retval
# check if there is enough free space in both tmp and cache
res = os.statvfs(self.tmpdir)
tmp_space = float(res.f_bsize * res.f_bavail) / (1024*1024)
if (todownload_size / (1024*1024)) > tmp_space:
question = _("Warning: Not enough free space in tmp dir '{0}'"
" ({1:.2f}Mb left). Continue?").format(
self.tmpdir, tmp_space)
if not self.noninteractive and not ask_yes_no(question):
print _("Download cancelled by user")
return RETURN_CANCEL_BY_USER
res = os.statvfs(self.cachedir)
cache_space = float(res.f_bsize * res.f_bavail) / (1024*1024)
if (installed_size / (1024*1024)) > cache_space:
question = _("Warning: Not enough free space in cache dir "
"'{0}' ({1:.2f}Mb left). Continue?").format(
self.cachedir, cache_space)
if not self.noninteractive and not ask_yes_no(question):
print _("Download cancelled by user")
return RETURN_CANCEL_BY_USER
for pkg, files in package_files_dict.iteritems():
dnlcb.downloaded_pkgs = downloaded_pkgs
repo.cache = 0
remote = pkg.returnSimple('relativepath')
local = os.path.basename(remote)
retval = self.setup_tmp_dirs()
# continue only if the tmp dirs are ok
if retval != RETURN_OK:
return retval
remote_path = pkg.returnSimple('remote_url')
# check if the pkg is in a local repo and copy it if it is
err = None
if remote_path.startswith('file:///'):
pkg_path = remote_path[7:]
log2("copying from local repo: %s", remote)
try:
shutil.copy(pkg_path, self.tmpdir)
except OSError, ex:
print _("Cannot copy file '{0}': {1}").format(pkg_path, ex)
continue
else:
# pkg is in a remote repo, we need to download it to tmpdir
local = os.path.join(self.tmpdir, local)
pkg.localpath = local # Hack: to set the localpath we want
err = self.downloadPkgs(pkglist=[pkg])
# normalize the name
# just str(pkg) doesn't work because it can have epoch
pkg_nvra = pkg.name + "-" + pkg.version + "-" + pkg.release + "." + pkg.arch
package_file_name = pkg_nvra + ".rpm"
if err:
# I observed a zero-length file left on error,
# which prevents cleanup later. Fix it:
try:
os.unlink(self.tmpdir + "/" + package_file_name)
except OSError:
pass
print (_("Downloading package {0} failed").format(pkg))
else:
unpack_result = unpack_rpm(package_file_name, files, self.tmpdir,
self.cachedir, self.keeprpms,
exact_files=download_exact_files)
if unpack_result == RETURN_FAILURE:
# recursively delete the temp dir on failure
print _("Unpacking failed, aborting download...")
clean_up()
return RETURN_FAILURE
downloaded_pkgs += 1
if not self.keeprpms and os.path.exists(self.tmpdir):
# Was: "All downloaded packages have been extracted, removing..."
# but it was appearing even if no packages were in fact extracted
# (say, when there was one package, and it has download error).
print (_("Removing {0}").format(self.tmpdir))
try:
os.rmdir(self.tmpdir)
except OSError:
error_msg(_("Can't remove %s, probably contains an error log").format(self.tmpdir))
return RETURN_OK
def build_ids_to_path(pfx, build_ids):
"""
Transforms build ids into a path.
build_id1=${build_id:0:2}
build_id2=${build_id:2}
file="usr/lib/debug/.build-id/$build_id1/$build_id2.debug"
"""
return ["%s/usr/lib/debug/.build-id/%s/%s.debug" % (pfx, b_id[:2], b_id[2:]) for b_id in build_ids]
# beware this finds only missing libraries, but not the executable itself ..
def filter_installed_debuginfos(build_ids, cache_dirs):
"""
Checks for installed debuginfos.
Arguments:
build_ids - string containing build ids
cache_dirs - list of cache directories
Returns:
List of missing debuginfo files.
"""
files = build_ids_to_path("", build_ids)
missing = []
# 1st pass -> search in /usr/lib
for debuginfo_path in files:
log2("looking: %s", debuginfo_path)
if os.path.exists(debuginfo_path):
log2("found: %s", debuginfo_path)
continue
log2("not found: %s", debuginfo_path)
missing.append(debuginfo_path)
if missing:
files = missing
missing = []
else: # nothing is missing, we can stop looking
return missing
for cache_dir in cache_dirs:
log2("looking in %s" % cache_dir)
for debuginfo_path in files:
cache_debuginfo_path = cache_dir + debuginfo_path
log2("looking: %s", cache_debuginfo_path)
if os.path.exists(cache_debuginfo_path):
log2("found: %s", cache_debuginfo_path)
continue
log2("not found: %s", debuginfo_path)
missing.append(debuginfo_path)
# in next iteration look only for files missing
# from previous iterations
if missing:
files = missing
missing = []
else: # nothing is missing, we can stop looking
return missing
return files