from __future__ import with_statement
import os
import re
import platform
import time
import json
import fnmatch
import tempfile
import random
import string
from getpass import getpass
from pwd import getpwuid
from sos.utilities import (ImporterHelper,
import_module,
shell_out,
sos_get_command_output,
is_executable)
from sos.plugins import IndependentPlugin, ExperimentalPlugin
from sos import _sos as _
from sos import SoSOptions, _arg_names
from textwrap import fill
from six import print_
from six.moves import input
PRESETS_PATH = "/var/lib/sos/presets"
try:
import requests
REQUESTS_LOADED = True
except ImportError:
REQUESTS_LOADED = False
def get_human_readable(size, precision=2):
# Credit to Pavan Gupta https://stackoverflow.com/questions/5194057/
suffixes = ['B', 'KiB', 'MiB', 'GiB', 'TiB']
suffixindex = 0
while size > 1024 and suffixindex < 4:
suffixindex += 1
size = size/1024.0
return "%.*f%s" % (precision, size, suffixes[suffixindex])
def import_policy(name):
policy_fqname = "sos.policies.%s" % name
try:
return import_module(policy_fqname, Policy)
except ImportError:
return None
def load(cache={}, sysroot=None):
if 'policy' in cache:
return cache.get('policy')
import sos.policies
helper = ImporterHelper(sos.policies)
for module in helper.get_modules():
for policy in import_policy(module):
if policy.check():
cache['policy'] = policy(sysroot=sysroot)
if 'policy' not in cache:
cache['policy'] = GenericPolicy()
return cache['policy']
class InitSystem(object):
"""Encapsulates an init system to provide service-oriented functions to
sos.
This should be used to query the status of services, such as if they are
enabled or disabled on boot, or if the service is currently running.
"""
def __init__(self, init_cmd=None, list_cmd=None, query_cmd=None):
self.services = {}
self.init_cmd = init_cmd
self.list_cmd = "%s %s" % (self.init_cmd, list_cmd) or None
self.query_cmd = "%s %s" % (self.init_cmd, query_cmd) or None
def is_enabled(self, name):
"""Check if given service name is enabled """
if self.services and name in self.services:
return self.services[name]['config'] == 'enabled'
return False
def is_disabled(self, name):
"""Check if a given service name is disabled """
if self.services and name in self.services:
return self.services[name]['config'] == 'disabled'
return False
def is_service(self, name):
"""Checks if the given service name exists on the system at all, this
does not check for the service status
"""
return name in self.services
def is_running(self, name):
"""Checks if the given service name is in a running state.
This should be overridden by initsystems that subclass InitSystem
"""
# This is going to be primarily used in gating if service related
# commands are going to be run or not. Default to always returning
# True when an actual init system is not specified by policy so that
# we don't inadvertantly restrict sosreports on those systems
return True
def load_all_services(self):
"""This loads all services known to the init system into a dict.
The dict should be keyed by the service name, and contain a dict of the
name and service status
"""
pass
def _query_service(self, name):
"""Query an individual service"""
if self.query_cmd:
try:
return sos_get_command_output("%s %s" % (self.query_cmd, name))
except Exception:
return None
return None
def parse_query(self, output):
"""Parses the output returned by the query command to make a
determination of what the state of the service is
This should be overriden by anything that subclasses InitSystem
"""
return output
def get_service_names(self, regex):
"""Get a list of all services discovered on the system that match the
given regex.
"""
reg = re.compile(regex, re.I)
return [s for s in self.services.keys() if reg.match(s)]
def get_service_status(self, name):
"""Returns the status for the given service name along with the output
of the query command
"""
_default = {
'name': name,
'status': 'missing',
'output': ''
}
if name not in self.services:
return _default
if 'status' in self.services[name]:
# service status has been queried before, return existing info
return self.services[name]
svc = self._query_service(name)
if svc is not None:
self.services[name]['status'] = self.parse_query(svc['output'])
self.services[name]['output'] = svc['output']
return self.services[name]
return _default
class SystemdInit(InitSystem):
def __init__(self):
super(SystemdInit, self).__init__(
init_cmd='systemctl',
list_cmd='list-unit-files --type=service',
query_cmd='status'
)
self.load_all_services()
def parse_query(self, output):
for line in output.splitlines():
if line.strip().startswith('Active:'):
return line.split()[1]
return 'unknown'
def load_all_services(self):
svcs = shell_out(self.list_cmd).splitlines()[1:]
for line in svcs:
try:
name = line.split('.service')[0]
config = line.split()[1]
self.services[name] = {
'name': name,
'config': config
}
except IndexError:
pass
def is_running(self, name):
svc = self.get_service_status(name)
return svc['status'] == 'active'
class PackageManager(object):
"""Encapsulates a package manager. If you provide a query_command to the
constructor it should print each package on the system in the following
format::
package name|package.version
You may also subclass this class and provide a get_pkg_list method to
build the list of packages and versions.
"""
query_command = None
verify_command = None
verify_filter = None
chroot = None
files = None
def __init__(self, chroot=None, query_command=None,
verify_command=None, verify_filter=None,
files_command=None):
self.packages = {}
self.files = []
self.query_command = query_command if query_command else None
self.verify_command = verify_command if verify_command else None
self.verify_filter = verify_filter if verify_filter else None
self.files_command = files_command if files_command else None
if chroot:
self.chroot = chroot
def all_pkgs_by_name(self, name):
"""
Return a list of packages that match name.
"""
return fnmatch.filter(self.all_pkgs().keys(), name)
def all_pkgs_by_name_regex(self, regex_name, flags=0):
"""
Return a list of packages that match regex_name.
"""
reg = re.compile(regex_name, flags)
return [pkg for pkg in self.all_pkgs().keys() if reg.match(pkg)]
def pkg_by_name(self, name):
"""
Return a single package that matches name.
"""
pkgmatches = self.all_pkgs_by_name(name)
if (len(pkgmatches) != 0):
return self.all_pkgs_by_name(name)[-1]
else:
return None
def get_pkg_list(self):
"""Returns a dictionary of packages in the following
format::
{'package_name': {'name': 'package_name',
'version': 'major.minor.version'}}
"""
if self.query_command:
cmd = self.query_command
pkg_list = shell_out(
cmd, timeout=0, chroot=self.chroot
).splitlines()
for pkg in pkg_list:
if '|' not in pkg:
continue
elif pkg.count("|") == 1:
name, version = pkg.split("|")
release = None
elif pkg.count("|") == 2:
name, version, release = pkg.split("|")
self.packages[name] = {
'name': name,
'version': version.split(".")
}
release = release if release else None
self.packages[name]['release'] = release
return self.packages
def all_pkgs(self):
"""
Return a list of all packages.
"""
if not self.packages:
self.packages = self.get_pkg_list()
return self.packages
def pkg_nvra(self, pkg):
fields = pkg.split("-")
version, release, arch = fields[-3:]
name = "-".join(fields[:-3])
return (name, version, release, arch)
def all_files(self):
"""
Returns a list of files known by the package manager
"""
if self.files_command and not self.files:
cmd = self.files_command
files = shell_out(cmd, timeout=0, chroot=self.chroot)
self.files = files.splitlines()
return self.files
def build_verify_command(self, packages):
"""build_verify_command(self, packages) -> str
Generate a command to verify the list of packages given
in ``packages`` using the native package manager's
verification tool.
The command to be executed is returned as a string that
may be passed to a command execution routine (for e.g.
``sos_get_command_output()``.
:param packages: a string, or a list of strings giving
package names to be verified.
:returns: a string containing an executable command
that will perform verification of the given
packages.
:returntype: str or ``NoneType``
"""
if not self.verify_command:
return None
# The re.match(pkg) used by all_pkgs_by_name_regex() may return
# an empty list (`[[]]`) when no package matches: avoid building
# an rpm -V command line with the empty string as the package
# list in this case.
by_regex = self.all_pkgs_by_name_regex
verify_list = filter(None, map(by_regex, packages))
# No packages after regex match?
if not verify_list:
return None
verify_packages = ""
for package_list in verify_list:
for package in package_list:
if any([f in package for f in self.verify_filter]):
continue
if len(verify_packages):
verify_packages += " "
verify_packages += package
return self.verify_command + " " + verify_packages
#: Constants for on-disk preset fields
DESC = "desc"
NOTE = "note"
OPTS = "args"
class PresetDefaults(object):
"""Preset command line defaults.
"""
#: Preset name, used for selection
name = None
#: Human readable preset description
desc = None
#: Notes on preset behaviour
note = None
#: Options set for this preset
opts = SoSOptions()
#: ``True`` if this preset if built-in or ``False`` otherwise.
builtin = True
def __str__(self):
"""Return a human readable string representation of this
``PresetDefaults`` object.
"""
return ("name=%s desc=%s note=%s opts=(%s)" %
(self.name, self.desc, self.note, str(self.opts)))
def __repr__(self):
"""Return a machine readable string representation of this
``PresetDefaults`` object.
"""
return ("PresetDefaults(name='%s' desc='%s' note='%s' opts=(%s)" %
(self.name, self.desc, self.note, repr(self.opts)))
def __init__(self, name="", desc="", note=None, opts=SoSOptions()):
"""Initialise a new ``PresetDefaults`` object with the specified
arguments.
:param name: The name of the new preset
:param desc: A description for the new preset
:param note: Note for the new preset
:param opts: Options set for the new preset
:returns: The newly initialised ``PresetDefaults``
"""
self.name = name
self.desc = desc
self.note = note
self.opts = opts
def write(self, presets_path):
"""Write this preset to disk in JSON notation.
:param presets_path: the directory where the preset will be
written.
"""
if self.builtin:
raise TypeError("Cannot write built-in preset")
# Make dictionaries of PresetDefaults values
odict = self.opts.dict()
pdict = {self.name: {DESC: self.desc, NOTE: self.note, OPTS: odict}}
if not os.path.exists(presets_path):
os.makedirs(presets_path, mode=0o755)
with open(os.path.join(presets_path, self.name), "w") as pfile:
json.dump(pdict, pfile)
def delete(self, presets_path):
os.unlink(os.path.join(presets_path, self.name))
NO_PRESET = 'none'
NO_PRESET_DESC = 'Do not load a preset'
NO_PRESET_NOTE = 'Use to disable automatically loaded presets'
GENERIC_PRESETS = {
NO_PRESET: PresetDefaults(name=NO_PRESET, desc=NO_PRESET_DESC,
note=NO_PRESET_NOTE, opts=SoSOptions())
}
class Policy(object):
msg = _("""\
This command will collect system configuration and diagnostic information \
from this %(distro)s system.
For more information on %(vendor)s visit:
%(vendor_url)s
The generated archive may contain data considered sensitive and its content \
should be reviewed by the originating organization before being passed to \
any third party.
%(changes_text)s
%(vendor_text)s
""")
distro = "Unknown"
vendor = "Unknown"
vendor_url = "http://www.example.com/"
vendor_text = ""
PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
default_scl_prefix = ""
name_pattern = 'legacy'
presets = {"": PresetDefaults()}
presets_path = PRESETS_PATH
_in_container = False
_host_sysroot = '/'
def __init__(self, sysroot=None):
"""Subclasses that choose to override this initializer should call
super() to ensure that they get the required platform bits attached.
super(SubClass, self).__init__(). Policies that require runtime
tests to construct PATH must call self.set_exec_path() after
modifying PATH in their own initializer."""
self._parse_uname()
self.case_id = None
self.package_manager = PackageManager()
self._valid_subclasses = []
self.set_exec_path()
self._host_sysroot = sysroot
self.register_presets(GENERIC_PRESETS)
def get_valid_subclasses(self):
return [IndependentPlugin] + self._valid_subclasses
def set_valid_subclasses(self, subclasses):
self._valid_subclasses = subclasses
def del_valid_subclasses(self):
del self._valid_subclasses
valid_subclasses = property(get_valid_subclasses,
set_valid_subclasses,
del_valid_subclasses,
"list of subclasses that this policy can "
"process")
def check(self):
"""
This function is responsible for determining if the underlying system
is supported by this policy.
"""
return False
def in_container(self):
""" Returns True if sos is running inside a container environment.
"""
return self._in_container
def host_sysroot(self):
return self._host_sysroot
def dist_version(self):
"""
Return the OS version
"""
pass
def get_preferred_archive(self):
"""
Return the class object of the prefered archive format for this
platform
"""
from sos.archive import TarFileArchive
return TarFileArchive
def get_archive_name(self):
"""
This function should return the filename of the archive without the
extension.
This uses the policy's name_pattern attribute to determine the name.
There are two pre-defined naming patterns - 'legacy' and 'friendly'
that give names like the following:
legacy - 'sosreport-tux.123456-20171224185433'
friendly - 'sosreport-tux-mylabel-123456-2017-12-24-ezcfcop.tar.xz'
A custom name_pattern can be used by a policy provided that it
defines name_pattern using a format() style string substitution.
Usable substitutions are:
name - the short hostname of the system
label - the label given by --label
case - the case id given by --case-id or --ticker-number
rand - a random string of 7 alpha characters
Note that if a datestamp is needed, the substring should be set
in the name_pattern in the format accepted by strftime().
"""
name = self.get_local_name().split('.')[0]
case = self.case_id
label = self.commons['cmdlineopts'].label
date = ''
rand = ''.join(random.choice(string.ascii_lowercase) for x in range(7))
if self.name_pattern == 'legacy':
nstr = "sosreport-{name}{case}{date}"
case = '.' + case if case else ''
date = '-%Y%m%d%H%M%S'
elif self.name_pattern == 'friendly':
nstr = "sosreport-{name}{label}{case}{date}-{rand}"
case = '-' + case if case else ''
label = '-' + label if label else ''
date = '-%Y-%m-%d'
else:
nstr = self.name_pattern
nstr = nstr.format(
name=name,
label=label,
case=case,
date=date,
rand=rand
)
return self.sanitize_filename(time.strftime(nstr))
# for some specific binaries like "xz", we need to determine package
# providing it; that is policy specific. By default return the binary
# name itself until particular policy overwrites it
def _get_pkg_name_for_binary(self, binary):
return binary
def get_cmd_for_compress_method(self, method, threads):
cmd = method
# use fast compression if using xz or bz2
if cmd != "gzip":
cmd = "%s -2" % cmd
# determine number of threads to use for compressing - applicable
# only for xz and of version 5.2 or higher
if cmd.startswith("xz"):
try:
xz_package = self._get_pkg_name_for_binary(method)
xz_version = self.package_manager\
.all_pkgs()[xz_package]["version"]
except Exception as e:
xz_version = [u'0'] # deal like xz version is really old
if xz_version >= [u'5', u'2']:
cmd = "%s -T%d" % (cmd, threads)
return cmd
def get_tmp_dir(self, opt_tmp_dir):
if not opt_tmp_dir:
return tempfile.gettempdir()
return opt_tmp_dir
def get_default_scl_prefix(self):
return self.default_scl_prefix
def match_plugin(self, plugin_classes):
if len(plugin_classes) > 1:
for p in plugin_classes:
# Give preference to the first listed tagging class
# so that e.g. UbuntuPlugin is chosen over DebianPlugin
# on an Ubuntu installation.
if issubclass(p, self.valid_subclasses[0]):
return p
return plugin_classes[0]
def validate_plugin(self, plugin_class, experimental=False):
"""
Verifies that the plugin_class should execute under this policy
"""
valid_subclasses = [IndependentPlugin] + self.valid_subclasses
if experimental:
valid_subclasses += [ExperimentalPlugin]
return any(issubclass(plugin_class, class_) for
class_ in valid_subclasses)
def pre_work(self):
"""
This function is called prior to collection.
"""
pass
def post_work(self):
"""
This function is called after the sosreport has been generated.
"""
pass
def pkg_by_name(self, pkg):
return self.package_manager.pkg_by_name(pkg)
def _parse_uname(self):
(system, node, release,
version, machine, processor) = platform.uname()
self.system = system
self.hostname = node
self.release = release
self.smp = version.split()[1] == "SMP"
self.machine = machine
def set_commons(self, commons):
self.commons = commons
def _set_PATH(self, path):
os.environ['PATH'] = path
def set_exec_path(self):
self._set_PATH(self.PATH)
def is_root(self):
"""This method should return true if the user calling the script is
considered to be a superuser"""
return (os.getuid() == 0)
def get_preferred_hash_name(self):
"""Returns the string name of the hashlib-supported checksum algorithm
to use"""
return "md5"
def display_results(self, archive, directory, checksum, archivestat=None):
# Display results is called from the tail of SoSReport.final_work()
#
# Logging is already shutdown and all terminal output must use the
# print() call.
# make sure a report exists
if not archive and not directory:
return False
self._print()
if archive:
self._print(_("Your sosreport has been generated and saved "
"in:\n %s\n") % archive, always=True)
self._print(_(" Size\t%s") %
get_human_readable(archivestat.st_size))
self._print(_(" Owner\t%s") %
getpwuid(archivestat.st_uid).pw_name)
else:
self._print(_("Your sosreport build tree has been generated "
"in:\n %s\n") % directory, always=True)
if checksum:
self._print(" " + self.get_preferred_hash_name() + "\t" + checksum)
self._print()
self._print(_("Please send this file to your support "
"representative."))
self._print()
def _print(self, msg=None, always=False):
"""A wrapper around print that only prints if we are not running in
quiet mode"""
if always or not self.commons['cmdlineopts'].quiet:
if msg:
print_(msg)
else:
print_()
def get_msg(self):
"""This method is used to prepare the preamble text to display to
the user in non-batch mode. If your policy sets self.distro that
text will be substituted accordingly. You can also override this
method to do something more complicated."""
if self.commons['cmdlineopts'].allow_system_changes:
changes_text = "Changes CAN be made to system configuration."
else:
changes_text = "No changes will be made to system configuration."
width = 72
_msg = self.msg % {'distro': self.distro, 'vendor': self.vendor,
'vendor_url': self.vendor_url,
'vendor_text': self.vendor_text,
'tmpdir': self.commons['tmpdir'],
'changes_text': changes_text}
_fmt = ""
for line in _msg.splitlines():
_fmt = _fmt + fill(line, width, replace_whitespace=False) + '\n'
return _fmt
def register_presets(self, presets, replace=False):
"""Add new presets to this policy object.
Merges the presets dictionary ``presets`` into this ``Policy``
object, or replaces the current presets if ``replace`` is
``True``.
``presets`` should be a dictionary mapping ``str`` preset names
to ``<class PresetDefaults>`` objects specifying the command
line defaults.
:param presets: dictionary of presets to add or replace
:param replace: replace presets rather than merge new presets.
"""
if replace:
self.presets = {}
self.presets.update(presets)
def find_preset(self, preset):
"""Find a preset profile matching the specified preset string.
:param preset: a string containing a preset profile name.
:returns: a matching PresetProfile.
"""
# FIXME: allow fuzzy matching?
for match in self.presets.keys():
if match == preset:
return self.presets[match]
return None
def probe_preset(self):
"""Return a ``PresetDefaults`` object matching the runing host.
Stub method to be implemented by derived policy classes.
:returns: a ``PresetDefaults`` object.
"""
return self.presets[NO_PRESET]
def load_presets(self, presets_path=None):
"""Load presets from disk.
Read JSON formatted preset data from the specified path,
or the default location at ``/var/lib/sos/presets``.
:param presets_path: a directory containing JSON presets.
"""
presets_path = presets_path or self.presets_path
if not os.path.exists(presets_path):
return
for preset_path in os.listdir(presets_path):
preset_path = os.path.join(presets_path, preset_path)
with open(preset_path) as pf:
try:
preset_data = json.load(pf)
except ValueError:
continue
for preset in preset_data.keys():
pd = PresetDefaults(preset, opts=SoSOptions())
data = preset_data[preset]
pd.desc = data[DESC] if DESC in data else ""
pd.note = data[NOTE] if NOTE in data else ""
if OPTS in data:
for arg in _arg_names:
if arg in data[OPTS]:
setattr(pd.opts, arg, data[OPTS][arg])
pd.builtin = False
self.presets[preset] = pd
def add_preset(self, name=None, desc=None, note=None, opts=SoSOptions()):
"""Add a new on-disk preset and write it to the configured
presets path.
:param preset: the new PresetDefaults to add
"""
presets_path = self.presets_path
if not name:
raise ValueError("Preset name cannot be empty")
if name in self.presets.keys():
raise ValueError("A preset with name '%s' already exists" % name)
preset = PresetDefaults(name=name, desc=desc, note=note, opts=opts)
preset.builtin = False
self.presets[preset.name] = preset
preset.write(presets_path)
def del_preset(self, name=""):
if not name or name not in self.presets.keys():
raise ValueError("Unknown profile: '%s'" % name)
preset = self.presets[name]
if preset.builtin:
raise ValueError("Cannot delete built-in preset '%s'" %
preset.name)
preset.delete(self.presets_path)
self.presets.pop(name)
class GenericPolicy(Policy):
"""This Policy will be returned if no other policy can be loaded. This
should allow for IndependentPlugins to be executed on any system"""
def get_msg(self):
return self.msg % {'distro': self.system}
class LinuxPolicy(Policy):
"""This policy is meant to be an abc class that provides common
implementations used in Linux distros"""
distro = "Linux"
vendor = "None"
PATH = "/bin:/sbin:/usr/bin:/usr/sbin"
init = None
# _ prefixed class attrs are used for storing any vendor-defined defaults
# the non-prefixed attrs are used by the upload methods, and will be set
# to the cmdline/config file values, if provided. If not provided, then
# those attrs will be set to the _ prefixed values as a fallback.
# TL;DR Use _upload_* for policy default values, use upload_* when wanting
# to actual use the value in a method/override
_upload_url = None
_upload_directory = '/'
_upload_user = None
_upload_password = None
_upload_method = None
_preferred_hash_name = None
upload_url = None
upload_user = None
upload_password = None
def __init__(self, sysroot=None, init=None):
super(LinuxPolicy, self).__init__(sysroot=sysroot)
self.init_kernel_modules()
if init is not None:
self.init_system = init
elif os.path.isdir("/run/systemd/system/"):
self.init_system = SystemdInit()
else:
self.init_system = InitSystem()
def get_preferred_hash_name(self):
if self._preferred_hash_name:
return self._preferred_hash_name
checksum = "md5"
try:
fp = open("/proc/sys/crypto/fips_enabled", "r")
except IOError:
self._preferred_hash_name = checksum
return checksum
fips_enabled = fp.read()
if fips_enabled.find("1") >= 0:
checksum = "sha256"
fp.close()
self._preferred_hash_name = checksum
return checksum
def default_runlevel(self):
try:
with open("/etc/inittab") as fp:
pattern = r"id:(\d{1}):initdefault:"
text = fp.read()
return int(re.findall(pattern, text)[0])
except (IndexError, IOError):
return 3
def kernel_version(self):
return self.release
def host_name(self):
return self.hostname
def is_kernel_smp(self):
return self.smp
def get_arch(self):
return self.machine
def get_local_name(self):
"""Returns the name usd in the pre_work step"""
return self.host_name()
def sanitize_filename(self, name):
return re.sub(r"[^-a-z,A-Z.0-9]", "", name)
def init_kernel_modules(self):
"""Obtain a list of loaded kernel modules to reference later for plugin
enablement and SoSPredicate checks
"""
lines = shell_out("lsmod", timeout=0).splitlines()
self.kernel_mods = [line.split()[0].strip() for line in lines]
def pre_work(self):
# this method will be called before the gathering begins
cmdline_opts = self.commons['cmdlineopts']
caseid = cmdline_opts.case_id if cmdline_opts.case_id else ""
# Set the cmdline settings to the class attrs that are referenced later
# The policy default '_' prefixed versions of these are untouched to
# allow fallback
self.upload_url = cmdline_opts.upload_url
self.upload_user = cmdline_opts.upload_user
self.upload_directory = cmdline_opts.upload_directory
self.upload_password = cmdline_opts.upload_pass
self.upload_archive_name = ''
if not cmdline_opts.batch and not \
cmdline_opts.quiet:
try:
if caseid:
self.case_id = caseid
else:
self.case_id = input(_("Please enter the case id "
"that you are generating this "
"report for [%s]: ") % caseid)
# Policies will need to handle the prompts for user information
if cmdline_opts.upload or self.upload_url:
self.prompt_for_upload_user()
self.prompt_for_upload_password()
self._print()
except KeyboardInterrupt:
self._print()
raise
if cmdline_opts.case_id:
self.case_id = cmdline_opts.case_id
return
def prompt_for_upload_user(self):
"""Should be overridden by policies to determine if a user needs to
be provided or not
"""
if not self.get_upload_password() and (self.get_upload_user() !=
self._upload_user):
msg = "Please provide upload user for %s: " % self.get_upload_url()
self.upload_user = input(_(msg))
def prompt_for_upload_password(self):
"""Should be overridden by policies to determine if a password needs to
be provided for upload or not
"""
if ((not self.upload_password and not self._upload_password) and
self.upload_user):
msg = (
"Please provide the upload password for %s: "
% self.upload_user
)
self.upload_password = getpass(msg)
def upload_archive(self, archive):
"""Entry point for sos attempts to upload the generated archive to a
policy or user specified location.
Curerntly there is support for HTTPS, SFTP, and FTP. HTTPS uploads are
preferred for policy-defined defaults.
Policies that need to override uploading methods should override the
respective upload_https(), upload_sftp(), and/or upload_ftp() methods
and should NOT override this method.
In order to enable this for a policy, that policy needs to implement
the following:
Required:
Class Attrs:
_upload_url The default location to use. Note
these MUST include protocol header
_upload_user Default username, if any else None
_upload_password Default password, if any else None
Optional:
Class Attrs:
_upload_directory Default FTP server directory, if any
Methods:
prompt_for_upload_user() Determines if sos should prompt
for a username or not.
get_upload_user() Determines if the default or a
different username should be used
get_upload_https_auth() Format authentication data for
HTTPS uploads
get_upload_url_string() If you want your policy to print
a string other than the default URL
for your vendor/distro, override
this method
"""
self.upload_archive_name = archive
if not self.upload_url:
self.upload_url = self.get_upload_url()
if not self.upload_url:
raise Exception("No upload destination provided by policy or by "
"--upload-url")
upload_func = self._determine_upload_type()
print(_("Attempting upload to %s" % self.get_upload_url_string()))
return upload_func()
def _determine_upload_type(self):
"""Based on the url provided, determine what type of upload to attempt.
Note that this requires users to provide a FQDN address, such as
https://myvendor.com/api or ftp://myvendor.com instead of
myvendor.com/api or myvendor.com
"""
prots = {
'ftp': self.upload_ftp,
'sftp': self.upload_sftp,
'https': self.upload_https
}
if self.commons['cmdlineopts'].upload_protocol in prots.keys():
return prots[self.commons['cmdlineopts'].upload_protocol]
elif '://' not in self.upload_url:
raise Exception("Must provide protocol in upload URL")
prot, url = self.upload_url.split('://')
if prot not in prots.keys():
raise Exception("Unsupported or unrecognized protocol: %s" % prot)
return prots[prot]
def get_upload_https_auth(self, user=None, password=None):
"""Formats the user/password credentials using basic auth
"""
if not user:
user = self.get_upload_user()
if not password:
password = self.get_upload_password()
return requests.auth.HTTPBasicAuth(user, password)
def get_upload_url(self):
"""Helper function to determine if we should use the policy default
upload url or one provided by the user
"""
return self.upload_url or self._upload_url
def get_upload_url_string(self):
"""Used by distro policies to potentially change the string used to
report upload location from the URL to a more human-friendly string
"""
return self.get_upload_url()
def get_upload_user(self):
"""Helper function to determine if we should use the policy default
upload user or one provided by the user
"""
return self.upload_user or self._upload_user
def get_upload_password(self):
"""Helper function to determine if we should use the policy default
upload password or one provided by the user
"""
return self.upload_password or self._upload_password
def upload_sftp(self, user=None, password=None):
"""Attempts to upload the archive to an SFTP location.
Due to the lack of well maintained, secure, and generally widespread
python libraries for SFTP, sos will shell-out to the system's local ssh
installation in order to handle these uploads.
Do not override this method with one that uses python-paramiko, as the
upstream sos team will reject any PR that includes that dependency.
"""
# if we somehow don't have sftp available locally, fail early
if not is_executable('sftp'):
raise Exception('SFTP is not locally supported')
# soft dependency on python-pexpect, which we need to use to control
# sftp login since as of this writing we don't have a viable solution
# via ssh python bindings commonly available among downstreams
try:
import pexpect
except ImportError:
raise Exception('SFTP upload requires python-pexpect, which is '
'not currently installed')
sftp_connected = False
if not user:
user = self.get_upload_user()
if not password:
password = self.get_upload_password()
# need to strip the protocol prefix here
sftp_url = self.get_upload_url().replace('sftp://', '')
sftp_cmd = "sftp -oStrictHostKeyChecking=no %s@%s" % (user, sftp_url)
if int(pexpect.__version__[0]) >= 4:
# newer expect requires decoding from subprocess
ret = pexpect.spawn(sftp_cmd, encoding='utf-8')
else:
# older pexpect does not
ret = pexpect.spawn(sftp_cmd)
sftp_expects = [
u'sftp>',
u'password:',
u'Connection refused',
pexpect.TIMEOUT,
pexpect.EOF
]
idx = ret.expect(sftp_expects, timeout=15)
if idx == 0:
sftp_connected = True
elif idx == 1:
ret.sendline(password)
pass_expects = [
u'sftp>',
u'Permission denied',
pexpect.TIMEOUT,
pexpect.EOF
]
sftp_connected = ret.expect(pass_expects, timeout=10) == 0
if not sftp_connected:
ret.close()
raise Exception("Incorrect username or password for %s"
% self.get_upload_url_string())
elif idx == 2:
raise Exception("Connection refused by %s. Incorrect port?"
% self.get_upload_url_string())
elif idx == 3:
raise Exception("Timeout hit trying to connect to %s"
% self.get_upload_url_string())
elif idx == 4:
raise Exception("Unexpected error trying to connect to sftp: %s"
% ret.before)
if not sftp_connected:
ret.close()
raise Exception("Unable to connect via SFTP to %s"
% self.get_upload_url_string())
put_cmd = 'put %s %s' % (self.upload_archive_name,
self._get_sftp_upload_name())
ret.sendline(put_cmd)
put_expects = [
u'100%',
pexpect.TIMEOUT,
pexpect.EOF,
u'No such file or directory'
]
put_success = ret.expect(put_expects, timeout=180)
if put_success == 0:
ret.sendline('bye')
return True
elif put_success == 1:
raise Exception("Timeout expired while uploading")
elif put_success == 2:
raise Exception("Unknown error during upload: %s" % ret.before)
elif put_success == 3:
raise Exception(
"Unable to write archive to destination %s" % ret.before
)
else:
raise Exception("Unexpected response from server: %s" % ret.before)
def _get_sftp_upload_name(self):
"""If a specific file name pattern is required by the SFTP server,
override this method in the relevant Policy. Otherwise the archive's
name on disk will be used
:returns: Filename as it will exist on the SFTP server
:rtype: ``str``
"""
return self.upload_archive_name.split('/')[-1]
def _upload_https_put(self, archive):
"""If upload_https() needs to use requests.put(), use this method.
Policies should override this method instead of the base upload_https()
Positional arguments:
:param archive: The open archive file object
"""
return requests.put(self.get_upload_url(), data=archive,
auth=self.get_upload_https_auth())
def _get_upload_headers(self):
"""Define any needed headers to be passed with the POST request here
"""
return {}
def _upload_https_post(self, archive):
"""If upload_https() needs to use requests.post(), use this method.
Policies should override this method instead of the base upload_https()
Positional arguments:
:param archive: The open archive file object
"""
files = {
'file': (archive.name.split('/')[-1], archive,
self._get_upload_headers())
}
return requests.post(self.get_upload_url(), files=files,
auth=self.get_upload_https_auth())
def upload_https(self):
"""Attempts to upload the archive to an HTTPS location.
"""
if not REQUESTS_LOADED:
raise Exception("Unable to upload due to missing python requests "
"library")
with open(self.upload_archive_name, 'rb') as arc:
if self.commons['cmdlineopts'].upload_method == 'auto':
method = self._upload_method
else:
method = self.commons['cmdlineopts'].upload_method
if method == 'put':
r = self._upload_https_put(arc)
else:
r = self._upload_https_post(arc)
if r.status_code != 200 and r.status_code != 201:
if r.status_code == 401:
raise Exception(
"Authentication failed: invalid user credentials"
)
raise Exception("POST request returned %s: %s"
% (r.status_code, r.reason))
return True
def upload_ftp(self, url=None, directory=None, user=None, password=None):
"""Attempts to upload the archive to either the policy defined or user
provided FTP location.
"""
try:
import ftplib
import socket
except ImportError:
# socket is part of the standard library, should only fail here on
# ftplib
raise Exception("missing python ftplib library")
if not url:
url = self.get_upload_url()
if url is None:
raise Exception("no FTP server specified by policy, use --upload-"
"url to specify a location")
url = url.replace('ftp://', '')
if not user:
user = self.get_upload_user()
if not password:
password = self.get_upload_password()
if not directory:
directory = self.upload_directory or self._upload_directory
try:
session = ftplib.FTP(url, user, password, timeout=15)
if not session:
raise Exception("connection failed, did you set a user and "
"password?")
session.cwd(directory)
except socket.timeout:
raise Exception("timeout hit while connecting to %s" % url)
except socket.gaierror:
raise Exception("unable to connect to %s" % url)
except ftplib.error_perm as err:
errno = str(err).split()[0]
if errno == '503':
raise Exception("could not login as '%s'" % user)
if errno == '530':
raise Exception("invalid password for user '%s'" % user)
if errno == '550':
raise Exception("could not set upload directory to %s"
% directory)
raise Exception("error trying to establish session: %s"
% str(err))
try:
with open(self.upload_archive_name, 'rb') as _arcfile:
session.storbinary(
"STOR %s" % self.upload_archive_name.split('/')[-1],
_arcfile
)
session.quit()
return True
except IOError:
raise Exception("could not open archive file")
# vim: set et ts=4 sw=4 :