# Copyright (c) 2014-2016 Hewlett Packard Enterprise Development LP
#
# Redistribution and use of this software in source and binary forms,
# with or without modification, are permitted provided that the following
# conditions are met:
#
# Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import contextlib
import hashlib
import os
import shutil
import signal
import subprocess
import tarfile
import tempfile
import euca2ools
from euca2ools.bundle.util import open_pipe_fileobjs
from euca2ools.commands.euimage.pack.metadata import (ImagePackMetadata,
ImageMetadata)
IMAGE_ARCNAME = 'image.xz'
IMAGE_MD_ARCNAME = 'image-md.yml'
PACK_MD_ARCNAME = 'pack-md.yml'
class ImagePack(object):
def __init__(self, filename=None):
self.pack_md = None
self.image_md = None
self.filename = filename
self.__tarball = None
@classmethod
def open(cls, filename):
with ImagePack(filename=filename) as pack:
member = pack.__tarball.getmember(PACK_MD_ARCNAME)
with contextlib.closing(pack.__tarball.extractfile(member)) \
as md_file:
pack.pack_md = ImagePackMetadata.from_fileobj(md_file)
member = pack.__tarball.getmember(IMAGE_MD_ARCNAME)
with contextlib.closing(pack.__tarball.extractfile(member)) \
as md_file:
pack.image_md = ImageMetadata.from_fileobj(md_file)
md_file.seek(0)
image_md_sha256sum = hashlib.sha256(md_file.read()).hexdigest()
if image_md_sha256sum != pack.pack_md.image_md_sha256sum:
raise RuntimeError('image metadata appears to be corrupt '
'(expected SHA256: {0}, actual: {1})',
pack.pack_md.image_md_sha256sum,
image_md_sha256sum)
return pack
@classmethod
def build(cls, image_md_filename, image_filename,
destdir='', progressbar=None):
pack = ImagePack()
pack.image_md = ImageMetadata.from_file(image_md_filename)
pack.pack_md = ImagePackMetadata()
if destdir != '' and not os.path.isdir(destdir):
raise ValueError('"{0}" is not a directory'.format(destdir))
pack.filename = os.path.join(destdir, '{0}.euimage'.format(
pack.image_md.get_nvra()))
with open(image_md_filename) as image_md_file:
digest = hashlib.sha256(image_md_file.read())
pack.pack_md.image_md_sha256sum = digest.hexdigest()
# Since we have to know the size of the compressed image ahead
# of time to write tarinfo headers we have to spool the whole
# thing to disk. :-\
with tempfile.NamedTemporaryFile() as compressed_image:
# Feed stuff to a subprocess to checksum and compress in one pass
digest = hashlib.sha256()
bytes_read = 0
with open(image_filename, 'rb') as original_image:
xz_proc = subprocess.Popen(('xz', '-c'), stdin=subprocess.PIPE,
stdout=compressed_image)
if progressbar:
progressbar.start()
while True:
chunk = original_image.read(euca2ools.BUFSIZE)
if not chunk:
break
digest.update(chunk)
xz_proc.stdin.write(chunk)
bytes_read += len(chunk)
if progressbar:
progressbar.update(bytes_read)
xz_proc.stdin.close()
xz_proc.wait()
if progressbar:
progressbar.finish()
pack.pack_md.image_sha256sum = digest.hexdigest()
pack.pack_md.image_size = bytes_read
# Write metadata and pack everything up
with contextlib.closing(tarfile.open(pack.filename, 'w',
dereference=True)) as tarball:
with tempfile.NamedTemporaryFile() as pack_md_file:
pack.pack_md.dump_to_fileobj(pack_md_file)
tarball.add(pack_md_file.name, arcname=PACK_MD_ARCNAME)
tarball.add(image_md_filename, arcname=IMAGE_MD_ARCNAME)
tarball.add(compressed_image.name, arcname=IMAGE_ARCNAME)
return pack
def close(self):
if self.__tarball:
self.__tarball.close()
self.__tarball = None
def __enter__(self):
assert self.filename
self.__tarball = tarfile.open(name=self.filename, mode='r')
return self
def __exit__(self, type_, value, tbk):
self.close()
def open_image(self):
"""
Return a file-like object that transparently yields the packed image.
"""
assert self.filename
with contextlib.closing(tarfile.open(name=self.filename, mode='r')) \
as tarball:
# This looks like it will return a file handle that will run out of
# data as soon as we leave this with block, but since what we
# return actually uses the read end of an os.pipe that reads from a
# forked process things should Just Work (tm).
return _PackedImageWrapper(tarball)
class _PackedImageWrapper(object):
"""
A file-like object that transparently unpacks and decompresses the
image from an image pack
"""
def __init__(self, tarball):
"""
This method takes a tarfile.TarFile object and spawns *two* new
processes: an xz process for decompression and an additional
python process that simply feeds data from the TarFile to it.
The latter is necessary because the file-like object we get from
TarFile.extractfile cannot be passed to a subprocess directly.
For that reason, one is also free to close the tarball after
this object is created.
"""
self.__subp_pid = None
self.__read_fh = None
member = tarball.getmember(IMAGE_ARCNAME)
compressed_image = tarball.extractfile(member)
pipe_r, pipe_w = open_pipe_fileobjs()
self.__subp_pid = os.fork()
if self.__subp_pid == 0:
os.setpgrp()
pipe_r.close()
self.__xz_proc = subprocess.Popen(
('xz', '-d'), stdin=subprocess.PIPE, stdout=pipe_w,
close_fds=True)
pipe_w.close()
shutil.copyfileobj(compressed_image, self.__xz_proc.stdin)
self.__xz_proc.stdin.close()
self.__xz_proc.wait()
os._exit(os.EX_OK)
else:
self.__read_fh = pipe_r
def close(self):
if self.__subp_pid:
# Kill the process group
os.kill(-os.getpgid(self.__subp_pid), signal.SIGTERM)
self.__read_fh.close()
else:
os._exit(os.EX_OK)
def __enter__(self):
return self
def __exit__(self, type_, value, tbk):
self.close()
def read(self, size=-1):
return self.__read_fh.read(size)