# Copyright 2013 Eucalyptus Systems, Inc.
#
# Redistribution and use of this software in source and binary forms,
# with or without modification, are permitted provided that the following
# conditions are met:
#
# Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import hashlib
import itertools
import multiprocessing
import os
import sys
import euca2ools.bundle.pipes
import euca2ools.bundle.util
def create_bundle_part_deleter(in_mpconn, out_mpconn=None):
del_p = multiprocessing.Process(target=_delete_part_files,
args=(in_mpconn,),
kwargs={'out_mpconn': out_mpconn})
del_p.start()
euca2ools.bundle.util.waitpid_in_thread(del_p.pid)
def create_bundle_part_writer(infile, part_prefix, part_size,
part_write_sem=None, debug=False):
partinfo_result_r, partinfo_result_w = multiprocessing.Pipe(duplex=False)
writer_p = multiprocessing.Process(
target=_write_parts,
args=(infile, part_prefix, part_size, partinfo_result_w),
kwargs={'part_write_sem': part_write_sem, 'debug': debug})
writer_p.start()
partinfo_result_w.close()
infile.close()
euca2ools.bundle.util.waitpid_in_thread(writer_p.pid)
return partinfo_result_r
def create_mpconn_aggregator(in_mpconn, out_mpconn=None, debug=False):
result_mpconn_r, result_mpconn_w = multiprocessing.Pipe(duplex=False)
agg_p = multiprocessing.Process(
target=_aggregate_mpconn_items, args=(in_mpconn, result_mpconn_w),
kwargs={'out_mpconn': out_mpconn, 'debug': debug})
agg_p.start()
result_mpconn_w.close()
euca2ools.bundle.util.waitpid_in_thread(agg_p.pid)
return result_mpconn_r
def _delete_part_files(in_mpconn, out_mpconn=None):
euca2ools.bundle.util.close_all_fds(except_fds=(in_mpconn, out_mpconn))
try:
while True:
part = in_mpconn.recv()
os.unlink(part.filename)
if out_mpconn is not None:
out_mpconn.send(part)
except EOFError:
return
finally:
in_mpconn.close()
if out_mpconn is not None:
out_mpconn.close()
def _aggregate_mpconn_items(in_mpconn, result_mpconn, out_mpconn=None,
debug=False):
euca2ools.bundle.util.close_all_fds(
except_fds=(in_mpconn, out_mpconn, result_mpconn))
results = []
try:
while True:
next_result = in_mpconn.recv()
results.append(next_result)
if out_mpconn is not None:
out_mpconn.send(next_result)
except EOFError:
try:
result_mpconn.send(results)
except IOError:
# HACK
if not debug:
return
raise
except IOError:
# HACK
if not debug:
return
raise
finally:
result_mpconn.close()
in_mpconn.close()
if out_mpconn is not None:
out_mpconn.close()
def _write_parts(infile, part_prefix, part_size, partinfo_mpconn,
part_write_sem=None, debug=False):
except_fds = [infile, partinfo_mpconn]
if part_write_sem is not None and sys.platform == 'darwin':
# When I ran close_all_fds on OS X and excluded only the FDs
# listed above, all attempts to use the semaphore resulted in
# complaints about bad file descriptors. The following code
# is a horrible hack that I stumbled upon while attempting
# to figure out what FD number I needed to avoid closing to
# preserve the semaphore. It is probably incorrect and reliant
# on implementation details, so I am happy to take a patch that
# manages to deal with this problem in a more reasonable way.
try:
except_fds.append(int(part_write_sem._semlock.handle))
except AttributeError:
part_write_sem = None
except ValueError:
part_write_sem = None
euca2ools.bundle.util.close_all_fds(except_fds=except_fds)
for part_no in itertools.count():
if part_write_sem is not None:
part_write_sem.acquire()
part_fname = '{0}.part.{1:02}'.format(part_prefix, part_no)
part_digest = hashlib.sha1()
with open(part_fname, 'w') as part:
bytes_written = 0
bytes_to_write = part_size
while bytes_to_write > 0:
try:
chunk = infile.read(min(bytes_to_write, euca2ools.BUFSIZE))
except ValueError: # I/O error on closed file
# HACK
if not debug:
partinfo_mpconn.close()
return
raise
if chunk:
part.write(chunk)
part_digest.update(chunk)
bytes_to_write -= len(chunk)
bytes_written += len(chunk)
else:
break
partinfo = euca2ools.bundle.BundlePart(
part_fname, part_digest.hexdigest(), 'SHA1', bytes_written)
partinfo_mpconn.send(partinfo)
if bytes_written < part_size:
# That's the last part
infile.close()
partinfo_mpconn.close()
return