from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from future import standard_library
standard_library.install_aliases()
from past.builtins import basestring
import os
import os.path
import stat
import logging
import hashlib
from io import BytesIO as StringIO
from zipfile import ZipFile, ZIP_DEFLATED
import botocore
import formic
from troposphere.awslambda import Code
from stacker.session_cache import get_session
from stacker.util import (
get_config_directory,
ensure_s3_bucket,
)
"""Mask to retrieve only UNIX file permissions from the external attributes
field of a ZIP entry.
"""
ZIP_PERMS_MASK = (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) << 16
logger = logging.getLogger(__name__)
def _zip_files(files, root):
"""Generates a ZIP file in-memory from a list of files.
Files will be stored in the archive with relative names, and have their
UNIX permissions forced to 755 or 644 (depending on whether they are
user-executable in the source filesystem).
Args:
files (list[str]): file names to add to the archive, relative to
``root``.
root (str): base directory to retrieve files from.
Returns:
str: content of the ZIP file as a byte string.
str: A calculated hash of all the files.
"""
zip_data = StringIO()
with ZipFile(zip_data, 'w', ZIP_DEFLATED) as zip_file:
for fname in files:
zip_file.write(os.path.join(root, fname), fname)
# Fix file permissions to avoid any issues - only care whether a file
# is executable or not, choosing between modes 755 and 644 accordingly.
for zip_entry in zip_file.filelist:
perms = (zip_entry.external_attr & ZIP_PERMS_MASK) >> 16
if perms & stat.S_IXUSR != 0:
new_perms = 0o755
else:
new_perms = 0o644
if new_perms != perms:
logger.debug("lambda: fixing perms: %s: %o => %o",
zip_entry.filename, perms, new_perms)
new_attr = ((zip_entry.external_attr & ~ZIP_PERMS_MASK) |
(new_perms << 16))
zip_entry.external_attr = new_attr
contents = zip_data.getvalue()
zip_data.close()
content_hash = _calculate_hash(files, root)
return contents, content_hash
def _calculate_hash(files, root):
""" Returns a hash of all of the given files at the given root.
Args:
files (list[str]): file names to include in the hash calculation,
relative to ``root``.
root (str): base directory to analyze files in.
Returns:
str: A hash of the hashes of the given files.
"""
file_hash = hashlib.md5()
for fname in sorted(files):
f = os.path.join(root, fname)
file_hash.update((fname + "\0").encode())
with open(f, "rb") as fd:
for chunk in iter(lambda: fd.read(4096), ""):
if not chunk:
break
file_hash.update(chunk)
file_hash.update("\0".encode())
return file_hash.hexdigest()
def _calculate_prebuilt_hash(f):
file_hash = hashlib.md5()
while True:
chunk = f.read(4096)
if not chunk:
break
file_hash.update(chunk)
return file_hash.hexdigest()
def _find_files(root, includes, excludes, follow_symlinks):
"""List files inside a directory based on include and exclude rules.
This is a more advanced version of `glob.glob`, that accepts multiple
complex patterns.
Args:
root (str): base directory to list files from.
includes (list[str]): inclusion patterns. Only files matching those
patterns will be included in the result.
excludes (list[str]): exclusion patterns. Files matching those
patterns will be excluded from the result. Exclusions take
precedence over inclusions.
follow_symlinks (bool): If true, symlinks will be included in the
resulting zip file
Yields:
str: a file name relative to the root.
Note:
Documentation for the patterns can be found at
http://www.aviser.asia/formic/doc/index.html
"""
root = os.path.abspath(root)
file_set = formic.FileSet(
directory=root, include=includes,
exclude=excludes, symlinks=follow_symlinks,
)
for filename in file_set.qualified_files(absolute=False):
yield filename
def _zip_from_file_patterns(root, includes, excludes, follow_symlinks):
"""Generates a ZIP file in-memory from file search patterns.
Args:
root (str): base directory to list files from.
includes (list[str]): inclusion patterns. Only files matching those
patterns will be included in the result.
excludes (list[str]): exclusion patterns. Files matching those
patterns will be excluded from the result. Exclusions take
precedence over inclusions.
follow_symlinks (bool): If true, symlinks will be included in the
resulting zip file
See Also:
:func:`_zip_files`, :func:`_find_files`.
Raises:
RuntimeError: when the generated archive would be empty.
"""
logger.info('lambda: base directory: %s', root)
files = list(_find_files(root, includes, excludes, follow_symlinks))
if not files:
raise RuntimeError('Empty list of files for Lambda payload. Check '
'your include/exclude options for errors.')
logger.info('lambda: adding %d files:', len(files))
for fname in files:
logger.debug('lambda: + %s', fname)
return _zip_files(files, root)
def _head_object(s3_conn, bucket, key):
"""Retrieve information about an object in S3 if it exists.
Args:
s3_conn (botocore.client.S3): S3 connection to use for operations.
bucket (str): name of the bucket containing the key.
key (str): name of the key to lookup.
Returns:
dict: S3 object information, or None if the object does not exist.
See the AWS documentation for explanation of the contents.
Raises:
botocore.exceptions.ClientError: any error from boto3 other than key
not found is passed through.
"""
try:
return s3_conn.head_object(Bucket=bucket, Key=key)
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == '404':
return None
else:
raise
def _upload_code(s3_conn, bucket, prefix, name, contents, content_hash,
payload_acl):
"""Upload a ZIP file to S3 for use by Lambda.
The key used for the upload will be unique based on the checksum of the
contents. No changes will be made if the contents in S3 already match the
expected contents.
Args:
s3_conn (botocore.client.S3): S3 connection to use for operations.
bucket (str): name of the bucket to create.
prefix (str): S3 prefix to prepend to the constructed key name for
the uploaded file
name (str): desired name of the Lambda function. Will be used to
construct a key name for the uploaded file.
contents (str): byte string with the content of the file upload.
content_hash (str): md5 hash of the contents to be uploaded.
payload_acl (str): The canned S3 object ACL to be applied to the
uploaded payload
Returns:
troposphere.awslambda.Code: CloudFormation Lambda Code object,
pointing to the uploaded payload in S3.
Raises:
botocore.exceptions.ClientError: any error from boto3 is passed
through.
"""
logger.debug('lambda: ZIP hash: %s', content_hash)
key = '{}lambda-{}-{}.zip'.format(prefix, name, content_hash)
if _head_object(s3_conn, bucket, key):
logger.info('lambda: object %s already exists, not uploading', key)
else:
logger.info('lambda: uploading object %s', key)
s3_conn.put_object(Bucket=bucket, Key=key, Body=contents,
ContentType='application/zip',
ACL=payload_acl)
return Code(S3Bucket=bucket, S3Key=key)
def _check_pattern_list(patterns, key, default=None):
"""Validates file search patterns from user configuration.
Acceptable input is a string (which will be converted to a singleton list),
a list of strings, or anything falsy (such as None or an empty dictionary).
Empty or unset input will be converted to a default.
Args:
patterns: input from user configuration (YAML).
key (str): name of the configuration key the input came from,
used for error display purposes.
Keyword Args:
default: value to return in case the input is empty or unset.
Returns:
list[str]: validated list of patterns
Raises:
ValueError: if the input is unacceptable.
"""
if not patterns:
return default
if isinstance(patterns, basestring):
return [patterns]
if isinstance(patterns, list):
if all(isinstance(p, basestring) for p in patterns):
return patterns
raise ValueError("Invalid file patterns in key '{}': must be a string or "
'list of strings'.format(key))
def _upload_prebuilt_zip(s3_conn, bucket, prefix, name, options, path,
payload_acl):
logging.debug('lambda: using prebuilt ZIP %s', path)
with open(path, 'rb') as zip_file:
# Default to the MD5 of the ZIP if no explicit version is provided
version = options.get('version')
if not version:
version = _calculate_prebuilt_hash(zip_file)
zip_file.seek(0)
return _upload_code(s3_conn, bucket, prefix, name, zip_file,
version, payload_acl)
def _build_and_upload_zip(s3_conn, bucket, prefix, name, options, path,
follow_symlinks, payload_acl):
includes = _check_pattern_list(options.get('include'), 'include',
default=['**'])
excludes = _check_pattern_list(options.get('exclude'), 'exclude',
default=[])
# os.path.join will ignore other parameters if the right-most one is an
# absolute path, which is exactly what we want.
zip_contents, zip_version = _zip_from_file_patterns(
path, includes, excludes, follow_symlinks)
version = options.get('version') or zip_version
return _upload_code(s3_conn, bucket, prefix, name, zip_contents, version,
payload_acl)
def _upload_function(s3_conn, bucket, prefix, name, options, follow_symlinks,
payload_acl):
"""Builds a Lambda payload from user configuration and uploads it to S3.
Args:
s3_conn (botocore.client.S3): S3 connection to use for operations.
bucket (str): name of the bucket to upload to.
prefix (str): S3 prefix to prepend to the constructed key name for
the uploaded file
name (str): desired name of the Lambda function. Will be used to
construct a key name for the uploaded file.
options (dict): configuration for how to build the payload.
Consists of the following keys:
* path:
base path to retrieve files from (mandatory). If not
absolute, it will be interpreted as relative to the stacker
configuration file directory, then converted to an absolute
path. See :func:`stacker.util.get_config_directory`.
* include:
file patterns to include in the payload (optional).
* exclude:
file patterns to exclude from the payload (optional).
follow_symlinks (bool): If true, symlinks will be included in the
resulting zip file
payload_acl (str): The canned S3 object ACL to be applied to the
uploaded payload
Returns:
troposphere.awslambda.Code: CloudFormation AWS Lambda Code object,
pointing to the uploaded object in S3.
Raises:
ValueError: if any configuration is invalid.
botocore.exceptions.ClientError: any error from boto3 is passed
through.
"""
try:
path = os.path.expanduser(options['path'])
except KeyError as e:
raise ValueError(
"missing required property '{}' in function '{}'".format(
e.args[0], name))
if not os.path.isabs(path):
path = os.path.abspath(os.path.join(get_config_directory(), path))
if path.endswith('.zip') and os.path.isfile(path):
logging.debug('lambda: using prebuilt zip: %s', path)
return _upload_prebuilt_zip(s3_conn, bucket, prefix, name, options,
path, payload_acl)
elif os.path.isdir(path):
logging.debug('lambda: building from directory: %s', path)
return _build_and_upload_zip(s3_conn, bucket, prefix, name, options,
path, follow_symlinks, payload_acl)
else:
raise ValueError('Path must be an existing ZIP file or directory')
[docs]def select_bucket_region(custom_bucket, hook_region, stacker_bucket_region,
provider_region):
"""Returns the appropriate region to use when uploading functions.
Select the appropriate region for the bucket where lambdas are uploaded in.
Args:
custom_bucket (str, None): The custom bucket name provided by the
`bucket` kwarg of the aws_lambda hook, if provided.
hook_region (str): The contents of the `bucket_region` argument to
the hook.
stacker_bucket_region (str): The contents of the
`stacker_bucket_region` global setting.
provider_region (str): The region being used by the provider.
Returns:
str: The appropriate region string.
"""
region = None
if custom_bucket:
region = hook_region
else:
region = stacker_bucket_region
return region or provider_region
[docs]def upload_lambda_functions(context, provider, **kwargs):
"""Builds Lambda payloads from user configuration and uploads them to S3.
Constructs ZIP archives containing files matching specified patterns for
each function, uploads the result to Amazon S3, then stores objects (of
type :class:`troposphere.awslambda.Code`) in the context's hook data,
ready to be referenced in blueprints.
Configuration consists of some global options, and a dictionary of function
specifications. In the specifications, each key indicating the name of the
function (used for generating names for artifacts), and the value
determines what files to include in the ZIP (see more details below).
Payloads are uploaded to either a custom bucket or stackers default bucket,
with the key containing it's checksum, to allow repeated uploads to be
skipped in subsequent runs.
The configuration settings are documented as keyword arguments below.
Keyword Arguments:
bucket (str, optional): Custom bucket to upload functions to.
Omitting it will cause the default stacker bucket to be used.
bucket_region (str, optional): The region in which the bucket should
exist. If not given, the region will be either be that of the
global `stacker_bucket_region` setting, or else the region in
use by the provider.
prefix (str, optional): S3 key prefix to prepend to the uploaded
zip name.
follow_symlinks (bool, optional): Will determine if symlinks should
be followed and included with the zip artifact. Default: False
payload_acl (str, optional): The canned S3 object ACL to be applied to
the uploaded payload. Default: private
functions (dict):
Configurations of desired payloads to build. Keys correspond to
function names, used to derive key names for the payload. Each
value should itself be a dictionary, with the following data:
* path (str):
Base directory or path of a ZIP file of the Lambda function
payload content.
If it not an absolute path, it will be considered relative
to the directory containing the stacker configuration file
in use.
When a directory, files contained will be added to the
payload ZIP, according to the include and exclude patterns.
If not patterns are provided, all files in the directory
(respecting default exclusions) will be used.
Files are stored in the archive with path names relative to
this directory. So, for example, all the files contained
directly under this directory will be added to the root of
the ZIP file.
When a ZIP file, it will be uploaded directly to S3.
The hash of whole ZIP file will be used as the version key
by default, which may cause spurious rebuilds when building
the ZIP in different environments. To avoid that,
explicitly provide a `version` option.
* include(str or list[str], optional):
Pattern or list of patterns of files to include in the
payload. If provided, only files that match these
patterns will be included in the payload.
Omitting it is equivalent to accepting all files that are
not otherwise excluded.
* exclude(str or list[str], optional):
Pattern or list of patterns of files to exclude from the
payload. If provided, any files that match will be ignored,
regardless of whether they match an inclusion pattern.
Commonly ignored files are already excluded by default,
such as ``.git``, ``.svn``, ``__pycache__``, ``*.pyc``,
``.gitignore``, etc.
* version(str, optional):
Value to use as the version for the current function, which
will be used to determine if a payload already exists in
S3. The value can be any string, such as a version number
or a git commit.
Note that when setting this value, to re-build/re-upload a
payload you must change the version manually.
Examples:
.. Hook configuration.
.. code-block:: yaml
pre_build:
- path: stacker.hooks.aws_lambda.upload_lambda_functions
required: true
enabled: true
data_key: lambda
args:
bucket: custom-bucket
follow_symlinks: true
prefix: cloudformation-custom-resources/
payload_acl: authenticated-read
functions:
MyFunction:
path: ./lambda_functions
include:
- '*.py'
- '*.txt'
exclude:
- '*.pyc'
- test/
.. Blueprint usage
.. code-block:: python
from troposphere.awslambda import Function
from stacker.blueprints.base import Blueprint
class LambdaBlueprint(Blueprint):
def create_template(self):
code = self.context.hook_data['lambda']['MyFunction']
self.template.add_resource(
Function(
'MyFunction',
Code=code,
Handler='my_function.handler',
Role='...',
Runtime='python2.7'
)
)
"""
custom_bucket = kwargs.get('bucket')
if not custom_bucket:
bucket_name = context.bucket_name
logger.info("lambda: using default bucket from stacker: %s",
bucket_name)
else:
bucket_name = custom_bucket
logger.info("lambda: using custom bucket: %s", bucket_name)
custom_bucket_region = kwargs.get("bucket_region")
if not custom_bucket and custom_bucket_region:
raise ValueError("Cannot specify `bucket_region` without specifying "
"`bucket`.")
bucket_region = select_bucket_region(
custom_bucket,
custom_bucket_region,
context.config.stacker_bucket_region,
provider.region
)
# Check if we should walk / follow symlinks
follow_symlinks = kwargs.get('follow_symlinks', False)
if not isinstance(follow_symlinks, bool):
raise ValueError('follow_symlinks option must be a boolean')
# Check for S3 object acl. Valid values from:
# https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
payload_acl = kwargs.get('payload_acl', 'private')
# Always use the global client for s3
session = get_session(bucket_region)
s3_client = session.client('s3')
ensure_s3_bucket(s3_client, bucket_name, bucket_region)
prefix = kwargs.get('prefix', '')
results = {}
for name, options in kwargs['functions'].items():
results[name] = _upload_function(s3_client, bucket_name, prefix, name,
options, follow_symlinks, payload_acl)
return results