Commit d39b8c19 authored by Kenneth Reitz's avatar Kenneth Reitz Committed by GitHub

basics of metrics (#374)

* basics of metrics

* pin to stdlib v2

* measure python installation size

* lots of additional metrics

* add old output methods

* $

* empty commit
parent 9a24c9d1
......@@ -15,6 +15,10 @@
# Fail fast and fail hard.
set -eo pipefail
# Standard Library.
export BPLOG_PREFIX="buildpack.python"
export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}
[ "$BUILDPACK_XTRACE" ] && set -o xtrace
# Prepend proper path for virtualenv hackery. This will be deprecated soon.
......@@ -137,10 +141,14 @@ if [[ $BUILD_DIR != '/app' ]]; then
fi
# Install Python.
let start=$(nowms)
source $BIN_DIR/steps/python
mtime "python.install.time" "${start}"
# Sanity check for setuptools/distribute.
let start=$(nowms)
source $BIN_DIR/steps/setuptools
mtime "setuptools.install.time" "${start}"
# Pipenv support.
source $BIN_DIR/steps/pipenv
......@@ -169,20 +177,28 @@ sub-env $BIN_DIR/steps/geo-libs
source $BIN_DIR/steps/gdal
# Install dependencies with Pip (where the magic happens).
let start=$(nowms)
source $BIN_DIR/steps/pip-install
mtime "pip.install.time" "${start}"
# Uninstall removed dependencies with Pip.
let start=$(nowms)
source $BIN_DIR/steps/pip-uninstall
mtime "pip.uninstall.time" "${start}"
# Support for NLTK corpora.
let start=$(nowms)
sub-env $BIN_DIR/steps/nltk
mtime "nltk.download.time" "${start}"
# Support for pip install -e.
rm -fr $BUILD_DIR/.heroku/src
deep-cp /app/.heroku/src $BUILD_DIR/.heroku/src
# Django collectstatic support.
let start=$(nowms)
sub-env $BIN_DIR/steps/collectstatic
mtime "collectstatic.time" "${start}"
# Create .profile script for application runtime environment variables.
set-env PATH '$HOME/.heroku/python/bin:$PATH'
......@@ -219,3 +235,6 @@ cp -R .heroku/vendor $CACHE_DIR/.heroku/ &> /dev/null || true
if [[ -d .heroku/src ]]; then
cp -R .heroku/src $CACHE_DIR/.heroku/ &> /dev/null || true
fi
# Measure the size of the Python installation.
mmeasure 'python.size' "$(measure-size)"
......@@ -26,7 +26,7 @@ pip-grep -s requirements.txt django Django && DJANGO_INSTALLED=1
if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALLED" ]; then
set +e
puts-cmd "python $MANAGE_FILE collectstatic --noinput"
puts-step "$ python $MANAGE_FILE collectstatic --noinput"
# Run collectstatic, cleanup some of the noisy output.
python $MANAGE_FILE collectstatic --noinput --traceback 2>&1 | sed '/^Post-processed/d;/^Copying/d;/^$/d' | indent
......
......@@ -25,6 +25,8 @@ if [ ! "$SKIP_INSTALL" ]; then
mkdir -p .heroku/python
curl https://lang-python.s3.amazonaws.com/$STACK/runtimes/$PYTHON_VERSION.tar.gz -s | tar zxv -C .heroku/python &> /dev/null
mcount "version.python.$PYTHON_VERSION"
if [[ $? != 0 ]] ; then
puts-warn "Requested runtime ($PYTHON_VERSION) is not available for this stack ($STACK)."
puts-warn "Aborting. More info: https://devcenter.heroku.com/articles/python-support"
......
#!/usr/bin/env bash
shopt -s extglob
# The standard library.
if [[ ! -f /tmp/stdlib.sh ]]; then
curl -s https://raw.githubusercontent.com/heroku/buildpack-stdlib/v2/stdlib.sh > /tmp/stdlib.sh
fi
source /tmp/stdlib.sh
if [ $(uname) == Darwin ]; then
sed() { command sed -l "$@"; }
else
......@@ -11,16 +18,12 @@ indent() {
sed "s/^/ /"
}
# Clean up pip output
cleanup() {
sed -e 's/\.\.\.\+/.../g' | sed -e '/already satisfied/Id' | sed -e '/No files were found to uninstall/Id' | sed -e '/Overwriting/Id' | sed -e '/python executable/Id' | sed -e '/no previously-included files/Id'
}
# Buildpack Indented line.
puts-line() {
echo " $@"
}
# Buildpack Steps.
puts-step() {
echo "-----> $@"
......@@ -31,28 +34,6 @@ puts-warn() {
echo " ! $@"
}
# Buildpack Commands.
puts-cmd() {
echo " $ $@"
}
# Usage: $ set-env key value
set-env() {
echo "export $1=$2" >> $PROFILE_PATH
echo "export $1=$2" >> $EXPORT_PATH
}
# Usage: $ set-default-env key value
set-default-env() {
echo "export $1=\${$1:-$2}" >> $PROFILE_PATH
echo "export $1=\${$1:-$2}" >> $EXPORT_PATH
}
# Usage: $ un-set-env key
un-set-env() {
echo "unset $1" >> $PROFILE_PATH
}
# Does some serious copying.
deep-cp() {
declare source="$1" target="$2"
......@@ -69,21 +50,6 @@ deep-cp() {
)
}
# Does some serious moving.
deep-mv() {
deep-cp "$1" "$2"
deep-rm "$1"
}
# Does some serious deleting.
deep-rm() {
# subshell to avoid surprising caller with shopts.
(
shopt -s dotglob
rm -rf "$1"/!(.curlrc|.netrc|tmp|.|..)
)
}
sub-env() {
......@@ -108,3 +74,8 @@ sub-env() {
)
}
# Measure the size of the Python installation.
measure-size() {
echo "$((du -s .heroku/python 2>/dev/null || echo 0) | awk '{print $1}')"
}
......@@ -7,6 +7,7 @@ old-platform() {
puts-warn "This caused the security warning you saw above during the 'pip install' step."
puts-warn "We recommend '$RECOMMENDED_PYTHON_VERSION', which you can specify in a 'runtime.txt' file."
puts-warn " -- Much Love, Heroku."
mcount 'warnings.python.old'
fi
}
......@@ -16,6 +17,7 @@ pylibmc-missing() {
puts-warn "Hello! There was a problem with your build related to libmemcache."
puts-warn "The Python library 'pylibmc' must be explicitly specified in 'requirements.txt' in order to build correctly."
puts-warn "Once you do that, everything should work as expected. -- Much Love, Heroku."
mcount 'warnings.libmemcache'
fi
}
......@@ -27,6 +29,7 @@ scipy-included() {
puts-warn "There is, however, a buildpack available that makes it possible to use it on Heroku."
puts-warn "You can learn more here: https://devcenter.heroku.com/articles/python-c-deps"
puts-warn "Sorry for the inconvenience. -- Much Love, Heroku."
mcount 'warnings.scipy'
fi
}
......@@ -37,6 +40,18 @@ distribute-included() {
puts-warn "This library is automatically installed by Heroku and shouldn't be in"
puts-warn "Your requirements.txt file. This can cause unexpected behavior."
puts-warn " -- Much Love, Heroku."
mcount 'warnings.distribute'
fi
}
six-included() {
if grep -qi 'Running setup.py install for six' "$WARNINGS_LOG"; then
echo
puts-warn "Hello! Your requirements.txt file contains the six package."
puts-warn "This library is automatically installed by Heroku and shouldn't be in"
puts-warn "Your requirements.txt file. This can cause unexpected behavior."
puts-warn " -- Much Love, Heroku."
mcount 'warnings.six'
fi
}
......@@ -45,5 +60,6 @@ show-warnings() {
pylibmc-missing
scipy-included
distribute-included
six-included
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment