Unverified Commit fe5da9a0 authored by Casey's avatar Casey Committed by GitHub

Merge branch 'master' into update-sqlite-tests

parents 0c78978e 4eed56e2
......@@ -4,7 +4,6 @@
- Sqlite3 Update:
- Add Tests
- Refactor: use variable rather than hardcoded /app
- Bug fix: pipenv no longer installs twice on CI
--------------------------------------------------------------------------------
......
......@@ -90,7 +90,6 @@ export PATH=$PATH:$ROOT_DIR/vendor/:$ROOT_DIR/vendor/pip-pop
unset GIT_DIR PYTHONHOME PYTHONPATH
unset RECEIVE_DATA RUN_KEY BUILD_INFO DEPLOY LOG_TOKEN
unset CYTOKINE_LOG_FILE GEM_PATH
export PYTHONPATH="$BUILD_DIR"
# Import the utils script, which contains helper functions used throughout the buildpack.
# shellcheck source=bin/utils
......@@ -105,7 +104,7 @@ source "$BIN_DIR/warnings"
# to `/app`.
# Symlinks are required, since Python is not a portable installation.
# More on this topic later.
mkdir -p "BUILD_DIR/.heroku"
mkdir -p /app/.heroku
# This buildpack programatically generates (or simply copies) a number of files for
# buildpack machinery: an export script, and a number of `.profile.d` scripts. This
......@@ -123,7 +122,7 @@ export BUILD_DIR CACHE_DIR BIN_DIR PROFILE_PATH EXPORT_PATH
# Notes on each variable included.
# PATH is relatively obvious, we need to be able to execute 'python'.
export PATH="$BUILD_DIR/.heroku/python/bin:$BUILD_DIR/.heroku/vendor/bin:$PATH"
export PATH=/app/.heroku/python/bin:/app/.heroku/vendor/bin:$PATH
# Tell Python to not buffer it's stdin/stdout.
export PYTHONUNBUFFERED=1
# Set the locale to a well-known and expected standard.
......@@ -131,11 +130,11 @@ export LANG=en_US.UTF-8
# `~/.heroku/vendor` is an place where the buildpack may stick pre-build binaries for known
# C dependencies (e.g. libmemcached on cedar-14). This section configures Python (GCC, more specifically)
# and pip to automatically include these paths when building binaries.
export C_INCLUDE_PATH="$BUILD_DIR/.heroku/vendor/include:$BUILD_DIR/.heroku/python/include:$C_INCLUDE_PATH"
export CPLUS_INCLUDE_PATH="$BUILD_DIR/.heroku/vendor/include:$BUILD_DIR/.heroku/python/include:$CPLUS_INCLUDE_PATH"
export LIBRARY_PATH="$BUILD_DIR/.heroku/vendor/lib:$BUILD_DIR/.heroku/python/lib:$LIBRARY_PATH"
export LD_LIBRARY_PATH="$BUILD_DIR/.heroku/vendor/lib:$BUILD_DIR/.heroku/python/lib:$LD_LIBRARY_PATH"
export PKG_CONFIG_PATH="$BUILD_DIR/.heroku/vendor/lib/pkg-config:$BUILD_DIR/.heroku/python/lib/pkg-config:$PKG_CONFIG_PATH"
export C_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$C_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$CPLUS_INCLUDE_PATH
export LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PATH
export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH
export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config:$PKG_CONFIG_PATH
# The Application Code
# --------------------
......@@ -212,16 +211,15 @@ fi
# Create the directory for .profile.d, if it doesn't exist.
mkdir -p "$(dirname "$PROFILE_PATH")"
# Create the directory for editable source code installation, if it doesn't exist.
mkdir -p "$BUILD_DIR/.heroku/src"
mkdir -p /app/.heroku/src
# On Heroku CI, builds happen in `/app`. Otherwise, on the Heroku platform,
# they occur in a temp directory. Beacuse Python is not portable, we must create
# symlinks to emulate that we are operating in `/app` during the build process.
# This is (hopefully obviously) because apps end up running from `/app` in production.
if [[ "$BUILD_DIR" != '/app' ]]; then
if [[ $BUILD_DIR != '/app' ]]; then
# python expects to reside in /app, so set up symlinks
# we will not remove these later so subsequent buildpacks can still invoke it
mkdir -p /app/.heroku
ln -nsf "$BUILD_DIR/.heroku/python" /app/.heroku/python
ln -nsf "$BUILD_DIR/.heroku/vendor" /app/.heroku/vendor
# Note: .heroku/src is copied in later.
......@@ -312,7 +310,8 @@ mtime "nltk.download.time" "${start}"
# and copying it into the proper place (the logical place to do this was early, but it must be done here).
# In CI, $BUILD_DIR is /app.
if [[ ! "$BUILD_DIR" == "/app" ]]; then
ln -nsf "$BUILD_DIR/.heroku/src" /app/.heroku/src
rm -fr "$BUILD_DIR/.heroku/src"
deep-cp /app/.heroku/src "$BUILD_DIR/.heroku/src"
fi
......@@ -344,12 +343,13 @@ set_default_env PYTHONHASHSEED random
# Tell Python to look for Python modules in the /app dir. Don't change this.
set_default_env PYTHONPATH "\$HOME"
# Python expects to be in "$BUILD_DIR," if at runtime, it is not, set
# Python expects to be in /app, if at runtime, it is not, set
# up symlinks… this can occur when the subdir buildpack is used.
cat <<EOT >> "$PROFILE_PATH"
if [[ \$HOME != "$BUILD_DIR" ]]; then
ln -nsf "\$HOME/.heroku/python" "$BUILD_DIR/.heroku/python"
ln -nsf "\$HOME/.heroku/vendor" "$BUILD_DIR/.heroku/vendor"
if [[ \$HOME != "/app" ]]; then
mkdir -p /app/.heroku
ln -nsf "\$HOME/.heroku/python" /app/.heroku/python
ln -nsf "\$HOME/.heroku/vendor" /app/.heroku/vendor
fi
EOT
......
......@@ -30,11 +30,12 @@ if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALL
puts-step "$ python $MANAGE_FILE collectstatic --noinput"
# Run collectstatic, cleanup some of the noisy output.
PYTHONPATH=${PYTHONPATH:-$BUILD_DIR}
PYTHONPATH=${PYTHONPATH:-.}
export PYTHONPATH
# Create a temporary file for collecting the collectstaic logs.
COLLECTSTATIC_LOG=$(mktemp)
python "$MANAGE_FILE" collectstatic --noinput --traceback 2>&1 | tee "$COLLECTSTATIC_LOG" | sed '/^Post-processed/d;/^Copying/d;/^$/d' | indent
COLLECTSTATIC_STATUS="${PIPESTATUS[0]}"
......
......@@ -12,7 +12,7 @@
# The location of the pre-compiled cryptography binary.
VENDORED_GDAL="${VENDOR_URL}/libraries/vendor/gdal.tar.gz"
PKG_CONFIG_PATH="$BUILD_DIR/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
# Syntax sugar.
# shellcheck source=bin/utils
......
......@@ -14,7 +14,7 @@ VENDORED_GDAL="${VENDOR_URL}/libraries/vendor/gdal.tar.gz"
VENDORED_GEOS="${VENDOR_URL}/libraries/vendor/geos.tar.gz"
VENDORED_PROJ="${VENDOR_URL}/libraries/vendor/proj.tar.gz"
PKG_CONFIG_PATH="$BUILD_DIR/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
# Syntax sugar.
# shellcheck source=bin/utils
......
......@@ -3,7 +3,7 @@
# Install Mercurial if it appears to be required.
if [[ -f "requirements.txt" ]]; then
if (grep -Fiq "hg+" requirements.txt) then
"$BUILD_DIR/.heroku/python/bin/pip" install mercurial | cleanup | indent
/app/.heroku/python/bin/pip install mercurial | cleanup | indent
mcount "steps.mercurial"
fi
fi
......@@ -41,8 +41,7 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
if [ ! -f "$BUILD_DIR/.heroku/python/bin/pip" ]; then
exit 1
fi
"$BUILD_DIR/.heroku/python/bin/pip" install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src="$BUILD_DIR/.heroku/src" --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent
/app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent
PIP_STATUS="${PIPESTATUS[0]}"
set -e
......@@ -55,7 +54,7 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
# Smart Requirements handling
cp requirements.txt .heroku/python/requirements-declared.txt
"$BUILD_DIR/.heroku/python/bin/pip" freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
/app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
echo
......@@ -63,7 +62,7 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
if [ "$INSTALL_TEST" ]; then
if [[ -f "$1/requirements-test.txt" ]]; then
puts-step "Installing test dependencies…"
"$BUILD_DIR/.heroku/python/bin/pip" install -r "$1/requirements-test.txt" --exists-action=w --src=./.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | cleanup | indent
/app/.heroku/python/bin/pip install -r "$1/requirements-test.txt" --exists-action=w --src=./.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | cleanup | indent
fi
fi
fi
......@@ -20,7 +20,7 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
if [[ -s .heroku/python/requirements-stale.txt ]]; then
puts-step "Uninstalling stale dependencies"
"$BUILD_DIR/.heroku/python/bin/pip" uninstall -r .heroku/python/requirements-stale.txt -y --exists-action=w --disable-pip-version-check | cleanup | indent
/app/.heroku/python/bin/pip uninstall -r .heroku/python/requirements-stale.txt -y --exists-action=w --disable-pip-version-check | cleanup | indent
fi
fi
......
......@@ -12,7 +12,7 @@ if [[ -f Pipfile.lock ]]; then
# Measure that we're using Pipenv.
mcount "tool.pipenv"
# Don't skip installation if there are git deps.
# Don't skip installation of there are git deps.
if ! grep -q 'git' Pipfile.lock; then
echo "Skipping installation, as Pipfile.lock hasn't changed since last deploy." | indent
......@@ -58,17 +58,18 @@ if [ ! "$SKIP_PIPENV_INSTALL" ]; then
# Due to weird old pip behavior and pipenv behavior, pipenv upgrades pip
# to latest if only --upgrade is specified. Specify upgrade strategy to
# avoid this eager behavior.
"$BUILD_DIR/.heroku/python/bin/pip" install pipenv==$PIPENV_VERSION --upgrade --upgrade-strategy only-if-needed &> /dev/null
/app/.heroku/python/bin/pip install pipenv==$PIPENV_VERSION --upgrade --upgrade-strategy only-if-needed &> /dev/null
# Install the test dependencies, for CI.
if [ "$INSTALL_TEST" ]; then
puts-step "Installing test dependencies…"
"$BUILD_DIR/.heroku/python/bin/pipenv" install --dev --system --deploy 2>&1 | cleanup | indent
/app/.heroku/python/bin/pipenv install --dev --system --deploy 2>&1 | cleanup | indent
# Install the dependencies.
elif [[ ! -f Pipfile.lock ]]; then
puts-step "Installing dependencies with Pipenv $PIPENV_VERSION…"
"$BUILD_DIR/.heroku/python/bin/pipenv" install --system --skip-lock 2>&1 | indent
/app/.heroku/python/bin/pipenv install --system --skip-lock 2>&1 | indent
else
pipenv-to-pip Pipfile.lock > requirements.txt
"$BIN_DIR/steps/pip-uninstall"
......@@ -76,7 +77,7 @@ if [ ! "$SKIP_PIPENV_INSTALL" ]; then
openssl dgst -sha256 Pipfile.lock > .heroku/python/Pipfile.lock.sha256
puts-step "Installing dependencies with Pipenv $PIPENV_VERSION…"
"$BUILD_DIR/.heroku/python/bin/pipenv" install --system --deploy 2>&1 | indent
/app/.heroku/python/bin/pipenv install --system --deploy 2>&1 | indent
fi
fi
else
......
......@@ -2,13 +2,13 @@
# Detect Python-version with Pipenv.
if [[ -f "$BUILD_DIR/Pipfile" ]]; then
if [[ -f $BUILD_DIR/Pipfile ]]; then
if [[ ! -f "$BUILD_DIR/runtime.txt" ]]; then
if [[ ! -f "$BUILD_DIR/Pipfile.lock" ]]; then
if [[ ! -f $BUILD_DIR/runtime.txt ]]; then
if [[ ! -f $BUILD_DIR/Pipfile.lock ]]; then
puts-warn "No 'Pipfile.lock' found! We recommend you commit this into your repository."
fi
if [[ -f "$BUILD_DIR/Pipfile.lock" ]]; then
if [[ -f $BUILD_DIR/Pipfile.lock ]]; then
set +e
PYTHON=$(jq -r '._meta.requires.python_full_version' "$BUILD_DIR/Pipfile.lock")
if [[ "$PYTHON" != "null" ]]; then
......
......@@ -113,11 +113,11 @@ if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
puts-step "Installing pip"
# Remove old installations.
rm -fr "$BUILD_DIR/.heroku/python/lib/python*/site-packages/pip-*"
rm -fr "$BUILD_DIR/.heroku/python/lib/python*/site-packages/setuptools-*"
rm -fr /app/.heroku/python/lib/python*/site-packages/pip-*
rm -fr /app/.heroku/python/lib/python*/site-packages/setuptools-*
"$BUILD_DIR/.heroku/python/bin/python" "$ROOT_DIR/get-pip.py" pip=="$PIP_UPDATE" &> /dev/null
"$BUILD_DIR/.heroku/python/bin/pip" install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null
/app/.heroku/python/bin/python "$ROOT_DIR/get-pip.py" pip=="$PIP_UPDATE" &> /dev/null
/app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null
fi
set -e
......
......@@ -4,7 +4,7 @@
OUT_PREFIX=$1
# Use new path, containing autoconf.
export PATH="$BUILD_DIR/.heroku/python/bin/:$PATH"
export PATH="/app/.heroku/python/bin/:$PATH"
hash -r
......
......@@ -4,7 +4,7 @@
OUT_PREFIX=$1
# Use new path, containing autoconf.
export PATH="$BUILD_DIR.heroku/python/bin/:$PATH"
export PATH="/app/.heroku/python/bin/:$PATH"
hash -r
......
......@@ -9,7 +9,7 @@ if [[ $S3_PREFIX == "heroku-16" ]]; then
fi
# Use new path, containing autoconf.
export PATH="$BUILD_DIR/.heroku/python/bin/:$PATH"
export PATH="/app/.heroku/python/bin/:$PATH"
hash -r
......
......@@ -4,7 +4,7 @@
OUT_PREFIX=$1
# Use new path, containing autoconf.
export PATH="$BUILD_DIR/.heroku/python/bin/:$PATH"
export PATH="/app/.heroku/python/bin/:$PATH"
hash -r
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment