Commit f74d4db8 authored by Kenneth Reitz's avatar Kenneth Reitz Committed by GitHub

Shellcheck Compliance (#438)

* fixed the bug for pypy-5.8.0

* enable shellcheck in makefile
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* make compile script shellcheck compatible
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* let's see what this says

* fix pip installation
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* shellcheck detect script

* shellcheck release
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* shellcheck test-compile
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* shellcheck warnings/utils

* updated makefile
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* shellcheck collectstatic
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* shellcheck cryptography
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* shellcheck eggpath fixes
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* shellcheck gdal

* shellcheck goes-libs
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* shellcheck mercurial
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* nltk shellcheck

* shellcheck pip-install
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* shellcheck pip-uninstall
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* shellcheck pipenv
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* shellcheck pylibmc
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* shellcheck python
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* update makefile
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* final update to makefile
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* fix tests
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* actually fix the tests
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* update pipenv-python-version
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* requested fixes
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* update spellcheck sources

* attempted fix for shell globbing
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* further attempted fix
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* switch nltk styling
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* no more need for temp app for installing pip
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* attempt at staged build for shellcheck
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* attempted fix of travis file
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* reorder travis file

* add shellcheck to install

* make things clearer in travis file
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* better travis file
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* attempt at better travis file
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>

* better travis file
Signed-off-by: default avatarKenneth Reitz <me@kennethreitz.org>
parent 2512694d
language: bash
sudo: required
# sudo: required
addons:
apt:
sources:
- debian-sid # Grab shellcheck from the Debian repo (o_O)
packages:
- shellcheck
services:
- docker
# install: docker pull heroku/cedar:14
script: ./tests.sh
env:
- STACK=heroku-16
- STACK=cedar-14
\ No newline at end of file
jobs:
include:
- stage: "Bash linting (shellcheck)"
script: make check
- stage: "Heroku-16 Stack Tests"
script: ./tests.sh
env:
- STACK=heroku-16
- stage: "Cedar-14 Stack Tests"
script: ./tests.sh
env:
- STACK=cedar-14
\ No newline at end of file
......@@ -3,6 +3,11 @@
test: test-heroku-16
check:
@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings
@shellcheck -x bin/steps/collectstatic bin/steps/cryptography bin/steps/eggpath-fix bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python
@shellcheck -x bin/steps/hooks/*
test-cedar-14:
@echo "Running tests in docker (cedar-14)..."
@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar:14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
......
......@@ -25,20 +25,24 @@ export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}
export PATH=:/usr/local/bin:$PATH
# Paths.
BIN_DIR=$(cd $(dirname $0); pwd) # absolute path
ROOT_DIR=$(dirname $BIN_DIR)
BIN_DIR=$(cd "$(dirname "$0")"; pwd) # absolute path
ROOT_DIR=$(dirname "$BIN_DIR")
BUILD_DIR=$1
CACHE_DIR=$2
ENV_DIR=$3
export BUILD_DIR CACHE_DIR ENV_DIR
# Python defaults
DEFAULT_PYTHON_VERSION="python-3.6.2"
DEFAULT_PYTHON_STACK="cedar-14"
PYTHON_EXE="/app/.heroku/python/bin/python"
PIP_VERSION="9.0.1"
PIP_UPDATE="9.0.1"
export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE
# Common Problem Warnings
export WARNINGS_LOG=$(mktemp)
WARNINGS_LOG=$(mktemp)
export WARNINGS_LOG
export RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION
# Setup vendored tools and pip-pop (pip-diff)
......@@ -55,10 +59,12 @@ unset RECEIVE_DATA RUN_KEY BUILD_INFO DEPLOY LOG_TOKEN
unset CYTOKINE_LOG_FILE GEM_PATH
# Syntax sugar.
source $BIN_DIR/utils
# shellcheck source=bin/utils
source "$BIN_DIR/utils"
# Import collection of warnings.
source $BIN_DIR/warnings
# shellcheck source=bin/warnings
source "$BIN_DIR/warnings"
# we need to put a bunch of symlinks in there later
mkdir -p /app/.heroku
......@@ -83,7 +89,7 @@ export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRA
export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config:$PKG_CONFIG_PATH
# Switch to the repo's context.
cd $BUILD_DIR
cd "$BUILD_DIR"
# Warn for lack of Procfile.
if [[ ! -f Procfile ]]; then
......@@ -92,60 +98,66 @@ if [[ ! -f Procfile ]]; then
fi
# Prepare the cache.
mkdir -p $CACHE_DIR
mkdir -p "$CACHE_DIR"
# Restore old artifacts from the cache.
mkdir -p .heroku
cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true
cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true
cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true
cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true
if [[ -d $CACHE_DIR/.heroku/src ]]; then
cp -R $CACHE_DIR/.heroku/src .heroku/ &> /dev/null || true
cp -R "$CACHE_DIR/.heroku/python" .heroku/ &> /dev/null || true
cp -R "$CACHE_DIR/.heroku/python-stack" .heroku/ &> /dev/null || true
cp -R "$CACHE_DIR/.heroku/python-version" .heroku/ &> /dev/null || true
cp -R "$CACHE_DIR/.heroku/vendor" .heroku/ &> /dev/null || true
if [[ -d "$CACHE_DIR/.heroku/src" ]]; then
cp -R "$CACHE_DIR/.heroku/src" .heroku/ &> /dev/null || true
fi
# Experimental pre_compile hook.
source $BIN_DIR/steps/hooks/pre_compile
# shellcheck source=bin/steps/hooks/pre_compile
source "$BIN_DIR/steps/hooks/pre_compile"
# Sticky runtimes.
if [ -f $CACHE_DIR/.heroku/python-version ]; then
DEFAULT_PYTHON_VERSION=$(cat $CACHE_DIR/.heroku/python-version)
if [ -f "$CACHE_DIR/.heroku/python-version" ]; then
DEFAULT_PYTHON_VERSION=$(cat "$CACHE_DIR/.heroku/python-version")
fi
# Stack fallback for non-declared caches.
if [ -f $CACHE_DIR/.heroku/python-stack ]; then
CACHED_PYTHON_STACK=$(cat $CACHE_DIR/.heroku/python-stack)
if [ -f "$CACHE_DIR/.heroku/python-stack" ]; then
CACHED_PYTHON_STACK=$(cat "$CACHE_DIR/.heroku/python-stack")
else
CACHED_PYTHON_STACK=$STACK
fi
export CACHED_PYTHON_STACK
# Pipenv Python version support.
source $BIN_DIR/steps/pipenv-python-version
# shellcheck source=bin/steps/pipenv-python-version
source "$BIN_DIR/steps/pipenv-python-version"
# If no runtime given, assume default version.
if [ ! -f runtime.txt ]; then
echo $DEFAULT_PYTHON_VERSION > runtime.txt
echo "$DEFAULT_PYTHON_VERSION" > runtime.txt
fi
mkdir -p $(dirname $PROFILE_PATH)
mkdir -p "$(dirname "$PROFILE_PATH")"
mkdir -p /app/.heroku/src
if [[ $BUILD_DIR != '/app' ]]; then
# python expects to reside in /app, so set up symlinks
# we will not remove these later so subsequent buildpacks can still invoke it
ln -nsf $BUILD_DIR/.heroku/python /app/.heroku/python
ln -nsf $BUILD_DIR/.heroku/vendor /app/.heroku/vendor
ln -nsf "$BUILD_DIR/.heroku/python" /app/.heroku/python
ln -nsf "$BUILD_DIR/.heroku/vendor" /app/.heroku/vendor
# Note: .heroku/src is copied in later.
fi
# Install Python.
let start=$(nowms)
source $BIN_DIR/steps/python
# shellcheck source=bin/steps/python
source "$BIN_DIR/steps/python"
mtime "python.install.time" "${start}"
# Pipenv support.
source $BIN_DIR/steps/pipenv
# shellcheck source=bin/steps/pipenv
source "$BIN_DIR/steps/pipenv"
# If no requirements.txt file given, assume `setup.py develop` is intended.
if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then
......@@ -153,88 +165,101 @@ if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then
fi
# Fix egg-links.
source $BIN_DIR/steps/eggpath-fix
# shellcheck source=bin/steps/eggpath-fix
source "$BIN_DIR/steps/eggpath-fix"
# Mercurial support.
source $BIN_DIR/steps/mercurial
# shellcheck source=bin/steps/mercurial
source "$BIN_DIR/steps/mercurial"
# Pylibmc support.
source $BIN_DIR/steps/pylibmc
# shellcheck source=bin/steps/pylibmc
source "$BIN_DIR/steps/pylibmc"
# Libffi support.
source $BIN_DIR/steps/cryptography
# shellcheck source=bin/steps/cryptography
source "$BIN_DIR/steps/cryptography"
# Support for Geo libraries.
sub-env $BIN_DIR/steps/geo-libs
# shellcheck source=bin/steps/geo-libs
sub-env "$BIN_DIR/steps/geo-libs"
# GDAL support.
source $BIN_DIR/steps/gdal
# shellcheck source=bin/steps/gdal
source "$BIN_DIR/steps/gdal"
# Uninstall removed dependencies with Pip.
let start=$(nowms)
source $BIN_DIR/steps/pip-uninstall
# shellcheck source=bin/steps/pip-uninstall
source "$BIN_DIR/steps/pip-uninstall"
mtime "pip.uninstall.time" "${start}"
# Install dependencies with Pip (where the magic happens).
let start=$(nowms)
source $BIN_DIR/steps/pip-install
# shellcheck source=bin/steps/pip-install
source "$BIN_DIR/steps/pip-install"
mtime "pip.install.time" "${start}"
# Support for NLTK corpora.
let start=$(nowms)
sub-env $BIN_DIR/steps/nltk
sub-env "$BIN_DIR/steps/nltk"
mtime "nltk.download.time" "${start}"
# Support for pip install -e.
# In CI, $BUILD_DIR is /app.
if [[ ! "$BUILD_DIR" == "/app" ]]; then
rm -fr $BUILD_DIR/.heroku/src
deep-cp /app/.heroku/src $BUILD_DIR/.heroku/src
rm -fr "$BUILD_DIR/.heroku/src"
deep-cp /app/.heroku/src "$BUILD_DIR/.heroku/src"
fi
# Django collectstatic support.
let start=$(nowms)
sub-env $BIN_DIR/steps/collectstatic
sub-env "$BIN_DIR/steps/collectstatic"
mtime "collectstatic.time" "${start}"
# Create .profile script for application runtime environment variables.
set-env PATH '$HOME/.heroku/python/bin:$PATH'
set-env PATH "\$HOME/.heroku/python/bin:\$PATH"
set-env PYTHONUNBUFFERED true
set-env PYTHONHOME /app/.heroku/python
set-env LIBRARY_PATH '/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PATH'
set-env LD_LIBRARY_PATH '/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH'
set-env LIBRARY_PATH "/app/.heroku/vendor/lib:/app/.heroku/python/lib:\$LIBRARY_PATH"
set-env LD_LIBRARY_PATH "/app/.heroku/vendor/lib:/app/.heroku/python/lib:\$LD_LIBRARY_PATH"
set-default-env LANG en_US.UTF-8
set-default-env PYTHONHASHSEED random
set-default-env PYTHONPATH /app/
# Install sane-default script for $WEB_CONCURRENCY and $FORWARDED_ALLOW_IPS.
cp $ROOT_DIR/vendor/WEB_CONCURRENCY.sh $WEB_CONCURRENCY_PROFILE_PATH
cp $ROOT_DIR/vendor/python.gunicorn.sh $GUNICORN_PROFILE_PATH
cp "$ROOT_DIR/vendor/WEB_CONCURRENCY.sh" "$WEB_CONCURRENCY_PROFILE_PATH"
cp "$ROOT_DIR/vendor/python.gunicorn.sh" "$GUNICORN_PROFILE_PATH"
# Experimental post_compile hook.
source $BIN_DIR/steps/hooks/post_compile
# shellcheck source=bin/steps/hooks/post_compile
source "$BIN_DIR/steps/hooks/post_compile"
# Fix egg-links, again.
source $BIN_DIR/steps/eggpath-fix2
# shellcheck source=bin/steps/eggpath-fix2
source "$BIN_DIR/steps/eggpath-fix2"
# Store new artifacts in cache.
rm -rf $CACHE_DIR/.heroku/python
rm -rf $CACHE_DIR/.heroku/python-version
rm -rf $CACHE_DIR/.heroku/python-stack
rm -rf $CACHE_DIR/.heroku/vendor
rm -rf $CACHE_DIR/.heroku/src
mkdir -p $CACHE_DIR/.heroku
cp -R .heroku/python $CACHE_DIR/.heroku/
cp -R .heroku/python-version $CACHE_DIR/.heroku/
cp -R .heroku/python-stack $CACHE_DIR/.heroku/ &> /dev/null || true
cp -R .heroku/vendor $CACHE_DIR/.heroku/ &> /dev/null || true
rm -rf "$CACHE_DIR/.heroku/python"
rm -rf "$CACHE_DIR/.heroku/python-version"
rm -rf "$CACHE_DIR/.heroku/python-stack"
rm -rf "$CACHE_DIR/.heroku/vendor"
rm -rf "$CACHE_DIR/.heroku/src"
mkdir -p "$CACHE_DIR/.heroku"
cp -R .heroku/python "$CACHE_DIR/.heroku/"
cp -R .heroku/python-version "$CACHE_DIR/.heroku/"
cp -R .heroku/python-stack "$CACHE_DIR/.heroku/" &> /dev/null || true
cp -R .heroku/vendor "$CACHE_DIR/.heroku/" &> /dev/null || true
if [[ -d .heroku/src ]]; then
cp -R .heroku/src $CACHE_DIR/.heroku/ &> /dev/null || true
cp -R .heroku/src "$CACHE_DIR/.heroku/" &> /dev/null || true
fi
# Measure the size of the Python installation.
# shellcheck disable=SC2119
mmeasure 'python.size' "$(measure-size)"
......@@ -15,7 +15,7 @@
BUILD_DIR=$1
# Exit early if app is clearly not Python.
if [ ! -f $BUILD_DIR/requirements.txt ] && [ ! -f $BUILD_DIR/setup.py ] && [ ! -f $BUILD_DIR/Pipfile ]; then
if [ ! -f "$BUILD_DIR/requirements.txt" ] && [ ! -f "$BUILD_DIR/setup.py" ] && [ ! -f "$BUILD_DIR/Pipfile" ]; then
exit 1
fi
......
#!/usr/bin/env bash
# bin/release <build-dir>
BIN_DIR=$(cd $(dirname $0); pwd) # absolute path
BUILD_DIR=$1
MANAGE_FILE=$(cd $BUILD_DIR && find . -maxdepth 3 -type f -name 'manage.py' | head -1)
MANAGE_FILE=$(cd "$BUILD_DIR" && find . -maxdepth 3 -type f -name 'manage.py' | head -1)
MANAGE_FILE=${MANAGE_FILE:2}
cat <<EOF
......
......@@ -10,6 +10,7 @@
# - $DISABLE_COLLECTSTATIC: disables this functionality.
# - $DEBUG_COLLECTSTATIC: upon failure, print out environment variables.
# shellcheck source=bin/utils
source $BIN_DIR/utils
# Location of 'manage.py', if it exists.
......@@ -29,13 +30,13 @@ if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALL
puts-step "$ python $MANAGE_FILE collectstatic --noinput"
# Run collectstatic, cleanup some of the noisy output.
python $MANAGE_FILE collectstatic --noinput --traceback 2>&1 | sed '/^Post-processed/d;/^Copying/d;/^$/d' | indent
python "$MANAGE_FILE" collectstatic --noinput --traceback 2>&1 | sed '/^Post-processed/d;/^Copying/d;/^$/d' | indent
COLLECTSTATIC_STATUS="${PIPESTATUS[0]}"
set -e
# Display a warning if collectstatic failed.
[ $COLLECTSTATIC_STATUS -ne 0 ] && {
[ "$COLLECTSTATIC_STATUS" -ne 0 ] && {
echo
echo " ! Error while running '$ python $MANAGE_FILE collectstatic --noinput'."
......
......@@ -15,7 +15,8 @@ VENDORED_LIBFFI="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/li
PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
# Syntax sugar.
source $BIN_DIR/utils
# shellcheck source=bin/utils
source "$BIN_DIR/utils"
# If a package using cffi exists within requirements, use vendored libffi.
if (pip-grep -s requirements.txt argon2-cffi bcrypt cffi cryptography django[argon2] Django[argon2] django[bcrypt] Django[bcrypt] PyNaCl pyOpenSSL PyOpenSSL requests[security] misaka &> /dev/null) then
......@@ -24,8 +25,9 @@ if (pip-grep -s requirements.txt argon2-cffi bcrypt cffi cryptography django[arg
echo "-----> Noticed cffi. Bootstrapping libffi."
mkdir -p .heroku/vendor
# Download and extract libffi into target vendor directory.
curl $VENDORED_LIBFFI -s | tar zxv -C .heroku/vendor &> /dev/null
curl "$VENDORED_LIBFFI" -s | tar zxv -C .heroku/vendor &> /dev/null
fi
export LIBFFI=$(pwd)/vendor
LIBFFI=$(pwd)/vendor
export LIBFFI
fi
#!/usr/bin/env bash
set +e
# delete any existing egg links, to uninstall exisisting installations.
find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -delete 2> /dev/null
......
#!/usr/bin/env bash
set +e
# rewrite build dir in egg links to /app so things are found at runtime
find .heroku/python/lib/python*/site-packages/ -name "*.pth" -print0 2> /dev/null | xargs -r -0 -n 1 sed -i -e "s#$(pwd)#/app#" &> /dev/null
......
......@@ -15,7 +15,8 @@ VENDORED_GDAL="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/gdal
PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
# Syntax sugar.
source $BIN_DIR/utils
# shellcheck source=bin/utils
source "$BIN_DIR/utils"
# If GDAL exists within requirements, use vendored gdal.
if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
......@@ -24,9 +25,10 @@ if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
echo "-----> Noticed GDAL. Bootstrapping gdal."
mkdir -p .heroku/vendor
# Download and extract cryptography into target vendor directory.
curl $VENDORED_GDAL -s | tar zxv -C .heroku/vendor &> /dev/null
curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null
fi
export GDAL=$(pwd)/vendor
GDAL=$(pwd)/vendor
export GDAL
fi
......@@ -17,7 +17,8 @@ VENDORED_PROJ="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/proj
PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
# Syntax sugar.
source $BIN_DIR/utils
# shellcheck source=bin/utils
source "$BIN_DIR/utils"
# If GDAL exists within requirements, use vendored gdal.
if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
......@@ -26,11 +27,12 @@ if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
echo "-----> Bootstrapping gdal, geos, proj."
mkdir -p .heroku/vendor
# Download and extract cryptography into target vendor directory.
curl $VENDORED_GDAL -s | tar zxv -C .heroku/vendor &> /dev/null
curl $VENDORED_GEOS -s | tar zxv -C .heroku/vendor &> /dev/null
curl $VENDORED_PROJ -s | tar zxv -C .heroku/vendor &> /dev/null
curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null
curl "$VENDORED_GEOS" -s | tar zxv -C .heroku/vendor &> /dev/null
curl "$VENDORED_PROJ" -s | tar zxv -C .heroku/vendor &> /dev/null
fi
export GDAL=$(pwd)/vendor
GDAL=$(pwd)/vendor
export GDAL
fi
#!/usr/bin/env bash
# Install Mercurial if it appears to be required.
if (grep -Fiq "hg+" requirements.txt) then
/app/.heroku/python/bin/pip install mercurial | cleanup | indent
......
......@@ -10,18 +10,23 @@
# This script is invoked by [`bin/compile`](/).
# Syntax sugar.
source $BIN_DIR/utils
# shellcheck source=bin/utils
source "$BIN_DIR/utils"
# Check that nltk was installed by pip, otherwise obviously not needed
python -m nltk.downloader -h >/dev/null 2>&1
if [ $? -eq 0 ]; then
if sp-grep -s nltk; then
puts-step "Downloading NLTK corpora..."
nltk_packages_definition="$BUILD_DIR/nltk.txt"
if [ -f "$nltk_packages_definition" ]; then
nltk_packages=$(tr "\n" " " < "$nltk_packages_definition")
puts-step "Downloading NLTK packages: $nltk_packages"
python -m nltk.downloader -d $BUILD_DIR/.heroku/python/nltk_data $nltk_packages | indent
python -m nltk.downloader -d "$BUILD_DIR/.heroku/python/nltk_data" "$nltk_packages" | indent
set-env NLTK_DATA "/app/.heroku/python/nltk_data"
else
puts-warn "'nltk.txt' not found, not downloading any corpora"
puts-warn "Learn more: https://devcenter.heroku.com/articles/python-nltk"
......
#!/usr/bin/env bash
if [ ! "$SKIP_PIP_INSTALL" ]; then
# Install dependencies with Pip.
puts-step "Installing requirements with pip"
set +e
/app/.heroku/python/bin/pip install -r $BUILD_DIR/requirements.txt --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent
/app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent
PIP_STATUS="${PIPESTATUS[0]}"
set -e
......
#!/usr/bin/env bash
set +e
# Install dependencies with Pip.
......
#!/usr/bin/env bash
# Pipenv support (Generate requriements.txt with pipenv).
if [[ -f Pipfile ]]; then
if [[ ! -f requirements.txt ]]; then
......
#!/usr/bin/env bash
# Detect Python-version with Pipenv.
if [[ -f $BUILD_DIR/Pipfile.lock ]]; then
......@@ -8,14 +10,14 @@ if [[ -f $BUILD_DIR/Pipfile.lock ]]; then
fi
if [[ -f $BUILD_DIR/Pipfile.lock ]]; then
set +e
PYTHON=$(cat $BUILD_DIR/Pipfile.lock | jq '._meta.requires.python_version' -r)
PYTHON=$(jq -r '._meta.requires.python_version' "$BUILD_DIR/Pipfile.lock")
set -e
if [ "$PYTHON" = 2.7 ]; then
echo "python-2.7.13" > $BUILD_DIR/runtime.txt
echo "python-2.7.13" > "$BUILD_DIR/runtime.txt"
fi
if [ "$PYTHON" = 3.6 ]; then
echo "python-3.6.0" > $BUILD_DIR/runtime.txt
echo "python-3.6.2" > "$BUILD_DIR/runtime.txt"
fi
fi
fi
......
......@@ -13,7 +13,8 @@
VENDORED_MEMCACHED="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/libmemcache.tar.gz"
# Syntax sugar.
source $BIN_DIR/utils
# shellcheck source=bin/utils
source "$BIN_DIR/utils"
# If pylibmc exists within requirements, use vendored libmemcached.
......@@ -23,8 +24,9 @@ if (pip-grep -s requirements.txt pylibmc &> /dev/null) then
echo "-----> Noticed pylibmc. Bootstrapping libmemcached."
mkdir -p .heroku/vendor
# Download and extract libmemcached into target vendor directory.
curl $VENDORED_MEMCACHED -s | tar zxv -C .heroku/vendor &> /dev/null
curl "$VENDORED_MEMCACHED" -s | tar zxv -C .heroku/vendor &> /dev/null
fi
export LIBMEMCACHED=$(pwd)/vendor
LIBMEMCACHED=$(pwd)/vendor
export LIBMEMCACHED
fi
#!/usr/bin/env bash
set +e
runtime-fixer runtime.txt
PYTHON_VERSION=$(cat runtime.txt)
# Install Python.
if [ -f .heroku/python-version ]; then
if [ ! $(cat .heroku/python-version) = $PYTHON_VERSION ]; then
if [ ! "$(cat .heroku/python-version)" = "$PYTHON_VERSION" ]; then
puts-step "Found $(cat .heroku/python-version), removing"
rm -fr .heroku/python
else
......@@ -12,7 +14,7 @@ if [ -f .heroku/python-version ]; then
fi
fi
if [ ! $STACK = $CACHED_PYTHON_STACK ]; then
if [ ! "$STACK" = "$CACHED_PYTHON_STACK" ]; then
rm -fr .heroku/python .heroku/python-stack .heroku/vendor
unset SKIP_INSTALL
fi
......@@ -24,29 +26,24 @@ if [ ! "$SKIP_INSTALL" ]; then
# Prepare destination directory.
mkdir -p .heroku/python
curl https://lang-python.s3.amazonaws.com/$STACK/runtimes/$PYTHON_VERSION.tar.gz -s | tar zxv -C .heroku/python &> /dev/null
mcount "version.python.$PYTHON_VERSION"
if [[ $? != 0 ]] ; then
if ! curl "https://lang-python.s3.amazonaws.com/$STACK/runtimes/$PYTHON_VERSION.tar.gz" -s | tar zxv -C .heroku/python &> /dev/null; then
puts-warn "Requested runtime ($PYTHON_VERSION) is not available for this stack ($STACK)."
puts-warn "Aborting. More info: https://devcenter.heroku.com/articles/python-support"
exit 1
fi
# Record for future reference.
echo $PYTHON_VERSION > .heroku/python-version
echo $STACK > .heroku/python-stack
echo "$PYTHON_VERSION" > .heroku/python-version
echo "$STACK" > .heroku/python-stack
FRESH_PYTHON=true
hash -r
fi
# If Pip isn't up to date:
if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_VERSION* ]]; then
WORKING_DIR=$(pwd)
TMPTARDIR=$(mktemp -d)
trap "rm -rf $TMPTARDIR" RETURN
if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
puts-step "Installing pip"
......@@ -54,7 +51,7 @@ if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_VERSION* ]]; then
rm -fr /app/.heroku/python/lib/python2.7/site-packages/pip-*
rm -fr /app/.heroku/python/lib/python2.7/site-packages/setuptools-*
/app/.heroku/python/bin/python $ROOT_DIR/vendor/get-pip.py &> /dev/null
/app/.heroku/python/bin/python "$ROOT_DIR/vendor/get-pip.py" &> /dev/null
fi
......
#!/usr/bin/env bash
# Syntax sugar.
BIN_DIR=$(cd $(dirname $0); pwd) # absolute path
source $BIN_DIR/utils
BIN_DIR=$(cd "$(dirname "$0")" || return; pwd) # absolute path
DISABLE_COLLECTSTATIC=1 "$(dirname ${0:-})/compile" "$1" "$2" "$3"
# shellcheck source=bin/utils
source "$BIN_DIR/utils"
DISABLE_COLLECTSTATIC=1 "$(dirname "${0:-}")/compile" "$1" "$2" "$3"
if [[ -f "$1/requirements-test.txt" ]]; then
/app/.heroku/python/bin/pip install -r "$1/requirements-test.txt" --exists-action=w --src=./.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | cleanup | indent
......
#!/usr/bin/env bash
shopt -s extglob
shopt -s nullglob
# The standard library.
if [[ ! -f /tmp/stdlib.sh ]]; then
curl --retry 3 -s https://lang-common.s3.amazonaws.com/buildpack-stdlib/v2/stdlib.sh > /tmp/stdlib.sh
fi
# shellcheck source=/dev/null
source /tmp/stdlib.sh