diff --git a/bin/compile b/bin/compile index db2d838bb..df9e79f54 100755 --- a/bin/compile +++ b/bin/compile @@ -257,6 +257,10 @@ mtime "python.install.time" "${start}" # shellcheck source=bin/steps/pipenv source "$BIN_DIR/steps/pipenv" +# Export requirements.txt from poetry.lock, if present. +# shellcheck source=bin/steps/poetry +source "$BIN_DIR/steps/poetry" + # Uninstall removed dependencies with Pip. # The buildpack will automatically remove any declared dependencies (in requirements.txt) # that were explicitly removed. This machinery is a bit complex, but it is not complicated. @@ -268,9 +272,8 @@ mtime "pip.uninstall.time" "${start}" # If no requirements.txt file given, assume `setup.py develop` is intended. # This allows for people to ship a setup.py application to Heroku # (which is rare, but I vouch that it should work!) - -if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then - echo "-e ." > requirements.txt +if [ ! -f requirements.txt ] && [ ! -f Pipfile ] && [ ! -f pyproject.toml ]; then + echo "-e ." > requirements.txt fi # Fix egg-links. diff --git a/bin/detect b/bin/detect index eeb965b0f..682585fc5 100755 --- a/bin/detect +++ b/bin/detect @@ -15,7 +15,7 @@ BUILD_DIR=$1 # Exit early if app is clearly not Python. -if [ ! -f "$BUILD_DIR/requirements.txt" ] && [ ! -f "$BUILD_DIR/setup.py" ] && [ ! -f "$BUILD_DIR/Pipfile" ]; then +if [ ! -f "$BUILD_DIR/requirements.txt" ] && [ ! -f "$BUILD_DIR/setup.py" ] && [ ! -f "$BUILD_DIR/Pipfile" ] && [ ! -f "$BUILD_DIR/pyproject.toml" ]; then exit 1 fi diff --git a/bin/steps/pip-install b/bin/steps/pip-install index b950feff0..7be26ebbf 100755 --- a/bin/steps/pip-install +++ b/bin/steps/pip-install @@ -34,15 +34,26 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then mcount "tool.pip" # Count expected build failures. - if grep -q '==0.0.0' requirements.txt; then + if [ -f requirements.txt ] && grep -q '==0.0.0' requirements.txt; then mcount "failure.none-version" fi if [ ! -f "$BUILD_DIR/.heroku/python/bin/pip" ]; then exit 1 fi - /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent - PIP_STATUS="${PIPESTATUS[0]}" + + if [ -f requirements.txt ]; then + /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent + PIP_STATUS="${PIPESTATUS[0]}" + else + PIP_STATUS=0 + fi + + if [ "$PIP_STATUS" -eq 0 ] && [ -f pyproject.toml ]; then + /app/.heroku/python/bin/pip install . --exists-action=w --disable-pip-version-check --no-cache-dir 2>&1 | tee -a "$WARNINGS_LOG" | cleanup | indent + PIP_STATUS="${PIPESTATUS[0]}" + fi + set -e show-warnings @@ -53,8 +64,10 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then fi # Smart Requirements handling - cp requirements.txt .heroku/python/requirements-declared.txt - /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt + if [ -f requirements.txt ]; then + cp requirements.txt .heroku/python/requirements-declared.txt + /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt + fi echo diff --git a/bin/steps/pip-uninstall b/bin/steps/pip-uninstall index 2e1ad8db7..ed5fae0cd 100755 --- a/bin/steps/pip-uninstall +++ b/bin/steps/pip-uninstall @@ -5,7 +5,7 @@ set +e # shellcheck source=bin/utils source "$BIN_DIR/utils" -if [ ! "$SKIP_PIP_INSTALL" ]; then +if [ ! "$SKIP_PIP_INSTALL" ] && [ -f requirements.txt ]; then if [[ -f .heroku/python/requirements-declared.txt ]]; then diff --git a/bin/steps/poetry b/bin/steps/poetry new file mode 100644 index 000000000..32ee98ab2 --- /dev/null +++ b/bin/steps/poetry @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +set -e + +# shellcheck source=bin/utils +source "$BIN_DIR/utils" + +if [ ! -f requirements.txt ] && [ -f pyproject.toml ] && [ -f poetry.lock ]; then + # Measure that we're using Poetry. + mcount "tool.poetry" + + # Hash poetry.lock to detect changes. + POETRY_LOCK_SHA=$(openssl dgst -sha256 poetry.lock) + + # Use cached requirements.txt if poetry.lock is unchanged. + CACHED_REQUIREMENTS=$CACHE_DIR/requirements.txt + CACHED_POETRY_LOCK_SHA=$CACHE_DIR/poetry.lock.sha256 + + if [ -f "$CACHED_REQUIREMENTS" ] && [ -f "$CACHED_POETRY_LOCK_SHA" ] && + [ "$POETRY_LOCK_SHA" == "$(cat "$CACHED_POETRY_LOCK_SHA")" ]; then + echo "Skipping requirements export, as poetry.lock hasn't changed since last deploy." | indent + cp "$CACHED_REQUIREMENTS" requirements.txt + else + # Set environment variables for pip + # This reads certain environment variables set on the Heroku app config + # and makes them accessible to the pip install process. + # + # PIP_EXTRA_INDEX_URL allows for an alternate pypi URL to be used. + if [[ -r "$ENV_DIR/PIP_EXTRA_INDEX_URL" ]]; then + PIP_EXTRA_INDEX_URL="$(cat "$ENV_DIR/PIP_EXTRA_INDEX_URL")" + export PIP_EXTRA_INDEX_URL + mcount "buildvar.PIP_EXTRA_INDEX_URL" + fi + + # Set SLUGIFY_USES_TEXT_UNIDECODE, required for Airflow versions >=1.10 + if [[ -r "$ENV_DIR/SLUGIFY_USES_TEXT_UNIDECODE" ]]; then + SLUGIFY_USES_TEXT_UNIDECODE="$(cat "$ENV_DIR/SLUGIFY_USES_TEXT_UNIDECODE")" + export SLUGIFY_USES_TEXT_UNIDECODE + mcount "buildvar.SLUGIFY_USES_TEXT_UNIDECODE" + fi + + # Install Poetry. + # + # Poetry is not used to install the project because it does not clean up + # stale requirements (see sdispater/poetry#648), so we need to export + # requirements.txt anyway for the pip-uninstall step. + # + # Since we only use Poetry to export a requirements.txt file, ignore the + # Poetry version specified in pyproject.toml. Install a pre-release of + # 1.0.0 because the export command is not available before 1.0.0a0. + export POETRY_VERSION="1.0.0b7" + puts-step "Exporting requirements with Poetry $POETRY_VERSION…" + /app/.heroku/python/bin/pip install "poetry==$POETRY_VERSION" \ + --disable-pip-version-check &> /dev/null + + # Export requirements. + /app/.heroku/python/bin/poetry export -f requirements.txt -o requirements.txt + + # Write SHA and requirements.txt to cache dir. + echo "$POETRY_LOCK_SHA" > "$CACHED_POETRY_LOCK_SHA" + cp requirements.txt "$CACHED_REQUIREMENTS" + fi +fi diff --git a/test/fixtures/flit-requires/foobar.py b/test/fixtures/flit-requires/foobar.py new file mode 100644 index 000000000..4b4c67818 --- /dev/null +++ b/test/fixtures/flit-requires/foobar.py @@ -0,0 +1,3 @@ +"""An amazing sample package!""" + +__version__ = '0.1' diff --git a/test/fixtures/flit-requires/pyproject.toml b/test/fixtures/flit-requires/pyproject.toml new file mode 100644 index 000000000..61cc974f1 --- /dev/null +++ b/test/fixtures/flit-requires/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +requires = ["flit"] +build-backend = "flit.buildapi" + +[tool.flit.metadata] +module = "foobar" +author = "Sir Robin" +author-email = "robin@camelot.uk" +home-page = "https://github.com/sirrobin/foobar" +requires = ["attrs >=19.1.0"] diff --git a/test/fixtures/flit/foobar.py b/test/fixtures/flit/foobar.py new file mode 100644 index 000000000..4b4c67818 --- /dev/null +++ b/test/fixtures/flit/foobar.py @@ -0,0 +1,3 @@ +"""An amazing sample package!""" + +__version__ = '0.1' diff --git a/test/fixtures/flit/pyproject.toml b/test/fixtures/flit/pyproject.toml new file mode 100644 index 000000000..3dca8c957 --- /dev/null +++ b/test/fixtures/flit/pyproject.toml @@ -0,0 +1,9 @@ +[build-system] +requires = ["flit"] +build-backend = "flit.buildapi" + +[tool.flit.metadata] +module = "foobar" +author = "Sir Robin" +author-email = "robin@camelot.uk" +home-page = "https://github.com/sirrobin/foobar" diff --git a/test/fixtures/poetry-lock/foobar.py b/test/fixtures/poetry-lock/foobar.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/fixtures/poetry-lock/poetry.lock b/test/fixtures/poetry-lock/poetry.lock new file mode 100644 index 000000000..99cd77b43 --- /dev/null +++ b/test/fixtures/poetry-lock/poetry.lock @@ -0,0 +1,20 @@ +[[package]] +category = "main" +description = "A lightweight library for converting complex datatypes to and from native Python datatypes." +name = "marshmallow" +optional = false +python-versions = ">=3.5" +version = "3.0.0" + +[package.extras] +dev = ["pytest", "pytz", "simplejson", "flake8 (3.7.8)", "flake8-bugbear (19.8.0)", "pre-commit (>=1.17,<2.0)", "tox"] +docs = ["sphinx (2.2.0)", "sphinx-issues (1.2.0)", "alabaster (0.7.12)", "sphinx-version-warning (1.1.2)"] +lint = ["flake8 (3.7.8)", "flake8-bugbear (19.8.0)", "pre-commit (>=1.17,<2.0)"] +tests = ["pytest", "pytz", "simplejson"] + +[metadata] +content-hash = "11b18d7787605b57a1019436950c60105518d3f3a8a60030493402031355e2f2" +python-versions = "^3.6" + +[metadata.hashes] +marshmallow = ["e5e9fd0c2e919b4ece915eb30808206349a49a45df72e99ed20e27a9053d574b", "fa2d8a4b61d09b0e161a14acc5ad8ab7aaaf1477f3dd52819ddd6c6c8275733a"] \ No newline at end of file diff --git a/test/fixtures/poetry-lock/pyproject.toml b/test/fixtures/poetry-lock/pyproject.toml new file mode 100644 index 000000000..85695c719 --- /dev/null +++ b/test/fixtures/poetry-lock/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "foobar" +version = "0.1.0" +description = "" +authors = ["Your Name "] + +[tool.poetry.dependencies] +python = "^3.6" +marshmallow = "^3.0.0" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" diff --git a/test/fixtures/poetry/foobar.py b/test/fixtures/poetry/foobar.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/fixtures/poetry/pyproject.toml b/test/fixtures/poetry/pyproject.toml new file mode 100644 index 000000000..4074cf7f7 --- /dev/null +++ b/test/fixtures/poetry/pyproject.toml @@ -0,0 +1,14 @@ +[tool.poetry] +name = "foobar" +version = "0.1.0" +description = "" +authors = ["Your Name "] + +[tool.poetry.dependencies] +python = "^3.6" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" diff --git a/test/fixtures/pyproject-toml/pyproject.toml b/test/fixtures/pyproject-toml/pyproject.toml new file mode 100644 index 000000000..864b334a8 --- /dev/null +++ b/test/fixtures/pyproject-toml/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta:__legacy__" diff --git a/test/fixtures/pyproject-toml/setup.py b/test/fixtures/pyproject-toml/setup.py new file mode 100644 index 000000000..4d986d25d --- /dev/null +++ b/test/fixtures/pyproject-toml/setup.py @@ -0,0 +1,4 @@ +from setuptools import setup + + +setup(name="foobar", version="1.0.0") diff --git a/test/run-features b/test/run-features index 66fac4ea7..e452a6cbe 100755 --- a/test/run-features +++ b/test/run-features @@ -70,6 +70,32 @@ testPipenvFullVersion() { assertCapturedSuccess } +testPyProjectToml() { + compile "pyproject-toml" + assertCapturedSuccess +} + +testPoetry() { + compile "poetry" + assertCapturedSuccess +} + +testPoetryLock() { + compile "poetry-lock" + assertCaptured "marshmallow==3.0.0" + assertCapturedSuccess +} + +testFlit() { + compile "flit" + assertCapturedSuccess +} + +testFlitRequires() { + compile "flit-requires" + assertCapturedSuccess +} + testNoRequirements() { compile "no-requirements" assertCapturedError