diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..a3abd994 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,25 @@ +name: Upload Python Package + +on: + release: + types: [created] + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + make release diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..2865cf85 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,25 @@ +name: Run Tests + +on: [push, pull_request] + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + max-parallel: 8 + matrix: + os: + - ubuntu-latest + python-version: [3.5, 3.6, 3.7, 3.8, 3.9, pypy3] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox tox-gh-actions + - name: Test with tox + run: tox diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 483f6d40..00000000 --- a/.travis.yml +++ /dev/null @@ -1,49 +0,0 @@ -language: python -cache: pip -os: linux -dist: xenial - -jobs: - include: - - python: "3.6" - env: TOXENV=lint - - python: "3.6" - env: TOXENV=manifest - - python: "2.7" - env: TOXENV=py27 - - python: "3.5" - env: TOXENV=py35 - - python: "3.6" - env: TOXENV=py36 - - python: "3.7" - env: TOXENV=py37 - - python: "3.8" - env: TOXENV=py38 - - python: "pypy" - env: TOXENV=pypy - - python: "pypy3" - env: TOXENV=pypy3 - -install: - - pip install tox - -script: - - tox - -before_install: - - pip install coveralls - -after_success: - - tox -e coverage-report - - coveralls - -deploy: - provider: pypi - username: theskumar - password: - secure: DXUkl4YSC2RCltChik1csvQulnVMQQpD/4i4u+6pEyUfBMYP65zFYSNwLh+jt+URyX+MpN/Er20+TZ/F/fu7xkru6/KBqKLugeXihNbwGhbHUIkjZT/0dNSo03uAz6s5fWgqr8EJk9Ll71GexAsBPx2yqsjc2BMgOjwcNly40Co= - distributions: "sdist bdist_wheel" - skip_existing: true - on: - tags: true - repo: theskumar/python-dotenv diff --git a/CHANGELOG.md b/CHANGELOG.md index fc2f7c9b..5da48f0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,88 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.19.0] - 2021-07-24 -*No unreleased change at this time.* +### Changed + +- Require Python 3.5 or a later version. Python 2 and 3.4 are no longer supported. (#341 + by [@bbc2]). + +### Added + +- The `dotenv_path` argument of `set_key` and `unset_key` now has a type of `Union[str, + os.PathLike]` instead of just `os.PathLike` (#347 by [@bbc2]). +- The `stream` argument of `load_dotenv` and `dotenv_values` can now be a text stream + (`IO[str]`), which includes values like `io.StringIO("foo")` and `open("file.env", + "r")` (#348 by [@bbc2]). + +## [0.18.0] - 2021-06-20 + +### Changed + +- Raise `ValueError` if `quote_mode` isn't one of `always`, `auto` or `never` in + `set_key` (#330 by [@bbc2]). +- When writing a value to a .env file with `set_key` or `dotenv set ` (#330 + by [@bbc2]): + - Use single quotes instead of double quotes. + - Don't strip surrounding quotes. + - In `auto` mode, don't add quotes if the value is only made of alphanumeric characters + (as determined by `string.isalnum`). + +## [0.17.1] - 2021-04-29 + +### Fixed + +- Fixed tests for build environments relying on `PYTHONPATH` (#318 by [@befeleme]). + +## [0.17.0] - 2021-04-02 + +### Changed + +- Make `dotenv get ` only show the value, not `key=value` (#313 by [@bbc2]). + +### Added + +- Add `--override`/`--no-override` option to `dotenv run` (#312 by [@zueve] and [@bbc2]). + +## [0.16.0] - 2021-03-27 + +### Changed + +- The default value of the `encoding` parameter for `load_dotenv` and `dotenv_values` is + now `"utf-8"` instead of `None` (#306 by [@bbc2]). +- Fix resolution order in variable expansion with `override=False` (#287 by [@bbc2]). + +## [0.15.0] - 2020-10-28 + +### Added + +- Add `--export` option to `set` to make it prepend the binding with `export` (#270 by + [@jadutter]). + +### Changed + +- Make `set` command create the `.env` file in the current directory if no `.env` file was + found (#270 by [@jadutter]). + +### Fixed + +- Fix potentially empty expanded value for duplicate key (#260 by [@bbc2]). +- Fix import error on Python 3.5.0 and 3.5.1 (#267 by [@gongqingkui]). +- Fix parsing of unquoted values containing several adjacent space or tab characters + (#277 by [@bbc2], review by [@x-yuri]). + +## [0.14.0] - 2020-07-03 + +### Changed + +- Privilege definition in file over the environment in variable expansion (#256 by + [@elbehery95]). + +### Fixed + +- Improve error message for when file isn't found (#245 by [@snobu]). +- Use HTTPS URL in package meta data (#251 by [@ekohl]). ## [0.13.0] - 2020-04-16 @@ -187,18 +266,33 @@ project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). [@andrewsmith]: https://github.com/andrewsmith [@asyncee]: https://github.com/asyncee [@bbc2]: https://github.com/bbc2 +[@befeleme]: https://github.com/befeleme [@cjauvin]: https://github.com/cjauvin [@earlbread]: https://github.com/earlbread +[@ekohl]: https://github.com/ekohl +[@elbehery95]: https://github.com/elbehery95 [@gergelyk]: https://github.com/gergelyk +[@gongqingkui]: https://github.com/gongqingkui [@greyli]: https://github.com/greyli +[@jadutter]: https://github.com/jadutter [@qnighy]: https://github.com/qnighy +[@snobu]: https://github.com/snobu [@techalchemy]: https://github.com/techalchemy [@theskumar]: https://github.com/theskumar [@ulyssessouza]: https://github.com/ulyssessouza [@venthur]: https://github.com/venthur +[@x-yuri]: https://github.com/x-yuri [@yannham]: https://github.com/yannham - -[Unreleased]: https://github.com/theskumar/python-dotenv/compare/v0.13.0...HEAD +[@zueve]: https://github.com/zueve + +[Unreleased]: https://github.com/theskumar/python-dotenv/compare/v0.19.0...HEAD +[0.19.0]: https://github.com/theskumar/python-dotenv/compare/v0.18.0...v0.19.0 +[0.18.0]: https://github.com/theskumar/python-dotenv/compare/v0.17.1...v0.18.0 +[0.17.1]: https://github.com/theskumar/python-dotenv/compare/v0.17.0...v0.17.1 +[0.17.0]: https://github.com/theskumar/python-dotenv/compare/v0.16.0...v0.17.0 +[0.16.0]: https://github.com/theskumar/python-dotenv/compare/v0.15.0...v0.16.0 +[0.15.0]: https://github.com/theskumar/python-dotenv/compare/v0.14.0...v0.15.0 +[0.14.0]: https://github.com/theskumar/python-dotenv/compare/v0.13.0...v0.14.0 [0.13.0]: https://github.com/theskumar/python-dotenv/compare/v0.12.0...v0.13.0 [0.12.0]: https://github.com/theskumar/python-dotenv/compare/v0.11.0...v0.12.0 [0.11.0]: https://github.com/theskumar/python-dotenv/compare/v0.10.5...v0.11.0 diff --git a/README.md b/README.md index 7374d05c..9b56b546 100644 --- a/README.md +++ b/README.md @@ -1,260 +1,204 @@ -``` - _______ .__ __. ____ ____ - | ____|| \ | | \ \ / / - | |__ | \| | \ \/ / - | __| | . ` | \ / - __ | |____ | |\ | \ / - (__)|_______||__| \__| \__/ -``` -python-dotenv | [![Build Status](https://travis-ci.org/theskumar/python-dotenv.svg?branch=master)](https://travis-ci.org/theskumar/python-dotenv) [![Coverage Status](https://coveralls.io/repos/theskumar/python-dotenv/badge.svg?branch=master)](https://coveralls.io/r/theskumar/python-dotenv?branch=master) [![PyPI version](https://badge.fury.io/py/python-dotenv.svg)](http://badge.fury.io/py/python-dotenv) [![Say Thanks!](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/theskumar) -=============================================================================== +# python-dotenv -Reads the key-value pair from `.env` file and adds them to environment -variable. It is great for managing app settings during development and -in production using [12-factor](http://12factor.net/) principles. +[![Build Status][build_status_badge]][build_status_link] +[![PyPI version][pypi_badge]][pypi_link] -> Do one thing, do it well! +Python-dotenv reads key-value pairs from a `.env` file and can set them as environment +variables. It helps in the development of applications following the +[12-factor](http://12factor.net/) principles. -## Usages +- [Getting Started](#getting-started) +- [Other Use Cases](#other-use-cases) + * [Load configuration without altering the environment](#load-configuration-without-altering-the-environment) + * [Parse configuration as a stream](#parse-configuration-as-a-stream) + * [Load .env files in IPython](#load-env-files-in-ipython) +- [Command-line Interface](#command-line-interface) +- [File format](#file-format) + * [Multiline values](#multiline-values) + * [Variable expansion](#variable-expansion) +- [Related Projects](#related-projects) +- [Acknowledgements](#acknowledgements) -The easiest and most common usage consists on calling `load_dotenv` when -the application starts, which will load environment variables from a -file named `.env` in the current directory or any of its parents or from -the path specificied; after that, you can just call the -environment-related method you need as provided by `os.getenv`. - -`.env` looks like this: +## Getting Started ```shell -# a comment that will be ignored. -REDIS_ADDRESS=localhost:6379 -MEANING_OF_LIFE=42 -MULTILINE_VAR="hello\nworld" +pip install python-dotenv ``` -You can optionally prefix each line with the word `export`, which is totally ignored by this library, but might allow you to [`source`](https://bash.cyberciti.biz/guide/Source_command) the file in bash. +If your application takes its configuration from environment variables, like a 12-factor +application, launching it in development is not very practical because you have to set +those environment variables yourself. -``` -export S3_BUCKET=YOURS3BUCKET -export SECRET_KEY=YOURSECRETKEYGOESHERE +To help you with that, you can add Python-dotenv to your application to make it load the +configuration from a `.env` file when it is present (e.g. in development) while remaining +configurable via the environment: + +```python +from dotenv import load_dotenv + +load_dotenv() # take environment variables from .env. + +# Code of your application, which uses environment variables (e.g. from `os.environ` or +# `os.getenv`) as if they came from the actual environment. ``` -Python-dotenv can interpolate variables using POSIX variable expansion. +By default, `load_dotenv` doesn't override existing environment variables. -The value of a variable is the first of the values defined in the following list: +To configure the development environment, add a `.env` in the root directory of your +project: -- Value of that variable in the environment. -- Value of that variable in the `.env` file. -- Default value, if provided. -- Empty string. +``` +. +├── .env +└── foo.py +``` -Ensure that variables are surrounded with `{}` like `${HOME}` as bare -variables such as `$HOME` are not expanded. +The syntax of `.env` files supported by python-dotenv is similar to that of Bash: -```shell -CONFIG_PATH=${HOME}/.config/foo +```bash +# Development settings DOMAIN=example.org -EMAIL=admin@${DOMAIN} -DEBUG=${DEBUG:-false} +ADMIN_EMAIL=admin@${DOMAIN} +ROOT_URL=${DOMAIN}/app ``` -## Getting started +If you use variables in values, ensure they are surrounded with `{` and `}`, like +`${DOMAIN}`, as bare variables such as `$DOMAIN` are not expanded. -Install the latest version with: +You will probably want to add `.env` to your `.gitignore`, especially if it contains +secrets like a password. -```shell -pip install -U python-dotenv -``` +See the section "File format" below for more information about what you can write in a +`.env` file. -Assuming you have created the `.env` file along-side your settings -module. +## Other Use Cases - . - ├── .env - └── settings.py +### Load configuration without altering the environment -Add the following code to your `settings.py`: +The function `dotenv_values` works more or less the same way as `load_dotenv`, except it +doesn't touch the environment, it just returns a `dict` with the values parsed from the +`.env` file. ```python -# settings.py -from dotenv import load_dotenv -load_dotenv() - -# OR, the same with increased verbosity -load_dotenv(verbose=True) +from dotenv import dotenv_values -# OR, explicitly providing path to '.env' -from pathlib import Path # python3 only -env_path = Path('.') / '.env' -load_dotenv(dotenv_path=env_path) +config = dotenv_values(".env") # config = {"USER": "foo", "EMAIL": "foo@example.org"} ``` -At this point, parsed key/value from the `.env` file is now present as -system environment variable and they can be conveniently accessed via -`os.getenv()`: +This notably enables advanced configuration management: ```python -# settings.py import os -SECRET_KEY = os.getenv("EMAIL") -DATABASE_PASSWORD = os.getenv("DATABASE_PASSWORD") -``` - -`load_dotenv` does not override existing System environment variables. To -override, pass `override=True` to `load_dotenv()`. - -`load_dotenv` also accepts `encoding` parameter to open the `.env` file. The default encoding is platform dependent (whatever `locale.getpreferredencoding()` returns), but any encoding supported by Python can be used. See the [codecs](https://docs.python.org/3/library/codecs.html#standard-encodings) module for the list of supported encodings. - -You can use `find_dotenv()` method that will try to find a `.env` file -by (a) guessing where to start using `__file__` or the working directory --- allowing this to work in non-file contexts such as IPython notebooks -and the REPL, and then (b) walking up the directory tree looking for the -specified file -- called `.env` by default. +from dotenv import dotenv_values -```python -from dotenv import load_dotenv, find_dotenv -load_dotenv(find_dotenv()) +config = { + **dotenv_values(".env.shared"), # load shared development variables + **dotenv_values(".env.secret"), # load sensitive variables + **os.environ, # override loaded values with environment variables +} ``` -### In-memory filelikes +### Parse configuration as a stream -It is possible to not rely on the filesystem to parse filelikes from -other sources (e.g. from a network storage). `load_dotenv` and -`dotenv_values` accepts a filelike `stream`. Just be sure to rewind it -before passing. +`load_dotenv` and `dotenv_values` accept [streams][python_streams] via their `stream` +argument. It is thus possible to load the variables from sources other than the +filesystem (e.g. the network). ```python ->>> from io import StringIO # Python2: from StringIO import StringIO ->>> from dotenv import dotenv_values ->>> filelike = StringIO('SPAM=EGGS\n') ->>> filelike.seek(0) ->>> parsed = dotenv_values(stream=filelike) ->>> parsed['SPAM'] -'EGGS' -``` - -The returned value is dictionary with key-value pairs. - -`dotenv_values` could be useful if you need to *consume* the envfile but -not *apply* it directly into the system environment. - -### Django - -If you are using Django, you should add the above loader script at the -top of `wsgi.py` and `manage.py`. +from io import StringIO +from dotenv import load_dotenv -## IPython Support +config = StringIO("USER=foo\nEMAIL=foo@example.org") +load_dotenv(stream=config) +``` -You can use dotenv with IPython. You can either let the dotenv search -for `.env` with `%dotenv` or provide the path to the `.env` file explicitly; see -below for usages. +### Load .env files in IPython - %load_ext dotenv +You can use dotenv in IPython. By default, it will use `find_dotenv` to search for a +`.env` file: - # Use find_dotenv to locate the file - %dotenv +```python +%load_ext dotenv +%dotenv +``` - # Specify a particular file - %dotenv relative/or/absolute/path/to/.env +You can also specify a path: - # Use '-o' to indicate override of existing variables - %dotenv -o +```python +%dotenv relative/or/absolute/path/to/.env +``` - # Use '-v' to turn verbose mode on - %dotenv -v +Optional flags: +- `-o` to override existing variables. +- `-v` for increased verbosity. ## Command-line Interface -For command-line support, use the CLI option during installation: +A CLI interface `dotenv` is also included, which helps you manipulate the `.env` file +without manually opening it. ```shell -pip install -U "python-dotenv[cli]" +$ pip install "python-dotenv[cli]" +$ dotenv set USER foo +$ dotenv set EMAIL foo@example.org +$ dotenv list +USER=foo +EMAIL=foo@example.org +$ dotenv run -- python foo.py ``` -A CLI interface `dotenv` is also included, which helps you manipulate -the `.env` file without manually opening it. The same CLI installed on -remote machine combined with fabric (discussed later) will enable you to -update your settings on a remote server; handy, isn't it! +Run `dotenv --help` for more information about the options and subcommands. -``` -Usage: dotenv [OPTIONS] COMMAND [ARGS]... - - This script is used to set, get or unset values from a .env file. - -Options: - -f, --file PATH Location of the .env file, defaults to .env - file in current working directory. - -q, --quote [always|never|auto] - Whether to quote or not the variable values. - Default mode is always. This does not affect - parsing. - --help Show this message and exit. - -Commands: - get Retrive the value for the given key. - list Display all the stored key/value. - run Run command with environment variables from .env file present - set Store the given key/value. - unset Removes the given key. -``` +## File format +The format is not formally specified and still improves over time. That being said, +`.env` files should mostly look like Bash files. -### Setting config on Remote Servers - -We make use of excellent [Fabric](http://www.fabfile.org/) to accomplish -this. Add a config task to your local fabfile; `dotenv_path` is the -location of the absolute path of `.env` file on the remote server. - -```python -# fabfile.py - -import dotenv -from fabric.api import task, run, env - -# absolute path to the location of .env on remote server. -env.dotenv_path = '/opt/myapp/.env' - -@task -def config(action=None, key=None, value=None): - '''Manage project configuration via .env - - e.g: fab config:set,, - fab config:get, - fab config:unset, - fab config:list - ''' - run('touch %(dotenv_path)s' % env) - command = dotenv.get_cli_string(env.dotenv_path, action, key, value) - run(command) -``` +Keys can be unquoted or single-quoted. Values can be unquoted, single- or double-quoted. +Spaces before and after keys, equal signs, and values are ignored. Values can be followed +by a comment. Lines can start with the `export` directive, which has no effect on their +interpretation. -Usage is designed to mirror the Heroku config API very closely. +Allowed escape sequences: -Get all your remote config info with `fab config`: +- in single-quoted values: `\\`, `\'` +- in double-quoted values: `\\`, `\'`, `\"`, `\a`, `\b`, `\f`, `\n`, `\r`, `\t`, `\v` - $ fab config - foo="bar" +### Multiline values -Set remote config variables with `fab config:set,,`: +It is possible for single- or double-quoted values to span multiple lines. The following +examples are equivalent: - $ fab config:set,hello,world +```bash +FOO="first line +second line" +``` -Get a single remote config variables with `fab config:get,`: +```bash +FOO="first line\nsecond line" +``` - $ fab config:get,hello +### Variable expansion -Delete a remote config variables with `fab config:unset,`: +Python-dotenv can interpolate variables using POSIX variable expansion. - $ fab config:unset,hello +With `load_dotenv(override=True)` or `dotenv_values()`, the value of a variable is the +first of the values defined in the following list: -Thanks entirely to fabric and not one bit to this project, you can chain -commands like so: -`fab config:set,, config:set,,` +- Value of that variable in the `.env` file. +- Value of that variable in the environment. +- Default value, if provided. +- Empty string. - $ fab config:set,hello,world config:set,foo,bar config:set,fizz=buzz +With `load_dotenv(override=False)`, the value of a variable is the first of the values +defined in the following list: +- Value of that variable in the environment. +- Value of that variable in the `.env` file. +- Default value, if provided. +- Empty string. ## Related Projects @@ -262,14 +206,21 @@ commands like so: Procfile-based applications. - [django-dotenv](https://github.com/jpadilla/django-dotenv) - [django-environ](https://github.com/joke2k/django-environ) +- [django-environ-2](https://github.com/sergeyklay/django-environ-2) - [django-configuration](https://github.com/jezdez/django-configurations) - [dump-env](https://github.com/sobolevn/dump-env) - [environs](https://github.com/sloria/environs) - [dynaconf](https://github.com/rochacbruno/dynaconf) - ## Acknowledgements -This project is currently maintained by [Saurabh Kumar](https://saurabh-kumar.com) and [Bertrand Bonnefoy-Claudet](https://github.com/bbc2) and would not -have been possible without the support of these [awesome +This project is currently maintained by [Saurabh Kumar](https://saurabh-kumar.com) and +[Bertrand Bonnefoy-Claudet](https://github.com/bbc2) and would not have been possible +without the support of these [awesome people](https://github.com/theskumar/python-dotenv/graphs/contributors). + +[build_status_badge]: https://github.com/theskumar/python-dotenv/actions/workflows/test.yml/badge.svg +[build_status_link]: https://github.com/theskumar/python-dotenv/actions/workflows/test.yml +[pypi_badge]: https://badge.fury.io/py/python-dotenv.svg +[pypi_link]: http://badge.fury.io/py/python-dotenv +[python_streams]: https://docs.python.org/3/library/io.html diff --git a/requirements.txt b/requirements.txt index e5e4de12..39302b21 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ bumpversion -typing; python_version<"3.5" click flake8>=2.2.3 ipython @@ -8,6 +7,7 @@ pytest-cov pytest>=3.9 sh>=1.09 tox +types-mock wheel twine portray diff --git a/setup.cfg b/setup.cfg index c19d6bb7..a20d2498 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.13.0 +current_version = 0.19.0 commit = True tag = True @@ -14,15 +14,17 @@ max-line-length = 120 exclude = .tox,.git,docs,venv,.venv [mypy] +check_untyped_defs = true ignore_missing_imports = true [metadata] -description-file = README.rst +description_file = README.md [tool:pytest] testpaths = tests [coverage:run] +relative_files = True source = dotenv [coverage:paths] @@ -33,6 +35,7 @@ source = [coverage:report] show_missing = True +include = */site-packages/dotenv/* exclude_lines = if IS_TYPE_CHECKING: pragma: no cover diff --git a/setup.py b/setup.py index db69a9ce..06ad2dd9 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import io from setuptools import setup @@ -19,13 +18,13 @@ def read_files(files): setup( name="python-dotenv", - description="Add .env support to your django/flask apps in development and deployments", + description="Read key-value pairs from a .env file and set them as environment variables", long_description=long_description, long_description_content_type='text/markdown', version=meta['__version__'], author="Saurabh Kumar", author_email="me+github@saurabh-kumar.com", - url="http://github.com/theskumar/python-dotenv", + url="https://github.com/theskumar/python-dotenv", keywords=['environment variables', 'deployments', 'settings', 'env', 'dotenv', 'configurations', 'python'], packages=['dotenv'], @@ -33,9 +32,7 @@ def read_files(files): package_data={ 'dotenv': ['py.typed'], }, - install_requires=[ - "typing; python_version<'3.5'", - ], + python_requires=">=3.5", extras_require={ 'cli': ['click>=5.0', ], }, @@ -43,16 +40,16 @@ def read_files(files): [console_scripts] dotenv=dotenv.cli:cli ''', + license='BSD-3-Clause', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: Implementation :: PyPy', 'Intended Audience :: Developers', 'Intended Audience :: System Administrators', diff --git a/src/dotenv/__init__.py b/src/dotenv/__init__.py index b88d9bc2..3512d101 100644 --- a/src/dotenv/__init__.py +++ b/src/dotenv/__init__.py @@ -1,18 +1,21 @@ -from .compat import IS_TYPE_CHECKING -from .main import load_dotenv, get_key, set_key, unset_key, find_dotenv, dotenv_values +from typing import Any, Optional -if IS_TYPE_CHECKING: - from typing import Any, Optional +from .main import (dotenv_values, find_dotenv, get_key, load_dotenv, set_key, + unset_key) -def load_ipython_extension(ipython): - # type: (Any) -> None +def load_ipython_extension(ipython: Any) -> None: from .ipython import load_ipython_extension load_ipython_extension(ipython) -def get_cli_string(path=None, action=None, key=None, value=None, quote=None): - # type: (Optional[str], Optional[str], Optional[str], Optional[str], Optional[str]) -> str +def get_cli_string( + path: Optional[str] = None, + action: Optional[str] = None, + key: Optional[str] = None, + value: Optional[str] = None, + quote: Optional[str] = None, +): """Returns a string suitable for running as a shell script. Useful for converting a arguments passed to a fabric task diff --git a/src/dotenv/cli.py b/src/dotenv/cli.py index 91a8e3d3..b7ae24af 100644 --- a/src/dotenv/cli.py +++ b/src/dotenv/cli.py @@ -1,6 +1,7 @@ import os import sys from subprocess import Popen +from typing import Any, Dict, List try: import click @@ -9,37 +10,40 @@ 'Run pip install "python-dotenv[cli]" to fix this.') sys.exit(1) -from .compat import IS_TYPE_CHECKING, to_env from .main import dotenv_values, get_key, set_key, unset_key from .version import __version__ -if IS_TYPE_CHECKING: - from typing import Any, List, Dict - @click.group() @click.option('-f', '--file', default=os.path.join(os.getcwd(), '.env'), - type=click.Path(exists=True), + type=click.Path(file_okay=True), help="Location of the .env file, defaults to .env file in current working directory.") @click.option('-q', '--quote', default='always', type=click.Choice(['always', 'never', 'auto']), help="Whether to quote or not the variable values. Default mode is always. This does not affect parsing.") +@click.option('-e', '--export', default=False, + type=click.BOOL, + help="Whether to write the dot file as an executable bash script.") @click.version_option(version=__version__) @click.pass_context -def cli(ctx, file, quote): - # type: (click.Context, Any, Any) -> None +def cli(ctx: click.Context, file: Any, quote: Any, export: Any) -> None: '''This script is used to set, get or unset values from a .env file.''' ctx.obj = {} - ctx.obj['FILE'] = file ctx.obj['QUOTE'] = quote + ctx.obj['EXPORT'] = export + ctx.obj['FILE'] = file @cli.command() @click.pass_context -def list(ctx): - # type: (click.Context) -> None +def list(ctx: click.Context) -> None: '''Display all the stored key/value.''' file = ctx.obj['FILE'] + if not os.path.isfile(file): + raise click.BadParameter( + 'Path "%s" does not exist.' % (file), + ctx=ctx + ) dotenv_as_dict = dotenv_values(file) for k, v in dotenv_as_dict.items(): click.echo('%s=%s' % (k, v)) @@ -49,12 +53,12 @@ def list(ctx): @click.pass_context @click.argument('key', required=True) @click.argument('value', required=True) -def set(ctx, key, value): - # type: (click.Context, Any, Any) -> None +def set(ctx: click.Context, key: Any, value: Any) -> None: '''Store the given key/value.''' file = ctx.obj['FILE'] quote = ctx.obj['QUOTE'] - success, key, value = set_key(file, key, value, quote) + export = ctx.obj['EXPORT'] + success, key, value = set_key(file, key, value, quote, export) if success: click.echo('%s=%s' % (key, value)) else: @@ -64,13 +68,17 @@ def set(ctx, key, value): @cli.command() @click.pass_context @click.argument('key', required=True) -def get(ctx, key): - # type: (click.Context, Any) -> None +def get(ctx: click.Context, key: Any) -> None: '''Retrieve the value for the given key.''' file = ctx.obj['FILE'] + if not os.path.isfile(file): + raise click.BadParameter( + 'Path "%s" does not exist.' % (file), + ctx=ctx + ) stored_value = get_key(file, key) if stored_value: - click.echo('%s=%s' % (key, stored_value)) + click.echo(stored_value) else: exit(1) @@ -78,8 +86,7 @@ def get(ctx, key): @cli.command() @click.pass_context @click.argument('key', required=True) -def unset(ctx, key): - # type: (click.Context, Any) -> None +def unset(ctx: click.Context, key: Any) -> None: '''Removes the given key.''' file = ctx.obj['FILE'] quote = ctx.obj['QUOTE'] @@ -92,12 +99,25 @@ def unset(ctx, key): @cli.command(context_settings={'ignore_unknown_options': True}) @click.pass_context +@click.option( + "--override/--no-override", + default=True, + help="Override variables from the environment file with those from the .env file.", +) @click.argument('commandline', nargs=-1, type=click.UNPROCESSED) -def run(ctx, commandline): - # type: (click.Context, List[str]) -> None +def run(ctx: click.Context, override: bool, commandline: List[str]) -> None: """Run command with environment variables present.""" file = ctx.obj['FILE'] - dotenv_as_dict = {to_env(k): to_env(v) for (k, v) in dotenv_values(file).items() if v is not None} + if not os.path.isfile(file): + raise click.BadParameter( + 'Invalid value for \'-f\' "%s" does not exist.' % (file), + ctx=ctx + ) + dotenv_as_dict = { + k: v + for (k, v) in dotenv_values(file).items() + if v is not None and (override or k not in os.environ) + } if not commandline: click.echo('No command given.') @@ -106,8 +126,7 @@ def run(ctx, commandline): exit(ret) -def run_command(command, env): - # type: (List[str], Dict[str, str]) -> int +def run_command(command: List[str], env: Dict[str, str]) -> int: """Run command in sub process. Runs the command in a sub process with the variables from `env` diff --git a/src/dotenv/compat.py b/src/dotenv/compat.py deleted file mode 100644 index f8089bf4..00000000 --- a/src/dotenv/compat.py +++ /dev/null @@ -1,49 +0,0 @@ -import sys - -PY2 = sys.version_info[0] == 2 # type: bool - -if PY2: - from StringIO import StringIO # noqa -else: - from io import StringIO # noqa - - -def is_type_checking(): - # type: () -> bool - try: - from typing import TYPE_CHECKING - except ImportError: - return False - return TYPE_CHECKING - - -IS_TYPE_CHECKING = is_type_checking() - - -if IS_TYPE_CHECKING: - from typing import Text - - -def to_env(text): - # type: (Text) -> str - """ - Encode a string the same way whether it comes from the environment or a `.env` file. - """ - if PY2: - return text.encode(sys.getfilesystemencoding() or "utf-8") - else: - return text - - -def to_text(string): - # type: (str) -> Text - """ - Make a string Unicode if it isn't already. - - This is useful for defining raw unicode strings because `ur"foo"` isn't valid in - Python 3. - """ - if PY2: - return string.decode("utf-8") - else: - return string diff --git a/src/dotenv/ipython.py b/src/dotenv/ipython.py index 7f1b13d6..7df727cd 100644 --- a/src/dotenv/ipython.py +++ b/src/dotenv/ipython.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from IPython.core.magic import Magics, line_magic, magics_class # type: ignore from IPython.core.magic_arguments import (argument, magic_arguments, # type: ignore parse_argstring) # type: ignore diff --git a/src/dotenv/main.py b/src/dotenv/main.py index 7fbd24f8..b8d0a4e0 100644 --- a/src/dotenv/main.py +++ b/src/dotenv/main.py @@ -1,50 +1,26 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, unicode_literals - import io import logging import os -import re import shutil import sys import tempfile from collections import OrderedDict from contextlib import contextmanager +from typing import (IO, Dict, Iterable, Iterator, Mapping, Optional, Tuple, + Union) -from .compat import IS_TYPE_CHECKING, PY2, StringIO, to_env from .parser import Binding, parse_stream +from .variables import parse_variables logger = logging.getLogger(__name__) -if IS_TYPE_CHECKING: - from typing import ( - Dict, Iterator, Match, Optional, Pattern, Union, Text, IO, Tuple - ) - if sys.version_info >= (3, 6): - _PathLike = os.PathLike - else: - _PathLike = Text +if sys.version_info >= (3, 6): + _PathLike = os.PathLike +else: + _PathLike = str - if sys.version_info >= (3, 0): - _StringIO = StringIO - else: - _StringIO = StringIO[Text] - -__posix_variable = re.compile( - r""" - \$\{ - (?P[^\}:]*) - (?::- - (?P[^\}]*) - )? - \} - """, - re.VERBOSE, -) # type: Pattern[Text] - - -def with_warn_for_invalid_lines(mappings): - # type: (Iterator[Binding]) -> Iterator[Binding] + +def with_warn_for_invalid_lines(mappings: Iterator[Binding]) -> Iterator[Binding]: for mapping in mappings: if mapping.error: logger.warning( @@ -55,60 +31,71 @@ def with_warn_for_invalid_lines(mappings): class DotEnv(): - - def __init__(self, dotenv_path, verbose=False, encoding=None, interpolate=True): - # type: (Union[Text, _PathLike, _StringIO], bool, Union[None, Text], bool) -> None - self.dotenv_path = dotenv_path # type: Union[Text,_PathLike, _StringIO] - self._dict = None # type: Optional[Dict[Text, Optional[Text]]] + def __init__( + self, + dotenv_path: Optional[Union[str, _PathLike]], + stream: Optional[IO[str]] = None, + verbose: bool = False, + encoding: Union[None, str] = None, + interpolate: bool = True, + override: bool = True, + ) -> None: + self.dotenv_path = dotenv_path # type: Optional[Union[str, _PathLike]] + self.stream = stream # type: Optional[IO[str]] + self._dict = None # type: Optional[Dict[str, Optional[str]]] self.verbose = verbose # type: bool - self.encoding = encoding # type: Union[None, Text] + self.encoding = encoding # type: Union[None, str] self.interpolate = interpolate # type: bool + self.override = override # type: bool @contextmanager - def _get_stream(self): - # type: () -> Iterator[IO[Text]] - if isinstance(self.dotenv_path, StringIO): - yield self.dotenv_path - elif os.path.isfile(self.dotenv_path): + def _get_stream(self) -> Iterator[IO[str]]: + if self.dotenv_path and os.path.isfile(self.dotenv_path): with io.open(self.dotenv_path, encoding=self.encoding) as stream: yield stream + elif self.stream is not None: + yield self.stream else: if self.verbose: - logger.warning("File doesn't exist %s", self.dotenv_path) - yield StringIO('') + logger.info( + "Python-dotenv could not find configuration file %s.", + self.dotenv_path or '.env', + ) + yield io.StringIO('') - def dict(self): - # type: () -> Dict[Text, Optional[Text]] + def dict(self) -> Dict[str, Optional[str]]: """Return dotenv as dict""" if self._dict: return self._dict - values = OrderedDict(self.parse()) - self._dict = resolve_nested_variables(values) if self.interpolate else values + raw_values = self.parse() + + if self.interpolate: + self._dict = OrderedDict(resolve_variables(raw_values, override=self.override)) + else: + self._dict = OrderedDict(raw_values) + return self._dict - def parse(self): - # type: () -> Iterator[Tuple[Text, Optional[Text]]] + def parse(self) -> Iterator[Tuple[str, Optional[str]]]: with self._get_stream() as stream: for mapping in with_warn_for_invalid_lines(parse_stream(stream)): if mapping.key is not None: yield mapping.key, mapping.value - def set_as_environment_variables(self, override=False): - # type: (bool) -> bool + def set_as_environment_variables(self) -> bool: """ - Load the current dotenv as system environemt variable. + Load the current dotenv as system environment variable. """ for k, v in self.dict().items(): - if k in os.environ and not override: + if k in os.environ and not self.override: continue if v is not None: - os.environ[to_env(k)] = to_env(v) + os.environ[k] = v return True - def get(self, key): - # type: (Text) -> Optional[Text] + def get(self, key: str) -> Optional[str]: """ """ data = self.dict() @@ -122,8 +109,7 @@ def get(self, key): return None -def get_key(dotenv_path, key_to_get): - # type: (Union[Text, _PathLike], Text) -> Optional[Text] +def get_key(dotenv_path: Union[str, _PathLike], key_to_get: str) -> Optional[str]: """ Gets the value of a given key from the given .env @@ -133,9 +119,11 @@ def get_key(dotenv_path, key_to_get): @contextmanager -def rewrite(path): - # type: (_PathLike) -> Iterator[Tuple[IO[Text], IO[Text]]] +def rewrite(path: Union[str, _PathLike]) -> Iterator[Tuple[IO[str], IO[str]]]: try: + if not os.path.isfile(path): + with io.open(path, "w+") as source: + source.write("") with tempfile.NamedTemporaryFile(mode="w+", delete=False) as dest: with io.open(path) as source: yield (source, dest) # type: ignore @@ -147,27 +135,35 @@ def rewrite(path): shutil.move(dest.name, path) -def set_key(dotenv_path, key_to_set, value_to_set, quote_mode="always"): - # type: (_PathLike, Text, Text, Text) -> Tuple[Optional[bool], Text, Text] +def set_key( + dotenv_path: Union[str, _PathLike], + key_to_set: str, + value_to_set: str, + quote_mode: str = "always", + export: bool = False, +) -> Tuple[Optional[bool], str, str]: """ Adds or Updates a key/value to the given .env If the .env path given doesn't exist, fails instead of risking creating an orphan .env somewhere in the filesystem """ - value_to_set = value_to_set.strip("'").strip('"') - if not os.path.exists(dotenv_path): - logger.warning("Can't write to %s - it doesn't exist.", dotenv_path) - return None, key_to_set, value_to_set + if quote_mode not in ("always", "auto", "never"): + raise ValueError("Unknown quote_mode: {}".format(quote_mode)) - if " " in value_to_set: - quote_mode = "always" + quote = ( + quote_mode == "always" + or (quote_mode == "auto" and not value_to_set.isalnum()) + ) - if quote_mode == "always": - value_out = '"{}"'.format(value_to_set.replace('"', '\\"')) + if quote: + value_out = "'{}'".format(value_to_set.replace("'", "\\'")) else: value_out = value_to_set - line_out = "{}={}\n".format(key_to_set, value_out) + if export: + line_out = 'export {}={}\n'.format(key_to_set, value_out) + else: + line_out = "{}={}\n".format(key_to_set, value_out) with rewrite(dotenv_path) as (source, dest): replaced = False @@ -183,8 +179,11 @@ def set_key(dotenv_path, key_to_set, value_to_set, quote_mode="always"): return True, key_to_set, value_to_set -def unset_key(dotenv_path, key_to_unset, quote_mode="always"): - # type: (_PathLike, Text, Text) -> Tuple[Optional[bool], Text] +def unset_key( + dotenv_path: Union[str, _PathLike], + key_to_unset: str, + quote_mode: str = "always", +) -> Tuple[Optional[bool], str]: """ Removes a given key from the given .env @@ -210,38 +209,32 @@ def unset_key(dotenv_path, key_to_unset, quote_mode="always"): return removed, key_to_unset -def resolve_nested_variables(values): - # type: (Dict[Text, Optional[Text]]) -> Dict[Text, Optional[Text]] - def _replacement(name, default): - # type: (Text, Optional[Text]) -> Text - """ - get appropriate value for a variable name. - first search in environ, if not found, - then look into the dotenv variables - """ - default = default if default is not None else "" - ret = os.getenv(name, new_values.get(name, default)) - return ret # type: ignore - - def _re_sub_callback(match): - # type: (Match[Text]) -> Text - """ - From a match object gets the variable name and returns - the correct replacement - """ - matches = match.groupdict() - return _replacement(name=matches["name"], default=matches["default"]) # type: ignore +def resolve_variables( + values: Iterable[Tuple[str, Optional[str]]], + override: bool, +) -> Mapping[str, Optional[str]]: + new_values = {} # type: Dict[str, Optional[str]] - new_values = {} + for (name, value) in values: + if value is None: + result = None + else: + atoms = parse_variables(value) + env = {} # type: Dict[str, Optional[str]] + if override: + env.update(os.environ) # type: ignore + env.update(new_values) + else: + env.update(new_values) + env.update(os.environ) # type: ignore + result = "".join(atom.resolve(env) for atom in atoms) - for k, v in values.items(): - new_values[k] = __posix_variable.sub(_re_sub_callback, v) if v is not None else None + new_values[name] = result return new_values -def _walk_to_root(path): - # type: (Text) -> Iterator[Text] +def _walk_to_root(path: str) -> Iterator[str]: """ Yield directories starting from the given directory up to the root """ @@ -259,8 +252,11 @@ def _walk_to_root(path): last_dir, current_dir = current_dir, parent_dir -def find_dotenv(filename='.env', raise_error_if_not_found=False, usecwd=False): - # type: (Text, bool, bool) -> Text +def find_dotenv( + filename: str = '.env', + raise_error_if_not_found: bool = False, + usecwd: bool = False, +) -> str: """ Search in increasingly higher folders for the given file @@ -278,13 +274,7 @@ def _is_interactive(): else: # will work for .py files frame = sys._getframe() - # find first frame that is outside of this file - if PY2 and not __file__.endswith('.py'): - # in Python2 __file__ extension could be .pyc or .pyo (this doesn't account - # for edge case of Python compiled for non-standard extension) - current_file = __file__.rsplit('.', 1)[0] + '.py' - else: - current_file = __file__ + current_file = __file__ while frame.f_code.co_filename == current_file: assert frame.f_back is not None @@ -303,21 +293,68 @@ def _is_interactive(): return '' -def load_dotenv(dotenv_path=None, stream=None, verbose=False, override=False, interpolate=True, **kwargs): - # type: (Union[Text, _PathLike, None], Optional[_StringIO], bool, bool, bool, Union[None, Text]) -> bool +def load_dotenv( + dotenv_path: Union[str, _PathLike, None] = None, + stream: Optional[IO[str]] = None, + verbose: bool = False, + override: bool = False, + interpolate: bool = True, + encoding: Optional[str] = "utf-8", +) -> bool: """Parse a .env file and then load all the variables found as environment variables. - *dotenv_path*: absolute or relative path to .env file. - - *stream*: `StringIO` object with .env content. - - *verbose*: whether to output the warnings related to missing .env file etc. Defaults to `False`. - - *override*: where to override the system environment variables with the variables in `.env` file. - Defaults to `False`. + - *stream*: Text stream (such as `io.StringIO`) with .env content, used if + `dotenv_path` is `None`. + - *verbose*: whether to output a warning the .env file is missing. Defaults to + `False`. + - *override*: whether to override the system environment variables with the variables + in `.env` file. Defaults to `False`. + - *encoding*: encoding to be used to read the file. + + If both `dotenv_path` and `stream`, `find_dotenv()` is used to find the .env file. """ - f = dotenv_path or stream or find_dotenv() - return DotEnv(f, verbose=verbose, interpolate=interpolate, **kwargs).set_as_environment_variables(override=override) + if dotenv_path is None and stream is None: + dotenv_path = find_dotenv() + + dotenv = DotEnv( + dotenv_path=dotenv_path, + stream=stream, + verbose=verbose, + interpolate=interpolate, + override=override, + encoding=encoding, + ) + return dotenv.set_as_environment_variables() + + +def dotenv_values( + dotenv_path: Union[str, _PathLike, None] = None, + stream: Optional[IO[str]] = None, + verbose: bool = False, + interpolate: bool = True, + encoding: Optional[str] = "utf-8", +) -> Dict[str, Optional[str]]: + """ + Parse a .env file and return its content as a dict. + - *dotenv_path*: absolute or relative path to .env file. + - *stream*: `StringIO` object with .env content, used if `dotenv_path` is `None`. + - *verbose*: whether to output a warning the .env file is missing. Defaults to + `False`. + in `.env` file. Defaults to `False`. + - *encoding*: encoding to be used to read the file. -def dotenv_values(dotenv_path=None, stream=None, verbose=False, interpolate=True, **kwargs): - # type: (Union[Text, _PathLike, None], Optional[_StringIO], bool, bool, Union[None, Text]) -> Dict[Text, Optional[Text]] # noqa: E501 - f = dotenv_path or stream or find_dotenv() - return DotEnv(f, verbose=verbose, interpolate=interpolate, **kwargs).dict() + If both `dotenv_path` and `stream`, `find_dotenv()` is used to find the .env file. + """ + if dotenv_path is None and stream is None: + dotenv_path = find_dotenv() + + return DotEnv( + dotenv_path=dotenv_path, + stream=stream, + verbose=verbose, + interpolate=interpolate, + override=True, + encoding=encoding, + ).dict() diff --git a/src/dotenv/parser.py b/src/dotenv/parser.py index 2c93cbd0..398bd49a 100644 --- a/src/dotenv/parser.py +++ b/src/dotenv/parser.py @@ -1,18 +1,11 @@ import codecs import re +from typing import (IO, Iterator, Match, NamedTuple, Optional, # noqa:F401 + Pattern, Sequence, Tuple) -from .compat import IS_TYPE_CHECKING, to_text -if IS_TYPE_CHECKING: - from typing import ( # noqa:F401 - IO, Iterator, Match, NamedTuple, Optional, Pattern, Sequence, Text, - Tuple - ) - - -def make_regex(string, extra_flags=0): - # type: (str, int) -> Pattern[Text] - return re.compile(to_text(string), re.UNICODE | extra_flags) +def make_regex(string: str, extra_flags: int = 0) -> Pattern[str]: + return re.compile(string, re.UNICODE | extra_flags) _newline = make_regex(r"(\r\n|\n|\r)") @@ -24,7 +17,7 @@ def make_regex(string, extra_flags=0): _equal_sign = make_regex(r"(=[^\S\r\n]*)") _single_quoted_value = make_regex(r"'((?:\\'|[^'])*)'") _double_quoted_value = make_regex(r'"((?:\\"|[^"])*)"') -_unquoted_value_part = make_regex(r"([^ \r\n]*)") +_unquoted_value = make_regex(r"([^\r\n]*)") _comment = make_regex(r"(?:[^\S\r\n]*#[^\r\n]*)?") _end_of_line = make_regex(r"[^\S\r\n]*(?:\r\n|\n|\r|$)") _rest_of_line = make_regex(r"[^\r\n]*(?:\r|\n|\r\n)?") @@ -32,67 +25,39 @@ def make_regex(string, extra_flags=0): _single_quote_escapes = make_regex(r"\\[\\']") -try: - # this is necessary because we only import these from typing - # when we are type checking, and the linter is upset if we - # re-import - import typing - - Original = typing.NamedTuple( - "Original", - [ - ("string", typing.Text), - ("line", int), - ], - ) - - Binding = typing.NamedTuple( - "Binding", - [ - ("key", typing.Optional[typing.Text]), - ("value", typing.Optional[typing.Text]), - ("original", Original), - ("error", bool), - ], - ) -except ImportError: - from collections import namedtuple - Original = namedtuple( # type: ignore - "Original", - [ - "string", - "line", - ], - ) - Binding = namedtuple( # type: ignore - "Binding", - [ - "key", - "value", - "original", - "error", - ], - ) +Original = NamedTuple( + "Original", + [ + ("string", str), + ("line", int), + ], +) + +Binding = NamedTuple( + "Binding", + [ + ("key", Optional[str]), + ("value", Optional[str]), + ("original", Original), + ("error", bool), + ], +) class Position: - def __init__(self, chars, line): - # type: (int, int) -> None + def __init__(self, chars: int, line: int) -> None: self.chars = chars self.line = line @classmethod - def start(cls): - # type: () -> Position + def start(cls) -> "Position": return cls(chars=0, line=1) - def set(self, other): - # type: (Position) -> None + def set(self, other: "Position") -> None: self.chars = other.chars self.line = other.line - def advance(self, string): - # type: (Text) -> None + def advance(self, string: str) -> None: self.chars += len(string) self.line += len(re.findall(_newline, string)) @@ -102,41 +67,34 @@ class Error(Exception): class Reader: - def __init__(self, stream): - # type: (IO[Text]) -> None + def __init__(self, stream: IO[str]) -> None: self.string = stream.read() self.position = Position.start() self.mark = Position.start() - def has_next(self): - # type: () -> bool + def has_next(self) -> bool: return self.position.chars < len(self.string) - def set_mark(self): - # type: () -> None + def set_mark(self) -> None: self.mark.set(self.position) - def get_marked(self): - # type: () -> Original + def get_marked(self) -> Original: return Original( string=self.string[self.mark.chars:self.position.chars], line=self.mark.line, ) - def peek(self, count): - # type: (int) -> Text + def peek(self, count: int) -> str: return self.string[self.position.chars:self.position.chars + count] - def read(self, count): - # type: (int) -> Text + def read(self, count: int) -> str: result = self.string[self.position.chars:self.position.chars + count] if len(result) < count: raise Error("read: End of string") self.position.advance(result) return result - def read_regex(self, regex): - # type: (Pattern[Text]) -> Sequence[Text] + def read_regex(self, regex: Pattern[str]) -> Sequence[str]: match = regex.match(self.string, self.position.chars) if match is None: raise Error("read_regex: Pattern not found") @@ -144,17 +102,14 @@ def read_regex(self, regex): return match.groups() -def decode_escapes(regex, string): - # type: (Pattern[Text], Text) -> Text - def decode_match(match): - # type: (Match[Text]) -> Text +def decode_escapes(regex: Pattern[str], string: str) -> str: + def decode_match(match: Match[str]) -> str: return codecs.decode(match.group(0), 'unicode-escape') # type: ignore return regex.sub(decode_match, string) -def parse_key(reader): - # type: (Reader) -> Optional[Text] +def parse_key(reader: Reader) -> Optional[str]: char = reader.peek(1) if char == "#": return None @@ -165,20 +120,12 @@ def parse_key(reader): return key -def parse_unquoted_value(reader): - # type: (Reader) -> Text - value = u"" - while True: - (part,) = reader.read_regex(_unquoted_value_part) - value += part - after = reader.peek(2) - if len(after) < 2 or after[0] in u"\r\n" or after[1] in u" #\r\n": - return value - value += reader.read(2) +def parse_unquoted_value(reader: Reader) -> str: + (part,) = reader.read_regex(_unquoted_value) + return re.sub(r"\s+#.*", "", part).rstrip() -def parse_value(reader): - # type: (Reader) -> Text +def parse_value(reader: Reader) -> str: char = reader.peek(1) if char == u"'": (value,) = reader.read_regex(_single_quoted_value) @@ -192,8 +139,7 @@ def parse_value(reader): return parse_unquoted_value(reader) -def parse_binding(reader): - # type: (Reader) -> Binding +def parse_binding(reader: Reader) -> Binding: reader.set_mark() try: reader.read_regex(_multiline_whitespace) @@ -209,7 +155,7 @@ def parse_binding(reader): reader.read_regex(_whitespace) if reader.peek(1) == "=": reader.read_regex(_equal_sign) - value = parse_value(reader) # type: Optional[Text] + value = parse_value(reader) # type: Optional[str] else: value = None reader.read_regex(_comment) @@ -230,8 +176,7 @@ def parse_binding(reader): ) -def parse_stream(stream): - # type: (IO[Text]) -> Iterator[Binding] +def parse_stream(stream: IO[str]) -> Iterator[Binding]: reader = Reader(stream) while reader.has_next(): yield parse_binding(reader) diff --git a/src/dotenv/variables.py b/src/dotenv/variables.py new file mode 100644 index 00000000..d77b700c --- /dev/null +++ b/src/dotenv/variables.py @@ -0,0 +1,88 @@ +import re +from abc import ABCMeta +from typing import Iterator, Mapping, Optional, Pattern + +_posix_variable = re.compile( + r""" + \$\{ + (?P[^\}:]*) + (?::- + (?P[^\}]*) + )? + \} + """, + re.VERBOSE, +) # type: Pattern[str] + + +class Atom(): + __metaclass__ = ABCMeta + + def __ne__(self, other: object) -> bool: + result = self.__eq__(other) + if result is NotImplemented: + return NotImplemented + return not result + + def resolve(self, env: Mapping[str, Optional[str]]) -> str: + raise NotImplementedError + + +class Literal(Atom): + def __init__(self, value: str) -> None: + self.value = value + + def __repr__(self) -> str: + return "Literal(value={})".format(self.value) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, self.__class__): + return NotImplemented + return self.value == other.value + + def __hash__(self) -> int: + return hash((self.__class__, self.value)) + + def resolve(self, env: Mapping[str, Optional[str]]) -> str: + return self.value + + +class Variable(Atom): + def __init__(self, name: str, default: Optional[str]) -> None: + self.name = name + self.default = default + + def __repr__(self) -> str: + return "Variable(name={}, default={})".format(self.name, self.default) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, self.__class__): + return NotImplemented + return (self.name, self.default) == (other.name, other.default) + + def __hash__(self) -> int: + return hash((self.__class__, self.name, self.default)) + + def resolve(self, env: Mapping[str, Optional[str]]) -> str: + default = self.default if self.default is not None else "" + result = env.get(self.name, default) + return result if result is not None else "" + + +def parse_variables(value: str) -> Iterator[Atom]: + cursor = 0 + + for match in _posix_variable.finditer(value): + (start, end) = match.span() + name = match.groupdict()["name"] + default = match.groupdict()["default"] + + if start > cursor: + yield Literal(value=value[cursor:start]) + + yield Variable(name=name, default=default) + cursor = end + + length = len(value) + if cursor < length: + yield Literal(value=value[cursor:length]) diff --git a/src/dotenv/version.py b/src/dotenv/version.py index f23a6b39..11ac8e1a 100644 --- a/src/dotenv/version.py +++ b/src/dotenv/version.py @@ -1 +1 @@ -__version__ = "0.13.0" +__version__ = "0.19.0" diff --git a/tests/conftest.py b/tests/conftest.py index 7a9ed7e5..24a82528 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,9 @@ @pytest.fixture def cli(): - yield CliRunner() + runner = CliRunner() + with runner.isolated_filesystem(): + yield runner @pytest.fixture diff --git a/tests/test_cli.py b/tests/test_cli.py index edc62fff..223476fe 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,4 +1,5 @@ -# -*- coding: utf-8 -*- +import os + import pytest import sh @@ -20,7 +21,7 @@ def test_list_non_existent_file(cli): result = cli.invoke(dotenv_cli, ['--file', 'nx_file', 'list']) assert result.exit_code == 2, result.output - assert "Invalid value for '-f'" in result.output + assert "does not exist" in result.output def test_list_no_file(cli): @@ -35,7 +36,7 @@ def test_get_existing_value(cli, dotenv_file): result = cli.invoke(dotenv_cli, ['--file', dotenv_file, 'get', 'a']) - assert (result.exit_code, result.output) == (0, "a=b\n") + assert (result.exit_code, result.output) == (0, "b\n") def test_get_non_existent_value(cli, dotenv_file): @@ -48,7 +49,7 @@ def test_get_no_file(cli): result = cli.invoke(dotenv_cli, ['--file', 'nx_file', 'get', 'a']) assert result.exit_code == 2 - assert "Invalid value for '-f'" in result.output + assert "does not exist" in result.output def test_unset_existing_value(cli, dotenv_file): @@ -71,16 +72,34 @@ def test_unset_non_existent_value(cli, dotenv_file): @pytest.mark.parametrize( "quote_mode,variable,value,expected", ( - ("always", "HELLO", "WORLD", 'HELLO="WORLD"\n'), - ("never", "HELLO", "WORLD", 'HELLO=WORLD\n'), - ("auto", "HELLO", "WORLD", 'HELLO=WORLD\n'), - ("auto", "HELLO", "HELLO WORLD", 'HELLO="HELLO WORLD"\n'), + ("always", "a", "x", "a='x'\n"), + ("never", "a", "x", 'a=x\n'), + ("auto", "a", "x", "a=x\n"), + ("auto", "a", "x y", "a='x y'\n"), + ("auto", "a", "$", "a='$'\n"), ) ) -def test_set_options(cli, dotenv_file, quote_mode, variable, value, expected): +def test_set_quote_options(cli, dotenv_file, quote_mode, variable, value, expected): result = cli.invoke( dotenv_cli, - ["--file", dotenv_file, "--quote", quote_mode, "set", variable, value] + ["--file", dotenv_file, "--export", "false", "--quote", quote_mode, "set", variable, value] + ) + + assert (result.exit_code, result.output) == (0, "{}={}\n".format(variable, value)) + assert open(dotenv_file, "r").read() == expected + + +@pytest.mark.parametrize( + "dotenv_file,export_mode,variable,value,expected", + ( + (".nx_file", "true", "a", "x", "export a='x'\n"), + (".nx_file", "false", "a", "x", "a='x'\n"), + ) +) +def test_set_export(cli, dotenv_file, export_mode, variable, value, expected): + result = cli.invoke( + dotenv_cli, + ["--file", dotenv_file, "--quote", "always", "--export", export_mode, "set", variable, value] ) assert (result.exit_code, result.output) == (0, "{}={}\n".format(variable, value)) @@ -97,7 +116,7 @@ def test_set_no_file(cli): result = cli.invoke(dotenv_cli, ["--file", "nx_file", "set"]) assert result.exit_code == 2 - assert "Invalid value for '-f'" in result.output + assert "Missing argument" in result.output def test_get_default_path(tmp_path): @@ -107,7 +126,7 @@ def test_get_default_path(tmp_path): result = sh.dotenv("get", "a") - assert result == "a=b\n" + assert result == "b\n" def test_run(tmp_path): @@ -121,6 +140,32 @@ def test_run(tmp_path): assert result == "b\n" +def test_run_with_existing_variable(tmp_path): + sh.cd(str(tmp_path)) + dotenv_file = str(tmp_path / ".env") + with open(dotenv_file, "w") as f: + f.write("a=b") + env = dict(os.environ) + env.update({"LANG": "en_US.UTF-8", "a": "c"}) + + result = sh.dotenv("run", "printenv", "a", _env=env) + + assert result == "b\n" + + +def test_run_with_existing_variable_not_overridden(tmp_path): + sh.cd(str(tmp_path)) + dotenv_file = str(tmp_path / ".env") + with open(dotenv_file, "w") as f: + f.write("a=b") + env = dict(os.environ) + env.update({"LANG": "en_US.UTF-8", "a": "c"}) + + result = sh.dotenv("run", "--no-override", "printenv", "a", _env=env) + + assert result == "c\n" + + def test_run_with_none_value(tmp_path): sh.cd(str(tmp_path)) dotenv_file = str(tmp_path / ".env") diff --git a/tests/test_ipython.py b/tests/test_ipython.py index afbf4797..8983bf13 100644 --- a/tests/test_ipython.py +++ b/tests/test_ipython.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import os import mock diff --git a/tests/test_main.py b/tests/test_main.py index f877d21a..13e2791c 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - +import io import logging import os import sys @@ -11,39 +9,34 @@ import sh import dotenv -from dotenv.compat import PY2, StringIO def test_set_key_no_file(tmp_path): nx_file = str(tmp_path / "nx") logger = logging.getLogger("dotenv.main") - with mock.patch.object(logger, "warning") as mock_warning: + with mock.patch.object(logger, "warning"): result = dotenv.set_key(nx_file, "foo", "bar") - assert result == (None, "foo", "bar") - assert not os.path.exists(nx_file) - mock_warning.assert_called_once_with( - "Can't write to %s - it doesn't exist.", - nx_file, - ) + assert result == (True, "foo", "bar") + assert os.path.exists(nx_file) @pytest.mark.parametrize( "before,key,value,expected,after", [ - ("", "a", "", (True, "a", ""), 'a=""\n'), - ("", "a", "b", (True, "a", "b"), 'a="b"\n'), - ("", "a", "'b'", (True, "a", "b"), 'a="b"\n'), - ("", "a", "\"b\"", (True, "a", "b"), 'a="b"\n'), - ("", "a", "b'c", (True, "a", "b'c"), 'a="b\'c"\n'), - ("", "a", "b\"c", (True, "a", "b\"c"), 'a="b\\\"c"\n'), - ("a=b", "a", "c", (True, "a", "c"), 'a="c"\n'), - ("a=b\n", "a", "c", (True, "a", "c"), 'a="c"\n'), - ("a=b\n\n", "a", "c", (True, "a", "c"), 'a="c"\n\n'), - ("a=b\nc=d", "a", "e", (True, "a", "e"), 'a="e"\nc=d'), - ("a=b\nc=d\ne=f", "c", "g", (True, "c", "g"), 'a=b\nc="g"\ne=f'), - ("a=b\n", "c", "d", (True, "c", "d"), 'a=b\nc="d"\n'), + ("", "a", "", (True, "a", ""), "a=''\n"), + ("", "a", "b", (True, "a", "b"), "a='b'\n"), + ("", "a", "'b'", (True, "a", "'b'"), "a='\\'b\\''\n"), + ("", "a", "\"b\"", (True, "a", '"b"'), "a='\"b\"'\n"), + ("", "a", "b'c", (True, "a", "b'c"), "a='b\\'c'\n"), + ("", "a", "b\"c", (True, "a", "b\"c"), "a='b\"c'\n"), + ("a=b", "a", "c", (True, "a", "c"), "a='c'\n"), + ("a=b\n", "a", "c", (True, "a", "c"), "a='c'\n"), + ("a=b\n\n", "a", "c", (True, "a", "c"), "a='c'\n\n"), + ("a=b\nc=d", "a", "e", (True, "a", "e"), "a='e'\nc=d"), + ("a=b\nc=d\ne=f", "c", "g", (True, "c", "g"), "a=b\nc='g'\ne=f"), + ("a=b\n", "c", "d", (True, "c", "d"), "a=b\nc='d'\n"), ], ) def test_set_key(dotenv_file, before, key, value, expected, after): @@ -74,14 +67,19 @@ def test_get_key_no_file(tmp_path): nx_file = str(tmp_path / "nx") logger = logging.getLogger("dotenv.main") - with mock.patch.object(logger, "warning") as mock_warning: + with mock.patch.object(logger, "info") as mock_info, \ + mock.patch.object(logger, "warning") as mock_warning: result = dotenv.get_key(nx_file, "foo") assert result is None + mock_info.assert_has_calls( + calls=[ + mock.call("Python-dotenv could not find configuration file %s.", nx_file) + ], + ) mock_warning.assert_has_calls( calls=[ - mock.call("File doesn't exist %s", nx_file), - mock.call("Key %s not found in %s.", "foo", nx_file), + mock.call("Key %s not found in %s.", "foo", nx_file) ], ) @@ -228,10 +226,10 @@ def test_load_dotenv_existing_file(dotenv_file): def test_load_dotenv_no_file_verbose(): logger = logging.getLogger("dotenv.main") - with mock.patch.object(logger, "warning") as mock_warning: + with mock.patch.object(logger, "info") as mock_info: dotenv.load_dotenv('.does_not_exist', verbose=True) - mock_warning.assert_called_once_with("File doesn't exist %s", ".does_not_exist") + mock_info.assert_called_once_with("Python-dotenv could not find configuration file %s.", ".does_not_exist") @mock.patch.dict(os.environ, {"a": "c"}, clear=True) @@ -256,17 +254,48 @@ def test_load_dotenv_existing_variable_override(dotenv_file): assert os.environ == {"a": "b"} +@mock.patch.dict(os.environ, {"a": "c"}, clear=True) +def test_load_dotenv_redefine_var_used_in_file_no_override(dotenv_file): + with open(dotenv_file, "w") as f: + f.write('a=b\nd="${a}"') + + result = dotenv.load_dotenv(dotenv_file) + + assert result is True + assert os.environ == {"a": "c", "d": "c"} + + +@mock.patch.dict(os.environ, {"a": "c"}, clear=True) +def test_load_dotenv_redefine_var_used_in_file_with_override(dotenv_file): + with open(dotenv_file, "w") as f: + f.write('a=b\nd="${a}"') + + result = dotenv.load_dotenv(dotenv_file, override=True) + + assert result is True + assert os.environ == {"a": "b", "d": "b"} + + @mock.patch.dict(os.environ, {}, clear=True) -def test_load_dotenv_utf_8(): - stream = StringIO("a=à") +def test_load_dotenv_string_io_utf_8(): + stream = io.StringIO("a=à") result = dotenv.load_dotenv(stream=stream) assert result is True - if PY2: - assert os.environ == {"a": "à".encode(sys.getfilesystemencoding())} - else: - assert os.environ == {"a": "à"} + assert os.environ == {"a": "à"} + + +@mock.patch.dict(os.environ, {}, clear=True) +def test_load_dotenv_file_stream(dotenv_file): + with open(dotenv_file, "w") as f: + f.write("a=b") + + with open(dotenv_file, "r") as f: + result = dotenv.load_dotenv(stream=f) + + assert result is True + assert os.environ == {"a": "b"} def test_load_dotenv_in_current_dir(tmp_path): @@ -329,13 +358,28 @@ def test_dotenv_values_file(dotenv_file): # Reused ({"b": "c"}, "a=${b}${b}", True, {"a": "cc"}), + + # Re-defined and used in file + ({"b": "c"}, "b=d\na=${b}", True, {"a": "d", "b": "d"}), + ({}, "a=b\na=c\nd=${a}", True, {"a": "c", "d": "c"}), + ({}, "a=b\nc=${a}\nd=e\nc=${d}", True, {"a": "b", "c": "e", "d": "e"}), ], ) -def test_dotenv_values_stream(env, string, interpolate, expected): +def test_dotenv_values_string_io(env, string, interpolate, expected): with mock.patch.dict(os.environ, env, clear=True): - stream = StringIO(string) + stream = io.StringIO(string) stream.seek(0) result = dotenv.dotenv_values(stream=stream, interpolate=interpolate) assert result == expected + + +def test_dotenv_values_file_stream(dotenv_file): + with open(dotenv_file, "w") as f: + f.write("a=b") + + with open(dotenv_file, "r") as f: + result = dotenv.dotenv_values(stream=f) + + assert result == {"a": "b"} diff --git a/tests/test_parser.py b/tests/test_parser.py index f8075138..b0621173 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,7 +1,7 @@ -# -*- coding: utf-8 -*- +import io + import pytest -from dotenv.compat import StringIO from dotenv.parser import Binding, Original, parse_stream @@ -19,20 +19,40 @@ (u"# a=b", [Binding(key=None, value=None, original=Original(string=u"# a=b", line=1), error=False)]), (u"a=b#c", [Binding(key=u"a", value=u"b#c", original=Original(string=u"a=b#c", line=1), error=False)]), ( - u'a=b # comment', - [Binding(key=u"a", value=u"b", original=Original(string=u"a=b # comment", line=1), error=False)], + u'a=b #c', + [Binding(key=u"a", value=u"b", original=Original(string=u"a=b #c", line=1), error=False)], + ), + ( + u'a=b\t#c', + [Binding(key=u"a", value=u"b", original=Original(string=u"a=b\t#c", line=1), error=False)], + ), + ( + u"a=b c", + [Binding(key=u"a", value=u"b c", original=Original(string=u"a=b c", line=1), error=False)], + ), + ( + u"a=b\tc", + [Binding(key=u"a", value=u"b\tc", original=Original(string=u"a=b\tc", line=1), error=False)], + ), + ( + u"a=b c", + [Binding(key=u"a", value=u"b c", original=Original(string=u"a=b c", line=1), error=False)], + ), + ( + u"a=b\u00a0 c", + [Binding(key=u"a", value=u"b\u00a0 c", original=Original(string=u"a=b\u00a0 c", line=1), error=False)], ), ( - u"a=b space ", - [Binding(key=u"a", value=u"b space", original=Original(string=u"a=b space ", line=1), error=False)], + u"a=b c ", + [Binding(key=u"a", value=u"b c", original=Original(string=u"a=b c ", line=1), error=False)], ), ( - u"a='b space '", - [Binding(key=u"a", value=u"b space ", original=Original(string=u"a='b space '", line=1), error=False)], + u"a='b c '", + [Binding(key=u"a", value=u"b c ", original=Original(string=u"a='b c '", line=1), error=False)], ), ( - u'a="b space "', - [Binding(key=u"a", value=u"b space ", original=Original(string=u'a="b space "', line=1), error=False)], + u'a="b c "', + [Binding(key=u"a", value=u"b c ", original=Original(string=u'a="b c "', line=1), error=False)], ), ( u"export export_a=1", @@ -146,6 +166,6 @@ ), ]) def test_parse_stream(test_input, expected): - result = parse_stream(StringIO(test_input)) + result = parse_stream(io.StringIO(test_input)) assert list(result) == expected diff --git a/tests/test_variables.py b/tests/test_variables.py new file mode 100644 index 00000000..86b06466 --- /dev/null +++ b/tests/test_variables.py @@ -0,0 +1,35 @@ +import pytest + +from dotenv.variables import Literal, Variable, parse_variables + + +@pytest.mark.parametrize( + "value,expected", + [ + ("", []), + ("a", [Literal(value="a")]), + ("${a}", [Variable(name="a", default=None)]), + ("${a:-b}", [Variable(name="a", default="b")]), + ( + "${a}${b}", + [ + Variable(name="a", default=None), + Variable(name="b", default=None), + ], + ), + ( + "a${b}c${d}e", + [ + Literal(value="a"), + Variable(name="b", default=None), + Literal(value="c"), + Variable(name="d", default=None), + Literal(value="e"), + ], + ), + ] +) +def test_parse_variables(value, expected): + result = parse_variables(value) + + assert list(result) == expected diff --git a/tox.ini b/tox.ini index 2dd61864..2cd63024 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,14 @@ [tox] -envlist = lint,py{27,34,35,36,37,38,34-no-typing},pypy,pypy3,manifest,coverage-report +envlist = lint,py{35,36,37,38,39},pypy3,manifest,coverage-report + +[gh-actions] +python = + 3.5: py35, coverage-report + 3.6: py36, coverage-report + 3.7: py37, coverage-report + 3.8: py38, coverage-report + 3.9: py39, lint, manifest, coverage-report + pypy3: pypy3, coverage-report [testenv] deps = @@ -8,29 +17,22 @@ deps = coverage sh click - py{27,py}: ipython<6.0.0 - py34{,-no-typing}: ipython<7.0.0 - py{35,36,37,38,py3}: ipython + py{35,36,37,38,39,py3}: ipython commands = coverage run --parallel -m pytest {posargs} -[testenv:py34-no-typing] -commands = - pip uninstall --yes typing - coverage run --parallel -m pytest -k 'not test_ipython' {posargs} - [testenv:lint] skip_install = true deps = flake8 mypy + types-mock commands = flake8 src tests + mypy --python-version=3.9 src tests mypy --python-version=3.8 src tests mypy --python-version=3.7 src tests mypy --python-version=3.6 src tests mypy --python-version=3.5 src tests - mypy --python-version=3.4 src tests - mypy --python-version=2.7 src tests [testenv:manifest] deps = check-manifest