diff --git a/docs/changelog.md b/docs/changelog.md index 8d01b63..03a9b71 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [UNRELEASED] + +### Added +- Support `DictConfigurator` prefixes for `rename_fields` and `static_fields`. [#45](https://github.com/nhairs/python-json-logger/pull/45) + - Allows using values like `ext://sys.stderr` in `fileConfig`/`dictConfig` value fields. + +Thanks @rubensa + ## [3.3.0](https://github.com/nhairs/python-json-logger/compare/v3.2.1...v3.3.0) - 2025-03-06 ### Added diff --git a/docs/contributing.md b/docs/contributing.md index 61ed3e0..9b58a93 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -25,7 +25,13 @@ The following are things that can be worked on without an existing issue: ### 2. Fork the repository and make your changes -We don't have styling documentation, so where possible try to match existing code. This includes the use of "headings" and "dividers" (this will make sense when you look at the code). +#### Coding Style + +Before writing any code, please familiarize yourself with our [Python Style Guide](style-guide.md). This document outlines our coding conventions, formatting expectations, and common patterns used in the project. Adhering to this guide is crucial for maintaining code consistency and readability. + +While the style guide covers detailed conventions, always try to match the style of existing code in the module you are working on, especially regarding local patterns and structure. + +#### Development Setup All devlopment tooling can be installed (usually into a virtual environment), using the `dev` optional dependency: @@ -47,10 +53,14 @@ mypy src tests pytest ``` -If making changes to the documentation you can preview the changes locally using `mkdocs`. Changes to the README can be previewed using [`grip`](https://github.com/joeyespo/grip) (not included in `dev` dependencies). +The above commands (`black`, `pylint`, `mypy`, `pytest`) should all be run before submitting a pull request. + +If making changes to the documentation you can preview the changes locally using `mkdocs`. Changes to the `README.md` can be previewed using a tool like [`grip`](https://github.com/joeyespo/grip) (installable via `pip install grip`). ```shell mkdocs serve +# For README preview (after installing grip): +# grip ``` !!! note diff --git a/docs/cookbook.md b/docs/cookbook.md index 11edc2a..4b747c5 100644 --- a/docs/cookbook.md +++ b/docs/cookbook.md @@ -32,7 +32,7 @@ You can modify the `dict` of data that will be logged by overriding the `process ```python class SillyFormatter(JsonFormatter): - def process_log_record(log_record): + def process_log_record(self, log_record): new_record = {k[::-1]: v for k, v in log_record.items()} return new_record ``` @@ -92,7 +92,7 @@ def generate_request_id(): class RequestIdFilter(logging.Filter): def filter(self, record): - record.record_id = get_request_id() + record.request_id = get_request_id() # Add request_id to the LogRecord return True request_id_filter = RequestIdFilter() @@ -139,35 +139,61 @@ main_3() ## Using `fileConfig` -To use the module with a config file using the [`fileConfig` function](https://docs.python.org/3/library/logging.config.html#logging.config.fileConfig), use the class `pythonjsonlogger.json.JsonFormatter`. Here is a sample config file. - -```ini -[loggers] -keys = root,custom +To use the module with a yaml config file using the [`fileConfig` function](https://docs.python.org/3/library/logging.config.html#logging.config.fileConfig), use the class `pythonjsonlogger.json.JsonFormatter`. Here is a sample config file: + +```yaml title="example_config.yaml" +version: 1 +disable_existing_loggers: False +formatters: + default: + "()": pythonjsonlogger.json.JsonFormatter + format: "%(asctime)s %(levelname)s %(name)s %(module)s %(funcName)s %(lineno)s %(message)s" + rename_fields: + "asctime": "timestamp" + "levelname": "status" + static_fields: + "service": ext://logging_config.PROJECT_NAME + "env": ext://logging_config.ENVIRONMENT + "version": ext://logging_config.PROJECT_VERSION + "app_log": "true" +handlers: + default: + formatter: default + class: logging.StreamHandler + stream: ext://sys.stderr + access: + formatter: default + class: logging.StreamHandler + stream: ext://sys.stdout +loggers: + uvicorn.error: + level: INFO + handlers: + - default + propagate: no + uvicorn.access: + level: INFO + handlers: + - access + propagate: no +``` -[logger_root] -handlers = +You'll notice that we are using `ext://...` for the `static_fields`. This will load data from other modules such as the one below. -[logger_custom] -level = INFO -handlers = custom -qualname = custom +```python title="logging_config.py" +import importlib.metadata +import os -[handlers] -keys = custom -[handler_custom] -class = StreamHandler -level = INFO -formatter = json -args = (sys.stdout,) +def get_version_metadata(): + # https://stackoverflow.com/a/78082532 + version = importlib.metadata.version(PROJECT_NAME) + return version -[formatters] -keys = json -[formatter_json] -format = %(message)s -class = pythonjsonlogger.jsonlogger.JsonFormatter +PROJECT_NAME = 'test-api' +PROJECT_VERSION = get_version_metadata() +ENVIRONMENT = os.environ.get('ENVIRONMENT', 'dev') ``` ## Logging Expensive to Compute Data diff --git a/docs/index.md b/docs/index.md index 6f15b2e..4c39bfb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -23,11 +23,13 @@ This library assumes that you are famliar with the `logging` standard library pa - **Fully Customizable Output Fields:** Control required, excluded, and static fields including automatically picking up custom attributes on `LogRecord` objects. Fields can be renamed before they are output. - **Encode Any Type:** Encoders are customized to ensure that something sane is logged for any input including those that aren't supported by default. For example formatting UUID objects into their string representation and bytes objects into a base 64 encoded string. -## Quick Start +## Getting Started -Follow our [Quickstart Guide](quickstart.md). +Jump right in with our [Quickstart Guide](quickstart.md) to get `python-json-logger` integrated into your project quickly. -```python title="TLDR" +Here's a small taste of what it looks like: + +```python title="Example Usage" import logging from pythonjsonlogger.json import JsonFormatter @@ -39,27 +41,32 @@ handler.setFormatter(JsonFormatter()) logger.addHandler(handler) -logger.info("Logging using pythonjsonlogger!", extra={"more_data": True}) - -# {"message": "Logging using pythonjsonlogger!", "more_data": true} +logger.info("Logging using python-json-logger!", extra={"more_data": True}) +# {"message": "Logging using python-json-logger!", "more_data": true} ``` +## Where to Go Next -## Bugs, Feature Requests etc -Please [submit an issue on github](https://github.com/nhairs/python-json-logger/issues). +* **[Quickstart Guide](quickstart.md):** For installation and basic setup. +* **[Cookbook](cookbook.md):** For more advanced usage patterns and recipes. +* **API Reference:** Dive into the details of specific formatters, functions, and classes (see navigation menu). +* **[Contributing Guidelines](contributing.md):** If you'd like to contribute to the project. +* **[Changelog](changelog.md):** To see what's new in recent versions. -In the case of bug reports, please help us help you by following best practices [^1^](https://marker.io/blog/write-bug-report/) [^2^](https://www.chiark.greenend.org.uk/~sgtatham/bugs.html). +## Project Information -In the case of feature requests, please provide background to the problem you are trying to solve so that we can a solution that makes the most sense for the library as well as your use case. +### Bugs, Feature Requests, etc. +Please [submit an issue on GitHub](https://github.com/nhairs/python-json-logger/issues). -## License +In the case of bug reports, please help us help you by following best practices [^1^](https://marker.io/blog/write-bug-report/) [^2^](https://www.chiark.greenend.org.uk/~sgtatham/bugs.html). -This project is licensed under the BSD 2 Clause License - see [`LICENSE`](https://github.com/nhairs/python-json-logger/blob/main/LICENSE) +In the case of feature requests, please provide background to the problem you are trying to solve so that we can find a solution that makes the most sense for the library as well as your use case. -## Authors and Maintainers +### License +This project is licensed under the BSD 2 Clause License - see the [LICENSE file](https://github.com/nhairs/python-json-logger/blob/main/LICENSE) on GitHub. -This project was originally authored by [Zakaria Zajac](https://github.com/madzak) and our wonderful [contributors](https://github.com/nhairs/python-json-logger/graphs/contributors) +### Authors and Maintainers +This project was originally authored by [Zakaria Zajac](https://github.com/madzak) and our wonderful [contributors](https://github.com/nhairs/python-json-logger/graphs/contributors). It is currently maintained by: - - [Nicholas Hairs](https://github.com/nhairs) - [nicholashairs.com](https://www.nicholashairs.com) diff --git a/docs/quickstart.md b/docs/quickstart.md index 7dab254..3613aa4 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -89,7 +89,8 @@ formatter = JsonFormatter( defaults={"environment": "dev"} ) # ... -logger.info("this overwrites the environment field", extras={"environment": "dev"}) +logger.info("this message will have environment=dev by default") +logger.info("this overwrites the environment field", extra={"environment": "prod"}) ``` #### Static Fields @@ -104,7 +105,7 @@ formatter = JsonFormatter( ### Excluding fields -You can prevent fields being added to the output data by adding them to `reserved_attrs`. By default all [`LogRecord` attributes](https://docs.python.org/3/library/logging.html#logrecord-attributes) are exluded. +You can prevent fields being added to the output data by adding them to `reserved_attrs`. By default all [`LogRecord` attributes](https://docs.python.org/3/library/logging.html#logrecord-attributes) are excluded. ```python from pythonjsonlogger.core import RESERVED_ATTRS diff --git a/docs/style-guide.md b/docs/style-guide.md new file mode 100644 index 0000000..ab217c5 --- /dev/null +++ b/docs/style-guide.md @@ -0,0 +1,131 @@ +# Python Style Guide + +This document outlines the coding style, conventions, and common patterns for the `python-json-logger` project. Adhering to this guide will help maintain code consistency, readability, and quality. + +## General Principles + +* **Readability Counts:** Write code that is easy for others (and your future self) to understand. This aligns with [PEP 20 (The Zen of Python)](https://peps.python.org/pep-0020/). +* **Consistency:** Strive for consistency in naming, formatting, and structure throughout the codebase. +* **Simplicity:** Prefer simple, straightforward solutions over overly complex ones. +* **PEP 8:** Follow [PEP 8 (Style Guide for Python Code)](https://peps.python.org/pep-0008/) for all Python code. The automated tools mentioned below will enforce many of these rules. This guide highlights project-specific conventions or particularly important PEP 8 aspects. + +## Formatting and Linting + +We use automated tools to enforce a consistent code style and catch potential errors. These include: + +* **Black:** For opinionated code formatting. +* **Pylint:** For static code analysis and error detection. +* **MyPy:** For static type checking. + +Ensure these tools are run before committing code. Configuration for these tools can be found in `pyproject.toml`, `pylintrc`, and `mypy.ini` respectively. This guide primarily focuses on conventions not automatically verifiable by these tools. + +## Imports + +Imports should be structured into the following groups, separated by a blank line, and generally alphabetized within each group: + +1. **Future Imports:** e.g., `from __future__ import annotations` +2. **Standard Library Imports:** e.g., `import sys`, `from datetime import datetime` +3. **Installed (Third-Party) Library Imports:** e.g., `import pytest` +4. **Application (Local) Imports:** e.g., `from .core import BaseJsonFormatter` (This project-specific pattern is crucial for internal organization). + +## Naming Conventions + +While PEP 8 covers most naming, we emphasize: + +* **Modules:** `lowercase_with_underscores.py` +* **Packages:** `lowercase` +* **Classes & Type Aliases:** `CapWords` (e.g., `BaseJsonFormatter`, `OptionalCallableOrStr`). This is standard, but explicitly stated for clarity. +* **Constants:** `UPPERCASE_WITH_UNDERSCORES` (e.g., `RESERVED_ATTRS`). This is a project convention for module-level constants. + +(Functions, methods, and variables follow standard PEP 8 `lowercase_with_underscores`). + +## Comments + +* Use comments to explain non-obvious code, complex logic, or important design decisions. Avoid comments that merely restate what the code does. +* For internal code organization within files, especially in longer modules or classes, use comments like `## Section Title ##` or `### Subsection Title ###` to delineate logical blocks of code (e.g., `## Parent Methods ##` as seen in `src/pythonjsonlogger/core.py`). This is distinct from Markdown headings used in this document. + +## Docstrings + +* All public modules, classes, functions, and methods **must** have docstrings. +* We use `mkdocstrings` for generating API documentation, which defaults to the **Google Python Style Guide** for docstrings. Please adhere to this style. You can find the guide [here](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings). +* Docstrings should clearly explain the purpose, arguments, return values, and any exceptions raised. +* **Project Convention:** Use the following markers to indicate changes over time: + * `*New in version_number*`: For features added in a specific version. + * `*Changed in version_number*`: For changes made in a specific version. + * `*Deprecated in version_number*`: For features deprecated in a specific version. + + Example: + ```python + def my_function(param1: str, param2: int) -> bool: + """Does something interesting. + + Args: + param1: The first parameter, a string. + param2: The second parameter, an integer. + + Returns: + True if successful, False otherwise. + + Raises: + ValueError: If param2 is negative. + + *New in 3.1* + """ + # ... function logic ... + return True # See 'Return Statements' + ``` + +## Type Hinting + +* All new code **must** include type hints for function arguments, return types, and variables where appropriate, as per PEP 484. +* Use standard types from the `typing` module. +* **Project Convention:** For Python versions older than 3.10, use `typing_extensions.TypeAlias` for creating type aliases. For Python 3.10+, use `typing.TypeAlias` (e.g., `OptionalCallableOrStr: TypeAlias = ...`). + +## Return Statements + +* **Project Convention:** All functions and methods **must** have an explicit `return` statement. +* If a function does not logically return a value, it should end with `return None` or simply `return`. This makes the intent clear and consistent across the codebase. + + Example: + ```python + def process_data(data: dict) -> None: + """Processes the given data.""" + # ... processing logic ... + print(data) + return # or return None + ``` + +## Class Structure + +* Group methods logically within a class (e.g., initialization, public, protected/private, special methods). +* The use of internal code comments like `## Parent Methods ##` (as seen in `src/pythonjsonlogger/core.py`) is encouraged for readability in complex classes. + +## Project-Specific Code Patterns and Idioms + +Familiarize yourself with these patterns commonly used in this project: + +* **Version-Specific Logic:** Using `sys.version_info` for compatibility: + ```python + if sys.version_info >= (3, 10): + # Python 3.10+ specific code + else: + # Code for older versions + ``` +* **Type Aliases for Clarity:** As mentioned in Type Hinting, using `TypeAlias` for complex type combinations improves readability. +* **Custom Exceptions:** Defining custom exception classes for application-specific error conditions (e.g., `MissingPackageError` in `src/pythonjsonlogger/exception.py`). +* **Helper/Utility Functions:** Encapsulating reusable logic in utility modules (e.g., functions in `src/pythonjsonlogger/utils.py`). +* **Conditional Imports for Optional Dependencies:** The pattern in `src/pythonjsonlogger/__init__.py` for checking optional dependencies like `orjson` and `msgspec` using `package_is_available` from `utils.py`. + +## Testing + +This project uses `pytest` for testing. Adherence to good testing practices is crucial. + +* **Test Location:** Tests are located in the `tests/` directory. +* **Test Naming:** Test files `test_*.py`; test functions `test_*`. +* **Fixtures:** Utilize `pytest` fixtures (`@pytest.fixture`) for setup. + * **Project Pattern:** The `LoggingEnvironment` dataclass and `env` fixture in `tests/test_formatters.py` is a key pattern for testing logger behavior. Adapt this for similar scenarios. +* **Parametrization:** Use `@pytest.mark.parametrize` extensively to cover multiple scenarios efficiently. +* **Clarity and Focus:** Each test should be focused and its name descriptive. +* **Assertions:** Use clear, specific `pytest` assertions. + +By following these guidelines, we can ensure that `python-json-logger` remains a high-quality, maintainable, and developer-friendly library. diff --git a/pyproject.toml b/pyproject.toml index 2381b3d..fd1856d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "python-json-logger" -version = "3.3.0" +version = "4.0.0.dev0" description = "JSON Log Formatter for the Python Logging Package" authors = [ {name = "Zakaria Zajac", email = "zak@madzak.com"}, diff --git a/src/pythonjsonlogger/core.py b/src/pythonjsonlogger/core.py index 1a4dee3..1d6c252 100644 --- a/src/pythonjsonlogger/core.py +++ b/src/pythonjsonlogger/core.py @@ -135,7 +135,7 @@ class BaseJsonFormatter(logging.Formatter): *Changed in 3.2*: `defaults` argument is no longer ignored. - *Added in UNRELEASED*: `exc_info_as_array` and `stack_info_as_array` options are added. + *Added in 3.3*: `exc_info_as_array` and `stack_info_as_array` options are added. """ _style: Union[logging.PercentStyle, str] # type: ignore[assignment] @@ -215,9 +215,18 @@ def __init__( ## JSON Logging specific ## --------------------------------------------------------------------- self.prefix = prefix - self.rename_fields = rename_fields if rename_fields is not None else {} + + # We recreate the dict in rename_fields and static_fields to support internal/external + # references which require getting the item to do the conversion. + # For more details see: https://github.com/nhairs/python-json-logger/pull/45 + self.rename_fields = ( + {key: rename_fields[key] for key in rename_fields} if rename_fields is not None else {} + ) + self.static_fields = ( + {key: static_fields[key] for key in static_fields} if static_fields is not None else {} + ) + self.rename_fields_keep_missing = rename_fields_keep_missing - self.static_fields = static_fields if static_fields is not None else {} self.reserved_attrs = set(reserved_attrs if reserved_attrs is not None else RESERVED_ATTRS) self.timestamp = timestamp diff --git a/tests/test_dictconfig.py b/tests/test_dictconfig.py new file mode 100644 index 0000000..e956c03 --- /dev/null +++ b/tests/test_dictconfig.py @@ -0,0 +1,80 @@ +### IMPORTS +### ============================================================================ +## Future +from __future__ import annotations + +## Standard Library +from dataclasses import dataclass +import io +import json +import logging +import logging.config +from typing import Any, Generator + +## Installed +import pytest + +### SETUP +### ============================================================================ +_LOGGER_COUNT = 0 +EXT_VAL = 999 + +LOGGING_CONFIG = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "default": { + "()": "pythonjsonlogger.json.JsonFormatter", + "static_fields": {"ext-val": "ext://tests.test_dictconfig.EXT_VAL"}, + } + }, + "handlers": { + "default": { + "level": "DEBUG", + "formatter": "default", + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout", # Default is stderr + }, + }, + "loggers": { + "": {"handlers": ["default"], "level": "WARNING", "propagate": False}, # root logger + }, +} + + +@dataclass +class LoggingEnvironment: + logger: logging.Logger + buffer: io.StringIO + + def load_json(self) -> Any: + return json.loads(self.buffer.getvalue()) + + +@pytest.fixture +def env() -> Generator[LoggingEnvironment, None, None]: + global _LOGGER_COUNT # pylint: disable=global-statement + _LOGGER_COUNT += 1 + logging.config.dictConfig(LOGGING_CONFIG) + default_formatter = logging.root.handlers[0].formatter + logger = logging.getLogger(f"pythonjsonlogger.tests.{_LOGGER_COUNT}") + logger.setLevel(logging.DEBUG) + buffer = io.StringIO() + handler = logging.StreamHandler(buffer) + handler.setFormatter(default_formatter) + logger.addHandler(handler) + yield LoggingEnvironment(logger=logger, buffer=buffer) + logger.removeHandler(handler) + logger.setLevel(logging.NOTSET) + buffer.close() + return + + +### TESTS +### ============================================================================ +def test_external_reference_support(env: LoggingEnvironment): + env.logger.info("hello") + log_json = env.load_json() + + assert log_json["ext-val"] == EXT_VAL + return