diff --git a/docs/changelog.md b/docs/changelog.md index 03a9b71..624cdf3 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -4,11 +4,28 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [UNRELEASED] +## [4.0.0](https://github.com/nhairs/python-json-logger/compare/v3.3.3...v4.0.0) - UNRELEASED ### Added - Support `DictConfigurator` prefixes for `rename_fields` and `static_fields`. [#45](https://github.com/nhairs/python-json-logger/pull/45) - Allows using values like `ext://sys.stderr` in `fileConfig`/`dictConfig` value fields. +- Support comma seperated lists for Formatter `fmt` (`style=","`) e.g. `"asctime,message,levelname"` [#15](https://github.com/nhairs/python-json-logger/issues/15) + - Note that this style is specific to `python-json-logger` and thus care should be taken not to pass this format to other logging Formatter implementations. + +### Changed +- Rename `pythonjsonlogger.core.LogRecord` and `log_record` arguemtns to avoid confusion / overlapping with `logging.LogRecord`. [#38](https://github.com/nhairs/python-json-logger/issues/38) + - Affects arguments to `pythonjsonlogger.core.BaseJsonFormatter` (and any child classes). + - `serialize_log_record` + - `add_fields` + - `jsonify_log_record` + - `process_log_record` + - Note: functions referring to `log_record` have **not** had their function name changed. + +### Removed +- Remove support for providing strings instead of objects when instantiating formatters. Instead use the `DictConfigurator` `ext://` prefix format when using `fileConfig`/`dictConfig`. [#47](https://github.com/nhairs/python-json-logger/issues/47) + - Affects `pythonjsonlogger.json.JsonFormatter`: `json_default`, `json_encoder`, `json_serializer`. + - Affects `pythonjsonlogger.orjson.OrjsonFormatter`: `json_default`. + - Affects `pythonjsonlogger.msgspec.MsgspecFormatter`: `json_default`. Thanks @rubensa diff --git a/docs/cookbook.md b/docs/cookbook.md index c0755d5..91a60ac 100644 --- a/docs/cookbook.md +++ b/docs/cookbook.md @@ -32,8 +32,8 @@ You can modify the `dict` of data that will be logged by overriding the `process ```python class SillyFormatter(JsonFormatter): - def process_log_record(log_record): - new_record = {k[::-1]: v for k, v in log_record.items()} + def process_log_record(log_data): + new_record = {k[::-1]: v for k, v in log_data.items()} return new_record ``` @@ -119,9 +119,9 @@ Another method would be to create a custom formatter class and override the `pro ## ----------------------------------------------------------------------------- # Reuse REQUEST_ID stuff from solution 2 class MyFormatter(JsonFormatter): - def process_log_record(self, log_record): - log_record["request_id"] = get_request_id() - return log_record + def process_log_record(self, log_data): + log_data["request_id"] = get_request_id() + return log_data handler.setFormatter(MyFormatter()) @@ -148,6 +148,7 @@ formatters: default: "()": pythonjsonlogger.json.JsonFormatter format: "%(asctime)s %(levelname)s %(name)s %(module)s %(funcName)s %(lineno)s %(message)s" + json_default: ext://logging_config.my_json_default rename_fields: "asctime": "timestamp" "levelname": "status" @@ -178,13 +179,23 @@ loggers: propagate: no ``` -You'll notice that we are using `ext://...` for the `static_fields`. This will load data from other modules such as the one below. +You'll notice that we are using `ext://...` for `json_default` and`static_fields`. This will load data from other modules such as the one below. ```python title="logging_config.py" import importlib.metadata import os +class Dummy: + pass + + +def my_json_default(obj: Any) -> Any: + if isinstance(obj, Dummy): + return "DUMMY" + return obj + + def get_version_metadata(): # https://stackoverflow.com/a/78082532 version = importlib.metadata.version(PROJECT_NAME) diff --git a/src/pythonjsonlogger/core.py b/src/pythonjsonlogger/core.py index a00510b..2f2bcd2 100644 --- a/src/pythonjsonlogger/core.py +++ b/src/pythonjsonlogger/core.py @@ -7,11 +7,10 @@ ## Standard Library from datetime import datetime, timezone -import importlib import logging import re import sys -from typing import Optional, Union, Callable, List, Dict, Container, Any, Sequence +from typing import Optional, Union, List, Dict, Container, Any, Sequence if sys.version_info >= (3, 10): from typing import TypeAlias @@ -72,31 +71,15 @@ ## Type Aliases ## ----------------------------------------------------------------------------- -OptionalCallableOrStr: TypeAlias = Optional[Union[Callable, str]] -"""Type alias""" +LogData: TypeAlias = Dict[str, Any] +"""Type alias -LogRecord: TypeAlias = Dict[str, Any] -"""Type alias""" +*Changed in 4.0*: renamed from `LogRecord` to `LogData` +""" ### FUNCTIONS ### ============================================================================ -def str_to_object(obj: Any) -> Any: - """Import strings to an object, leaving non-strings as-is. - - Args: - obj: the object or string to process - - *New in 3.1* - """ - - if not isinstance(obj, str): - return obj - - module_name, attribute_name = obj.rsplit(".", 1) - return getattr(importlib.import_module(module_name), attribute_name) - - def merge_record_extra( record: logging.LogRecord, target: Dict, @@ -135,7 +118,7 @@ class BaseJsonFormatter(logging.Formatter): *Changed in 3.2*: `defaults` argument is no longer ignored. - *Added in UNRELEASED*: `exc_info_as_array` and `stack_info_as_array` options are added. + *Added in 4.0*: `exc_info_as_array` and `stack_info_as_array` options are added. """ _style: Union[logging.PercentStyle, str] # type: ignore[assignment] @@ -192,6 +175,12 @@ def __init__( - Renaming fields now preserves the order that fields were added in and avoids adding missing fields. The original behaviour, missing fields have a value of `None`, is still available by setting `rename_fields_keep_missing` to `True`. + + *Added in 4.0*: + + - `fmt` now supports comma seperated lists (`style=","`). Note that this style is specific + to `python-json-logger` and thus care should be taken to not to pass this format to other + logging Formatter implementations. """ ## logging.Formatter compatibility ## --------------------------------------------------------------------- @@ -203,7 +192,7 @@ def __init__( self._style = _style self._fmt = _style._fmt - elif not validate: + elif style == "," or not validate: self._style = style self._fmt = fmt @@ -269,11 +258,11 @@ def format(self, record: logging.LogRecord) -> str: if record.stack_info and not message_dict.get("stack_info"): message_dict["stack_info"] = self.formatStack(record.stack_info) - log_record: LogRecord = {} - self.add_fields(log_record, record, message_dict) - log_record = self.process_log_record(log_record) + log_data: LogData = {} + self.add_fields(log_data, record, message_dict) + log_data = self.process_log_record(log_data) - return self.serialize_log_record(log_record) + return self.serialize_log_record(log_data) ## JSON Formatter Specific Methods ## ------------------------------------------------------------------------- @@ -288,6 +277,18 @@ def parse(self) -> List[str]: Returns: list of fields to be extracted and serialized """ + if self._fmt is None: + # TODO: does it matter that we do this before checking if the style is valid? + # (we already (mostly) check for valid style names in __init__ + return [] + + if isinstance(self._style, str) and self._style == ",": + # TODO: should we check that there are no empty fields? + # If yes we should do this in __init__ where we validate other styles? + # Do we drop empty fields? + # etc + return [field.strip() for field in self._fmt.split(",") if field.strip()] + if isinstance(self._style, logging.StringTemplateStyle): formatter_style_pattern = STYLE_STRING_TEMPLATE_REGEX @@ -302,22 +303,21 @@ def parse(self) -> List[str]: else: raise ValueError(f"Style {self._style!r} is not supported") - if self._fmt: - return formatter_style_pattern.findall(self._fmt) - - return [] + return formatter_style_pattern.findall(self._fmt) - def serialize_log_record(self, log_record: LogRecord) -> str: - """Returns the final representation of the log record. + def serialize_log_record(self, log_data: LogData) -> str: + """Returns the final representation of the data to be logged Args: - log_record: the log record + log_data: the data + + *Changed in 4.0*: `log_record` renamed to `log_data` """ - return self.prefix + self.jsonify_log_record(log_record) + return self.prefix + self.jsonify_log_record(log_data) def add_fields( self, - log_record: Dict[str, Any], + log_data: Dict[str, Any], record: logging.LogRecord, message_dict: Dict[str, Any], ) -> None: @@ -326,38 +326,40 @@ def add_fields( This method can be overridden to implement custom logic for adding fields. Args: - log_record: data that will be logged + log_data: data that will be logged record: the record to extract data from message_dict: dictionary that was logged instead of a message. e.g `logger.info({"is_this_message_dict": True})` + + *Changed in 4.0*: `log_record` renamed to `log_data` """ for field in self.defaults: - log_record[self._get_rename(field)] = self.defaults[field] + log_data[self._get_rename(field)] = self.defaults[field] for field in self._required_fields: - log_record[self._get_rename(field)] = record.__dict__.get(field) + log_data[self._get_rename(field)] = record.__dict__.get(field) for data_dict in [self.static_fields, message_dict]: for key, value in data_dict.items(): - log_record[self._get_rename(key)] = value + log_data[self._get_rename(key)] = value merge_record_extra( record, - log_record, + log_data, reserved=self._skip_fields, rename_fields=self.rename_fields, ) if self.timestamp: key = self.timestamp if isinstance(self.timestamp, str) else "timestamp" - log_record[self._get_rename(key)] = datetime.fromtimestamp( + log_data[self._get_rename(key)] = datetime.fromtimestamp( record.created, tz=timezone.utc ) if self.rename_fields_keep_missing: for field in self.rename_fields.values(): - if field not in log_record: - log_record[field] = None + if field not in log_data: + log_data[field] = None return def _get_rename(self, key: str) -> str: @@ -365,26 +367,30 @@ def _get_rename(self, key: str) -> str: # Child Methods # .......................................................................... - def jsonify_log_record(self, log_record: LogRecord) -> str: - """Convert this log record into a JSON string. + def jsonify_log_record(self, log_data: LogData) -> str: + """Convert the log data into a JSON string. Child classes MUST override this method. Args: - log_record: the data to serialize + log_data: the data to serialize + + *Changed in 4.0*: `log_record` renamed to `log_data` """ raise NotImplementedError() - def process_log_record(self, log_record: LogRecord) -> LogRecord: - """Custom processing of the log record. + def process_log_record(self, log_data: LogData) -> LogData: + """Custom processing of the data to be logged. Child classes can override this method to alter the log record before it is serialized. Args: - log_record: incoming data + log_data: incoming data + + *Changed in 4.0*: `log_record` renamed to `log_data` """ - return log_record + return log_data def formatException(self, ei) -> Union[str, list[str]]: # type: ignore """Format and return the specified exception information. diff --git a/src/pythonjsonlogger/json.py b/src/pythonjsonlogger/json.py index 21e78d0..cce67c6 100644 --- a/src/pythonjsonlogger/json.py +++ b/src/pythonjsonlogger/json.py @@ -67,9 +67,9 @@ class JsonFormatter(core.BaseJsonFormatter): def __init__( self, *args, - json_default: core.OptionalCallableOrStr = None, - json_encoder: core.OptionalCallableOrStr = None, - json_serializer: Union[Callable, str] = json.dumps, + json_default: Optional[Callable] = None, + json_encoder: Optional[Callable] = None, + json_serializer: Callable = json.dumps, json_indent: Optional[Union[int, str]] = None, json_ensure_ascii: bool = True, **kwargs, @@ -87,19 +87,19 @@ def __init__( """ super().__init__(*args, **kwargs) - self.json_default = core.str_to_object(json_default) - self.json_encoder = core.str_to_object(json_encoder) - self.json_serializer = core.str_to_object(json_serializer) + self.json_default = json_default + self.json_encoder = json_encoder + self.json_serializer = json_serializer self.json_indent = json_indent self.json_ensure_ascii = json_ensure_ascii if not self.json_encoder and not self.json_default: self.json_encoder = JsonEncoder return - def jsonify_log_record(self, log_record: core.LogRecord) -> str: - """Returns a json string of the log record.""" + def jsonify_log_record(self, log_data: core.LogData) -> str: + """Returns a json string of the log data.""" return self.json_serializer( - log_record, + log_data, default=self.json_default, cls=self.json_encoder, indent=self.json_indent, diff --git a/src/pythonjsonlogger/msgspec.py b/src/pythonjsonlogger/msgspec.py index 8646f85..4f17391 100644 --- a/src/pythonjsonlogger/msgspec.py +++ b/src/pythonjsonlogger/msgspec.py @@ -6,7 +6,7 @@ from __future__ import annotations ## Standard Library -from typing import Any +from typing import Any, Optional, Callable ## Installed @@ -43,7 +43,7 @@ class MsgspecFormatter(core.BaseJsonFormatter): def __init__( self, *args, - json_default: core.OptionalCallableOrStr = msgspec_default, + json_default: Optional[Callable] = msgspec_default, **kwargs, ) -> None: """ @@ -54,10 +54,10 @@ def __init__( """ super().__init__(*args, **kwargs) - self.json_default = core.str_to_object(json_default) + self.json_default = json_default self._encoder = msgspec.json.Encoder(enc_hook=self.json_default) return - def jsonify_log_record(self, log_record: core.LogRecord) -> str: - """Returns a json string of the log record.""" - return self._encoder.encode(log_record).decode("utf8") + def jsonify_log_record(self, log_data: core.LogData) -> str: + """Returns a json string of the log data.""" + return self._encoder.encode(log_data).decode("utf8") diff --git a/src/pythonjsonlogger/orjson.py b/src/pythonjsonlogger/orjson.py index 16db842..af726b3 100644 --- a/src/pythonjsonlogger/orjson.py +++ b/src/pythonjsonlogger/orjson.py @@ -6,7 +6,7 @@ from __future__ import annotations ## Standard Library -from typing import Any +from typing import Any, Optional, Callable ## Installed @@ -45,7 +45,7 @@ class OrjsonFormatter(core.BaseJsonFormatter): def __init__( self, *args, - json_default: core.OptionalCallableOrStr = orjson_default, + json_default: Optional[Callable] = orjson_default, json_indent: bool = False, **kwargs, ) -> None: @@ -58,14 +58,14 @@ def __init__( """ super().__init__(*args, **kwargs) - self.json_default = core.str_to_object(json_default) + self.json_default = json_default self.json_indent = json_indent return - def jsonify_log_record(self, log_record: core.LogRecord) -> str: - """Returns a json string of the log record.""" + def jsonify_log_record(self, log_data: core.LogData) -> str: + """Returns a json string of the log data.""" opt = orjson.OPT_NON_STR_KEYS if self.json_indent: opt |= orjson.OPT_INDENT_2 - return orjson.dumps(log_record, default=self.json_default, option=opt).decode("utf8") + return orjson.dumps(log_data, default=self.json_default, option=opt).decode("utf8") diff --git a/tests/test_dictconfig.py b/tests/test_dictconfig.py index e956c03..c14cc75 100644 --- a/tests/test_dictconfig.py +++ b/tests/test_dictconfig.py @@ -19,12 +19,24 @@ _LOGGER_COUNT = 0 EXT_VAL = 999 + +class Dummy: + pass + + +def my_json_default(obj: Any) -> Any: + if isinstance(obj, Dummy): + return "DUMMY" + return obj + + LOGGING_CONFIG = { "version": 1, "disable_existing_loggers": False, "formatters": { "default": { "()": "pythonjsonlogger.json.JsonFormatter", + "json_default": "ext://tests.test_dictconfig.my_json_default", "static_fields": {"ext-val": "ext://tests.test_dictconfig.EXT_VAL"}, } }, @@ -73,8 +85,12 @@ def env() -> Generator[LoggingEnvironment, None, None]: ### TESTS ### ============================================================================ def test_external_reference_support(env: LoggingEnvironment): - env.logger.info("hello") + + assert logging.root.handlers[0].formatter.json_default is my_json_default # type: ignore[union-attr] + + env.logger.info("hello", extra={"dummy": Dummy()}) log_json = env.load_json() assert log_json["ext-val"] == EXT_VAL + assert log_json["dummy"] == "DUMMY" return diff --git a/tests/test_formatters.py b/tests/test_formatters.py index 050fc5e..0a0f458 100644 --- a/tests/test_formatters.py +++ b/tests/test_formatters.py @@ -158,12 +158,22 @@ def test_default_format(env: LoggingEnvironment, class_: type[BaseJsonFormatter] @pytest.mark.parametrize("class_", ALL_FORMATTERS) def test_percentage_format(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): - env.set_formatter( - class_( - # All kind of different styles to check the regex - "[%(levelname)8s] %(message)s %(filename)s:%(lineno)d %(asctime)" - ) - ) + # Note: All kind of different %s styles to check the regex + env.set_formatter(class_("[%(levelname)8s] %(message)s %(filename)s:%(lineno)d %(asctime)")) + + msg = "testing logging format" + env.logger.info(msg) + log_json = env.load_json() + + assert log_json["message"] == msg + assert log_json.keys() == {"levelname", "message", "filename", "lineno", "asctime"} + return + + +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_comma_format(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): + # Note: we have double comma `,,` to test handling "empty" names + env.set_formatter(class_("levelname,,message,filename,lineno,asctime,", style=",")) msg = "testing logging format" env.logger.info(msg) @@ -380,9 +390,9 @@ def test_log_extra(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): def test_custom_logic_adds_field(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): class CustomJsonFormatter(class_): # type: ignore[valid-type,misc] - def process_log_record(self, log_record): - log_record["custom"] = "value" - return super().process_log_record(log_record) + def process_log_record(self, log_data): + log_data["custom"] = "value" + return super().process_log_record(log_data) env.set_formatter(CustomJsonFormatter()) env.logger.info("message")