feat: Use most compact JSON encoding (getsentry#746)

rhcarvalho · web-flow · commit 7d482b5bfa1d · 2020-06-29T21:50:42.000+02:00
This shrinks event sizes a bit, even when gzip'ed.
The compact representation is documented in the json module.

Alternatively, we can also look into using a custom encoder (that could
also handle datetime objects, instead of the current manual
serialization of those).

In the absence of proper benchmark data, consider a random transaction
event t:

&gt;&gt;&gt; len(json.dumps(t)), len(json.dumps(t, separators=(',', ':')))
(82174, 78516)

That is 95.5% of the original size.

With gzip compression:

&gt;&gt;&gt; len(gzips(json.dumps(t))), len(gzips(json.dumps(t, separators=(',', ':'))))
(13093, 12988)

That is 99.2% of the original size.
diff --git a/sentry_sdk/envelope.py b/sentry_sdk/envelope.py
@@ -6,6 +6,7 @@
 from sentry_sdk._compat import text_type
 from sentry_sdk._types import MYPY
 from sentry_sdk.sessions import Session
+from sentry_sdk.utils import json_dumps
 
 if MYPY:
     from typing import Any
@@ -86,7 +87,7 @@ def serialize_into(
         self, f  # type: Any
     ):
         # type: (...) -> None
-        f.write(json.dumps(self.headers, allow_nan=False).encode("utf-8"))
+        f.write(json_dumps(self.headers))
         f.write(b"\n")
         for item in self.items:
             item.serialize_into(f)
@@ -142,7 +143,7 @@ def get_bytes(self):
                 with open(self.path, "rb") as f:
                     self.bytes = f.read()
             elif self.json is not None:
-                self.bytes = json.dumps(self.json, allow_nan=False).encode("utf-8")
+                self.bytes = json_dumps(self.json)
             else:
                 self.bytes = b""
         return self.bytes
@@ -256,7 +257,7 @@ def serialize_into(
         headers = dict(self.headers)
         length, writer = self.payload._prepare_serialize()
         headers["length"] = length
-        f.write(json.dumps(headers, allow_nan=False).encode("utf-8"))
+        f.write(json_dumps(headers))
         f.write(b"\n")
         writer(f)
         f.write(b"\n")
diff --git a/sentry_sdk/transport.py b/sentry_sdk/transport.py
@@ -1,14 +1,13 @@
 from __future__ import print_function
 
-import json
 import io
 import urllib3  # type: ignore
 import certifi
 import gzip
 
 from datetime import datetime, timedelta
 
-from sentry_sdk.utils import Dsn, logger, capture_internal_exceptions
+from sentry_sdk.utils import Dsn, logger, capture_internal_exceptions, json_dumps
 from sentry_sdk.worker import BackgroundWorker
 from sentry_sdk.envelope import Envelope, get_event_data_category
 
@@ -214,7 +213,7 @@ def _send_event(
 
         body = io.BytesIO()
         with gzip.GzipFile(fileobj=body, mode="w") as f:
-            f.write(json.dumps(event, allow_nan=False).encode("utf-8"))
+            f.write(json_dumps(event))
 
         assert self.parsed_dsn is not None
         logger.debug(
diff --git a/sentry_sdk/utils.py b/sentry_sdk/utils.py
@@ -1,7 +1,8 @@
-import os
-import sys
+import json
 import linecache
 import logging
+import os
+import sys
 
 from datetime import datetime
 
@@ -37,6 +38,12 @@
 MAX_FORMAT_PARAM_LENGTH = 128
 
 
+def json_dumps(data):
+    # type: (Any) -> bytes
+    """Serialize data into a compact JSON representation encoded as UTF-8."""
+    return json.dumps(data, allow_nan=False, separators=(",", ":")).encode("utf-8")
+
+
 def _get_debug_hub():
     # type: () -> Optional[sentry_sdk.Hub]
     # This function is replaced by debug.py
