During decompression python tries to read sys.maxsize - which fail on start with MemoryError in case of any resource memory limit set. #119881
Description
Bug report
Bug description:
import logging
import resource
import sys
import boto3
logger = logging.getLogger(__name__)
def format_b(n_bytes):
""" Format a number of bytes into a human-readable format. """
for unit in ['B', 'KB', 'MB', 'GB', 'TB', 'PB']:
if n_bytes < 1024:
return "%3.1f %s" % (n_bytes, unit)
n_bytes /= 1024
return "%.1f %s" % (n_bytes, 'EB')
def set_memory_limit_as(n_bytes):
_, hard = resource.getrlimit(resource.RLIMIT_AS)
logger.info(f"Setting memory limit to {format_b(n_bytes)}")
resource.setrlimit(resource.RLIMIT_AS, (n_bytes, hard))
def set_memory_limit_rss(n_bytes):
_, hard = resource.getrlimit(resource.RLIMIT_RSS)
logger.info(f"Setting RSS memory limit to {format_b(n_bytes)}")
resource.setrlimit(resource.RLIMIT_RSS, (n_bytes, hard))
def set_memory_limit_data(n_bytes):
_, hard = resource.getrlimit(resource.RLIMIT_DATA)
logger.info(f"Setting heap memory limit to {format_b(n_bytes)}")
resource.setrlimit(resource.RLIMIT_DATA, (n_bytes, hard))
def set_memory_limit_memlock(n_bytes):
_, hard = resource.getrlimit(resource.RLIMIT_MEMLOCK)
logger.info(f"Setting virtual memory limit to {format_b(n_bytes)}")
resource.setrlimit(resource.RLIMIT_MEMLOCK, (n_bytes, hard))
def set_memory_limit_stack(n_bytes):
_, hard = resource.getrlimit(resource.RLIMIT_STACK)
logger.info(f"Setting stack memory limit to {format_b(n_bytes)}")
resource.setrlimit(resource.RLIMIT_STACK, (n_bytes, hard))
def main():
logger.info(f"sys maxsize {sys.maxsize}")
limit = 1e9
set_memory_limit_as(limit)
try:
ecs_client = boto3.client("ecs")
except MemoryError as e:
logger.error(f"Got oom limit exception {e}")
if __name__ == "__main__":
main() File "/home/ds/projects/cox/OracleReplicatorToS3/.venv/lib/python3.10/site-packages/boto3/__init__.py", line 92, in client
return _get_default_session().client(*args, **kwargs)
File "/home/ds/projects/cox/OracleReplicatorToS3/.venv/lib/python3.10/site-packages/boto3/session.py", line 299, in client
return self._session.create_client(
File "/home/ds/projects/cox/OracleReplicatorToS3/.venv/lib/python3.10/site-packages/botocore/session.py", line 997, in create_client
client = client_creator.create_client(
File "/home/ds/projects/cox/OracleReplicatorToS3/.venv/lib/python3.10/site-packages/botocore/client.py", line 132, in create_client
service_model = self._load_service_model(service_name, api_version)
File "/home/ds/projects/cox/OracleReplicatorToS3/.venv/lib/python3.10/site-packages/botocore/client.py", line 235, in _load_service_model
json_model = self._loader.load_service_model(
File "/home/ds/projects/cox/OracleReplicatorToS3/.venv/lib/python3.10/site-packages/botocore/loaders.py", line 142, in _wrapper
data = func(self, *args, **kwargs)
File "/home/ds/projects/cox/OracleReplicatorToS3/.venv/lib/python3.10/site-packages/botocore/loaders.py", line 417, in load_service_model
model = self.load_data(full_path)
File "/home/ds/projects/cox/OracleReplicatorToS3/.venv/lib/python3.10/site-packages/botocore/loaders.py", line 471, in load_data
data, _ = self.load_data_with_path(name)
File "/home/ds/projects/cox/OracleReplicatorToS3/.venv/lib/python3.10/site-packages/botocore/loaders.py", line 142, in _wrapper
data = func(self, *args, **kwargs)
File "/home/ds/projects/cox/OracleReplicatorToS3/.venv/lib/python3.10/site-packages/botocore/loaders.py", line 448, in load_data_with_path
found = self.file_loader.load_file(possible_path)
File "/home/ds/projects/cox/OracleReplicatorToS3/.venv/lib/python3.10/site-packages/botocore/loaders.py", line 194, in load_file
data = self._load_file(file_path + ext, open_method)
File "/home/ds/projects/cox/OracleReplicatorToS3/.venv/lib/python3.10/site-packages/botocore/loaders.py", line 178, in _load_file
payload = fp.read().decode('utf-8')
File "/usr/lib/python3.10/gzip.py", line 301, in read
return self._buffer.read(size)
File "/usr/lib/python3.10/_compression.py", line 118, in readall
while data := self.read(sys.maxsize):
File "/usr/lib/python3.10/gzip.py", line 496, in read
uncompress = self._decompressor.decompress(buf, size)
MemoryError: Unable to allocate output buffer.
Should be used value for out buffer be limited by memory settings instead of maxsize?
CPython versions tested on:
3.10
Operating systems tested on:
Linux
Metadata
Metadata
Assignees
Projects
Status