Skip to content

S3: implement TransitionDefaultMinimumObjectSize for Lifecycle Configuration #12212

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Feb 3, 2025
Merged

S3: implement TransitionDefaultMinimumObjectSize for Lifecycle Configuration #12212

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
85fc8ac
implement TransitionDefaultMinimumObjectSize for Lifecycle Configuration
bentsku Jan 30, 2025
31ba4f5
use generated StrEnum instead of hardcoded string
bentsku Feb 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions localstack-core/localstack/services/s3/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
SSECustomerKeyMD5,
SSEKMSKeyId,
StorageClass,
TransitionDefaultMinimumObjectSize,
WebsiteConfiguration,
WebsiteRedirectLocation,
)
Expand Down Expand Up @@ -98,6 +99,7 @@ class S3Bucket:
objects: Union["KeyStore", "VersionedKeyStore"]
versioning_status: BucketVersioningStatus | None
lifecycle_rules: Optional[LifecycleRules]
transition_default_minimum_object_size: Optional[TransitionDefaultMinimumObjectSize]
policy: Optional[Policy]
website_configuration: Optional[WebsiteConfiguration]
acl: AccessControlPolicy
Expand Down Expand Up @@ -145,6 +147,7 @@ def __init__(
self.logging = {}
self.cors_rules = None
self.lifecycle_rules = None
self.transition_default_minimum_object_size = None
self.website_configuration = None
self.policy = None
self.accelerate_status = None
Expand Down
28 changes: 24 additions & 4 deletions localstack-core/localstack/services/s3/provider.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2285,8 +2285,6 @@ def upload_part(
if s3_multipart.object.checksum_algorithm
else "null"
)
# TODO: properly fix this, this is to unblock default behavior of boto adding checksums and it being
# accepted by AWS
if not error_mp_checksum == "null":
raise InvalidRequest(
f"Checksum Type mismatch occurred, expected checksum Type: {error_mp_checksum}, actual checksum Type: {error_req_checksum}"
Expand Down Expand Up @@ -3201,7 +3199,15 @@ def get_bucket_lifecycle_configuration(
BucketName=bucket,
)

return GetBucketLifecycleConfigurationOutput(Rules=s3_bucket.lifecycle_rules)
return GetBucketLifecycleConfigurationOutput(
Rules=s3_bucket.lifecycle_rules,
# TODO: remove for next major version, safe access to new attribute
TransitionDefaultMinimumObjectSize=getattr(
s3_bucket,
"transition_default_minimum_object_size",
TransitionDefaultMinimumObjectSize.all_storage_classes_128K,
),
)

def put_bucket_lifecycle_configuration(
self,
Expand All @@ -3215,14 +3221,28 @@ def put_bucket_lifecycle_configuration(
) -> PutBucketLifecycleConfigurationOutput:
store, s3_bucket = self._get_cross_account_bucket(context, bucket)

transition_min_obj_size = (
transition_default_minimum_object_size
or TransitionDefaultMinimumObjectSize.all_storage_classes_128K
)

if transition_min_obj_size not in (
TransitionDefaultMinimumObjectSize.all_storage_classes_128K,
TransitionDefaultMinimumObjectSize.varies_by_storage_class,
):
raise InvalidRequest(
f"Invalid TransitionDefaultMinimumObjectSize found: {transition_min_obj_size}"
)

validate_lifecycle_configuration(lifecycle_configuration)
# TODO: we either apply the lifecycle to existing objects when we set the new rules, or we need to apply them
# everytime we get/head an object
# for now, we keep a cache and get it everytime we fetch an object
s3_bucket.lifecycle_rules = lifecycle_configuration["Rules"]
s3_bucket.transition_default_minimum_object_size = transition_min_obj_size
self._expiration_cache[bucket].clear()
return PutBucketLifecycleConfigurationOutput(
TransitionDefaultMinimumObjectSize=transition_default_minimum_object_size
TransitionDefaultMinimumObjectSize=transition_min_obj_size
)

def delete_bucket_lifecycle(
Expand Down
53 changes: 50 additions & 3 deletions tests/aws/services/s3/test_s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
import localstack.config
from localstack import config
from localstack.aws.api.lambda_ import Runtime
from localstack.aws.api.s3 import StorageClass
from localstack.aws.api.s3 import StorageClass, TransitionDefaultMinimumObjectSize
from localstack.config import S3_VIRTUAL_HOSTNAME
from localstack.constants import (
AWS_REGION_US_EAST_1,
Expand Down Expand Up @@ -8280,8 +8280,6 @@ def test_access_favicon_via_aws_endpoints(
assert exc.value.response["Error"]["Message"] == "Not Found"


# TODO: implement TransitionDefaultMinimumObjectSize
@markers.snapshot.skip_snapshot_verify(paths=["$..TransitionDefaultMinimumObjectSize"])
class TestS3BucketLifecycle:
@markers.aws.validated
def test_delete_bucket_lifecycle_configuration(self, s3_bucket, snapshot, aws_client):
Expand Down Expand Up @@ -8947,6 +8945,55 @@ def test_lifecycle_expired_object_delete_marker(self, s3_bucket, snapshot, aws_c
response = aws_client.s3.head_object(Bucket=s3_bucket, Key=key)
snapshot.match("head-object", response)

@markers.aws.validated
def test_s3_transition_default_minimum_object_size(self, aws_client, s3_bucket, snapshot):
lfc = {
"Rules": [
{
"Expiration": {"Days": 7},
"ID": "wholebucket",
"Filter": {"Prefix": ""},
"Status": "Enabled",
}
]
}
put_lifecycle_varies = aws_client.s3.put_bucket_lifecycle_configuration(
Bucket=s3_bucket,
LifecycleConfiguration=lfc,
TransitionDefaultMinimumObjectSize=TransitionDefaultMinimumObjectSize.varies_by_storage_class,
)
snapshot.match("varies-by-storage", put_lifecycle_varies)

get_lifecycle_varies = aws_client.s3.get_bucket_lifecycle_configuration(Bucket=s3_bucket)
snapshot.match("get-varies-by-storage", get_lifecycle_varies)

put_lifecycle_default = aws_client.s3.put_bucket_lifecycle_configuration(
Bucket=s3_bucket,
LifecycleConfiguration=lfc,
)
snapshot.match("default", put_lifecycle_default)

get_default = aws_client.s3.get_bucket_lifecycle_configuration(Bucket=s3_bucket)
snapshot.match("get-default", get_default)

put_lifecycle_all_storage = aws_client.s3.put_bucket_lifecycle_configuration(
Bucket=s3_bucket,
LifecycleConfiguration=lfc,
TransitionDefaultMinimumObjectSize=TransitionDefaultMinimumObjectSize.all_storage_classes_128K,
)
snapshot.match("all-storage", put_lifecycle_all_storage)

get_all_storage = aws_client.s3.get_bucket_lifecycle_configuration(Bucket=s3_bucket)
snapshot.match("get-all-storage", get_all_storage)

with pytest.raises(ClientError) as e:
aws_client.s3.put_bucket_lifecycle_configuration(
Bucket=s3_bucket,
LifecycleConfiguration=lfc,
TransitionDefaultMinimumObjectSize="value",
)
snapshot.match("bad-value", e.value.response)


class TestS3ObjectLockRetention:
@markers.aws.validated
Expand Down
93 changes: 93 additions & 0 deletions tests/aws/services/s3/test_s3.snapshot.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16468,5 +16468,98 @@
}
}
}
},
"tests/aws/services/s3/test_s3.py::TestS3BucketLifecycle::test_s3_transition_default_minimum_object_size": {
"recorded-date": "03-02-2025, 10:15:23",
"recorded-content": {
"varies-by-storage": {
"TransitionDefaultMinimumObjectSize": "varies_by_storage_class",
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 200
}
},
"get-varies-by-storage": {
"Rules": [
{
"Expiration": {
"Days": 7
},
"Filter": {
"Prefix": ""
},
"ID": "wholebucket",
"Status": "Enabled"
}
],
"TransitionDefaultMinimumObjectSize": "varies_by_storage_class",
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 200
}
},
"default": {
"TransitionDefaultMinimumObjectSize": "all_storage_classes_128K",
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 200
}
},
"get-default": {
"Rules": [
{
"Expiration": {
"Days": 7
},
"Filter": {
"Prefix": ""
},
"ID": "wholebucket",
"Status": "Enabled"
}
],
"TransitionDefaultMinimumObjectSize": "all_storage_classes_128K",
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 200
}
},
"all-storage": {
"TransitionDefaultMinimumObjectSize": "all_storage_classes_128K",
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 200
}
},
"get-all-storage": {
"Rules": [
{
"Expiration": {
"Days": 7
},
"Filter": {
"Prefix": ""
},
"ID": "wholebucket",
"Status": "Enabled"
}
],
"TransitionDefaultMinimumObjectSize": "all_storage_classes_128K",
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 200
}
},
"bad-value": {
"Error": {
"Code": "InvalidRequest",
"Message": "Invalid TransitionDefaultMinimumObjectSize found: value"
},
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 400
}
}
}
}
}
3 changes: 3 additions & 0 deletions tests/aws/services/s3/test_s3.validation.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -536,6 +536,9 @@
"tests/aws/services/s3/test_s3.py::TestS3BucketLifecycle::test_put_bucket_lifecycle_conf_exc": {
"last_validated_date": "2025-01-21T18:18:24+00:00"
},
"tests/aws/services/s3/test_s3.py::TestS3BucketLifecycle::test_s3_transition_default_minimum_object_size": {
"last_validated_date": "2025-02-03T10:15:22+00:00"
},
"tests/aws/services/s3/test_s3.py::TestS3BucketLogging::test_put_bucket_logging": {
"last_validated_date": "2023-08-12T17:54:07+00:00"
},
Expand Down
Loading