Skip to content

[3.14] GH-132983: PEP 7 and Argument Clinic changes for zstd (GH-133791) #133792

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 10, 2025
Merged

[3.14] GH-132983: PEP 7 and Argument Clinic changes for zstd (GH-133791) #133792

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions Lib/compression/zstd/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ def get_frame_info(frame_buffer):
the frame may or may not need a dictionary to be decoded,
and the ID of such a dictionary is not specified.
"""
return FrameInfo(*_zstd._get_frame_info(frame_buffer))
return FrameInfo(*_zstd.get_frame_info(frame_buffer))


def train_dict(samples, dict_size):
Expand All @@ -91,7 +91,7 @@ def train_dict(samples, dict_size):
chunk_sizes = tuple(_nbytes(sample) for sample in samples)
if not chunks:
raise ValueError("samples contained no data; can't train dictionary.")
dict_content = _zstd._train_dict(chunks, chunk_sizes, dict_size)
dict_content = _zstd.train_dict(chunks, chunk_sizes, dict_size)
return ZstdDict(dict_content)


Expand Down Expand Up @@ -127,7 +127,7 @@ def finalize_dict(zstd_dict, /, samples, dict_size, level):
if not chunks:
raise ValueError("The samples are empty content, can't finalize the"
"dictionary.")
dict_content = _zstd._finalize_dict(zstd_dict.dict_content,
dict_content = _zstd.finalize_dict(zstd_dict.dict_content,
chunks, chunk_sizes,
dict_size, level)
return ZstdDict(dict_content)
Expand Down Expand Up @@ -201,7 +201,7 @@ def bounds(self):

Both the lower and upper bounds are inclusive.
"""
return _zstd._get_param_bounds(self.value, is_compress=True)
return _zstd.get_param_bounds(self.value, is_compress=True)


class DecompressionParameter(enum.IntEnum):
Expand All @@ -214,7 +214,7 @@ def bounds(self):

Both the lower and upper bounds are inclusive.
"""
return _zstd._get_param_bounds(self.value, is_compress=False)
return _zstd.get_param_bounds(self.value, is_compress=False)


class Strategy(enum.IntEnum):
Expand All @@ -237,4 +237,4 @@ class Strategy(enum.IntEnum):


# Check validity of the CompressionParameter & DecompressionParameter types
_zstd._set_parameter_types(CompressionParameter, DecompressionParameter)
_zstd.set_parameter_types(CompressionParameter, DecompressionParameter)
34 changes: 17 additions & 17 deletions Lib/test/test_zstd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1174,43 +1174,43 @@ def test_finalize_dict_arguments(self):
def test_train_dict_c(self):
# argument wrong type
with self.assertRaises(TypeError):
_zstd._train_dict({}, (), 100)
_zstd.train_dict({}, (), 100)
with self.assertRaises(TypeError):
_zstd._train_dict(b'', 99, 100)
_zstd.train_dict(b'', 99, 100)
with self.assertRaises(TypeError):
_zstd._train_dict(b'', (), 100.1)
_zstd.train_dict(b'', (), 100.1)

# size > size_t
with self.assertRaises(ValueError):
_zstd._train_dict(b'', (2**64+1,), 100)
_zstd.train_dict(b'', (2**64+1,), 100)

# dict_size <= 0
with self.assertRaises(ValueError):
_zstd._train_dict(b'', (), 0)
_zstd.train_dict(b'', (), 0)

def test_finalize_dict_c(self):
with self.assertRaises(TypeError):
_zstd._finalize_dict(1, 2, 3, 4, 5)
_zstd.finalize_dict(1, 2, 3, 4, 5)

# argument wrong type
with self.assertRaises(TypeError):
_zstd._finalize_dict({}, b'', (), 100, 5)
_zstd.finalize_dict({}, b'', (), 100, 5)
with self.assertRaises(TypeError):
_zstd._finalize_dict(TRAINED_DICT.dict_content, {}, (), 100, 5)
_zstd.finalize_dict(TRAINED_DICT.dict_content, {}, (), 100, 5)
with self.assertRaises(TypeError):
_zstd._finalize_dict(TRAINED_DICT.dict_content, b'', 99, 100, 5)
_zstd.finalize_dict(TRAINED_DICT.dict_content, b'', 99, 100, 5)
with self.assertRaises(TypeError):
_zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (), 100.1, 5)
_zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100.1, 5)
with self.assertRaises(TypeError):
_zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5.1)
_zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5.1)

# size > size_t
with self.assertRaises(ValueError):
_zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (2**64+1,), 100, 5)
_zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (2**64+1,), 100, 5)

# dict_size <= 0
with self.assertRaises(ValueError):
_zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (), 0, 5)
_zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 0, 5)

def test_train_buffer_protocol_samples(self):
def _nbytes(dat):
Expand All @@ -1232,19 +1232,19 @@ def _nbytes(dat):
# wrong size list
with self.assertRaisesRegex(ValueError,
"The samples size tuple doesn't match the concatenation's size"):
_zstd._train_dict(concatenation, tuple(wrong_size_lst), 100*_1K)
_zstd.train_dict(concatenation, tuple(wrong_size_lst), 100*_1K)

# correct size list
_zstd._train_dict(concatenation, tuple(correct_size_lst), 3*_1K)
_zstd.train_dict(concatenation, tuple(correct_size_lst), 3*_1K)

# wrong size list
with self.assertRaisesRegex(ValueError,
"The samples size tuple doesn't match the concatenation's size"):
_zstd._finalize_dict(TRAINED_DICT.dict_content,
_zstd.finalize_dict(TRAINED_DICT.dict_content,
concatenation, tuple(wrong_size_lst), 300*_1K, 5)

# correct size list
_zstd._finalize_dict(TRAINED_DICT.dict_content,
_zstd.finalize_dict(TRAINED_DICT.dict_content,
concatenation, tuple(correct_size_lst), 300*_1K, 5)

def test_as_prefix(self):
Expand Down
67 changes: 32 additions & 35 deletions Modules/_zstd/_zstdmodule.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,7 @@ typedef struct {
char parameter_name[32];
} ParameterInfo;

static const ParameterInfo cp_list[] =
{
static const ParameterInfo cp_list[] = {
{ZSTD_c_compressionLevel, "compression_level"},
{ZSTD_c_windowLog, "window_log"},
{ZSTD_c_hashLog, "hash_log"},
Expand All @@ -95,8 +94,7 @@ static const ParameterInfo cp_list[] =
{ZSTD_c_overlapLog, "overlap_log"}
};

static const ParameterInfo dp_list[] =
{
static const ParameterInfo dp_list[] = {
{ZSTD_d_windowLogMax, "window_log_max"}
};

Expand Down Expand Up @@ -173,7 +171,7 @@ get_zstd_state(PyObject *module)


/*[clinic input]
_zstd._train_dict
_zstd.train_dict

samples_bytes: PyBytesObject
Concatenation of samples.
Expand All @@ -187,9 +185,9 @@ Internal function, train a zstd dictionary on sample data.
[clinic start generated code]*/

static PyObject *
_zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
PyObject *samples_sizes, Py_ssize_t dict_size)
/*[clinic end generated code: output=b5b4f36347c0addd input=2dce5b57d63923e2]*/
_zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
PyObject *samples_sizes, Py_ssize_t dict_size)
/*[clinic end generated code: output=8e87fe43935e8f77 input=70fcd8937f2528b6]*/
{
// TODO(emmatyping): The preamble and suffix to this function and _finalize_dict
// are pretty similar. We should see if we can refactor them to share that code.
Expand Down Expand Up @@ -277,7 +275,7 @@ _zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
}

/*[clinic input]
_zstd._finalize_dict
_zstd.finalize_dict

custom_dict_bytes: PyBytesObject
Custom dictionary content.
Expand All @@ -295,11 +293,11 @@ Internal function, finalize a zstd dictionary.
[clinic start generated code]*/

static PyObject *
_zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
PyBytesObject *samples_bytes,
PyObject *samples_sizes, Py_ssize_t dict_size,
int compression_level)
/*[clinic end generated code: output=5dc5b520fddba37f input=8afd42a249078460]*/
_zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
PyBytesObject *samples_bytes,
PyObject *samples_sizes, Py_ssize_t dict_size,
int compression_level)
/*[clinic end generated code: output=f91821ba5ae85bda input=130d1508adb55ba1]*/
{
Py_ssize_t chunks_number;
size_t *chunk_sizes = NULL;
Expand Down Expand Up @@ -396,7 +394,7 @@ _zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,


/*[clinic input]
_zstd._get_param_bounds
_zstd.get_param_bounds

parameter: int
The parameter to get bounds.
Expand All @@ -407,9 +405,8 @@ Internal function, get CompressionParameter/DecompressionParameter bounds.
[clinic start generated code]*/

static PyObject *
_zstd__get_param_bounds_impl(PyObject *module, int parameter,
int is_compress)
/*[clinic end generated code: output=9892cd822f937e79 input=884cd1a01125267d]*/
_zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress)
/*[clinic end generated code: output=4acf5a876f0620ca input=84e669591e487008]*/
{
ZSTD_bounds bound;
if (is_compress) {
Expand Down Expand Up @@ -466,7 +463,7 @@ _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer)
}

/*[clinic input]
_zstd._get_frame_info
_zstd.get_frame_info

frame_buffer: Py_buffer
A bytes-like object, containing the header of a zstd frame.
Expand All @@ -475,8 +472,8 @@ Internal function, get zstd frame infomation from a frame header.
[clinic start generated code]*/

static PyObject *
_zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
/*[clinic end generated code: output=5462855464ecdf81 input=67f1f8e4b7b89c4d]*/
_zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
/*[clinic end generated code: output=56e033cf48001929 input=1816f14656b6aa22]*/
{
uint64_t decompressed_size;
uint32_t dict_id;
Expand Down Expand Up @@ -508,7 +505,7 @@ _zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
}

/*[clinic input]
_zstd._set_parameter_types
_zstd.set_parameter_types

c_parameter_type: object(subclass_of='&PyType_Type')
CompressionParameter IntEnum type object
Expand All @@ -519,9 +516,9 @@ Internal function, set CompressionParameter/DecompressionParameter types for val
[clinic start generated code]*/

static PyObject *
_zstd__set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type,
PyObject *d_parameter_type)
/*[clinic end generated code: output=a13d4890ccbd2873 input=4535545d903853d3]*/
_zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type,
PyObject *d_parameter_type)
/*[clinic end generated code: output=f3313b1294f19502 input=30402523871b8280]*/
{
_zstd_state* const mod_state = get_zstd_state(module);

Expand All @@ -544,14 +541,13 @@ _zstd__set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type,
}

static PyMethodDef _zstd_methods[] = {
_ZSTD__TRAIN_DICT_METHODDEF
_ZSTD__FINALIZE_DICT_METHODDEF
_ZSTD__GET_PARAM_BOUNDS_METHODDEF
_ZSTD_TRAIN_DICT_METHODDEF
_ZSTD_FINALIZE_DICT_METHODDEF
_ZSTD_GET_PARAM_BOUNDS_METHODDEF
_ZSTD_GET_FRAME_SIZE_METHODDEF
_ZSTD__GET_FRAME_INFO_METHODDEF
_ZSTD__SET_PARAMETER_TYPES_METHODDEF

{0}
_ZSTD_GET_FRAME_INFO_METHODDEF
_ZSTD_SET_PARAMETER_TYPES_METHODDEF
{NULL, NULL}
};

static int
Expand Down Expand Up @@ -595,7 +591,7 @@ do { \
ADD_TYPE(mod_state->ZstdCompressor_type, zstd_compressor_type_spec);
ADD_TYPE(mod_state->ZstdDecompressor_type, zstd_decompressor_type_spec);
mod_state->ZstdError = PyErr_NewExceptionWithDoc(
"_zstd.ZstdError",
"compression.zstd.ZstdError",
"An error occurred in the zstd library.",
NULL, NULL);
if (mod_state->ZstdError == NULL) {
Expand Down Expand Up @@ -732,14 +728,15 @@ static struct PyModuleDef_Slot _zstd_slots[] = {
};

struct PyModuleDef _zstdmodule = {
PyModuleDef_HEAD_INIT,
.m_base = PyModuleDef_HEAD_INIT,
.m_name = "_zstd",
.m_doc = "Implementation module for Zstandard compression.",
.m_size = sizeof(_zstd_state),
.m_slots = _zstd_slots,
.m_methods = _zstd_methods,
.m_traverse = _zstd_traverse,
.m_clear = _zstd_clear,
.m_free = _zstd_free
.m_free = _zstd_free,
};

PyMODINIT_FUNC
Expand Down
16 changes: 4 additions & 12 deletions Modules/_zstd/_zstdmodule.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ extern PyModuleDef _zstdmodule;

/* For clinic type calculations */
static inline _zstd_state *
get_zstd_state_from_type(PyTypeObject *type) {
get_zstd_state_from_type(PyTypeObject *type)
{
PyObject *module = PyType_GetModuleByDef(type, &_zstdmodule);
if (module == NULL) {
return NULL;
Expand Down Expand Up @@ -149,7 +150,8 @@ typedef enum {
} dictionary_type;

static inline int
mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out) {
mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out)
{
return in->size == in->pos && out->size != out->pos;
}

Expand All @@ -163,13 +165,3 @@ set_parameter_error(const _zstd_state* const state, int is_compress,
int key_v, int value_v);

static const char init_twice_msg[] = "__init__ method is called twice.";

extern PyObject *
decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in,
Py_ssize_t max_length,
Py_ssize_t initial_size,
decompress_type type);

extern PyObject *
compress_impl(ZstdCompressor *self, Py_buffer *data,
ZSTD_EndDirective end_directive);
Loading
Loading