gh-132983: Convert dict_content to take Py_buffer #133924
+90
−63
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
|
|
@@ -25,7 +25,7 @@ class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec" | |||||||
| /*[clinic input] | ||||||||
| @classmethod | ||||||||
| _zstd.ZstdDict.__new__ as _zstd_ZstdDict_new | ||||||||
| dict_content: Py_buffer | ||||||||
| The content of a Zstandard dictionary as a bytes-like object. | ||||||||
| / | ||||||||
| * | ||||||||
|
|
@@ -41,16 +41,15 @@ by multiple ZstdCompressor or ZstdDecompressor objects. | |||||||
| [clinic start generated code]*/ | ||||||||
|
|
||||||||
| static PyObject * | ||||||||
| _zstd_ZstdDict_new_impl(PyTypeObject *type, Py_buffer *dict_content, | ||||||||
| int is_raw) | ||||||||
| /*[clinic end generated code: output=685b7406a48b0949 input=9e8c493e31c98383]*/ | ||||||||
| { | ||||||||
| ZstdDict* self = PyObject_GC_New(ZstdDict, type); | ||||||||
| if (self == NULL) { | ||||||||
| return NULL; | ||||||||
| } | ||||||||
|
|
||||||||
| self->d_dict = NULL; | ||||||||
|
There was a problem hiding this comment.
Suggested change
Otherwise |
||||||||
|
|
||||||||
| /* ZSTD_CDict dict */ | ||||||||
|
|
@@ -60,24 +59,28 @@ _zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content, | |||||||
| } | ||||||||
|
|
||||||||
| /* Check dict_content's type */ | ||||||||
| if (dict_content == NULL) { | ||||||||
|
There was a problem hiding this comment. It is never NULL. BTW, overriding arbitrary exception with a TypeError in the current code is wrong. |
||||||||
| PyErr_SetString(PyExc_TypeError, | ||||||||
| "dict_content argument should be bytes-like object."); | ||||||||
| goto error; | ||||||||
| } | ||||||||
|
|
||||||||
| self->dict_buffer = PyMem_RawMalloc(dict_content->len); | ||||||||
| if (!self->dict_buffer) { | ||||||||
| return PyErr_NoMemory(); | ||||||||
|
|
||||||||
| } | ||||||||
| memcpy(self->dict_buffer, dict_content->buf, dict_content->len); | ||||||||
| self->dict_len = dict_content->len; | ||||||||
|
Comment on lines
+68
to
+73
There was a problem hiding this comment. cc @vstinner, I think There was a problem hiding this comment. It seems like you told the GIL in this code, so you can use PyMem_Malloc() instead. The Raw variant is when you don't hold the GIL. |
||||||||
|
|
||||||||
| /* Both ordinary and "raw content" dictionaries must be 8 bytes minimum */ | ||||||||
| if (self->dict_len < 8) { | ||||||||
|
There was a problem hiding this comment. This can be checked at the beginning, before any allocations. |
||||||||
| PyErr_SetString(PyExc_ValueError, | ||||||||
| "Zstandard dictionary content should at least 8 bytes."); | ||||||||
| goto error; | ||||||||
| } | ||||||||
|
|
||||||||
| /* Get dict_id, 0 means "raw content" dictionary. */ | ||||||||
| self->dict_id = ZSTD_getDictID_fromDict(self->dict_buffer, self->dict_len); | ||||||||
|
|
||||||||
| /* Check validity for ordinary dictionary */ | ||||||||
| if (!is_raw && self->dict_id == 0) { | ||||||||
|
|
@@ -86,15 +89,12 @@ _zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content, | |||||||
| goto error; | ||||||||
| } | ||||||||
|
|
||||||||
| PyObject_GC_Track(self); | ||||||||
|
|
||||||||
| return (PyObject*)self; | ||||||||
|
|
||||||||
| error: | ||||||||
| PyObject_GC_Del(self); | ||||||||
| return NULL; | ||||||||
| } | ||||||||
|
|
||||||||
|
|
@@ -108,12 +108,12 @@ ZstdDict_dealloc(PyObject *ob) | |||||||
| /* Free ZSTD_DDict instance */ | ||||||||
| ZSTD_freeDDict(self->d_dict); | ||||||||
|
|
||||||||
| /* Release dict_buffer after Free ZSTD_CDict/ZSTD_DDict instances */ | ||||||||
| PyMem_RawFree(self->dict_buffer); | ||||||||
| Py_CLEAR(self->c_dicts); | ||||||||
|
|
||||||||
| PyTypeObject *tp = Py_TYPE(self); | ||||||||
| tp->tp_free(self); | ||||||||
| Py_DECREF(tp); | ||||||||
| } | ||||||||
|
|
||||||||
|
|
@@ -124,23 +124,33 @@ PyDoc_STRVAR(ZstdDict_dictid_doc, | |||||||
| "The special value '0' means a 'raw content' dictionary," | ||||||||
| "without any restrictions on format or content."); | ||||||||
|
|
||||||||
| static PyObject * | ||||||||
| ZstdDict_str(PyObject *ob) | ||||||||
|
There was a problem hiding this comment. It is wrong to implement |
||||||||
| { | ||||||||
| ZstdDict *dict = ZstdDict_CAST(ob); | ||||||||
| return PyUnicode_FromFormat("<ZstdDict dict_id=%u dict_size=%zd>", | ||||||||
| dict->dict_id, dict->dict_len); | ||||||||
|
There was a problem hiding this comment.
Suggested change
There was a problem hiding this comment. dict_id is already a uint32_t, is there a reason for adding this cast? There was a problem hiding this comment. On every supported platform int is 32-bit, but it is not harm to be pedantic. This just goes to show that we didn't miss it. |
||||||||
| } | ||||||||
|
|
||||||||
| static PyMemberDef ZstdDict_members[] = { | ||||||||
| {"dict_id", Py_T_UINT, offsetof(ZstdDict, dict_id), Py_READONLY, ZstdDict_dictid_doc}, | ||||||||
| {NULL} | ||||||||
| }; | ||||||||
|
|
||||||||
| /*[clinic input] | ||||||||
| @getter | ||||||||
| _zstd.ZstdDict.dict_content | ||||||||
|
|
||||||||
| The content of a Zstandard dictionary, as a bytes object. | ||||||||
| [clinic start generated code]*/ | ||||||||
|
|
||||||||
| static PyObject * | ||||||||
| _zstd_ZstdDict_dict_content_get_impl(ZstdDict *self) | ||||||||
| /*[clinic end generated code: output=0d05caa5b550eabb input=4ed526d1c151c596]*/ | ||||||||
| { | ||||||||
| return PyBytes_FromStringAndSize(self->dict_buffer, self->dict_len); | ||||||||
| } | ||||||||
|
|
||||||||
| /*[clinic input] | ||||||||
| @critical_section | ||||||||
| @getter | ||||||||
|
|
@@ -207,6 +217,7 @@ _zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self) | |||||||
| } | ||||||||
|
|
||||||||
| static PyGetSetDef ZstdDict_getset[] = { | ||||||||
| _ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF | ||||||||
| _ZSTD_ZSTDDICT_AS_DIGESTED_DICT_GETSETDEF | ||||||||
| _ZSTD_ZSTDDICT_AS_UNDIGESTED_DICT_GETSETDEF | ||||||||
| _ZSTD_ZSTDDICT_AS_PREFIX_GETSETDEF | ||||||||
|
|
@@ -217,24 +228,14 @@ static Py_ssize_t | |||||||
| ZstdDict_length(PyObject *ob) | ||||||||
| { | ||||||||
| ZstdDict *self = ZstdDict_CAST(ob); | ||||||||
| return self->dict_len; | ||||||||
| } | ||||||||
|
|
||||||||
| static int | ||||||||
| ZstdDict_traverse(PyObject *ob, visitproc visit, void *arg) | ||||||||
| { | ||||||||
| ZstdDict *self = ZstdDict_CAST(ob); | ||||||||
| Py_VISIT(self->c_dicts); | ||||||||
| return 0; | ||||||||
| } | ||||||||
|
|
||||||||
|
|
@@ -247,7 +248,6 @@ static PyType_Slot zstddict_slots[] = { | |||||||
| {Py_tp_doc, (void *)_zstd_ZstdDict_new__doc__}, | ||||||||
| {Py_sq_length, ZstdDict_length}, | ||||||||
| {Py_tp_traverse, ZstdDict_traverse}, | ||||||||
| {0, 0} | ||||||||
| }; | ||||||||
|
|
||||||||
|
|
||||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -15,8 +15,10 @@ typedef struct { | |
| ZSTD_DDict *d_dict; | ||
| PyObject *c_dicts; | ||
|
|
||
| /* Dictionary content. */ | ||
| char *dict_buffer; | ||
| Py_ssize_t dict_len; | ||
|
|
||
| /* Dictionary id */ | ||
| uint32_t dict_id; | ||
| } ZstdDict; | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.