Skip to content

Major overhaul of mbstring (part 2) #6402

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 6 commits into from
Closed

Major overhaul of mbstring (part 2) #6402

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
856ea31
Fix issues with mbstring encoding tests
alexdowad Nov 5, 2020
a03dafd
Remove useless byte{2,4}{be,le} encodings from mbstring
alexdowad Nov 4, 2020
79e8c23
Fix mbstring support for Shift-JIS
alexdowad Oct 19, 2020
0ac660a
Fix mbstring support for EUC-JP text encoding
alexdowad Oct 1, 2020
ab37929
Treat non-ASCII characters as erroneous when converting ASCII text
alexdowad Nov 5, 2020
1c4a8d1
Remove mbstring identify filters
alexdowad Nov 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Zend/tests/multibyte/multibyte_encoding_001.phpt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ internal_encoding=SJIS
<?php
declare(encoding='Shift_JIS');
$s = "�\"; // 0x95+0x5c in script, not somewhere else "
printf("%x:%x\n", ord($s[0]), ord($s[1]));
printf("%x:%x", ord($s[0]), ord($s[1]));
?>
--EXPECT--
95:5c
5 changes: 1 addition & 4 deletions ext/mbstring/config.m4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,6 @@ AC_DEFUN([PHP_MBSTRING_SETUP_LIBMBFL], [
libmbfl/filters/mbfilter_ascii.c
libmbfl/filters/mbfilter_base64.c
libmbfl/filters/mbfilter_big5.c
libmbfl/filters/mbfilter_byte2.c
libmbfl/filters/mbfilter_byte4.c
libmbfl/filters/mbfilter_cp1251.c
libmbfl/filters/mbfilter_cp1252.c
libmbfl/filters/mbfilter_cp1254.c
Expand Down Expand Up @@ -163,7 +161,6 @@ AC_DEFUN([PHP_MBSTRING_SETUP_LIBMBFL], [
libmbfl/mbfl/mbfl_convert.c
libmbfl/mbfl/mbfl_encoding.c
libmbfl/mbfl/mbfl_filter_output.c
libmbfl/mbfl/mbfl_ident.c
libmbfl/mbfl/mbfl_language.c
libmbfl/mbfl/mbfl_memory_device.c
libmbfl/mbfl/mbfl_string.c
Expand All @@ -179,7 +176,7 @@ AC_DEFUN([PHP_MBSTRING_SETUP_LIBMBFL], [
libmbfl/nls/nls_tr.c
libmbfl/nls/nls_ua.c
])
PHP_MBSTRING_ADD_INSTALL_HEADERS([libmbfl/config.h libmbfl/mbfl/eaw_table.h libmbfl/mbfl/mbfilter.h libmbfl/mbfl/mbfilter_8bit.h libmbfl/mbfl/mbfilter_pass.h libmbfl/mbfl/mbfilter_wchar.h libmbfl/mbfl/mbfl_consts.h libmbfl/mbfl/mbfl_convert.h libmbfl/mbfl/mbfl_defs.h libmbfl/mbfl/mbfl_encoding.h libmbfl/mbfl/mbfl_filter_output.h libmbfl/mbfl/mbfl_ident.h libmbfl/mbfl/mbfl_language.h libmbfl/mbfl/mbfl_memory_device.h libmbfl/mbfl/mbfl_string.h])
PHP_MBSTRING_ADD_INSTALL_HEADERS([libmbfl/config.h libmbfl/mbfl/eaw_table.h libmbfl/mbfl/mbfilter.h libmbfl/mbfl/mbfilter_8bit.h libmbfl/mbfl/mbfilter_pass.h libmbfl/mbfl/mbfilter_wchar.h libmbfl/mbfl/mbfl_consts.h libmbfl/mbfl/mbfl_convert.h libmbfl/mbfl/mbfl_defs.h libmbfl/mbfl/mbfl_encoding.h libmbfl/mbfl/mbfl_filter_output.h libmbfl/mbfl/mbfl_language.h libmbfl/mbfl/mbfl_memory_device.h libmbfl/mbfl/mbfl_string.h])
])

dnl
Expand Down
6 changes: 3 additions & 3 deletions ext/mbstring/config.w32
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ if (PHP_MBSTRING != "no") {

ADD_SOURCES("ext/mbstring/libmbfl/filters", "html_entities.c \
mbfilter_7bit.c mbfilter_ascii.c mbfilter_base64.c mbfilter_big5.c \
mbfilter_byte2.c mbfilter_byte4.c mbfilter_cp1251.c mbfilter_cp1252.c \
mbfilter_cp1251.c mbfilter_cp1252.c \
mbfilter_cp866.c mbfilter_cp932.c mbfilter_cp936.c mbfilter_cp51932.c \
mbfilter_euc_cn.c mbfilter_euc_jp.c mbfilter_euc_jp_win.c mbfilter_euc_kr.c \
mbfilter_euc_tw.c mbfilter_htmlent.c mbfilter_hz.c mbfilter_iso2022_kr.c \
Expand All @@ -40,14 +40,14 @@ if (PHP_MBSTRING != "no") {

ADD_SOURCES("ext/mbstring/libmbfl/mbfl", "mbfilter.c mbfilter_8bit.c \
mbfilter_pass.c mbfilter_wchar.c mbfl_convert.c mbfl_encoding.c \
mbfl_filter_output.c mbfl_ident.c mbfl_language.c mbfl_memory_device.c \
mbfl_filter_output.c mbfl_language.c mbfl_memory_device.c \
mbfl_string.c", "mbstring");

ADD_SOURCES("ext/mbstring/libmbfl/nls", "nls_de.c nls_en.c nls_ja.c \
nls_kr.c nls_neutral.c nls_ru.c nls_uni.c nls_zh.c nls_hy.c \
nls_ua.c nls_tr.c", "mbstring");

PHP_INSTALL_HEADERS("ext/mbstring", "mbstring.h libmbfl/config.h libmbfl/mbfl/eaw_table.h libmbfl/mbfl/mbfilter.h libmbfl/mbfl/mbfilter_8bit.h libmbfl/mbfl/mbfilter_pass.h libmbfl/mbfl/mbfilter_wchar.h libmbfl/mbfl/mbfl_consts.h libmbfl/mbfl/mbfl_convert.h libmbfl/mbfl/mbfl_defs.h libmbfl/mbfl/mbfl_encoding.h libmbfl/mbfl/mbfl_filter_output.h libmbfl/mbfl/mbfl_ident.h libmbfl/mbfl/mbfl_language.h libmbfl/mbfl/mbfl_memory_device.h libmbfl/mbfl/mbfl_string.h");
PHP_INSTALL_HEADERS("ext/mbstring", "mbstring.h libmbfl/config.h libmbfl/mbfl/eaw_table.h libmbfl/mbfl/mbfilter.h libmbfl/mbfl/mbfilter_8bit.h libmbfl/mbfl/mbfilter_pass.h libmbfl/mbfl/mbfilter_wchar.h libmbfl/mbfl/mbfl_consts.h libmbfl/mbfl/mbfl_convert.h libmbfl/mbfl/mbfl_defs.h libmbfl/mbfl/mbfl_encoding.h libmbfl/mbfl/mbfl_filter_output.h libmbfl/mbfl/mbfl_language.h libmbfl/mbfl/mbfl_memory_device.h libmbfl/mbfl/mbfl_string.h");

AC_DEFINE('HAVE_MBSTRING', 1, 'Have mbstring support');

Expand Down
16 changes: 0 additions & 16 deletions ext/mbstring/libmbfl/filters/mbfilter_7bit.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@
#include "mbfilter.h"
#include "mbfilter_7bit.h"

static int mbfl_filt_ident_7bit(int c, mbfl_identify_filter *filter);

const mbfl_encoding mbfl_encoding_7bit = {
mbfl_no_encoding_7bit,
"7bit",
Expand All @@ -44,12 +42,6 @@ const mbfl_encoding mbfl_encoding_7bit = {
NULL
};

const struct mbfl_identify_vtbl vtbl_identify_7bit = {
mbfl_no_encoding_7bit,
mbfl_filt_ident_common_ctor,
mbfl_filt_ident_7bit
};

const struct mbfl_convert_vtbl vtbl_8bit_7bit = {
mbfl_no_encoding_8bit,
mbfl_no_encoding_7bit,
Expand Down Expand Up @@ -88,11 +80,3 @@ int mbfl_filt_conv_any_7bit(int c, mbfl_convert_filter *filter)
}
return c;
}

static int mbfl_filt_ident_7bit(int c, mbfl_identify_filter *filter)
{
if (c >= 0x80) {
filter->flag = 1;
}
return c;
}
1 change: 0 additions & 1 deletion ext/mbstring/libmbfl/filters/mbfilter_7bit.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
#include "mbfilter.h"

extern const mbfl_encoding mbfl_encoding_7bit;
extern const struct mbfl_identify_vtbl vtbl_identify_7bit;
extern const struct mbfl_convert_vtbl vtbl_8bit_7bit;
extern const struct mbfl_convert_vtbl vtbl_7bit_8bit;

Expand Down
16 changes: 0 additions & 16 deletions ext/mbstring/libmbfl/filters/mbfilter_armscii8.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,6 @@
#include "mbfilter_armscii8.h"
#include "unicode_table_armscii8.h"

static int mbfl_filt_ident_armscii8(int c, mbfl_identify_filter *filter);

static const char *mbfl_encoding_armscii8_aliases[] = {"ArmSCII-8", "ArmSCII8", "ARMSCII-8", "ARMSCII8", NULL};

const mbfl_encoding mbfl_encoding_armscii8 = {
Expand All @@ -45,12 +43,6 @@ const mbfl_encoding mbfl_encoding_armscii8 = {
&vtbl_wchar_armscii8
};

const struct mbfl_identify_vtbl vtbl_identify_armscii8 = {
mbfl_no_encoding_armscii8,
mbfl_filt_ident_common_ctor,
mbfl_filt_ident_armscii8
};

const struct mbfl_convert_vtbl vtbl_wchar_armscii8 = {
mbfl_no_encoding_wchar,
mbfl_no_encoding_armscii8,
Expand Down Expand Up @@ -108,11 +100,3 @@ int mbfl_filt_conv_wchar_armscii8(int c, mbfl_convert_filter *filter)

return c;
}

static int mbfl_filt_ident_armscii8(int c, mbfl_identify_filter *filter)
{
if (c >= armscii8_ucs_table_min && !armscii8_ucs_table[c - armscii8_ucs_table_min]) {
filter->flag = 1;
}
return c;
}
1 change: 0 additions & 1 deletion ext/mbstring/libmbfl/filters/mbfilter_armscii8.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
#include "mbfilter.h"

extern const mbfl_encoding mbfl_encoding_armscii8;
extern const struct mbfl_identify_vtbl vtbl_identify_armscii8;
extern const struct mbfl_convert_vtbl vtbl_wchar_armscii8;
extern const struct mbfl_convert_vtbl vtbl_armscii8_wchar;

Expand Down
28 changes: 6 additions & 22 deletions ext/mbstring/libmbfl/filters/mbfilter_ascii.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@
#include "mbfilter.h"
#include "mbfilter_ascii.h"

static int mbfl_filt_ident_ascii(int c, mbfl_identify_filter *filter);

static const char *mbfl_encoding_ascii_aliases[] = {"ANSI_X3.4-1968", "iso-ir-6", "ANSI_X3.4-1986", "ISO_646.irv:1991", "US-ASCII", "ISO646-US", "us", "IBM367", "IBM-367", "cp367", "csASCII", NULL};

const mbfl_encoding mbfl_encoding_ascii = {
Expand All @@ -46,12 +44,6 @@ const mbfl_encoding mbfl_encoding_ascii = {
&vtbl_wchar_ascii
};

const struct mbfl_identify_vtbl vtbl_identify_ascii = {
mbfl_no_encoding_ascii,
mbfl_filt_ident_common_ctor,
mbfl_filt_ident_ascii
};

const struct mbfl_convert_vtbl vtbl_ascii_wchar = {
mbfl_no_encoding_ascii,
mbfl_no_encoding_wchar,
Expand Down Expand Up @@ -79,7 +71,12 @@ const struct mbfl_convert_vtbl vtbl_wchar_ascii = {
*/
int mbfl_filt_conv_ascii_wchar(int c, mbfl_convert_filter *filter)
{
return (*filter->output_function)(c, filter->data);
if (c < 0x80) {
(*filter->output_function)(c, filter->data);
} else {
(*filter->output_function)(c | MBFL_WCSGROUP_THROUGH, filter->data);
}
return c;
}


Expand All @@ -96,16 +93,3 @@ int mbfl_filt_conv_wchar_ascii(int c, mbfl_convert_filter *filter)

return c;
}

static int mbfl_filt_ident_ascii(int c, mbfl_identify_filter *filter)
{
if (c >= 0x20 && c < 0x80) {
;
} else if (c == 0x0d || c == 0x0a || c == 0x09 || c == 0) { /* CR or LF or HTAB or null */
;
} else {
filter->flag = 1;
}

return c;
}
1 change: 0 additions & 1 deletion ext/mbstring/libmbfl/filters/mbfilter_ascii.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
#include "mbfilter.h"

extern const mbfl_encoding mbfl_encoding_ascii;
extern const struct mbfl_identify_vtbl vtbl_identify_ascii;
extern const struct mbfl_convert_vtbl vtbl_ascii_wchar;
extern const struct mbfl_convert_vtbl vtbl_wchar_ascii;

Expand Down
39 changes: 0 additions & 39 deletions ext/mbstring/libmbfl/filters/mbfilter_big5.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@

#include "unicode_table_big5.h"

static int mbfl_filt_ident_big5(int c, mbfl_identify_filter *filter);

static const unsigned char mblen_table_big5[] = { /* 0x81-0xFE */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
Expand Down Expand Up @@ -77,18 +75,6 @@ const mbfl_encoding mbfl_encoding_cp950 = {
&vtbl_wchar_cp950
};

const struct mbfl_identify_vtbl vtbl_identify_big5 = {
mbfl_no_encoding_big5,
mbfl_filt_ident_common_ctor,
mbfl_filt_ident_big5
};

const struct mbfl_identify_vtbl vtbl_identify_cp950 = {
mbfl_no_encoding_cp950,
mbfl_filt_ident_common_ctor,
mbfl_filt_ident_big5
};

const struct mbfl_convert_vtbl vtbl_big5_wchar = {
mbfl_no_encoding_big5,
mbfl_no_encoding_wchar,
Expand Down Expand Up @@ -322,28 +308,3 @@ mbfl_filt_conv_wchar_big5(int c, mbfl_convert_filter *filter)

return c;
}

static int mbfl_filt_ident_big5(int c, mbfl_identify_filter *filter)
{
int c1;
if (filter->encoding->no_encoding == mbfl_no_encoding_cp950) {
c1 = 0x80;
} else {
c1 = 0xa0;
}

if (filter->status) { /* kanji second char */
if (c < 0x40 || (c > 0x7e && c < 0xa1) ||c > 0xfe) { /* bad */
filter->flag = 1;
}
filter->status = 0;
} else if (c >= 0 && c < 0x80) { /* latin ok */
;
} else if (c > c1 && c < 0xff) { /* DBCS lead byte */
filter->status = 1;
} else { /* bad */
filter->flag = 1;
}

return c;
}
2 changes: 0 additions & 2 deletions ext/mbstring/libmbfl/filters/mbfilter_big5.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,10 @@
#include "mbfilter.h"

extern const mbfl_encoding mbfl_encoding_big5;
extern const struct mbfl_identify_vtbl vtbl_identify_big5;
extern const struct mbfl_convert_vtbl vtbl_big5_wchar;
extern const struct mbfl_convert_vtbl vtbl_wchar_big5;

extern const mbfl_encoding mbfl_encoding_cp950;
extern const struct mbfl_identify_vtbl vtbl_identify_cp950;
extern const struct mbfl_convert_vtbl vtbl_cp950_wchar;
extern const struct mbfl_convert_vtbl vtbl_wchar_cp950;

Expand Down
Loading