Skip to content

Commit d40d564

Browse files
peterepetere
committed
Add support for other normal forms to Unicode normalization API
It previously only supported NFKC, for use by SASLprep. This expands the API to offer the choice of all four normalization forms. Right now, there are no internal users of the forms other than NFKC. Reviewed-by: Daniel Verite <daniel@manitou-mail.org> Reviewed-by: Andreas Karlsson <andreas@proxel.se> Discussion: https://www.postgresql.org/message-id/flat/c1909f27-c269-2ed9-12f8-3ab72c8caf7a@2ndquadrant.com
1 parent cedffbd commit d40d564

File tree

7 files changed

+3727
-3702
lines changed

7 files changed

+3727
-3702
lines changed

‎src/common/saslprep.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1156,7 +1156,7 @@ pg_saslprep(const char *input, char **output)
11561156
* 2) Normalize -- Normalize the result of step 1 using Unicode
11571157
* normalization.
11581158
*/
1159-
output_chars = unicode_normalize_kc(input_chars);
1159+
output_chars = unicode_normalize(UNICODE_NFKC, input_chars);
11601160
if (!output_chars)
11611161
goto oom;
11621162

‎src/common/unicode/generate-norm_test_table.pl

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848
{
4949
int linenum;
5050
pg_wchar input[50];
51-
pg_wchar output[50];
51+
pg_wchar output[4][50];
5252
} pg_unicode_test;
5353
5454
/* test table */
@@ -89,13 +89,16 @@ sub codepoint_string_to_hex
8989
my ($source, $nfc, $nfd, $nfkc, $nfkd) = split(';', $line);
9090

9191
my $source_utf8 = codepoint_string_to_hex($source);
92+
my $nfc_utf8 = codepoint_string_to_hex($nfc);
93+
my $nfd_utf8 = codepoint_string_to_hex($nfd);
9294
my $nfkc_utf8 = codepoint_string_to_hex($nfkc);
95+
my $nfkd_utf8 = codepoint_string_to_hex($nfkd);
9396

94-
print $OUTPUT "\t{ $linenum, { $source_utf8 }, { $nfkc_utf8 } },\n";
97+
print $OUTPUT "\t{ $linenum, { $source_utf8 }, { { $nfc_utf8 }, { $nfd_utf8 }, { $nfkc_utf8 }, { $nfkd_utf8 } } },\n";
9598
}
9699

97100
# Output terminator entry
98-
print $OUTPUT "\t{ 0, { 0 }, { 0 } }";
101+
print $OUTPUT "\t{ 0, { 0 }, { { 0 }, { 0 }, { 0 }, { 0 } } }";
99102
print $OUTPUT "\n};\n";
100103

101104
close $OUTPUT;

‎src/common/unicode/generate-unicode_norm_table.pl

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -99,10 +99,12 @@
9999
#define DECOMP_NO_COMPOSE 0x80 /* don't use for re-composition */
100100
#define DECOMP_INLINE 0x40 /* decomposition is stored inline in
101101
* dec_index */
102+
#define DECOMP_COMPAT 0x20 /* compatibility mapping */
102103
103-
#define DECOMPOSITION_SIZE(x) ((x)->dec_size_flags & 0x3F)
104-
#define DECOMPOSITION_NO_COMPOSE(x) (((x)->dec_size_flags & DECOMP_NO_COMPOSE) != 0)
104+
#define DECOMPOSITION_SIZE(x) ((x)->dec_size_flags & 0x1F)
105+
#define DECOMPOSITION_NO_COMPOSE(x) (((x)->dec_size_flags & (DECOMP_NO_COMPOSE | DECOMP_COMPAT)) != 0)
105106
#define DECOMPOSITION_IS_INLINE(x) (((x)->dec_size_flags & DECOMP_INLINE) != 0)
107+
#define DECOMPOSITION_IS_COMPAT(x) (((x)->dec_size_flags & DECOMP_COMPAT) != 0)
106108
107109
/* Table of Unicode codepoints and their decompositions */
108110
static const pg_unicode_decomposition UnicodeDecompMain[$num_characters] =
@@ -136,22 +138,22 @@
136138
# Decomposition size
137139
# Print size of decomposition
138140
my $decomp_size = scalar(@decomp_elts);
141+
die if $decomp_size > 0x1F; # to not overrun bitmask
139142

140143
my $first_decomp = shift @decomp_elts;
141144

142145
my $flags = "";
143146
my $comment = "";
144147

145-
if ($decomp_size == 2)
148+
if ($compat)
146149
{
150+
$flags .= " | DECOMP_COMPAT";
151+
}
147152

153+
if ($decomp_size == 2)
154+
{
148155
# Should this be used for recomposition?
149-
if ($compat)
150-
{
151-
$flags .= " | DECOMP_NO_COMPOSE";
152-
$comment = "compatibility mapping";
153-
}
154-
elsif ($character_hash{$first_decomp}
156+
if ($character_hash{$first_decomp}
155157
&& $character_hash{$first_decomp}->{class} != 0)
156158
{
157159
$flags .= " | DECOMP_NO_COMPOSE";

‎src/common/unicode/norm_test.c

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -63,18 +63,21 @@ main(int argc, char **argv)
6363

6464
for (test = UnicodeNormalizationTests; test->input[0] != 0; test++)
6565
{
66-
pg_wchar *result;
66+
for (int form = 0; form < 4; form++)
67+
{
68+
pg_wchar *result;
6769

68-
result = unicode_normalize_kc(test->input);
70+
result = unicode_normalize(form, test->input);
6971

70-
if (pg_wcscmp(test->output, result) != 0)
71-
{
72-
printf("FAILURE (NormalizationTest.txt line %d):\n", test->linenum);
73-
printf("input: %s\n", print_wchar_str(test->input));
74-
printf("expected: %s\n", print_wchar_str(test->output));
75-
printf("got: %s\n", print_wchar_str(result));
76-
printf("\n");
77-
exit(1);
72+
if (pg_wcscmp(test->output[form], result) != 0)
73+
{
74+
printf("FAILURE (NormalizationTest.txt line %d form %d):\n", test->linenum, form);
75+
printf("input: %s\n", print_wchar_str(test->input));
76+
printf("expected: %s\n", print_wchar_str(test->output[form]));
77+
printf("got: %s\n", print_wchar_str(result));
78+
printf("\n");
79+
exit(1);
80+
}
7881
}
7982
}
8083

‎src/common/unicode_norm.c

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/*-------------------------------------------------------------------------
22
* unicode_norm.c
3-
* Normalize a Unicode string to NFKC form
3+
* Normalize a Unicode string
44
*
55
* This implements Unicode normalization, per the documentation at
66
* https://www.unicode.org/reports/tr15/.
@@ -98,7 +98,7 @@ get_code_decomposition(pg_unicode_decomposition *entry, int *dec_size)
9898
* are, in turn, decomposable.
9999
*/
100100
static int
101-
get_decomposed_size(pg_wchar code)
101+
get_decomposed_size(pg_wchar code, bool compat)
102102
{
103103
pg_unicode_decomposition *entry;
104104
int size = 0;
@@ -131,7 +131,8 @@ get_decomposed_size(pg_wchar code)
131131
* Just count current code if no other decompositions. A NULL entry is
132132
* equivalent to a character with class 0 and no decompositions.
133133
*/
134-
if (entry == NULL || DECOMPOSITION_SIZE(entry) == 0)
134+
if (entry == NULL || DECOMPOSITION_SIZE(entry) == 0 ||
135+
(!compat && DECOMPOSITION_IS_COMPAT(entry)))
135136
return 1;
136137

137138
/*
@@ -143,7 +144,7 @@ get_decomposed_size(pg_wchar code)
143144
{
144145
uint32 lcode = decomp[i];
145146

146-
size += get_decomposed_size(lcode);
147+
size += get_decomposed_size(lcode, compat);
147148
}
148149

149150
return size;
@@ -224,7 +225,7 @@ recompose_code(uint32 start, uint32 code, uint32 *result)
224225
* in the array result.
225226
*/
226227
static void
227-
decompose_code(pg_wchar code, pg_wchar **result, int *current)
228+
decompose_code(pg_wchar code, bool compat, pg_wchar **result, int *current)
228229
{
229230
pg_unicode_decomposition *entry;
230231
int i;
@@ -272,7 +273,8 @@ decompose_code(pg_wchar code, pg_wchar **result, int *current)
272273
* character with class 0 and no decompositions, so just leave also in
273274
* this case.
274275
*/
275-
if (entry == NULL || DECOMPOSITION_SIZE(entry) == 0)
276+
if (entry == NULL || DECOMPOSITION_SIZE(entry) == 0 ||
277+
(!compat && DECOMPOSITION_IS_COMPAT(entry)))
276278
{
277279
pg_wchar *res = *result;
278280

@@ -290,12 +292,12 @@ decompose_code(pg_wchar code, pg_wchar **result, int *current)
290292
pg_wchar lcode = (pg_wchar) decomp[i];
291293

292294
/* Leave if no more decompositions */
293-
decompose_code(lcode, result, current);
295+
decompose_code(lcode, compat, result, current);
294296
}
295297
}
296298

297299
/*
298-
* unicode_normalize_kc - Normalize a Unicode string to NFKC form.
300+
* unicode_normalize - Normalize a Unicode string to the specified form.
299301
*
300302
* The input is a 0-terminated array of codepoints.
301303
*
@@ -304,8 +306,10 @@ decompose_code(pg_wchar code, pg_wchar **result, int *current)
304306
* string is palloc'd instead, and OOM is reported with ereport().
305307
*/
306308
pg_wchar *
307-
unicode_normalize_kc(const pg_wchar *input)
309+
unicode_normalize(UnicodeNormalizationForm form, const pg_wchar *input)
308310
{
311+
bool compat = (form == UNICODE_NFKC || form == UNICODE_NFKD);
312+
bool recompose = (form == UNICODE_NFC || form == UNICODE_NFKC);
309313
pg_wchar *decomp_chars;
310314
pg_wchar *recomp_chars;
311315
int decomp_size,
@@ -326,7 +330,7 @@ unicode_normalize_kc(const pg_wchar *input)
326330
*/
327331
decomp_size = 0;
328332
for (p = input; *p; p++)
329-
decomp_size += get_decomposed_size(*p);
333+
decomp_size += get_decomposed_size(*p, compat);
330334

331335
decomp_chars = (pg_wchar *) ALLOC((decomp_size + 1) * sizeof(pg_wchar));
332336
if (decomp_chars == NULL)
@@ -338,7 +342,7 @@ unicode_normalize_kc(const pg_wchar *input)
338342
*/
339343
current_size = 0;
340344
for (p = input; *p; p++)
341-
decompose_code(*p, &decomp_chars, &current_size);
345+
decompose_code(*p, compat, &decomp_chars, &current_size);
342346
decomp_chars[decomp_size] = '\0';
343347
Assert(decomp_size == current_size);
344348

@@ -385,8 +389,11 @@ unicode_normalize_kc(const pg_wchar *input)
385389
count -= 2;
386390
}
387391

392+
if (!recompose)
393+
return decomp_chars;
394+
388395
/*
389-
* The last phase of NFKC is the recomposition of the reordered Unicode
396+
* The last phase of NFC and NFKC is the recomposition of the reordered Unicode
390397
* string using combining classes. The recomposed string cannot be longer
391398
* than the decomposed one, so make the allocation of the output string
392399
* based on that assumption.

‎src/include/common/unicode_norm.h

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,14 @@
1616

1717
#include "mb/pg_wchar.h"
1818

19-
extern pg_wchar *unicode_normalize_kc(const pg_wchar *input);
19+
typedef enum
20+
{
21+
UNICODE_NFC = 0,
22+
UNICODE_NFD = 1,
23+
UNICODE_NFKC = 2,
24+
UNICODE_NFKD = 3,
25+
} UnicodeNormalizationForm;
26+
27+
extern pg_wchar *unicode_normalize(UnicodeNormalizationForm form, const pg_wchar *input);
2028

2129
#endif /* UNICODE_NORM_H */

0 commit comments

Comments
 (0)