Skip to content

Commit 419fe7c

Browse files
feodorfeodor
committed
Fix string's length calculation for recoding, fix strlower() to avoid wrong assumption about length of recoded string (was: recoded string is no greater that source, it may not true for multibyte encodings) Thanks to Thomas H. <me@alternize.com> and Magnus Hagander <mha@sollentuna.net>
1 parent 1a5c450 commit 419fe7c

File tree

6 files changed

+131
-62
lines changed

6 files changed

+131
-62
lines changed

‎contrib/tsearch2/dict_ex.c

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* $PostgreSQL: pgsql/contrib/tsearch2/dict_ex.c,v 1.8 2006/03/11 04:38:30 momjian Exp $ */
1+
/* $PostgreSQL: pgsql/contrib/tsearch2/dict_ex.c,v 1.9 2006/11/20 14:03:30 teodor Exp $ */
22

33
/*
44
* example of dictionary
@@ -52,9 +52,11 @@ dex_lexize(PG_FUNCTION_ARGS)
5252
{
5353
DictExample *d = (DictExample *) PG_GETARG_POINTER(0);
5454
char *in = (char *) PG_GETARG_POINTER(1);
55-
char *txt = pnstrdup(in, PG_GETARG_INT32(2));
55+
char *utxt = pnstrdup(in, PG_GETARG_INT32(2));
5656
TSLexeme *res = palloc(sizeof(TSLexeme) * 2);
57+
char *txt = lowerstr(utxt);
5758

59+
pfree(utxt);
5860
memset(res, 0, sizeof(TSLexeme) * 2);
5961

6062
if (*txt == '\0' || searchstoplist(&(d->stoplist), txt))

‎contrib/tsearch2/dict_snowball.c

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* $PostgreSQL: pgsql/contrib/tsearch2/dict_snowball.c,v 1.12 2006/07/11 16:35:31 momjian Exp $ */
1+
/* $PostgreSQL: pgsql/contrib/tsearch2/dict_snowball.c,v 1.13 2006/11/20 14:03:30 teodor Exp $ */
22

33
/*
44
* example of Snowball dictionary
@@ -142,9 +142,11 @@ snb_lexize(PG_FUNCTION_ARGS)
142142
{
143143
DictSnowball *d = (DictSnowball *) PG_GETARG_POINTER(0);
144144
char *in = (char *) PG_GETARG_POINTER(1);
145-
char *txt = pnstrdup(in, PG_GETARG_INT32(2));
145+
char *utxt = pnstrdup(in, PG_GETARG_INT32(2));
146146
TSLexeme *res = palloc(sizeof(TSLexeme) * 2);
147-
147+
char *txt = lowerstr(utxt);
148+
149+
pfree(utxt);
148150
memset(res, 0, sizeof(TSLexeme) * 2);
149151
if (*txt == '\0' || searchstoplist(&(d->stoplist), txt))
150152
{

‎contrib/tsearch2/dict_syn.c

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* $PostgreSQL: pgsql/contrib/tsearch2/dict_syn.c,v 1.9 2006/03/11 04:38:30 momjian Exp $ */
1+
/* $PostgreSQL: pgsql/contrib/tsearch2/dict_syn.c,v 1.10 2006/11/20 14:03:30 teodor Exp $ */
22

33
/*
44
* ISpell interface
@@ -132,8 +132,8 @@ syn_init(PG_FUNCTION_ARGS)
132132
continue;
133133
*end = '\0';
134134

135-
d->syn[cur].in = strdup(lowerstr(starti));
136-
d->syn[cur].out = strdup(lowerstr(starto));
135+
d->syn[cur].in = lowerstr(starti);
136+
d->syn[cur].out = lowerstr(starto);
137137
if (!(d->syn[cur].in && d->syn[cur].out))
138138
{
139139
fclose(fin);
@@ -163,12 +163,15 @@ syn_lexize(PG_FUNCTION_ARGS)
163163
Syn key,
164164
*found;
165165
TSLexeme *res = NULL;
166+
char *wrd;
166167

167168
if (!PG_GETARG_INT32(2))
168169
PG_RETURN_POINTER(NULL);
169170

170171
key.out = NULL;
171-
key.in = lowerstr(pnstrdup(in, PG_GETARG_INT32(2)));
172+
wrd = pnstrdup(in, PG_GETARG_INT32(2));
173+
key.in = lowerstr(wrd);
174+
pfree(wrd);
172175

173176
found = (Syn *) bsearch(&key, d->syn, d->len, sizeof(Syn), compareSyn);
174177
pfree(key.in);

‎contrib/tsearch2/ispell/spell.c

Lines changed: 47 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ NIAddSpell(IspellDict * Conf, const char *word, const char *flag)
147147
int
148148
NIImportDictionary(IspellDict * Conf, const char *filename)
149149
{
150-
char str[BUFSIZ];
150+
char str[BUFSIZ], *pstr;
151151
FILE *dict;
152152

153153
if (!(dict = fopen(filename, "r")))
@@ -190,9 +190,10 @@ NIImportDictionary(IspellDict * Conf, const char *filename)
190190
}
191191
s += pg_mblen(s);
192192
}
193-
lowerstr(str);
193+
pstr = lowerstr(str);
194194

195-
NIAddSpell(Conf, str, flag);
195+
NIAddSpell(Conf, pstr, flag);
196+
pfree(pstr);
196197
}
197198
fclose(dict);
198199
return (0);
@@ -418,8 +419,7 @@ parse_affentry(char *str, char *mask, char *find, char *repl, int line)
418419
int
419420
NIImportAffixes(IspellDict * Conf, const char *filename)
420421
{
421-
char str[BUFSIZ];
422-
char tmpstr[BUFSIZ];
422+
char str[BUFSIZ], *pstr = NULL;
423423
char mask[BUFSIZ];
424424
char find[BUFSIZ];
425425
char repl[BUFSIZ];
@@ -439,11 +439,14 @@ NIImportAffixes(IspellDict * Conf, const char *filename)
439439
while (fgets(str, sizeof(str), affix))
440440
{
441441
line++;
442+
if ( *str == '#' || *str == '\n' )
443+
continue;
444+
442445
pg_verifymbstr(str, strlen(str), false);
443-
memcpy(tmpstr, str, 32); /* compoundwords... */
444-
tmpstr[32] = '\0';
445-
lowerstr(tmpstr);
446-
if (STRNCMP(tmpstr, "compoundwords") == 0)
446+
if ( pstr )
447+
pfree( pstr );
448+
pstr = lowerstr(str);
449+
if (STRNCMP(pstr, "compoundwords") == 0)
447450
{
448451
s = findchar(str, 'l');
449452
if (s)
@@ -458,21 +461,21 @@ NIImportAffixes(IspellDict * Conf, const char *filename)
458461
continue;
459462
}
460463
}
461-
if (STRNCMP(tmpstr, "suffixes") == 0)
464+
if (STRNCMP(pstr, "suffixes") == 0)
462465
{
463466
suffixes = 1;
464467
prefixes = 0;
465468
oldformat++;
466469
continue;
467470
}
468-
if (STRNCMP(tmpstr, "prefixes") == 0)
471+
if (STRNCMP(pstr, "prefixes") == 0)
469472
{
470473
suffixes = 0;
471474
prefixes = 1;
472475
oldformat++;
473476
continue;
474477
}
475-
if (STRNCMP(tmpstr, "flag") == 0)
478+
if (STRNCMP(pstr, "flag") == 0)
476479
{
477480
s = str + 4;
478481
flagflags = 0;
@@ -523,26 +526,28 @@ NIImportAffixes(IspellDict * Conf, const char *filename)
523526
if ((!suffixes) && (!prefixes))
524527
continue;
525528

526-
lowerstr(str);
527-
if (!parse_affentry(str, mask, find, repl, line))
529+
if (!parse_affentry(pstr, mask, find, repl, line))
528530
continue;
529531

530532
NIAddAffix(Conf, flag, flagflags, mask, find, repl, suffixes ? FF_SUFFIX : FF_PREFIX);
531533
}
532534
fclose(affix);
533535

536+
if ( pstr )
537+
pfree( pstr );
538+
534539
return (0);
535540
}
536541

537542
int
538543
NIImportOOAffixes(IspellDict * Conf, const char *filename)
539544
{
540545
char str[BUFSIZ];
541-
char type[BUFSIZ];
546+
char type[BUFSIZ], *ptype = NULL;
542547
char sflag[BUFSIZ];
543-
char mask[BUFSIZ];
544-
char find[BUFSIZ];
545-
char repl[BUFSIZ];
548+
char mask[BUFSIZ], *pmask;
549+
char find[BUFSIZ], *pfind;
550+
char repl[BUFSIZ], *prepl;
546551
bool isSuffix = false;
547552
int flag = 0;
548553
char flagflags = 0;
@@ -577,38 +582,46 @@ NIImportOOAffixes(IspellDict * Conf, const char *filename)
577582

578583
scanread = sscanf(str, scanbuf, type, sflag, find, repl, mask);
579584

580-
lowerstr(type);
581-
if (scanread < 4 || (STRNCMP(type, "sfx") && STRNCMP(type, "pfx")))
585+
if (ptype)
586+
pfree(ptype);
587+
ptype = lowerstr(type);
588+
if (scanread < 4 || (STRNCMP(ptype, "sfx") && STRNCMP(ptype, "pfx")))
582589
continue;
583590

584591
if (scanread == 4)
585592
{
586593
if (strlen(sflag) != 1)
587594
continue;
588595
flag = *sflag;
589-
isSuffix = (STRNCMP(type, "sfx") == 0) ? true : false;
590-
lowerstr(find);
596+
isSuffix = (STRNCMP(ptype, "sfx") == 0) ? true : false;
597+
pfind = lowerstr(find);
591598
if (t_iseq(find, 'y'))
592599
flagflags |= FF_CROSSPRODUCT;
593600
else
594601
flagflags = 0;
602+
pfree(pfind);
595603
}
596604
else
597605
{
598606
if (strlen(sflag) != 1 || flag != *sflag || flag == 0)
599607
continue;
600-
lowerstr(repl);
601-
lowerstr(find);
602-
lowerstr(mask);
608+
prepl = lowerstr(repl);
609+
pfind = lowerstr(find);
610+
pmask = lowerstr(mask);
603611
if (t_iseq(find, '0'))
604612
*find = '\0';
605613
if (t_iseq(repl, '0'))
606614
*repl = '\0';
607615

608616
NIAddAffix(Conf, flag, flagflags, mask, find, repl, isSuffix ? FF_SUFFIX : FF_PREFIX);
617+
pfree(prepl);
618+
pfree(pfind);
619+
pfree(pmask);
609620
}
610621
}
611622

623+
if (ptype)
624+
pfree(ptype);
612625
fclose(affix);
613626

614627
return 0;
@@ -1053,7 +1066,6 @@ NormalizeSubWord(IspellDict * Conf, char *word, char flag)
10531066

10541067
if (wrdlen > MAXNORMLEN)
10551068
return NULL;
1056-
lowerstr(word);
10571069
cur = forms = (char **) palloc(MAX_NORM * sizeof(char *));
10581070
*cur = NULL;
10591071

@@ -1354,13 +1366,17 @@ SplitToVariants(IspellDict * Conf, SPNode * snode, SplitVar * orig, char *word,
13541366
}
13551367

13561368
TSLexeme *
1357-
NINormalizeWord(IspellDict * Conf, char *word)
1369+
NINormalizeWord(IspellDict * Conf, char *uword)
13581370
{
1359-
char **res = NormalizeSubWord(Conf, word, 0);
1371+
char **res;
1372+
char *word;
13601373
TSLexeme *lcur = NULL,
13611374
*lres = NULL;
13621375
uint16 NVariant = 1;
13631376

1377+
word = lowerstr(uword);
1378+
res = NormalizeSubWord(Conf, word, 0);
1379+
13641380
if (res)
13651381
{
13661382
char **ptr = res;
@@ -1431,6 +1447,9 @@ NINormalizeWord(IspellDict * Conf, char *word)
14311447
var = ptr;
14321448
}
14331449
}
1450+
1451+
pfree(word);
1452+
14341453
return lres;
14351454
}
14361455

‎contrib/tsearch2/stopword.c

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ readstoplist(text *in, StopList * s)
3636
{
3737
char *filename = to_absfilename(text2char(in));
3838
FILE *hin;
39-
char buf[STOPBUFLEN];
39+
char buf[STOPBUFLEN], *pbuf;
4040
int reallen = 0;
4141

4242
if ((hin = fopen(filename, "r")) == NULL)
@@ -49,7 +49,6 @@ readstoplist(text *in, StopList * s)
4949
{
5050
buf[strlen(buf) - 1] = '\0';
5151
pg_verifymbstr(buf, strlen(buf), false);
52-
lowerstr(buf);
5352
if (*buf == '\0')
5453
continue;
5554

@@ -70,7 +69,14 @@ readstoplist(text *in, StopList * s)
7069
stop = tmp;
7170
}
7271

73-
stop[s->len] = strdup(buf);
72+
if (s->wordop)
73+
{
74+
pbuf = s->wordop(buf);
75+
stop[s->len] = strdup(pbuf);
76+
pfree(pbuf);
77+
} else
78+
stop[s->len] = strdup(buf);
79+
7480
if (!stop[s->len])
7581
{
7682
freestoplist(s);
@@ -79,8 +85,6 @@ readstoplist(text *in, StopList * s)
7985
(errcode(ERRCODE_OUT_OF_MEMORY),
8086
errmsg("out of memory")));
8187
}
82-
if (s->wordop)
83-
stop[s->len] = (s->wordop) (stop[s->len]);
8488

8589
(s->len)++;
8690
}
@@ -106,7 +110,5 @@ sortstoplist(StopList * s)
106110
bool
107111
searchstoplist(StopList * s, char *key)
108112
{
109-
if (s->wordop)
110-
key = (*(s->wordop)) (key);
111113
return (s->stop && s->len > 0 && bsearch(&key, s->stop, s->len, sizeof(char *), comparestr)) ? true : false;
112114
}

0 commit comments

Comments
 (0)