Skip to content

Commit 7351b5f

Browse files
tglsfdctglsfdc
committed
Cleanup for some problems in tsearch patch:
- ispell initialization crashed on empty dictionary file - ispell initialization crashed on affix file with prefixes but no suffixes - stop words file was run through pg_verify_mbstr, with database encoding, but it's supposed to be UTF-8; similar bug for synonym files - bunch of comments added, typos fixed, and other cleanup Introduced consistent encoding checking/conversion of data read from tsearch configuration files, by doing this in a single t_readline() subroutine (replacing direct usages of fgets). Cleaned up API for readstopwords too. Heikki Linnakangas
1 parent b918bf8 commit 7351b5f

File tree

14 files changed

+341
-328
lines changed

14 files changed

+341
-328
lines changed

‎src/backend/snowball/dict_snowball.c

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
* Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
77
*
88
* IDENTIFICATION
9-
* $PostgreSQL: pgsql/src/backend/snowball/dict_snowball.c,v 1.2 2007/08/22 01:39:44 tgl Exp $
9+
* $PostgreSQL: pgsql/src/backend/snowball/dict_snowball.c,v 1.3 2007/08/25 00:03:59 tgl Exp $
1010
*
1111
*-------------------------------------------------------------------------
1212
*/
@@ -192,7 +192,6 @@ dsnowball_init(PG_FUNCTION_ARGS)
192192
ListCell *l;
193193

194194
d = (DictSnowball *) palloc0(sizeof(DictSnowball));
195-
d->stoplist.wordop = recode_and_lowerstr;
196195

197196
foreach(l, dictoptions)
198197
{
@@ -204,8 +203,7 @@ dsnowball_init(PG_FUNCTION_ARGS)
204203
ereport(ERROR,
205204
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
206205
errmsg("multiple StopWords parameters")));
207-
readstoplist(defGetString(defel), &d->stoplist);
208-
sortstoplist(&d->stoplist);
206+
readstoplist(defGetString(defel), &d->stoplist, lowerstr);
209207
stoploaded = true;
210208
}
211209
else if (pg_strcasecmp("Language", defel->defname) == 0)

‎src/backend/tsearch/dict_ispell.c

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
*
88
*
99
* IDENTIFICATION
10-
* $PostgreSQL: pgsql/src/backend/tsearch/dict_ispell.c,v 1.2 2007/08/22 01:39:44 tgl Exp $
10+
* $PostgreSQL: pgsql/src/backend/tsearch/dict_ispell.c,v 1.3 2007/08/25 00:03:59 tgl Exp $
1111
*
1212
*-------------------------------------------------------------------------
1313
*/
@@ -39,7 +39,6 @@ dispell_init(PG_FUNCTION_ARGS)
3939
ListCell *l;
4040

4141
d = (DictISpell *) palloc0(sizeof(DictISpell));
42-
d->stoplist.wordop = recode_and_lowerstr;
4342

4443
foreach(l, dictoptions)
4544
{
@@ -73,8 +72,7 @@ dispell_init(PG_FUNCTION_ARGS)
7372
ereport(ERROR,
7473
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
7574
errmsg("multiple StopWords parameters")));
76-
readstoplist(defGetString(defel), &(d->stoplist));
77-
sortstoplist(&(d->stoplist));
75+
readstoplist(defGetString(defel), &(d->stoplist), lowerstr);
7876
stoploaded = true;
7977
}
8078
else

‎src/backend/tsearch/dict_simple.c

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
*
88
*
99
* IDENTIFICATION
10-
* $PostgreSQL: pgsql/src/backend/tsearch/dict_simple.c,v 1.2 2007/08/22 01:39:44 tgl Exp $
10+
* $PostgreSQL: pgsql/src/backend/tsearch/dict_simple.c,v 1.3 2007/08/25 00:03:59 tgl Exp $
1111
*
1212
*-------------------------------------------------------------------------
1313
*/
@@ -23,19 +23,17 @@
2323
typedef struct
2424
{
2525
StopList stoplist;
26-
} DictExample;
26+
} DictSimple;
2727

2828

2929
Datum
3030
dsimple_init(PG_FUNCTION_ARGS)
3131
{
3232
List *dictoptions = (List *) PG_GETARG_POINTER(0);
33-
DictExample *d = (DictExample *) palloc0(sizeof(DictExample));
33+
DictSimple *d = (DictSimple *) palloc0(sizeof(DictSimple));
3434
bool stoploaded = false;
3535
ListCell *l;
3636

37-
d->stoplist.wordop = recode_and_lowerstr;
38-
3937
foreach(l, dictoptions)
4038
{
4139
DefElem *defel = (DefElem *) lfirst(l);
@@ -46,8 +44,7 @@ dsimple_init(PG_FUNCTION_ARGS)
4644
ereport(ERROR,
4745
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
4846
errmsg("multiple StopWords parameters")));
49-
readstoplist(defGetString(defel), &d->stoplist);
50-
sortstoplist(&d->stoplist);
47+
readstoplist(defGetString(defel), &d->stoplist, lowerstr);
5148
stoploaded = true;
5249
}
5350
else
@@ -65,16 +62,16 @@ dsimple_init(PG_FUNCTION_ARGS)
6562
Datum
6663
dsimple_lexize(PG_FUNCTION_ARGS)
6764
{
68-
DictExample *d = (DictExample *) PG_GETARG_POINTER(0);
65+
DictSimple *d = (DictSimple *) PG_GETARG_POINTER(0);
6966
char *in = (char *) PG_GETARG_POINTER(1);
7067
int32 len = PG_GETARG_INT32(2);
71-
char *txt = lowerstr_with_len(in, len);
68+
char *txt;
7269
TSLexeme *res = palloc0(sizeof(TSLexeme) * 2);
7370

71+
txt = lowerstr_with_len(in, len);
72+
7473
if (*txt == '\0' || searchstoplist(&(d->stoplist), txt))
75-
{
7674
pfree(txt);
77-
}
7875
else
7976
res[0].lexeme = txt;
8077

‎src/backend/tsearch/dict_synonym.c

Lines changed: 48 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
*
88
*
99
* IDENTIFICATION
10-
* $PostgreSQL: pgsql/src/backend/tsearch/dict_synonym.c,v 1.2 2007/08/22 04:13:15 tgl Exp $
10+
* $PostgreSQL: pgsql/src/backend/tsearch/dict_synonym.c,v 1.3 2007/08/25 00:03:59 tgl Exp $
1111
*
1212
*-------------------------------------------------------------------------
1313
*/
@@ -20,9 +20,6 @@
2020
#include "tsearch/ts_utils.h"
2121
#include "utils/builtins.h"
2222

23-
24-
#define SYNBUFLEN 4096
25-
2623
typedef struct
2724
{
2825
char *in;
@@ -31,23 +28,34 @@ typedef struct
3128

3229
typedef struct
3330
{
34-
int len;
31+
int len; /* length of syn array */
3532
Syn *syn;
3633
} DictSyn;
3734

35+
/*
36+
* Finds the next whitespace-delimited word within the 'in' string.
37+
* Returns a pointer to the first character of the word, and a pointer
38+
* to the next byte after the last character in the word (in *end).
39+
*/
3840
static char *
3941
findwrd(char *in, char **end)
4042
{
4143
char *start;
4244

43-
*end = NULL;
45+
/* Skip leading spaces */
4446
while (*in && t_isspace(in))
4547
in += pg_mblen(in);
4648

49+
/* Return NULL on empty lines */
4750
if (*in == '\0')
51+
{
52+
*end = NULL;
4853
return NULL;
54+
}
55+
4956
start = in;
5057

58+
/* Find end of word */
5159
while (*in && !t_isspace(in))
5260
in += pg_mblen(in);
5361

@@ -70,12 +78,11 @@ dsynonym_init(PG_FUNCTION_ARGS)
7078
ListCell *l;
7179
char *filename = NULL;
7280
FILE *fin;
73-
char buf[SYNBUFLEN];
7481
char *starti,
7582
*starto,
7683
*end = NULL;
7784
int cur = 0;
78-
int slen;
85+
char *line = NULL;
7986

8087
foreach(l, dictoptions)
8188
{
@@ -105,10 +112,33 @@ dsynonym_init(PG_FUNCTION_ARGS)
105112

106113
d = (DictSyn *) palloc0(sizeof(DictSyn));
107114

108-
while (fgets(buf, SYNBUFLEN, fin))
115+
while ((line = t_readline(fin)) != NULL)
109116
{
110-
slen = strlen(buf);
111-
pg_verifymbstr(buf, slen, false);
117+
starti = findwrd(line, &end);
118+
if (!starti)
119+
{
120+
/* Empty line */
121+
goto skipline;
122+
}
123+
*end = '\0';
124+
if (end >= line + strlen(line))
125+
{
126+
/* A line with only one word. Ignore silently. */
127+
goto skipline;
128+
}
129+
130+
starto = findwrd(end + 1, &end);
131+
if (!starto)
132+
{
133+
/* A line with only one word. Ignore silently. */
134+
goto skipline;
135+
}
136+
*end = '\0';
137+
138+
/* starti now points to the first word, and starto to the second
139+
* word on the line, with a \0 terminator at the end of both words.
140+
*/
141+
112142
if (cur == d->len)
113143
{
114144
if (d->len == 0)
@@ -123,36 +153,19 @@ dsynonym_init(PG_FUNCTION_ARGS)
123153
}
124154
}
125155

126-
starti = findwrd(buf, &end);
127-
if (!starti)
128-
continue;
129-
*end = '\0';
130-
if (end >= buf + slen)
131-
continue;
132-
133-
starto = findwrd(end + 1, &end);
134-
if (!starto)
135-
continue;
136-
*end = '\0';
137-
138-
d->syn[cur].in = recode_and_lowerstr(starti);
139-
d->syn[cur].out = recode_and_lowerstr(starto);
140-
if (!(d->syn[cur].in && d->syn[cur].out))
141-
{
142-
FreeFile(fin);
143-
ereport(ERROR,
144-
(errcode(ERRCODE_OUT_OF_MEMORY),
145-
errmsg("out of memory")));
146-
}
156+
d->syn[cur].in = lowerstr(starti);
157+
d->syn[cur].out = lowerstr(starto);
147158

148159
cur++;
160+
161+
skipline:
162+
pfree(line);
149163
}
150164

151165
FreeFile(fin);
152166

153167
d->len = cur;
154-
if (cur > 1)
155-
qsort(d->syn, d->len, sizeof(Syn), compareSyn);
168+
qsort(d->syn, d->len, sizeof(Syn), compareSyn);
156169

157170
PG_RETURN_POINTER(d);
158171
}
@@ -179,8 +192,7 @@ dsynonym_lexize(PG_FUNCTION_ARGS)
179192
if (!found)
180193
PG_RETURN_POINTER(NULL);
181194

182-
res = palloc(sizeof(TSLexeme) * 2);
183-
memset(res, 0, sizeof(TSLexeme) * 2);
195+
res = palloc0(sizeof(TSLexeme) * 2);
184196
res[0].lexeme = pstrdup(found->out);
185197

186198
PG_RETURN_POINTER(res);

‎src/backend/tsearch/dict_thesaurus.c

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
*
88
*
99
* IDENTIFICATION
10-
* $PostgreSQL: pgsql/src/backend/tsearch/dict_thesaurus.c,v 1.2 2007/08/22 01:39:44 tgl Exp $
10+
* $PostgreSQL: pgsql/src/backend/tsearch/dict_thesaurus.c,v 1.3 2007/08/25 00:03:59 tgl Exp $
1111
*
1212
*-------------------------------------------------------------------------
1313
*/
@@ -170,10 +170,10 @@ static void
170170
thesaurusRead(char *filename, DictThesaurus * d)
171171
{
172172
FILE *fh;
173-
char str[BUFSIZ];
174173
int lineno = 0;
175174
uint16 idsubst = 0;
176175
bool useasis = false;
176+
char *line;
177177

178178
filename = get_tsearch_config_filename(filename, "ths");
179179
fh = AllocateFile(filename, "r");
@@ -183,27 +183,28 @@ thesaurusRead(char *filename, DictThesaurus * d)
183183
errmsg("could not open thesaurus file \"%s\": %m",
184184
filename)));
185185

186-
while (fgets(str, sizeof(str), fh))
186+
while ((line = t_readline(fh)) != NULL)
187187
{
188-
char *ptr,
189-
*recoded;
188+
char *ptr;
190189
int state = TR_WAITLEX;
191190
char *beginwrd = NULL;
192191
uint16 posinsubst = 0;
193192
uint16 nwrd = 0;
194193

195-
ptr = recoded = (char *) pg_do_encoding_conversion((unsigned char *) str, strlen(str),
196-
GetDatabaseEncoding(), PG_UTF8);
197-
if (recoded == NULL)
198-
elog(ERROR, "encoding conversion failed");
199-
200194
lineno++;
201195

202-
/* is it comment ? */
203-
while (t_isspace(ptr))
196+
ptr = line;
197+
198+
/* is it a comment? */
199+
while (*ptr && t_isspace(ptr))
204200
ptr += pg_mblen(ptr);
205-
if (t_iseq(recoded, '#') || *recoded == '\0' || t_iseq(recoded, '\n') || t_iseq(recoded, '\r'))
201+
202+
if (t_iseq(ptr, '#') || *ptr == '\0' ||
203+
t_iseq(ptr, '\n') || t_iseq(ptr, '\r'))
204+
{
205+
pfree(line);
206206
continue;
207+
}
207208

208209
while (*ptr)
209210
{
@@ -301,8 +302,7 @@ thesaurusRead(char *filename, DictThesaurus * d)
301302
lineno, filename)));
302303
}
303304

304-
if (recoded != str)
305-
pfree(recoded);
305+
pfree(line);
306306
}
307307

308308
d->nsubst = idsubst;

0 commit comments

Comments
 (0)