Skip to content

Commit eb62398

Browse files
peterepetere
committed
Fix Unicode support in PL/Python
Check calls of PyUnicode_AsEncodedString() for NULL return, probably because the encoding name is not known. Add special treatment for SQL_ASCII, which Python definitely does not know. Since using SQL_ASCII produces errors in the regression tests when non-ASCII characters are involved, we have to put back various regression test result variants.
1 parent 6689ce3 commit eb62398

File tree

5 files changed

+175
-4
lines changed

5 files changed

+175
-4
lines changed

‎src/pl/plpython/expected/README

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,8 @@ Guide to alternative expected files:
22

33
plpython_error_2.out Python 2.2, 2.3, 2.4
44
plpython_error.out Python 2.5, 2.6
5+
6+
plpython_unicode.out any version, when server encoding != SQL_ASCII; else ...
7+
plpython_unicode_2.out Python 2.2
8+
plpython_unicode_3.out Python 2.3, 2.4
9+
plpython_unicode_5.out Python 2.5, 2.6

‎src/pl/plpython/expected/plpython_unicode_2.out

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
--
2+
-- Unicode handling
3+
--
4+
CREATE TABLE unicode_test (
5+
testvalue text NOT NULL
6+
);
7+
CREATE FUNCTION unicode_return() RETURNS text AS E'
8+
return u"\\x80"
9+
' LANGUAGE plpythonu;
10+
CREATE FUNCTION unicode_trigger() RETURNS trigger AS E'
11+
TD["new"]["testvalue"] = u"\\x80"
12+
return "MODIFY"
13+
' LANGUAGE plpythonu;
14+
CREATE TRIGGER unicode_test_bi BEFORE INSERT ON unicode_test
15+
FOR EACH ROW EXECUTE PROCEDURE unicode_trigger();
16+
CREATE FUNCTION unicode_plan1() RETURNS text AS E'
17+
plan = plpy.prepare("SELECT $1 AS testvalue", ["text"])
18+
rv = plpy.execute(plan, [u"\\x80"], 1)
19+
return rv[0]["testvalue"]
20+
' LANGUAGE plpythonu;
21+
CREATE FUNCTION unicode_plan2() RETURNS text AS E'
22+
plan = plpy.prepare("SELECT $1 || $2 AS testvalue", ["text", u"text"])
23+
rv = plpy.execute(plan, ["foo", "bar"], 1)
24+
return rv[0]["testvalue"]
25+
' LANGUAGE plpythonu;
26+
SELECT unicode_return();
27+
ERROR: PL/Python: could not convert Python Unicode object to PostgreSQL server encoding
28+
DETAIL: exceptions.UnicodeError: ASCII encoding error: ordinal not in range(128)
29+
CONTEXT: while creating return value
30+
PL/Python function "unicode_return"
31+
INSERT INTO unicode_test (testvalue) VALUES ('test');
32+
ERROR: PL/Python: could not convert Python Unicode object to PostgreSQL server encoding
33+
DETAIL: exceptions.UnicodeError: ASCII encoding error: ordinal not in range(128)
34+
CONTEXT: while modifying trigger row
35+
PL/Python function "unicode_trigger"
36+
SELECT * FROM unicode_test;
37+
testvalue
38+
-----------
39+
(0 rows)
40+
41+
SELECT unicode_plan1();
42+
WARNING: PL/Python: plpy.Error: unrecognized error in PLy_spi_execute_plan
43+
CONTEXT: PL/Python function "unicode_plan1"
44+
ERROR: PL/Python: could not convert Python Unicode object to PostgreSQL server encoding
45+
DETAIL: exceptions.UnicodeError: ASCII encoding error: ordinal not in range(128)
46+
CONTEXT: PL/Python function "unicode_plan1"
47+
SELECT unicode_plan2();
48+
unicode_plan2
49+
---------------
50+
foobar
51+
(1 row)
52+

‎src/pl/plpython/expected/plpython_unicode_3.out

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
--
2+
-- Unicode handling
3+
--
4+
CREATE TABLE unicode_test (
5+
testvalue text NOT NULL
6+
);
7+
CREATE FUNCTION unicode_return() RETURNS text AS E'
8+
return u"\\x80"
9+
' LANGUAGE plpythonu;
10+
CREATE FUNCTION unicode_trigger() RETURNS trigger AS E'
11+
TD["new"]["testvalue"] = u"\\x80"
12+
return "MODIFY"
13+
' LANGUAGE plpythonu;
14+
CREATE TRIGGER unicode_test_bi BEFORE INSERT ON unicode_test
15+
FOR EACH ROW EXECUTE PROCEDURE unicode_trigger();
16+
CREATE FUNCTION unicode_plan1() RETURNS text AS E'
17+
plan = plpy.prepare("SELECT $1 AS testvalue", ["text"])
18+
rv = plpy.execute(plan, [u"\\x80"], 1)
19+
return rv[0]["testvalue"]
20+
' LANGUAGE plpythonu;
21+
CREATE FUNCTION unicode_plan2() RETURNS text AS E'
22+
plan = plpy.prepare("SELECT $1 || $2 AS testvalue", ["text", u"text"])
23+
rv = plpy.execute(plan, ["foo", "bar"], 1)
24+
return rv[0]["testvalue"]
25+
' LANGUAGE plpythonu;
26+
SELECT unicode_return();
27+
ERROR: PL/Python: could not convert Python Unicode object to PostgreSQL server encoding
28+
DETAIL: exceptions.UnicodeEncodeError: 'ascii' codec can't encode character u'\x80' in position 0: ordinal not in range(128)
29+
CONTEXT: while creating return value
30+
PL/Python function "unicode_return"
31+
INSERT INTO unicode_test (testvalue) VALUES ('test');
32+
ERROR: PL/Python: could not convert Python Unicode object to PostgreSQL server encoding
33+
DETAIL: exceptions.UnicodeEncodeError: 'ascii' codec can't encode character u'\x80' in position 0: ordinal not in range(128)
34+
CONTEXT: while modifying trigger row
35+
PL/Python function "unicode_trigger"
36+
SELECT * FROM unicode_test;
37+
testvalue
38+
-----------
39+
(0 rows)
40+
41+
SELECT unicode_plan1();
42+
WARNING: PL/Python: plpy.Error: unrecognized error in PLy_spi_execute_plan
43+
CONTEXT: PL/Python function "unicode_plan1"
44+
ERROR: PL/Python: could not convert Python Unicode object to PostgreSQL server encoding
45+
DETAIL: exceptions.UnicodeEncodeError: 'ascii' codec can't encode character u'\x80' in position 0: ordinal not in range(128)
46+
CONTEXT: PL/Python function "unicode_plan1"
47+
SELECT unicode_plan2();
48+
unicode_plan2
49+
---------------
50+
foobar
51+
(1 row)
52+

‎src/pl/plpython/expected/plpython_unicode_5.out

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
--
2+
-- Unicode handling
3+
--
4+
CREATE TABLE unicode_test (
5+
testvalue text NOT NULL
6+
);
7+
CREATE FUNCTION unicode_return() RETURNS text AS E'
8+
return u"\\x80"
9+
' LANGUAGE plpythonu;
10+
CREATE FUNCTION unicode_trigger() RETURNS trigger AS E'
11+
TD["new"]["testvalue"] = u"\\x80"
12+
return "MODIFY"
13+
' LANGUAGE plpythonu;
14+
CREATE TRIGGER unicode_test_bi BEFORE INSERT ON unicode_test
15+
FOR EACH ROW EXECUTE PROCEDURE unicode_trigger();
16+
CREATE FUNCTION unicode_plan1() RETURNS text AS E'
17+
plan = plpy.prepare("SELECT $1 AS testvalue", ["text"])
18+
rv = plpy.execute(plan, [u"\\x80"], 1)
19+
return rv[0]["testvalue"]
20+
' LANGUAGE plpythonu;
21+
CREATE FUNCTION unicode_plan2() RETURNS text AS E'
22+
plan = plpy.prepare("SELECT $1 || $2 AS testvalue", ["text", u"text"])
23+
rv = plpy.execute(plan, ["foo", "bar"], 1)
24+
return rv[0]["testvalue"]
25+
' LANGUAGE plpythonu;
26+
SELECT unicode_return();
27+
ERROR: PL/Python: could not convert Python Unicode object to PostgreSQL server encoding
28+
DETAIL: <type 'exceptions.UnicodeEncodeError'>: 'ascii' codec can't encode character u'\x80' in position 0: ordinal not in range(128)
29+
CONTEXT: while creating return value
30+
PL/Python function "unicode_return"
31+
INSERT INTO unicode_test (testvalue) VALUES ('test');
32+
ERROR: PL/Python: could not convert Python Unicode object to PostgreSQL server encoding
33+
DETAIL: <type 'exceptions.UnicodeEncodeError'>: 'ascii' codec can't encode character u'\x80' in position 0: ordinal not in range(128)
34+
CONTEXT: while modifying trigger row
35+
PL/Python function "unicode_trigger"
36+
SELECT * FROM unicode_test;
37+
testvalue
38+
-----------
39+
(0 rows)
40+
41+
SELECT unicode_plan1();
42+
WARNING: PL/Python: <class 'plpy.Error'>: unrecognized error in PLy_spi_execute_plan
43+
CONTEXT: PL/Python function "unicode_plan1"
44+
ERROR: PL/Python: could not convert Python Unicode object to PostgreSQL server encoding
45+
DETAIL: <type 'exceptions.UnicodeEncodeError'>: 'ascii' codec can't encode character u'\x80' in position 0: ordinal not in range(128)
46+
CONTEXT: PL/Python function "unicode_plan1"
47+
SELECT unicode_plan2();
48+
unicode_plan2
49+
---------------
50+
foobar
51+
(1 row)
52+

‎src/pl/plpython/plpython.c

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/**********************************************************************
22
* plpython.c - python as a procedural language for PostgreSQL
33
*
4-
* $PostgreSQL: pgsql/src/pl/plpython/plpython.c,v 1.129 2009/09/12 22:13:12 petere Exp $
4+
* $PostgreSQL: pgsql/src/pl/plpython/plpython.c,v 1.130 2009/09/13 22:07:06 petere Exp $
55
*
66
*********************************************************************
77
*/
@@ -3343,11 +3343,21 @@ PLy_free(void *ptr)
33433343
static PyObject*
33443344
PLyUnicode_Str(PyObject *unicode)
33453345
{
3346+
PyObject *rv;
3347+
const char *serverenc;
3348+
33463349
/*
3347-
* This assumes that the PostgreSQL encoding names are acceptable
3348-
* to Python, but that appears to be the case.
3350+
* Python understands almost all PostgreSQL encoding names, but it
3351+
* doesn't know SQL_ASCII.
33493352
*/
3350-
return PyUnicode_AsEncodedString(unicode, GetDatabaseEncodingName(), "strict");
3353+
if (GetDatabaseEncoding() == PG_SQL_ASCII)
3354+
serverenc = "ascii";
3355+
else
3356+
serverenc = GetDatabaseEncodingName();
3357+
rv = PyUnicode_AsEncodedString(unicode, serverenc, "strict");
3358+
if (rv == NULL)
3359+
PLy_elog(ERROR, "could not convert Python Unicode object to PostgreSQL server encoding");
3360+
return rv;
33513361
}
33523362

33533363
/*

0 commit comments

Comments
 (0)