From 0eea1362bc38c5e0e3a1caa2358c472c8f6eb3bd Mon Sep 17 00:00:00 2001 From: Kyle Maxwell Date: Thu, 24 Dec 2009 13:49:23 -0800 Subject: [PATCH] added unicode tests --- pyparsleymodule.c | 6 ++++-- test/test_pyparsley.py | 10 ++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pyparsleymodule.c b/pyparsleymodule.c index 07ef4ed..eed962d 100644 --- a/pyparsleymodule.c +++ b/pyparsleymodule.c @@ -188,16 +188,17 @@ PyParsley_parse(PyParsley *self, PyObject *args, PyObject *keywords) int collate = 1; int allow_net = 1; int allow_local = 1; + int utf8 = 0; int len; int flags = 0; parsedParsleyPtr ptr; static char * list[] = { "file", "string", "input", "output", "base", "prune", "collate", - "allow_net", "allow_local", NULL }; + "allow_net", "allow_local", "utf8", NULL }; if (!PyArg_ParseTupleAndKeywords(args, keywords, - "|ss#sssiiii", list, &file, &string, &len, &input, &output, &base, &prune, &collate, &allow_net, &allow_local)) { + "|ss#sssiiiii", list, &file, &string, &len, &input, &output, &base, &prune, &collate, &allow_net, &allow_local, &utf8)) { return NULL; } @@ -206,6 +207,7 @@ PyParsley_parse(PyParsley *self, PyObject *args, PyObject *keywords) SET_FLAG(PARSLEY_OPTIONS_COLLATE, collate); SET_FLAG(PARSLEY_OPTIONS_ALLOW_NET, allow_net); SET_FLAG(PARSLEY_OPTIONS_ALLOW_LOCAL, allow_local); + SET_FLAG(PARSLEY_OPTIONS_FORCE_UTF8, utf8); // printf("%d %d %d %d %d \n", prune, collate, allow_net, allow_local, flags); diff --git a/test/test_pyparsley.py b/test/test_pyparsley.py index c5291b5..d26d3bd 100755 --- a/test/test_pyparsley.py +++ b/test/test_pyparsley.py @@ -16,14 +16,20 @@ class TestPyParsley(unittest.TestCase): self.json = '{ "title": "\\t\\tNick\'s Crispy Tacos - Russian Hill - San Francisco, CA\\n" }' self.native = { "title": "\t\tNick's Crispy Tacos - Russian Hill - San Francisco, CA\n" } self.links = '{ "links": [ "\\/signup?return_url=%2Fuser_details", "\\/signup?return_url=%2Fwriteareview", "\\/signup?return_url=%2Finvite_friends", "\\/signup?return_url=%2Fmail", "\\/signup?return_url=%2Fprofile", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup", "\\/signup" ] }' + self.unicode_string = u'\u2019blah blah blah\u2019' + self.unicode_document = u'\u2019blah blah blah\u2019' + + def test_unicode(self): + parsed = self.parsley.parse(string = self.unicode_document.encode("utf-8"), output = "python", utf8 = 1) + self.assertEquals(parsed['title'].decode("utf-8"), self.unicode_string) def test_file_xml(self): parsed = self.parsley.parse(file = self.file, output = "json") self.assertEquals(self.json, parsed) def test_pruning(self): - parsed = self.a_parsley.parse(file = self.file, output = "json") - self.assertEquals(self.links, parsed) + parsed = self.a_parsley.parse(file = self.file, output = "json") + self.assertEquals(self.links, parsed) def test_json_file_xml(self): parsed = self.alt_parsley.parse(file = self.file, output = "json")