rtyler
/
py-yajl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Allow yajl to be compiled out of the box for Python 2.x and 3.x 

Most of this work was done by Travis Parker (teepark), I based
this changeset off his changes but #ifdef'd things such that
both 2 and 3 could compile off the same source.
This commit is contained in:
R. Tyler Ballance 2009-12-05 23:59:54 -08:00
parent 7c57eaf4fa
commit 99e912651e
6 changed files with 144 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
compare.py
View File

@ -58,5 +58,5 @@ if json:
for name, args in contenders:
test(*args)
x, y = profile(*args)
print "%-11s serialize: %0.3f deserialize: %0.3f total: %0.3f" % (
name, x, y, x+y)
print("%-11s serialize: %0.3f deserialize: %0.3f total: %0.3f" % (
name, x, y, x+y))

31
decoder.c
View File

@ -93,7 +93,12 @@ static int handle_bool(void *ctx, int value)
static int handle_number(void *ctx, const char *value, unsigned int length)
{
_YajlDecoder *self = (_YajlDecoder *)(ctx);
PyObject *string, *object;
PyObject *object;
#ifdef IS_PYTHON3
PyBytesObject *string;
#else
PyObject *string;
#endif
int floaty_char;
@ -106,20 +111,28 @@ static int handle_number(void *ctx, const char *value, unsigned int length)
}
floatin:
#ifdef IS_PYTHON3
string = (PyBytesObject *)PyBytes_FromStringAndSize(value, length);
if (floaty_char >= length) {
object = PyLong_FromString(string->ob_sval, NULL, 10);
} else {
object = PyFloat_FromString((PyObject *)string);
}
#else
string = PyString_FromStringAndSize(value, length);
if (floaty_char >= length) {
object = PyInt_FromString(PyString_AS_STRING(string), NULL, 10);
} else {
object = PyFloat_FromString(string, NULL);
}
#endif
Py_XDECREF(string);
return PlaceObject(self, object);
}
static int handle_string(void *ctx, const unsigned char *value, unsigned int length)
{
return PlaceObject(ctx, PyString_FromStringAndSize((char *)value, length));
return PlaceObject(ctx, PyUnicode_FromStringAndSize((char *)value, length));
}
static int handle_start_dict(void *ctx)
@ -134,7 +147,7 @@ static int handle_start_dict(void *ctx)
static int handle_dict_key(void *ctx, const unsigned char *value, unsigned int length)
{
PyObject *object = PyString_FromStringAndSize((const char *) value, length);
PyObject *object = PyUnicode_FromStringAndSize((const char *) value, length);
if (object == NULL)
return failure;
@ -243,13 +256,13 @@ PyObject *_internal_decode(_YajlDecoder *self, char *buffer, unsigned int buflen
if (yrc != yajl_status_ok) {
PyErr_SetObject(PyExc_ValueError,
PyString_FromString(yajl_status_to_string(yrc)));
PyUnicode_FromString(yajl_status_to_string(yrc)));
return NULL;
}
if (self->root == NULL) {
PyErr_SetObject(PyExc_ValueError,
PyString_FromString("The root object is NULL"));
PyUnicode_FromString("The root object is NULL"));
return NULL;
}
@ -271,7 +284,7 @@ PyObject *py_yajldecoder_decode(PYARGS)
if (!buflen) {
PyErr_SetObject(PyExc_ValueError,
PyString_FromString("Cannot parse an empty buffer"));
PyUnicode_FromString("Cannot parse an empty buffer"));
return NULL;
}
return _internal_decode(decoder, buffer, buflen);
@ -296,5 +309,9 @@ void yajldecoder_dealloc(_YajlDecoder *self)
if (self->root) {
Py_XDECREF(self->root);
}
#ifdef IS_PYTHON3
Py_TYPE(self)->tp_free((PyObject*)self);
#else
self->ob_type->tp_free((PyObject*)self);
#endif
}

44
encoder.c
View File

@ -60,19 +60,29 @@ static yajl_gen_status ProcessObject(_YajlEncoder *self, PyObject *object)
object = PyUnicode_AsUTF8String(object);
decref = 1;
}
#ifdef IS_PYTHON3
if (PyBytes_Check(object)) {
#else
if (PyString_Check(object)) {
#endif
const unsigned char *buffer = NULL;
Py_ssize_t length;
#ifdef IS_PYTHON3
PyBytes_AsStringAndSize(object, (char **)&buffer, &length);
#else
PyString_AsStringAndSize(object, (char **)&buffer, &length);
#endif
status = yajl_gen_string(handle, buffer, (unsigned int)(length));
if (decref) {
Py_XDECREF(object);
}
return status;
}
#ifndef IS_PYTHON3
if (PyInt_Check(object)) {
return yajl_gen_integer(handle, PyInt_AsLong(object));
}
#endif
if (PyLong_Check(object)) {
return yajl_gen_integer(handle, PyLong_AsLong(object));
}
@ -135,13 +145,22 @@ static void py_yajl_printer(void * ctx,
newsize = Py_SIZE(sauc->str);
while (sauc->used + len > newsize) newsize *= 2;
if (newsize != Py_SIZE(sauc->str)) {
#ifdef IS_PYTHON3
_PyBytes_Resize(&(sauc->str), newsize);
#else
_PyString_Resize(&(sauc->str), newsize);
if (!sauc->str) return;
#endif
if (!sauc->str)
return;
}
/* and append data if available */
if (len && str) {
#ifdef IS_PYTHON3
memcpy((void *)(((PyBytesObject *)sauc->str)->ob_sval + sauc->used), str, len);
#else
memcpy((void *) (((PyStringObject *) sauc->str)->ob_sval + sauc->used), str, len);
#endif
sauc->used += len;
}
}
@ -149,12 +168,19 @@ static void py_yajl_printer(void * ctx,
/* Efficiently allocate a python string of a fixed size containing uninitialized memory */
static PyObject * lowLevelStringAlloc(Py_ssize_t size)
{
#ifdef IS_PYTHON3
PyBytesObject * op = (PyBytesObject *)PyObject_MALLOC(sizeof(PyBytesObject) + size);
if (op) {
PyObject_INIT_VAR(op, &PyBytes_Type, size);
}
#else
PyStringObject * op = (PyStringObject *)PyObject_MALLOC(sizeof(PyStringObject) + size);
if (op) {
PyObject_INIT_VAR(op, &PyString_Type, size);
op->ob_shash = -1;
op->ob_sstate = SSTATE_NOT_INTERNED;
}
#endif
return (PyObject *) op;
}
@ -164,6 +190,7 @@ PyObject *_internal_encode(_YajlEncoder *self, PyObject *obj)
yajl_gen_config genconfig = { 0, NULL};
yajl_gen_status status;
struct StringAndUsedCount sauc;
PyObject *result = NULL;
/* initialize context for our printer function which
* performs low level string appending, using the python
@ -182,20 +209,25 @@ PyObject *_internal_encode(_YajlEncoder *self, PyObject *obj)
/* if resize failed inside our printer function we'll have a null sauc.str */
if (!sauc.str) {
PyErr_SetObject(PyExc_ValueError, PyString_FromString("Allocation failure"));
PyErr_SetObject(PyExc_ValueError, PyUnicode_FromString("Allocation failure"));
return NULL;
}
if (status != yajl_gen_status_ok) {
PyErr_SetObject(PyExc_ValueError, PyString_FromString("Failed to process"));
PyErr_SetObject(PyExc_ValueError, PyUnicode_FromString("Failed to process"));
Py_XDECREF(sauc.str);
return NULL;
}
#ifdef IS_PYTHON3
result = PyUnicode_DecodeUTF8(((PyBytesObject *)sauc.str)->ob_sval, sauc.used, "strict");
Py_XDECREF(sauc.str);
return result;
#else
/* truncate to used size, and resize will handle the null plugging */
_PyString_Resize(&sauc.str, sauc.used);
return sauc.str;
#endif
}
PyObject *py_yajlencoder_encode(PYARGS)
@ -220,5 +252,9 @@ int yajlencoder_init(PYARGS)
void yajlencoder_dealloc(_YajlEncoder *self)
{
#ifdef IS_PYTHON3
Py_TYPE(self)->tp_free((PyObject*)self);
#else
self->ob_type->tp_free((PyObject*)self);
#endif
}

4
py_yajl.h
View File

@ -36,6 +36,10 @@
#include <Python.h>
#include "ptrstack.h"
#if PY_MAJOR_VERSION >= 3
#define IS_PYTHON3
#endif
typedef struct {
PyObject_HEAD

19
tests.py
View File

@ -1,9 +1,15 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from StringIO import StringIO
import sys
import unittest
if sys.version_info[0] == 3:
from io import StringIO
else:
from StringIO import StringIO
import yajl
class BasicJSONDecodeTests(unittest.TestCase):
@ -73,8 +79,13 @@ class BasicJSONEncodeTests(unittest.TestCase):
def test_Dict(self):
self.assertEncodesTo({'key' : 'value'}, '{"key":"value"}')
def test_UnicodeDict(self):
self.assertEncodesTo({u'foō' : u'bār'}, '{"foō":"bār"}')
# Python 3 version
#def test_UnicodeDict(self):
# self.assertEncodesTo({'foō' : 'bār'}, '{"foō":"bār"}')
# Python 2 version
#def test_UnicodeDict(self):
# self.assertEncodesTo({u'foō' : u'bār'}, '{"foō":"bār"}')
def test_NestedDictAndList(self):
self.assertEncodesTo({'key' : {'subkey' : [1,2,3]}},
@ -126,7 +137,7 @@ class StreamIterDecodingTests(object): # TODO: Change to unittest.TestCase when
def test_simple_decode(self):
for k, v in yajl.iterload(self.stream):
print k, v
print(k, v)
class StreamEncodingTests(unittest.TestCase):

69
yajl.c
View File

@ -44,8 +44,12 @@ static PyMethodDef yajlencoder_methods[] = {
};
static PyTypeObject YajlDecoderType = {
#ifdef IS_PYTHON3
PyVarObject_HEAD_INIT(NULL, 0)
#else
PyObject_HEAD_INIT(NULL)
0, /*ob_size*/
#endif
"yajl.YajlDecoder", /*tp_name*/
sizeof(_YajlDecoder), /*tp_basicsize*/
0, /*tp_itemsize*/
@ -85,8 +89,12 @@ static PyTypeObject YajlDecoderType = {
};
static PyTypeObject YajlEncoderType = {
#ifdef IS_PYTHON3
PyVarObject_HEAD_INIT(NULL, 0)
#else
PyObject_HEAD_INIT(NULL)
0, /*ob_size*/
#endif
"yajl.YajlEncoder", /*tp_name*/
sizeof(_YajlEncoder), /*tp_basicsize*/
0, /*tp_itemsize*/
@ -173,13 +181,14 @@ static PyObject *_internal_stream_load(PyObject *args, unsigned int blocking)
PyObject *stream = NULL;
PyObject *buffer = NULL;
PyObject *result = NULL;
PyObject *bufferstring = NULL;
if (!PyArg_ParseTuple(args, "O", &stream)) {
goto bad_type;
}
if (__read == NULL) {
__read = PyString_FromString("read");
__read = PyUnicode_FromString("read");
}
if (!PyObject_HasAttr(stream, __read)) {
@ -191,19 +200,31 @@ static PyObject *_internal_stream_load(PyObject *args, unsigned int blocking)
if (!buffer)
return NULL;
#ifdef IS_PYTHON3
bufferstring = PyUnicode_AsUTF8String(buffer);
if (!bufferstring)
return NULL;
#endif
decoder = PyObject_Call((PyObject *)(&YajlDecoderType), NULL, NULL);
if (decoder == NULL) {
return NULL;
}
#ifdef IS_PYTHON3
result = _internal_decode((_YajlDecoder *)decoder, PyBytes_AsString(bufferstring),
PyBytes_Size(bufferstring));
Py_XDECREF(bufferstring);
#else
result = _internal_decode((_YajlDecoder *)decoder, PyString_AsString(buffer),
PyString_Size(buffer));
#endif
Py_XDECREF(decoder);
Py_XDECREF(buffer);
return result;
bad_type:
PyErr_SetObject(PyExc_TypeError, PyString_FromString("Must pass a single stream object"));
PyErr_SetObject(PyExc_TypeError, PyUnicode_FromString("Must pass a single stream object"));
return NULL;
}
@ -229,7 +250,7 @@ static PyObject *_internal_stream_dump(PyObject *args, unsigned int blocking)
}
if (__write == NULL) {
__write = PyString_FromString("write");
__write = PyUnicode_FromString("write");
}
if (!PyObject_HasAttr(stream, __write)) {
@ -247,7 +268,7 @@ static PyObject *_internal_stream_dump(PyObject *args, unsigned int blocking)
return Py_True;
bad_type:
PyErr_SetObject(PyExc_TypeError, PyString_FromString("Must pass a stream object"));
PyErr_SetObject(PyExc_TypeError, PyUnicode_FromString("Must pass a stream object"));
return NULL;
}
static PyObject *py_dump(PYARGS)
@ -277,11 +298,16 @@ Encodes the given `obj` and writes it to the `fp` stream-like object. \n\
};
#ifdef IS_PYTHON3
static struct PyModuleDef yajlmodule = {
PyModuleDef_HEAD_INIT,
"yajl",
#else
PyMODINIT_FUNC inityajl(void)
{
PyObject *module = Py_InitModule3("yajl", yajl_methods,
#endif
"Providing a pythonic interface to the yajl (Yet Another JSON Library) parser\n\n\
The interface is similar to that of simplejson or jsonlib providing a consistent syntax for JSON\n\
encoding and decoding. Unlike simplejson or jsonlib, yajl is **fast** :)\n\n\
@ -292,20 +318,43 @@ yajl.loads():\t\t502.4572ms\n\
\n\
json.dumps():\t\t7760.6348ms\n\
simplejson.dumps():\t930.9748ms\n\
yajl.dumps():\t\t681.0221ms");
yajl.dumps():\t\t681.0221ms"
#ifdef IS_PYTHON3
, -1, yajl_methods, NULL, NULL, NULL, NULL
};
PyMODINIT_FUNC PyInit_yajl(void)
{
PyObject *module = PyModule_Create(&yajlmodule);
#else
);
#endif
YajlDecoderType.tp_new = PyType_GenericNew;
if (PyType_Ready(&YajlDecoderType) < 0)
return;
if (PyType_Ready(&YajlDecoderType) < 0) {
goto bad_exit;
}
Py_INCREF(&YajlDecoderType);
PyModule_AddObject(module, "Decoder", (PyObject *)(&YajlDecoderType));
YajlEncoderType.tp_new = PyType_GenericNew;
if (PyType_Ready(&YajlEncoderType) < 0)
return;
if (PyType_Ready(&YajlEncoderType) < 0) {
goto bad_exit;
}
Py_INCREF(&YajlEncoderType);
PyModule_AddObject(module, "Encoder", (PyObject *)(&YajlEncoderType));
#ifdef IS_PYTHON3
return module;
#endif
bad_exit:
#ifdef IS_PYTHON3
return NULL;
#else
return;
#endif
}