Rev 4730: Add a _static_tuple_c.pxd file to define the C api to pyrex code. in http://bazaar.launchpad.net/~jameinel/bzr/2.1-static-tuple
John Arbash Meinel
john at arbash-meinel.com
Thu Oct 1 20:06:43 BST 2009
At http://bazaar.launchpad.net/~jameinel/bzr/2.1-static-tuple
------------------------------------------------------------
revno: 4730
revision-id: john at arbash-meinel.com-20091001190635-m0531rj2mebmhjq0
parent: john at arbash-meinel.com-20091001150005-sos54ozqd4w5psam
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: 2.1-static-tuple
timestamp: Thu 2009-10-01 14:06:35 -0500
message:
Add a _static_tuple_c.pxd file to define the C api to pyrex code.
I had to drop the notion of void*items[1] and then subtracting it from tp_basicsize
because Pyrex directly checks that sizeof(struct) == tp_basicsize. I understand
why it was doing so, but it was very hard to track down. The error during
import was basically suppressed, and then I just got segfaults because
the type wasn't actually available.
Anyway, I'm reasonably happy with using a .pxd file, because then I don't have
to worry about getting the invocation right in all callers, though I still
have to repeat the 'from x cimport a,b,c,d' which is a bit annoying.
-------------- next part --------------
=== modified file 'bzrlib/_btree_serializer_pyx.pyx'
--- a/bzrlib/_btree_serializer_pyx.pyx 2009-10-01 15:00:05 +0000
+++ b/bzrlib/_btree_serializer_pyx.pyx 2009-10-01 19:06:35 +0000
@@ -57,18 +57,11 @@
# void *memrchr(void *s, int c, size_t n)
int strncmp(char *s1, char *s2, size_t n)
-cdef extern from "_static_tuple_c.h":
- void **StaticTuple_API
- int import_static_tuple()
- # ctypedef object (*st_new_type)(Py_ssize_t)
- # st_new_type st_new
-
- object StaticTuple_New(Py_ssize_t)
- object StaticTuple_intern(object)
- # Steals a reference and Val must be a PyStringObject, no checking is done
- void StaticTuple_SET_ITEM(object key, Py_ssize_t offset, object val)
- object StaticTuple_GET_ITEM(object key, Py_ssize_t offset)
- int StaticTuple_CheckExact(object)
+# It seems we need to import the definitions so that the pyrex compiler has
+# local names to access them.
+from _static_tuple_c cimport StaticTuple, StaticTuple_API,\
+ import_static_tuple, STATIC_TUPLE_ALL_STRING, StaticTuple_New, \
+ StaticTuple_intern, StaticTuple_SET_ITEM, StaticTuple_CheckExact
# TODO: Find some way to import this from _dirstate_helpers
@@ -110,11 +103,13 @@
return result
from bzrlib import _static_tuple_c
+cdef object _ST
+_ST = _static_tuple_c.StaticTuple
# This sets up the StaticTuple C_API functionality
-if import_static_tuple() == -1 or StaticTuple_API == NULL:
+if import_static_tuple() == -1:
raise ImportError('failed to import_static_tuple()')
-cdef object StaticTuple
-StaticTuple = _static_tuple_c.StaticTuple
+if StaticTuple_API == NULL:
+ raise ImportError('StaticTuple_API failed to be initialized.')
cdef class BTreeLeafParser:
@@ -167,6 +162,7 @@
"""
cdef char *temp_ptr
cdef int loop_counter
+ cdef StaticTuple key
key = StaticTuple_New(self.key_length)
for loop_counter from 0 <= loop_counter < self.key_length:
@@ -195,6 +191,7 @@
self._start = temp_ptr + 1
Py_INCREF(key_element)
StaticTuple_SET_ITEM(key, loop_counter, key_element)
+ # key->flags = key->flags | STATIC_TUPLE_ALL_STRING
key = StaticTuple_intern(key)
return key
=== modified file 'bzrlib/_chk_map_pyx.pyx'
--- a/bzrlib/_chk_map_pyx.pyx 2009-09-30 21:53:59 +0000
+++ b/bzrlib/_chk_map_pyx.pyx 2009-10-01 19:06:35 +0000
@@ -59,25 +59,17 @@
uLong crc32(uLong crc, Bytef *buf, uInt len)
-cdef extern from "_static_tuple_c.h":
- void **StaticTuple_API
- int import_static_tuple()
-
- object StaticTuple_New(Py_ssize_t)
- object StaticTuple_intern(object)
- # Steals a reference and val must be a String or StaticTuple, no checking
- # is done
- void StaticTuple_SET_ITEM(object key, Py_ssize_t offset, object val)
- object StaticTuple_GET_ITEM(object key, Py_ssize_t offset)
- int StaticTuple_CheckExact(object)
+# It seems we need to import the definitions so that the pyrex compiler has
+# local names to access them.
+from _static_tuple_c cimport StaticTuple, StaticTuple_API,\
+ import_static_tuple, STATIC_TUPLE_ALL_STRING, StaticTuple_New, \
+ StaticTuple_intern, StaticTuple_SET_ITEM, StaticTuple_CheckExact
from bzrlib import _static_tuple_c
# This sets up the StaticTuple C_API functionality
if import_static_tuple() == -1 or StaticTuple_API == NULL:
raise ImportError('failed to import_static_tuple()')
-cdef object StaticTuple
-StaticTuple = _static_tuple_c.StaticTuple
cdef object _LeafNode
_LeafNode = None
@@ -219,6 +211,7 @@
cdef char *prefix, *value_start, *prefix_tail
cdef char *next_null, *last_null, *line_start
cdef char *c_entry, *entry_start
+ cdef StaticTuple entry_bits
if _LeafNode is None:
from bzrlib import chk_map
=== modified file 'bzrlib/_static_tuple_c.c'
--- a/bzrlib/_static_tuple_c.c 2009-09-30 21:53:59 +0000
+++ b/bzrlib/_static_tuple_c.c 2009-10-01 19:06:35 +0000
@@ -35,7 +35,7 @@
/* The one and only StaticTuple with no values */
static StaticTuple *_empty_tuple = NULL;
-static PyObject *_interned_keys = NULL;
+static PyObject *_interned_tuples = NULL;
static inline int
@@ -59,7 +59,7 @@
return NULL;
}
for (i = 0; i < len; ++i) {
- obj = (PyObject *)self->key_bits[i];
+ obj = (PyObject *)self->items[i];
Py_INCREF(obj);
PyTuple_SET_ITEM(tpl, i, obj);
}
@@ -74,7 +74,8 @@
{
PyObject *unique_key = NULL;
- if (_interned_keys == NULL) {
+ if (_interned_tuples == NULL) {
+ Py_INCREF(self);
return self;
}
if (_StaticTuple_is_interned(self)) {
@@ -82,16 +83,17 @@
Py_INCREF(self);
return self;
}
- unique_key = PyDict_GetItem((PyObject *)_interned_keys, (PyObject *)self);
+ unique_key = PyDict_GetItem((PyObject *)_interned_tuples, (PyObject *)self);
if (unique_key) {
// An entry already existed, return it, instead of self
Py_INCREF(unique_key);
return (StaticTuple *)unique_key;
}
// An entry did not exist, make 'self' the unique item
- if (PyDict_SetItem(_interned_keys, (PyObject *)self, (PyObject *)self) < 0) {
+ if (PyDict_SetItem(_interned_tuples, (PyObject *)self, (PyObject *)self) < 0) {
// Suppress an error
PyErr_Clear();
+ Py_INCREF(self);
return self;
}
// self was added to the dict, return it.
@@ -119,32 +121,31 @@
if (_StaticTuple_is_interned(self)) {
/* revive dead object temporarily for DelItem */
Py_REFCNT(self) = 3;
- if (PyDict_DelItem(_interned_keys, (PyObject *)self) != 0) {
+ if (PyDict_DelItem(_interned_tuples, (PyObject *)self) != 0) {
Py_FatalError("deletion of interned StaticTuple failed");
}
}
len = self->size;
for (i = 0; i < len; ++i) {
- Py_XDECREF(self->key_bits[i]);
+ Py_XDECREF(self->items[i]);
}
Py_TYPE(self)->tp_free((PyObject *)self);
}
/* Similar to PyTuple_New() */
-static PyObject *
+static StaticTuple *
StaticTuple_New(Py_ssize_t size)
{
- StaticTuple *key;
+ StaticTuple *stuple;
if (size < 0) {
PyErr_BadInternalCall();
return NULL;
}
- if (size == 0) {
- assert(_empty_tuple != NULL);
+ if (size == 0 && _empty_tuple != NULL) {
Py_INCREF(_empty_tuple);
- return (PyObject *)_empty_tuple;
+ return _empty_tuple;
}
/* Note that we use PyObject_NewVar because we want to allocate a variable
* width entry. However we *aren't* truly a PyVarObject because we don't
@@ -153,19 +154,21 @@
* As such we do the alloc, and then have to clean up anything it does
* incorrectly.
*/
- key = PyObject_NewVar(StaticTuple, &StaticTuple_Type, size);
- if (key == NULL) {
+ stuple = PyObject_NewVar(StaticTuple, &StaticTuple_Type, size);
+ if (stuple == NULL) {
return NULL;
}
- key->size = size;
- key->flags = 0;
- key->_unused0 = 0;
- key->_unused1 = 0;
- memset(key->key_bits, 0, sizeof(PyObject *) * size);
+ stuple->size = size;
+ stuple->flags = 0;
+ stuple->_unused0 = 0;
+ stuple->_unused1 = 0;
+ if (size > 0) {
+ memset(stuple->items, 0, sizeof(PyObject *) * size);
+ }
#if STATIC_TUPLE_HAS_HASH
- key->hash = -1;
+ stuple->hash = -1;
#endif
- return (PyObject *)key;
+ return stuple;
}
@@ -210,7 +213,7 @@
}
}
Py_INCREF(obj);
- self->key_bits[i] = obj;
+ self->items[i] = obj;
}
if (is_all_str) {
self->flags |= STATIC_TUPLE_ALL_STRING;
@@ -249,7 +252,7 @@
}
#endif
x = 0x345678L;
- p = self->key_bits;
+ p = self->items;
if (self->flags & STATIC_TUPLE_ALL_STRING
&& self->flags & STATIC_TUPLE_DID_HASH) {
/* If we know that we only reference strings, and we've already
@@ -498,7 +501,7 @@
PyErr_SetString(PyExc_IndexError, "StaticTuple index out of range");
return NULL;
}
- obj = (PyObject *)self->key_bits[offset];
+ obj = (PyObject *)self->items[offset];
Py_INCREF(obj);
return obj;
}
@@ -522,7 +525,7 @@
{
Py_ssize_t i;
for (i = self->size; --i >= 0;) {
- Py_VISIT(self->key_bits[i]);
+ Py_VISIT(self->items[i]);
}
return 0;
}
@@ -557,7 +560,7 @@
PyObject_HEAD_INIT(NULL)
0, /* ob_size */
"StaticTuple", /* tp_name */
- sizeof(StaticTuple) - sizeof(PyObject *), /* tp_basicsize */
+ sizeof(StaticTuple), /* tp_basicsize */
sizeof(PyObject *), /* tp_itemsize */
(destructor)StaticTuple_dealloc, /* tp_dealloc */
0, /* tp_print */
@@ -678,12 +681,12 @@
static void
-setup_interned_keys(PyObject *m)
+setup_interned_tuples(PyObject *m)
{
- _interned_keys = PyDict_New();
- if (_interned_keys != NULL) {
- Py_INCREF(_interned_keys);
- PyModule_AddObject(m, "_interned_keys", _interned_keys);
+ _interned_tuples = PyDict_New();
+ if (_interned_tuples != NULL) {
+ Py_INCREF(_interned_tuples);
+ PyModule_AddObject(m, "_interned_tuples", _interned_tuples);
}
}
@@ -691,24 +694,19 @@
static void
setup_empty_tuple(PyObject *m)
{
- StaticTuple *key;
+ StaticTuple *stuple;
+ if (_interned_tuples == NULL) {
+ fprintf(stderr, "You need to call setup_interned_tuples() before"
+ " setup_empty_tuple, because we intern it.\n");
+ }
// We need to create the empty tuple
- key = PyObject_NewVar(StaticTuple, &StaticTuple_Type, 0);
- if (key == NULL) {
- return;
- }
- key->size = 0;
- /* This is all strings, because it has no entries */
- key->flags = STATIC_TUPLE_ALL_STRING;
- key->_unused0 = 0;
- key->_unused1 = 0;
-#if STATIC_TUPLE_HAS_HASH
- key->hash = -1;
-#endif
- _empty_tuple = StaticTuple_intern(key);
- assert(_empty_tuple == key);
- Py_INCREF(_empty_tuple); // Never dies
- Py_INCREF(_empty_tuple); // for the module
+ stuple = (StaticTuple *)StaticTuple_New(0);
+ stuple->flags = STATIC_TUPLE_ALL_STRING;
+ _empty_tuple = StaticTuple_intern(stuple);
+ assert(_empty_tuple == stuple);
+ // At this point, refcnt is 2: 1 from New(), and 1 from the return from
+ // intern(). We will keep 1 for the _empty_tuple global, and use the other
+ // for the module reference.
PyModule_AddObject(m, "_empty_tuple", (PyObject *)_empty_tuple);
}
@@ -752,7 +750,7 @@
Py_INCREF(&StaticTuple_Type);
PyModule_AddObject(m, "StaticTuple", (PyObject *)&StaticTuple_Type);
- setup_interned_keys(m);
+ setup_interned_tuples(m);
setup_empty_tuple(m);
setup_c_api(m);
}
=== modified file 'bzrlib/_static_tuple_c.h'
--- a/bzrlib/_static_tuple_c.h 2009-09-30 21:53:59 +0000
+++ b/bzrlib/_static_tuple_c.h 2009-10-01 19:06:35 +0000
@@ -52,24 +52,24 @@
unsigned char _unused0;
unsigned char _unused1;
// Note that on 64-bit, we actually have 4-more unused bytes
- // because key_bits will always be aligned to a 64-bit boundary
+ // because items will always be aligned to a 64-bit boundary
#if STATIC_TUPLE_HAS_HASH
long hash;
#endif
- PyObject *key_bits[1];
+ PyObject *items[0];
} StaticTuple;
extern PyTypeObject StaticTuple_Type;
typedef struct {
PyObject_VAR_HEAD
- PyObject *table[1];
+ PyObject *table[0];
} KeyIntern;
// extern PyTypeObject StaticTuple_Type;
#define StaticTuple_CheckExact(op) (Py_TYPE(op) == &StaticTuple_Type)
#define StaticTuple_SET_ITEM(key, offset, val) \
- ((((StaticTuple*)(key))->key_bits[(offset)]) = ((PyObject *)(val)))
-#define StaticTuple_GET_ITEM(key, offset) (((StaticTuple*)key)->key_bits[offset])
+ ((((StaticTuple*)(key))->items[(offset)]) = ((PyObject *)(val)))
+#define StaticTuple_GET_ITEM(key, offset) (((StaticTuple*)key)->items[offset])
/* C API Functions */
@@ -83,15 +83,15 @@
#ifdef STATIC_TUPLE_MODULE
/* Used when compiling _static_tuple_c.c */
-static PyObject * StaticTuple_New(Py_ssize_t);
+static StaticTuple * StaticTuple_New(Py_ssize_t);
static StaticTuple * StaticTuple_intern(StaticTuple *self);
#else
/* Used by foriegn callers */
static void **StaticTuple_API;
-static PyObject *(*StaticTuple_New)(Py_ssize_t);
-static PyObject *(*StaticTuple_intern)(PyObject *);
+static StaticTuple *(*StaticTuple_New)(Py_ssize_t);
+static StaticTuple *(*StaticTuple_intern)(StaticTuple *);
#undef StaticTuple_CheckExact
static int (*StaticTuple_CheckExact)(PyObject *);
=== added file 'bzrlib/_static_tuple_c.pxd'
--- a/bzrlib/_static_tuple_c.pxd 1970-01-01 00:00:00 +0000
+++ b/bzrlib/_static_tuple_c.pxd 2009-10-01 19:06:35 +0000
@@ -0,0 +1,46 @@
+# Copyright (C) 2009 Canonical Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+"""The interface definition file for the StaticTuple class."""
+
+
+cdef extern from "Python.h":
+ ctypedef int Py_ssize_t # Required for older pyrex versions
+ ctypedef struct PyObject:
+ pass
+
+cdef extern from "_static_tuple_c.h":
+ ctypedef class bzrlib._static_tuple_c.StaticTuple [object StaticTuple]:
+ cdef unsigned char size
+ cdef unsigned char flags
+ # We don't need to define _unused attributes, because the raw
+ # StaticTuple structure will be referenced
+ # cdef unsigned char _unused0
+ # cdef unsigned char _unused1
+ cdef PyObject *items[0]
+
+ void **StaticTuple_API
+ int import_static_tuple()
+ # ctypedef object (*st_new_type)(Py_ssize_t)
+ # st_new_type st_new
+ int STATIC_TUPLE_ALL_STRING
+
+ StaticTuple StaticTuple_New(Py_ssize_t)
+ StaticTuple StaticTuple_intern(StaticTuple)
+ # Steals a reference and Val must be a PyStringObject, no checking is done
+ void StaticTuple_SET_ITEM(StaticTuple key, Py_ssize_t offset, object val)
+ object StaticTuple_GET_ITEM(StaticTuple key, Py_ssize_t offset)
+ int StaticTuple_CheckExact(object)
=== modified file 'bzrlib/_static_tuple_py.py'
--- a/bzrlib/_static_tuple_py.py 2009-09-30 19:43:34 +0000
+++ b/bzrlib/_static_tuple_py.py 2009-10-01 19:06:35 +0000
@@ -67,9 +67,9 @@
return self._tuple
def intern(self):
- return _interned_keys.setdefault(self, self)
+ return _interned_tuples.setdefault(self, self)
_empty_tuple = None
_empty_tuple = StaticTuple()
-_interned_keys = {}
+_interned_tuples = {}
=== modified file 'bzrlib/tests/test__static_tuple.py'
--- a/bzrlib/tests/test__static_tuple.py 2009-09-30 21:17:48 +0000
+++ b/bzrlib/tests/test__static_tuple.py 2009-10-01 19:06:35 +0000
@@ -308,14 +308,14 @@
unique_str1 = 'unique str ' + osutils.rand_chars(20)
unique_str2 = 'unique str ' + osutils.rand_chars(20)
key = self.module.StaticTuple(unique_str1, unique_str2)
- self.assertFalse(key in self.module._interned_keys)
+ self.assertFalse(key in self.module._interned_tuples)
key2 = self.module.StaticTuple(unique_str1, unique_str2)
self.assertEqual(key, key2)
self.assertIsNot(key, key2)
key3 = key.intern()
self.assertIs(key, key3)
- self.assertTrue(key in self.module._interned_keys)
- self.assertEqual(key, self.module._interned_keys[key])
+ self.assertTrue(key in self.module._interned_tuples)
+ self.assertEqual(key, self.module._interned_tuples[key])
key2 = key2.intern()
self.assertIs(key, key2)
@@ -325,7 +325,7 @@
unique_str1 = 'unique str ' + osutils.rand_chars(20)
unique_str2 = 'unique str ' + osutils.rand_chars(20)
key = self.module.StaticTuple(unique_str1, unique_str2)
- self.assertFalse(key in self.module._interned_keys)
+ self.assertFalse(key in self.module._interned_tuples)
self.assertFalse(key._is_interned())
key2 = self.module.StaticTuple(unique_str1, unique_str2)
self.assertEqual(key, key2)
@@ -335,8 +335,8 @@
key3 = key.intern()
self.assertIs(key, key3)
- self.assertTrue(key in self.module._interned_keys)
- self.assertEqual(key, self.module._interned_keys[key])
+ self.assertTrue(key in self.module._interned_tuples)
+ self.assertEqual(key, self.module._interned_tuples[key])
del key3
# We should not increase the refcount just via 'intern'
self.assertEqual(2, sys.getrefcount(key))
@@ -352,18 +352,18 @@
unique_str1 = 'unique str ' + osutils.rand_chars(20)
unique_str2 = 'unique str ' + osutils.rand_chars(20)
key = self.module.StaticTuple(unique_str1, unique_str2)
- self.assertFalse(key in self.module._interned_keys)
+ self.assertFalse(key in self.module._interned_tuples)
self.assertEqual(2, sys.getrefcount(key))
key = key.intern()
self.assertEqual(2, sys.getrefcount(key))
- self.assertTrue(key in self.module._interned_keys)
+ self.assertTrue(key in self.module._interned_tuples)
self.assertTrue(key._is_interned())
del key
# Create a new entry, which would point to the same location
key = self.module.StaticTuple(unique_str1, unique_str2)
self.assertEqual(2, sys.getrefcount(key))
- # This old entry in _interned_keys should be gone
- self.assertFalse(key in self.module._interned_keys)
+ # This old entry in _interned_tuples should be gone
+ self.assertFalse(key in self.module._interned_tuples)
self.assertFalse(key._is_interned())
def test__c_has_C_API(self):
More information about the bazaar-commits
mailing list