Rev 4726: Work out how to expose the C api using the Python PyCObject interface. in http://bazaar.launchpad.net/~jameinel/bzr/2.1-static-tuple
John Arbash Meinel
john at arbash-meinel.com
Wed Sep 30 22:17:55 BST 2009
At http://bazaar.launchpad.net/~jameinel/bzr/2.1-static-tuple
------------------------------------------------------------
revno: 4726
revision-id: john at arbash-meinel.com-20090930211748-8er5cdw0cf5l01mo
parent: john at arbash-meinel.com-20090930201508-3qgsaulrjaql1c83
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: 2.1-static-tuple
timestamp: Wed 2009-09-30 16:17:48 -0500
message:
Work out how to expose the C api using the Python PyCObject interface.
At this point, we are at the same speed as bzr.dev at doing 'bzr log -n0 -r-1 bzr.dev'
We are faster at parsing all keys, and about 20% lower in memory.
-------------- next part --------------
=== modified file 'bzrlib/_btree_serializer_pyx.pyx'
--- a/bzrlib/_btree_serializer_pyx.pyx 2009-09-30 19:15:34 +0000
+++ b/bzrlib/_btree_serializer_pyx.pyx 2009-09-30 21:17:48 +0000
@@ -55,13 +55,19 @@
# void *memrchr(void *s, int c, size_t n)
int strncmp(char *s1, char *s2, size_t n)
-## cdef extern from "_keys_type_c.h":
-## cdef struct Key:
-## pass
-## object Key_New(Py_ssize_t)
-## # Steals a reference and Val must be a PyStringObject, no checking is done
-## void Key_SET_ITEM(object key, Py_ssize_t offset, object val)
-## object Key_GET_ITEM(object key, Py_ssize_t offset)
+cdef extern from "_static_tuple_c.h":
+ cdef struct StaticTuple:
+ pass
+ void **StaticTuple_API
+ int import_static_tuple()
+ # ctypedef object (*st_new_type)(Py_ssize_t)
+ # st_new_type st_new
+
+ object StaticTuple_New(Py_ssize_t)
+ object StaticTuple_intern(object)
+ # Steals a reference and Val must be a PyStringObject, no checking is done
+ void StaticTuple_SET_ITEM(object key, Py_ssize_t offset, object val)
+ object StaticTuple_GET_ITEM(object key, Py_ssize_t offset)
# TODO: Find some way to import this from _dirstate_helpers
@@ -103,8 +109,11 @@
return result
from bzrlib import _static_tuple_c
-cdef object StaticTuple
-StaticTuple = _static_tuple_c.StaticTuple
+# This sets up the StaticTuple C_API functionality
+if import_static_tuple() == -1 or StaticTuple_API == NULL:
+ raise ImportError('failed to import_static_tuple()')
+cdef object _ST
+_ST = _static_tuple_c.StaticTuple
cdef class BTreeLeafParser:
@@ -145,6 +154,9 @@
self._cur_str = NULL
self._end_str = NULL
self._header_found = 0
+ # keys are tuples
+ if StaticTuple_API == NULL:
+ raise ImportError('failed to import_static_tuple()')
cdef extract_key(self, char * last):
"""Extract a key.
@@ -154,8 +166,8 @@
"""
cdef char *temp_ptr
cdef int loop_counter
- # keys are tuples
- key = PyTuple_New(self.key_length)# PyTuple_New(self.key_length)
+
+ key = StaticTuple_New(self.key_length)
for loop_counter from 0 <= loop_counter < self.key_length:
# grab a key segment
temp_ptr = <char*>memchr(self._start, c'\0', last - self._start)
@@ -170,18 +182,15 @@
last - self._start)))
raise AssertionError(failure_string)
# capture the key string
- # TODO: Consider using PyIntern_FromString, the only caveat is that
- # it assumes a NULL-terminated string, so we have to check if
- # temp_ptr[0] == c'\0' or some other char.
key_element = safe_interned_string_from_size(self._start,
temp_ptr - self._start)
# advance our pointer
self._start = temp_ptr + 1
Py_INCREF(key_element)
# PyTuple_SET_ITEM(key, loop_counter, key_element)
- PyTuple_SET_ITEM(key, loop_counter, key_element)
+ StaticTuple_SET_ITEM(key, loop_counter, key_element)
# return _keys_type_c.Key(*key)
- return StaticTuple(*key).intern()
+ return StaticTuple_intern(key)
cdef int process_line(self) except -1:
"""Process a line in the bytes."""
@@ -265,18 +274,18 @@
# key runs to the end
temp_ptr = ref_ptr
PyList_Append(ref_list, self.extract_key(temp_ptr))
- ref_list = StaticTuple(*ref_list).intern()
+ ref_list = _ST(*ref_list).intern()
PyList_Append(ref_lists, ref_list)
# prepare for the next reference list
self._start = next_start
- ref_lists = StaticTuple(*ref_lists)
- node_value = StaticTuple(value, ref_lists)
+ ref_lists = _ST(*ref_lists)
+ node_value = _ST(value, ref_lists)
else:
if last != self._start:
# unexpected reference data present
return -1
- node_value = StaticTuple(value, StaticTuple())
- PyList_Append(self.keys, StaticTuple(key, node_value))
+ node_value = _ST(value, _ST())
+ PyList_Append(self.keys, _ST(key, node_value))
return 0
def parse(self):
=== modified file 'bzrlib/_static_tuple_c.c'
--- a/bzrlib/_static_tuple_c.c 2009-09-30 20:15:08 +0000
+++ b/bzrlib/_static_tuple_c.c 2009-09-30 21:17:48 +0000
@@ -18,9 +18,7 @@
/* Must be defined before importing _static_tuple_c.h so that we get the right
* linkage.
*/
-#if defined(_WIN32)
-# define StaticTupleAPI_FUNC(RTYPE) __declspec(dllexport) RTYPE
-#endif
+#define STATIC_TUPLE_MODULE
#include "_static_tuple_c.h"
@@ -134,7 +132,7 @@
/* Similar to PyTuple_New() */
-PyObject *
+static PyObject *
StaticTuple_New(Py_ssize_t size)
{
StaticTuple *key;
@@ -679,33 +677,21 @@
};
-PyMODINIT_FUNC
-init_static_tuple_c(void)
+static void
+setup_interned_keys(PyObject *m)
{
- PyObject* m;
- StaticTuple *key;
-
- if (PyType_Ready(&StaticTuple_Type) < 0)
- return;
- //if (PyType_Ready(&KeyIntern_Type) < 0)
- // return;
-
- m = Py_InitModule3("_static_tuple_c", static_tuple_c_methods,
- "C implementation of a StaticTuple structure");
- if (m == NULL)
- return;
-
- Py_INCREF(&StaticTuple_Type);
- PyModule_AddObject(m, "StaticTuple", (PyObject *)&StaticTuple_Type);
- // Py_INCREF(&KeyIntern_Type);
- // PyModule_AddObject(m, "KeyIntern", (PyObject *)&KeyIntern_Type);
- // _interned_keys = PyObject_NewVar(KeyIntern, &KeyIntern_Type, 10);
_interned_keys = PyDict_New();
if (_interned_keys != NULL) {
Py_INCREF(_interned_keys);
PyModule_AddObject(m, "_interned_keys", _interned_keys);
}
-
+}
+
+
+static void
+setup_empty_tuple(PyObject *m)
+{
+ StaticTuple *key;
// We need to create the empty tuple
key = PyObject_NewVar(StaticTuple, &StaticTuple_Type, 0);
if (key == NULL) {
@@ -725,3 +711,41 @@
Py_INCREF(_empty_tuple); // for the module
PyModule_AddObject(m, "_empty_tuple", (PyObject *)_empty_tuple);
}
+
+
+static void
+setup_c_api(PyObject *m)
+{
+ static void *StaticTuple_API[StaticTuple_API_pointers];
+ PyObject *c_api_object;
+
+ StaticTuple_API[StaticTuple_New_NUM] = (void *)StaticTuple_New;
+ StaticTuple_API[StaticTuple_intern_NUM] = (void *)StaticTuple_intern;
+ c_api_object = PyCObject_FromVoidPtr((void *)StaticTuple_API, NULL);
+ if (c_api_object != NULL) {
+ PyModule_AddObject(m, "_C_API", c_api_object);
+ }
+}
+
+
+PyMODINIT_FUNC
+init_static_tuple_c(void)
+{
+ PyObject* m;
+
+ if (PyType_Ready(&StaticTuple_Type) < 0)
+ return;
+ //if (PyType_Ready(&KeyIntern_Type) < 0)
+ // return;
+
+ m = Py_InitModule3("_static_tuple_c", static_tuple_c_methods,
+ "C implementation of a StaticTuple structure");
+ if (m == NULL)
+ return;
+
+ Py_INCREF(&StaticTuple_Type);
+ PyModule_AddObject(m, "StaticTuple", (PyObject *)&StaticTuple_Type);
+ setup_interned_keys(m);
+ setup_empty_tuple(m);
+ setup_c_api(m);
+}
=== modified file 'bzrlib/_static_tuple_c.h'
--- a/bzrlib/_static_tuple_c.h 2009-09-30 20:15:08 +0000
+++ b/bzrlib/_static_tuple_c.h 2009-09-30 21:17:48 +0000
@@ -15,16 +15,10 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
+#ifndef _STATIC_TUPLE_H_
+#define _STATIC_TUPLE_H_
#include <Python.h>
-#if !defined(StaticTupleAPI_FUNC)
-# if defined(_WIN32)
-# define StaticTupleAPI_FUNC(RTYPE) __declspec(dllexport) RTYPE
-# else
-# define StaticTupleAPI_FUNC(RTYPE) RTYPE
-# endif
-#endif
-
#define STATIC_TUPLE_HAS_HASH 0
/* Caching the hash adds memory, but allows us to save a little time during
* lookups. TIMEIT hash(key) shows it as
@@ -66,11 +60,6 @@
} StaticTuple;
extern PyTypeObject StaticTuple_Type;
-/* TODO: we need to change this into an api table, look at the python extension
- * docs.
- */
-StaticTupleAPI_FUNC(PyObject *) StaticTuple_New(Py_ssize_t size);
-
typedef struct {
PyObject_VAR_HEAD
PyObject *table[1];
@@ -79,6 +68,57 @@
#define StaticTuple_CheckExact(op) (Py_TYPE(op) == &StaticTuple_Type)
#define StaticTuple_SET_ITEM(key, offset, val) \
- ((((StaticTuple*)(key))->key_bits[(offset)]) = ((PyObject *)(val))
+ ((((StaticTuple*)(key))->key_bits[(offset)]) = ((PyObject *)(val)))
#define StaticTuple_GET_ITEM(key, offset) (((StaticTuple*)key)->key_bits[offset])
+
+/* C API Functions */
+#define StaticTuple_New_NUM 0
+#define StaticTuple_intern_NUM 1
+
+/* Total number of C API Pointers */
+#define StaticTuple_API_pointers 2
+
+#ifdef STATIC_TUPLE_MODULE
+/* Used when compiling _static_tuple_c.c */
+
+static PyObject * StaticTuple_New(Py_ssize_t);
+static StaticTuple * StaticTuple_intern(StaticTuple *self);
+
+#else
+/* Used by foriegn callers */
+static void **StaticTuple_API;
+
+static PyObject *(*StaticTuple_New)(Py_ssize_t);
+static PyObject *(*StaticTuple_intern)(PyObject *);
+
+/* Return -1 and set exception on error, 0 on success */
+static int
+import_static_tuple(void)
+{
+ PyObject *module = PyImport_ImportModule("bzrlib._static_tuple_c");
+ PyObject *c_api_object;
+
+ if (module == NULL) {
+ fprintf(stderr, "Failed to find module _static_tuple_c.\n");
+ return -1;
+ }
+ c_api_object = PyObject_GetAttrString(module, "_C_API");
+ if (c_api_object == NULL) {
+ fprintf(stderr, "Failed to find _static_tuple_c._C_API.\n");
+ return -1;
+ }
+ if (!PyCObject_Check(c_api_object)) {
+ fprintf(stderr, "_static_tuple_c._C_API not a CObject.\n");
+ Py_DECREF(c_api_object);
+ return -1;
+ }
+ StaticTuple_API = (void **)PyCObject_AsVoidPtr(c_api_object);
+ StaticTuple_New = StaticTuple_API[StaticTuple_New_NUM];
+ StaticTuple_intern = StaticTuple_API[StaticTuple_intern_NUM];
+ Py_DECREF(c_api_object);
+ return 0;
+}
+
+#endif
+#endif // _STATIC_TUPLE_H_
=== modified file 'bzrlib/tests/test__static_tuple.py'
--- a/bzrlib/tests/test__static_tuple.py 2009-09-30 19:43:34 +0000
+++ b/bzrlib/tests/test__static_tuple.py 2009-09-30 21:17:48 +0000
@@ -365,3 +365,8 @@
# This old entry in _interned_keys should be gone
self.assertFalse(key in self.module._interned_keys)
self.assertFalse(key._is_interned())
+
+ def test__c_has_C_API(self):
+ if self.module is _static_tuple_py:
+ return
+ self.assertIsNot(None, self.module._C_API)
More information about the bazaar-commits
mailing list