Rev 4704: Using Key_New() rather than Key(PyTuple_New()) gets us to 720ms. in http://bazaar.launchpad.net/~jameinel/bzr/2.1-memory-consumption

[John Arbash Meinel]

At http://bazaar.launchpad.net/~jameinel/bzr/2.1-memory-consumption

------------------------------------------------------------
revno: 4704
revision-id: john at arbash-meinel.com-20090914044222-a1nnzosoosefb2sd
parent: john at arbash-meinel.com-20090913202746-2uu54no9eh659glh
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: 2.1-memory-consumption
timestamp: Sun 2009-09-13 23:42:22 -0500
message:
  Using Key_New() rather than Key(PyTuple_New()) gets us to 720ms.
  It takes some gymnastics to get it to work, because we have to export the
  symbol and then find the .lib file to use it from the pyrex code.
  setup.py syntax doesn't seem to support that cleanly :(
  Going further, we'll have to update _flatten_node to support Key nodes,
  since right now it has some explicit 'is tuple' checks that fail.
-------------- next part --------------
=== modified file 'bzrlib/_btree_serializer_pyx.pyx'
--- a/bzrlib/_btree_serializer_pyx.pyx	2009-09-10 19:14:07 +0000
+++ b/bzrlib/_btree_serializer_pyx.pyx	2009-09-14 04:42:22 +0000
@@ -55,6 +55,14 @@
     # void *memrchr(void *s, int c, size_t n)
     int strncmp(char *s1, char *s2, size_t n)
 
+cdef extern from "_keys_type_c.h":
+    cdef struct Key:
+        pass
+    object Key_New(Py_ssize_t)
+    # Steals a reference and Val must be a PyStringObject, no checking is done
+    void Key_SET_ITEM(object key, Py_ssize_t offset, object val)
+    object Key_GET_ITEM(object key, Py_ssize_t offset)
+
 
 # TODO: Find some way to import this from _dirstate_helpers
 cdef void* _my_memrchr(void *s, int c, size_t n):
@@ -145,7 +153,7 @@
         cdef char *temp_ptr
         cdef int loop_counter
         # keys are tuples
-        key = PyTuple_New(self.key_length)
+        key = Key_New(self.key_length)# PyTuple_New(self.key_length)
         for loop_counter from 0 <= loop_counter < self.key_length:
             # grab a key segment
             temp_ptr = <char*>memchr(self._start, c'\0', last - self._start)
@@ -168,8 +176,10 @@
             # advance our pointer
             self._start = temp_ptr + 1
             Py_INCREF(key_element)
-            PyTuple_SET_ITEM(key, loop_counter, key_element)
-        return _keys_type_c.Key(*key)
+            # PyTuple_SET_ITEM(key, loop_counter, key_element)
+            Key_SET_ITEM(key, loop_counter, key_element)
+        # return _keys_type_c.Key(*key)
+        return key
 
     cdef int process_line(self) except -1:
         """Process a line in the bytes."""

=== modified file 'bzrlib/_keys_type_c.c'
--- a/bzrlib/_keys_type_c.c	2009-09-13 20:27:46 +0000
+++ b/bzrlib/_keys_type_c.c	2009-09-14 04:42:22 +0000
@@ -15,6 +15,13 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
+/* Must be defined before importing _keys_type_c.h so that we get the right
+ * linkage.
+ */
+#if defined(_WIN32)
+#  define KeysAPI_FUNC(RTYPE) __declspec(dllexport) RTYPE
+#endif
+
 #include "_keys_type_c.h"
 
 #include "python-compat.h"

=== modified file 'bzrlib/_keys_type_c.h'
--- a/bzrlib/_keys_type_c.h	2009-09-13 20:27:46 +0000
+++ b/bzrlib/_keys_type_c.h	2009-09-14 04:42:22 +0000
@@ -17,6 +17,13 @@
 
 #include <Python.h>
 
+#if !defined(KeysAPI_FUNC)
+#  if defined(_WIN32)
+#    define KeysAPI_FUNC(RTYPE) __declspec(dllexport) RTYPE
+#  else
+#    define KeysAPI_FUNC(RTYPE) RTYPE
+#  endif
+#endif
 
 /* This defines a single variable-width key.
  * It is basically the same as a tuple, but
@@ -35,7 +42,7 @@
 extern PyTypeObject Key_Type;
 
 /* Do we need a PyAPI_FUNC sort of wrapper? */
-PyObject * Key_New(Py_ssize_t size);
+KeysAPI_FUNC(PyObject *) Key_New(Py_ssize_t size);
 
 /* Because of object alignment, it seems that using unsigned char doesn't make
  * things any smaller than using an 'int'... :(
@@ -52,3 +59,7 @@
 
 /* Forward declaration */
 extern PyTypeObject Keys_Type;
+
+#define Key_SET_ITEM(key, offset, val) \
+    ((((Key*)key)->key_bits[offset]) = (PyStringObject *)val)
+#define Key_GET_ITEM(key, offset) (((Key*)key)->key_bits[offset])

=== modified file 'setup.py'
--- a/setup.py	2009-09-08 20:43:04 +0000
+++ b/setup.py	2009-09-14 04:42:22 +0000
@@ -265,7 +265,6 @@
 
 add_pyrex_extension('bzrlib._annotator_pyx')
 add_pyrex_extension('bzrlib._bencode_pyx')
-add_pyrex_extension('bzrlib._btree_serializer_pyx')
 add_pyrex_extension('bzrlib._chunks_to_lines_pyx')
 add_pyrex_extension('bzrlib._groupcompress_pyx',
                     extra_source=['bzrlib/diff-delta.c'])
@@ -297,6 +296,9 @@
                              ['bzrlib/_patiencediff_c.c']))
 ext_modules.append(Extension('bzrlib._keys_type_c',
                              ['bzrlib/_keys_type_c.c']))
+add_pyrex_extension('bzrlib._btree_serializer_pyx',
+                    libraries=['_keys_type_c'])
+ext_modules[-1].library_dirs.append('build/temp.win32-2.6/Release/bzrlib')
 
 
 if unavailable_files: