Rev 4699: Implement tp_traverse, and add a soft dependency on meliae to test it. in http://bazaar.launchpad.net/~jameinel/bzr/2.1-memory-consumption

John Arbash Meinel john at arbash-meinel.com
Thu Sep 10 21:15:06 BST 2009


At http://bazaar.launchpad.net/~jameinel/bzr/2.1-memory-consumption

------------------------------------------------------------
revno: 4699
revision-id: john at arbash-meinel.com-20090910201450-opufy4lpfyqovzkf
parent: john at arbash-meinel.com-20090910191407-ig2769a7dvbwfzva
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: 2.1-memory-consumption
timestamp: Thu 2009-09-10 15:14:50 -0500
message:
  Implement tp_traverse, and add a soft dependency on meliae to test it.
  
  Basically, gc.get_referents doesn't use tp_traverse if the object isn't being
  tracked. Which is ok, but not what we want here.
  Key isn't in GC because it is overkill for something that can never have
  cyclical references.
  So I implemented a get_referents in meliae that ignores IS_GC.
-------------- next part --------------
=== modified file 'bzrlib/_keys_type_c.c'
--- a/bzrlib/_keys_type_c.c	2009-09-10 19:14:07 +0000
+++ b/bzrlib/_keys_type_c.c	2009-09-10 20:14:50 +0000
@@ -39,7 +39,7 @@
  */
 typedef struct {
     PyObject_VAR_HEAD
-    PyStringObject *key_strings[1];
+    PyStringObject *key_bits[1];
 } Key;
 extern PyTypeObject KeyType;
 
@@ -53,7 +53,7 @@
     // unsigned char num_keys;
     // unsigned char flags; /* not used yet */
     unsigned int info; /* Broken down into 4 1-byte fields */
-    PyStringObject *key_strings[1]; /* key_width * num_keys entries */
+    PyStringObject *key_bits[1]; /* key_width * num_keys entries */
 } Keys;
 
 /* Forward declaration */
@@ -75,7 +75,7 @@
         return NULL;
     }
     for (i = 0; i < self->ob_size; ++i) {
-        obj = (PyObject *)self->key_strings[i];
+        obj = (PyObject *)self->key_bits[i];
         Py_INCREF(obj);
         PyTuple_SET_ITEM(tpl, i, obj);
     }
@@ -90,7 +90,7 @@
     Py_ssize_t i;
 
     for (i = 0; i < self->ob_size; ++i) {
-        Py_XDECREF(self->key_strings[i]);
+        Py_XDECREF(self->key_bits[i]);
     }
     Py_TYPE(self)->tp_free((PyObject *)self);
 }
@@ -133,7 +133,7 @@
             return NULL;
         }
         Py_INCREF(obj);
-        self->key_strings[i] = (PyStringObject *)obj;
+        self->key_bits[i] = (PyStringObject *)obj;
     }
     return (PyObject *)self;
 }
@@ -224,7 +224,7 @@
         PyErr_SetString(PyExc_IndexError, "Key index out of range");
         return NULL;
     }
-    obj = (PyObject *)self->key_strings[offset];
+    obj = (PyObject *)self->key_bits[offset];
     Py_INCREF(obj);
     return obj;
 }
@@ -243,6 +243,16 @@
     return result;
 }
 
+static int
+Key_traverse(Key *self, visitproc visit, void *arg)
+{
+    Py_ssize_t i;
+    for (i = Py_SIZE(self); --i >= 0;) {
+        Py_VISIT(self->key_bits[i]);
+    }
+    return 0;
+}
+
 static char Key_doc[] =
     "C implementation of a Key structure."
     "\n This is used as Key(key_bit_1, key_bit_2, key_bit_3, ...)"
@@ -288,9 +298,12 @@
     0,                                           /* tp_as_buffer */
     Py_TPFLAGS_DEFAULT,                          /* tp_flags*/
     Key_doc,                                     /* tp_doc */
-    // might want to set this, except we don't participate in gc, so it might
-    // confuse things
-    0,                                           /* tp_traverse */
+    /* gc.get_referents checks the IS_GC flag before it calls tp_traverse
+     * And we don't include this object in the garbage collector because we
+     * know it doesn't create cycles. However, 'meliae' will follow
+     * tp_traverse, even if the object isn't GC, and we want that.
+     */
+    (traverseproc)Key_traverse,                  /* tp_traverse */
     0,                                           /* tp_clear */
     // TODO: implement richcompare, we should probably be able to compare vs an
     //       tuple, as well as versus another Keys object.
@@ -353,7 +366,7 @@
          */
         int i;
         for(i = num_keys - 1; i >= 0; --i) {
-            Py_XDECREF(self->key_strings[i]);
+            Py_XDECREF(self->key_bits[i]);
         }
     }
     Py_TYPE(self)->tp_free((PyObject *)self);
@@ -431,7 +444,7 @@
             return NULL;
         }
         Py_INCREF(obj);
-        self->key_strings[i] = (PyStringObject *)obj;
+        self->key_bits[i] = (PyStringObject *)obj;
     }
     return (PyObject *)self;
 }
@@ -535,6 +548,17 @@
     return result;
 }
 
+static int
+Keys_traverse(Keys *self, visitproc visit, void *arg)
+{
+    Py_ssize_t i, num_key_bits;
+    num_key_bits = Keys_get_key_width(self) * Keys_get_num_keys(self);
+    for (i = num_key_bits; --i >= 0;) {
+        Py_VISIT(self->key_bits[i]);
+    }
+    return 0;
+}
+
 static char Keys_doc[] =
     "C implementation of a Keys structure."
     "\n This is used as Keys(width, key_bit_1, key_bit_2, key_bit_3, ...)"
@@ -572,7 +596,7 @@
     }
     start = offset * key_width;
     for (i = 0; i < key_width; ++i) {
-        obj = (PyObject *)self->key_strings[start + i];
+        obj = (PyObject *)self->key_bits[start + i];
         Py_INCREF(obj);
         PyTuple_SET_ITEM(tpl, i, obj);
     }
@@ -622,9 +646,8 @@
     0,                                           /* tp_as_buffer */
     Py_TPFLAGS_DEFAULT,                          /* tp_flags*/
     Keys_doc,                                    /* tp_doc */
-    // might want to set this, except we don't participate in gc, so it might
-    // confuse things
-    0,                                           /* tp_traverse */
+    /* See Key_traverse for why we have this, even though we aren't GC */
+    Keys_traverse,                               /* tp_traverse */
     0,                                           /* tp_clear */
     // TODO: implement richcompare, we should probably be able to compare vs an
     //       tuple, as well as versus another Keys object.

=== modified file 'bzrlib/tests/test__keys_type.py'
--- a/bzrlib/tests/test__keys_type.py	2009-09-10 19:14:07 +0000
+++ b/bzrlib/tests/test__keys_type.py	2009-09-10 20:14:50 +0000
@@ -16,6 +16,7 @@
 
 """Tests for the Keys type."""
 
+import gc
 import sys
 
 from bzrlib import (
@@ -163,6 +164,19 @@
         self.assertEqual(('foo', 'bar'), k[:2])
         self.assertEqual(('baz',), k[2:-1])
 
+    def test_referents(self):
+        # We implement tp_traverse so that things like 'meliae' can measure the
+        # amount of referenced memory. Unfortunately gc.get_referents() first
+        # checks the IS_GC flag before it traverses anything. So there isn't a
+        # way to expose it that I can see.
+        try:
+            from meliae import scanner
+        except ImportError:
+            return
+        strs = ['foo', 'bar', 'baz', 'bing']
+        k = self.module.Key(*strs)
+        self.assertEqual(sorted(strs), sorted(scanner.get_referents(k)))
+
 
 class TestKeysType(tests.TestCase):
 
@@ -264,3 +278,20 @@
         self.assertEqual('foo', x[as_tuple])
         x[as_tuple] = 'bar'
         self.assertEqual({as_tuple: 'bar'}, x)
+
+    def test_referents(self):
+        # We implement tp_traverse so that things like 'meliae' can measure the
+        # amount of referenced memory. Unfortunately gc.get_referents() first
+        # checks the IS_GC flag before it traverses anything. So there isn't a
+        # way to expose it that I can see.
+        try:
+            from meliae import scanner
+        except ImportError:
+            return
+        strs = ['foo', 'bar', 'baz', 'bing']
+        k = self.module.Keys(2, *strs)
+        if type(k) == tuple:
+            self.assertEqual(sorted([('foo', 'bar'), ('baz', 'bing')]),
+                             sorted(scanner.get_referents(k)))
+        else:
+            self.assertEqual(sorted(strs), sorted(scanner.get_referents(k)))



More information about the bazaar-commits mailing list