Rev 146: Intern the addresses in references and we are down to 350MB. in http://bazaar.launchpad.net/~jameinel/meliae/mem-object-collection

[John Arbash Meinel]

At http://bazaar.launchpad.net/~jameinel/meliae/mem-object-collection

------------------------------------------------------------
revno: 146
revision-id: john at arbash-meinel.com-20091229032855-0cmiei00xq2bj3rj
parent: john at arbash-meinel.com-20091229031233-07plrvl9nsyudipo
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: mem-object-collection
timestamp: Mon 2009-12-28 21:28:55 -0600
message:
  Intern the addresses in references and we are down to 350MB.
-------------- next part --------------
=== modified file 'meliae/_loader.pyx'
--- a/meliae/_loader.pyx	2009-12-29 03:12:33 +0000
+++ b/meliae/_loader.pyx	2009-12-29 03:28:55 +0000
@@ -24,7 +24,10 @@
     long PyObject_Hash(PyObject *) except -1
 
     PyObject *PyDict_GetItem(object d, object key)
+    PyObject *PyDict_GetItem_ptr "PyDict_GetItem" (object d, PyObject *key)
     int PyDict_SetItem(object d, object key, object val) except -1
+    int PyDict_SetItem_ptr "PyDict_SetItem" (object d, PyObject *key,
+                                             PyObject *val) except -1
     void Py_INCREF(PyObject*)
     void Py_XDECREF(PyObject*)
     void Py_DECREF(PyObject*)
@@ -61,6 +64,23 @@
     return val
 
 
+cdef int _set_default_ptr(object d, PyObject **val) except -1:
+    """Similar to _set_default, only it sets the val in place"""
+    cdef PyObject *tmp
+
+    tmp = PyDict_GetItem_ptr(d, val[0])
+    if tmp == NULL:
+        # val is unchanged, so we don't change the refcounts
+        PyDict_SetItem_ptr(d, val[0], val[0])
+        return 0
+    else:
+        # We will be pointing val to something new, so fix up the refcounts
+        Py_INCREF(tmp)
+        Py_DECREF(val[0])
+        val[0] = tmp
+        return 1
+
+
 cdef _free_ref_list(RefList *ref_list):
     """Decref and free the list."""
     cdef long i
@@ -269,16 +289,16 @@
             return self.__len__()
 
     def _intern_from_cache(self, cache):
-        address = _set_default(cache, <object>self._obj.address)
-        if (<PyObject *>address) != self._obj.address:
-            Py_DECREF(self._obj.address)
-            self._obj.address = <PyObject *>address
-            Py_INCREF(self._obj.address)
-        type_str = _set_default(cache, <object>self.type_str)
-        if (<PyObject *>type_str) != self._obj.type_str:
-            Py_DECREF(self._obj.type_str)
-            self._obj.type_str = <PyObject *>type_str
-            Py_INCREF(self._obj.type_str)
+        cdef long i
+        _set_default_ptr(cache, &self._obj.address)
+        _set_default_ptr(cache, &self._obj.type_str)
+        if self._obj.ref_list != NULL:
+            for i from 0 <= i < self._obj.ref_list.size:
+                _set_default_ptr(cache, &self._obj.ref_list.refs[i])
+        if self._obj.referrer_list != NULL:
+            for i from 0 <= i < self._obj.referrer_list.size:
+                _set_default_ptr(cache, &self._obj.referrer_list.refs[i])
+
 
     property ref_list:
         """The list of objects referenced by this object."""

=== modified file 'meliae/tests/test__loader.py'
--- a/meliae/tests/test__loader.py	2009-12-28 19:58:48 +0000
+++ b/meliae/tests/test__loader.py	2009-12-29 03:28:55 +0000
@@ -336,11 +336,29 @@
         self.assertTrue(mop.type_str is t)
         del self.moc[addr]
         mop = self.moc.add(1234566+1, 'my ' + ' ty' + 'pe', 256)
+        addr876543 = 876543
+        cache[addr876543] = addr876543
+        addr654321 = 654321
+        cache[addr654321] = addr654321
+        mop.ref_list = [876542+1, 654320+1]
+        mop.referrers = [876542+1, 654320+1]
         self.assertFalse(mop.address is addr)
         self.assertFalse(mop.type_str is t)
+        rl = mop.ref_list
+        self.assertFalse(rl[0] is addr876543)
+        self.assertFalse(rl[1] is addr654321)
+        rfrs = mop.referrers
+        self.assertFalse(rl[0] is addr876543)
+        self.assertFalse(rl[1] is addr654321)
         mop._intern_from_cache(cache)
         self.assertTrue(mop.address is addr)
         self.assertTrue(mop.type_str is t)
+        rl = mop.ref_list
+        self.assertTrue(rl[0] is addr876543)
+        self.assertTrue(rl[1] is addr654321)
+        rfrs = mop.referrers
+        self.assertTrue(rfrs[0] is addr876543)
+        self.assertTrue(rfrs[1] is addr654321)
 
     def test_ref_list(self):
         mop = self.moc.add(1234567, 'type', 256, ref_list=[1, 2, 3])