Rev 144: Finally, significantly reduced memory. in http://bazaar.launchpad.net/~jameinel/meliae/mem-object-collection

[John Arbash Meinel]

At http://bazaar.launchpad.net/~jameinel/meliae/mem-object-collection

------------------------------------------------------------
revno: 144
revision-id: john at arbash-meinel.com-20091229031026-sfhgv08w1djtoaiu
parent: john at arbash-meinel.com-20091229030220-nl220knepib7l4d2
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: mem-object-collection
timestamp: Mon 2009-12-28 21:10:26 -0600
message:
  Finally, significantly reduced memory.
  
  Memory after loading large dump is now 494 rather than 623 when
  replacing _load with _load_moc. I had hoped for a bigger result,
  but we're getting there.
-------------- next part --------------
=== modified file 'meliae/_loader.pyx'
--- a/meliae/_loader.pyx	2009-12-29 03:02:20 +0000
+++ b/meliae/_loader.pyx	2009-12-29 03:10:26 +0000
@@ -37,7 +37,6 @@
     # void fprintf(void *, char *, ...)
     # void *stderr
 
-import weakref
 
 ctypedef struct RefList:
     long size
@@ -146,7 +145,7 @@
     PyObject *name
     RefList *referrer_list
     unsigned long total_size
-    PyObject *proxy_ref
+    PyObject *proxy
 
 
 cdef int _free_mem_object(_MemObject *cur) except -1:
@@ -168,8 +167,7 @@
     cur.name = NULL
     _free_ref_list(cur.referrer_list)
     cur.referrer_list = NULL
-    Py_XDECREF(cur.proxy_ref)
-    cur.proxy_ref = NULL
+    cur.proxy = NULL
     PyMem_Free(cur)
     return 1
 
@@ -196,7 +194,6 @@
     cdef _MemObject *_obj
     # If not NULL, this will be freed when this object is deallocated
     cdef _MemObject *_managed_obj
-    cdef object __weakref__
 
     def __init__(self, collection):
         self.collection = collection
@@ -204,6 +201,14 @@
         self._managed_obj = NULL
 
     def __dealloc__(self):
+        if self._obj != NULL:
+            if self._obj.proxy == <PyObject *>self:
+                # This object is going away, remove the reference
+                self._obj.proxy = NULL
+            else:
+                fprintf(stderr, "obj at address %x referenced"
+                    " a proxy that was not self\n",
+                    <int><object>self._obj.address)
         if self._managed_obj != NULL:
             _free_mem_object(self._managed_obj)
             self._managed_obj = NULL
@@ -394,18 +399,12 @@
     cdef _MemObjectProxy _proxy_for(self, address, _MemObject *val):
         cdef _MemObjectProxy proxy
 
-        proxy = None
-        if val.proxy_ref != NULL:
-            proxy = (<object>val.proxy_ref)()
-            if proxy is None:
-                Py_DECREF(val.proxy_ref)
-                val.proxy_ref = NULL
-        if val.proxy_ref == NULL:
+        if val.proxy == NULL:
             proxy = _MemObjectProxy(self)
             proxy._obj = val
-            proxy_ref = weakref.ref(proxy)
-            val.proxy_ref = <PyObject *>proxy_ref
-            Py_INCREF(val.proxy_ref)
+            val.proxy = <PyObject *>proxy
+        else:
+            proxy = <object>val.proxy
         return proxy
 
     def __getitem__(self, at):
@@ -446,13 +445,13 @@
         slot = self._lookup(address)
         if slot[0] == NULL or slot[0] == _dummy:
             raise KeyError('address %s not present' % (at,))
-        proxy = None
-        if slot[0].proxy_ref != NULL:
-            # Have the proxy take over the memory lifetime
-            proxy = (<object>slot[0].proxy_ref)()
-            if proxy is not None:
-                proxy._managed_obj = proxy._obj
-        if proxy is None:
+        if slot[0].proxy != NULL:
+            # Have the proxy take over the memory lifetime. At the same time,
+            # we break the reference cycle, so that the proxy will get cleaned
+            # up properly
+            proxy = <object>slot[0].proxy
+            proxy._managed_obj = proxy._obj
+        else:
             # Without a proxy, we just nuke the object
             self._clear_slot(slot)
         slot[0] = _dummy