Rev 147: Removing .name as a separate attribute from .value brisg us to 322MB. in http://bazaar.launchpad.net/~jameinel/meliae/mem-object-collection

John Arbash Meinel john at arbash-meinel.com
Tue Dec 29 03:44:59 GMT 2009


At http://bazaar.launchpad.net/~jameinel/meliae/mem-object-collection

------------------------------------------------------------
revno: 147
revision-id: john at arbash-meinel.com-20091229034439-otwc36e6zas9ddkv
parent: john at arbash-meinel.com-20091229032855-0cmiei00xq2bj3rj
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: mem-object-collection
timestamp: Mon 2009-12-28 21:44:39 -0600
message:
  Removing .name as a separate attribute from .value brisg us to 322MB.
  
  With 3.6M objects, every 4 byte pointer removed saves 14.4MB.
  Which is what this experiment is all about. Decreasing the per-object overhead
  in memory. Getting rid of a full python object saves a 16B gc header, plus
  the 4 byte refcount and the 4 byte type pointer.
  Using a custom dict saves 3 4B pointers down to 1 per slot. And at an average
  overhead of 2:1 slots/entries, that is another 16B per object.
  
  Finally we can analyze a dump file in less than its uncompressed size, and
  less than the original memory consumption. Now we just need to clean up
  all of the other test code, etc.
  
  Also, use the nicer repr from the ObjProxy experiment.
-------------- next part --------------
=== modified file 'meliae/_loader.pyx'
--- a/meliae/_loader.pyx	2009-12-29 03:28:55 +0000
+++ b/meliae/_loader.pyx	2009-12-29 03:44:39 +0000
@@ -162,7 +162,7 @@
     # TODO: I don't think I've found an object that has both a value and a
     #       name. As such, I should probably remove the redundancy, as it saves
     #       a pointer
-    PyObject *name
+    # PyObject *name
     RefList *referrer_list
     unsigned long total_size
     # This is an uncounted ref to a _MemObjectProxy. _MemObjectProxy also has a
@@ -186,8 +186,8 @@
     cur.ref_list = NULL
     Py_XDECREF(cur.value)
     cur.value = NULL
-    Py_XDECREF(cur.name)
-    cur.name = NULL
+    # Py_XDECREF(cur.name)
+    # cur.name = NULL
     _free_ref_list(cur.referrer_list)
     cur.referrer_list = NULL
     cur.proxy = NULL
@@ -253,10 +253,10 @@
         def __set__(self, value):
             self._obj.size = value
 
-    property name:
-        """Name associated with this object."""
-        def __get__(self):
-            return <object>self._obj.name
+    # property name:
+    #     """Name associated with this object."""
+    #     def __get__(self):
+    #         return <object>self._obj.name
 
     property value:
         """Value for this object (for strings and ints)"""
@@ -594,10 +594,14 @@
         #     new_entry.length = -1
         # else:
         #     new_entry.length = length
-        new_entry.value = <PyObject *>value
+        if value is not None and name is not None:
+            raise RuntimeError("We currently only support one of value or name"
+                " per object.")
+        if value is not None:
+            new_entry.value = <PyObject *>value
+        else:
+            new_entry.value = <PyObject *>name
         Py_INCREF(new_entry.value)
-        new_entry.name = <PyObject *>name
-        Py_INCREF(new_entry.name)
         new_entry.referrer_list = _list_to_ref_list(referrer_list)
         new_entry.total_size = total_size
 

=== modified file 'meliae/loader.py'
--- a/meliae/loader.py	2009-12-29 03:12:33 +0000
+++ b/meliae/loader.py	2009-12-29 03:44:39 +0000
@@ -564,7 +564,7 @@
     # _fill_total_size(objs)
     return ObjManager(objs, show_progress=show_prog)
 
-#_load = _load_moc
+_load = _load_moc
 
 
 def remove_expensive_references(source, total_objs=0, show_progress=False):

=== modified file 'meliae/tests/test__loader.py'
--- a/meliae/tests/test__loader.py	2009-12-29 03:28:55 +0000
+++ b/meliae/tests/test__loader.py	2009-12-29 03:44:39 +0000
@@ -316,12 +316,8 @@
 
     def test_name(self):
         mop = self.moc.add(1234, 'type', 256, name='the name')
-        self.assertEqual('the name', mop.name)
-        # TODO: We may remove name as a separate attribute from value, but for
-        #       now, 'name' is not settable
-        def set(value):
-            mop.name = value
-        self.assertRaises(AttributeError, set, 'test')
+        # 'name' entries get mapped as value
+        self.assertEqual('the name', mop.value)
 
     def test__intern_from_cache(self):
         cache = {}



More information about the bazaar-commits mailing list