Rev 142: (bug #581918) Avoid pre-allocating a very large list during collapse_instance_dicts. in http://bazaar.launchpad.net/~meliae-dev/meliae/trunk

[John Arbash Meinel]

At http://bazaar.launchpad.net/~meliae-dev/meliae/trunk

------------------------------------------------------------
revno: 142
revision-id: john at arbash-meinel.com-20100520155812-hccxcwfu9z8u4u5a
parent: john at arbash-meinel.com-20100520140853-ob49gh91dx1g2bhg
fixes bug(s): https://launchpad.net/bugs/581918
author: Gary Poster <gary.poster at canonical.com>
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: trunk
timestamp: Thu 2010-05-20 10:58:12 -0500
message:
  (bug #581918) Avoid pre-allocating a very large list during collapse_instance_dicts.
  
  Instead, we use iteritems() and then delete everything at the end.
-------------- next part --------------
=== modified file 'meliae/loader.py'
--- a/meliae/loader.py	2010-01-08 23:00:40 +0000
+++ b/meliae/loader.py	2010-05-20 15:58:12 +0000
@@ -395,7 +395,8 @@
         collapsed = 0
         total = len(self.objs)
         tlast = timer()-20
-        for item_idx, (address, obj) in enumerate(self.objs.items()):
+        to_be_removed = set()
+        for item_idx, (address, obj) in enumerate(self.objs.iteritems()):
             if obj.type_str in ('str', 'dict', 'tuple', 'list', 'type',
                                 'function', 'wrapper_descriptor',
                                 'code', 'classobj', 'int',
@@ -441,9 +442,14 @@
             obj.total_size = 0
             if obj.type_str == 'instance':
                 obj.type_str = type_obj.value
-            # Now that all the data has been moved into the instance, remove
-            # the dict from the collection
-            del self.objs[dict_obj.address]
+            # Now that all the data has been moved into the instance, we
+            # will want to remove the dict from the collection.  We'll do the
+            # actual deletion later, since we are using iteritems for this
+            # loop.
+            to_be_removed.add(dict_obj.address)
+        # Now we can do the actual deletion.
+        for address in to_be_removed:
+            del self.objs[address]
         if self.show_progress:
             sys.stderr.write('checked %8d / %8d collapsed %8d    \n'
                              % (item_idx, total, collapsed))