Rev 2439: More inner loop tuning of walkdirs, can save as much as 5% in http://bazaar.launchpad.net/%7Ebzr/bzr/dirstate

[John Arbash Meinel]

At http://bazaar.launchpad.net/%7Ebzr/bzr/dirstate

------------------------------------------------------------
revno: 2439
revision-id: john at arbash-meinel.com-20070228221101-c3szee94o5h3uvt7
parent: john at arbash-meinel.com-20070228215011-hfmfc13mswbubn18
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: dirstate
timestamp: Wed 2007-02-28 16:11:01 -0600
message:
  More inner loop tuning of walkdirs, can save as much as 5%
  (drops time on LP tree from 100-105ms, to 95-100ms)
  Also, unify all walkdirs variants to have minimal variation.
modified:
  bzrlib/osutils.py              osutils.py-20050309040759-eeaff12fbf77ac86
-------------- next part --------------
=== modified file 'bzrlib/osutils.py'
--- a/bzrlib/osutils.py	2007-02-28 21:50:11 +0000
+++ b/bzrlib/osutils.py	2007-02-28 22:11:01 +0000
@@ -1079,29 +1079,29 @@
     # depending on top and prefix - i.e. ./foo and foo as a pair leads to
     # potentially confusing output. We should make this more robust - but
     # not at a speed cost. RBC 20060731
-    lstat = os.lstat
-    pending = []
+    _lstat = os.lstat
     _directory = _directory_kind
     _listdir = os.listdir
+    _kind_from_mode = _formats.get
     pending = [(safe_unicode(prefix), "", _directory, None, safe_unicode(top))]
     while pending:
-        dirblock = []
-        currentdir = pending.pop()
         # 0 - relpath, 1- basename, 2- kind, 3- stat, 4-toppath
-        relroot = currentdir[0]
-        top = currentdir[4]
+        relroot, _, _, _, top = pending.pop()
         if relroot:
             relprefix = relroot + u'/'
         else:
-            relprefix = ""
+            relprefix = ''
         top_slash = top + u'/'
+
+        dirblock = []
+        append = dirblock.append
         for name in sorted(_listdir(top)):
             abspath = top_slash + name
-            statvalue = lstat(abspath)
-            dirblock.append((relprefix + name, name,
-                file_kind_from_stat_mode(statvalue.st_mode),
-                statvalue, abspath))
+            statvalue = _lstat(abspath)
+            kind = _kind_from_mode(statvalue.st_mode & 0170000, 'unknown')
+            append((relprefix + name, name, kind, statvalue, abspath))
         yield (relroot, top), dirblock
+
         # push the user specified dirs from dirblock
         pending.extend(d for d in reversed(dirblock) if d[2] == _directory)
 
@@ -1135,38 +1135,32 @@
     encoding. So we don't need to transcode filenames.
     """
     _lstat = os.lstat
-    pending = []
     _directory = _directory_kind
     _listdir = os.listdir
-    _kind_from_mode = file_kind_from_stat_mode
-    # TODO: make these assert instead
-    if isinstance(top, unicode):
-        top = top.encode('utf8')
-    if isinstance(prefix, unicode):
-        prefix = prefix.encode('utf8')
+    _kind_from_mode = _formats.get
 
-    pending = [(prefix, top)]
+    # 0 - relpath, 1- basename, 2- kind, 3- stat, 4-toppath
+    # But we don't actually uses 1-3 in pending, so set them to None
+    pending = [(safe_utf8(prefix), None, None, None, safe_utf8(top))]
     while pending:
-        relroot, top = pending.pop()
-        if relroot == '':
-            rel_prefix = ''
+        relroot, _, _, _, top = pending.pop()
+        if relroot:
+            relprefix = relroot + '/'
         else:
-            rel_prefix = relroot + '/'
+            relprefix = ''
         top_slash = top + '/'
-        # In plain for loop form
+
         dirblock = []
+        append = dirblock.append
         for name in sorted(_listdir(top)):
             abspath = top_slash + name
             statvalue = _lstat(abspath)
-            kind = _kind_from_mode(statvalue.st_mode)
-            dirblock.append((rel_prefix + name, name,
-                             kind, statvalue, abspath))
-
+            kind = _kind_from_mode(statvalue.st_mode & 0170000, 'unknown')
+            append((relprefix + name, name, kind, statvalue, abspath))
         yield (relroot, top), dirblock
+
         # push the user specified dirs from dirblock
-        pending.extend((d[0], d[4])
-                       for d in reversed(dirblock)
-                       if d[2] == _directory)
+        pending.extend(d for d in reversed(dirblock) if d[2] == _directory)
 
 
 def _walkdirs_unicode_to_utf8(top, prefix=""):
@@ -1181,33 +1175,31 @@
     """
     _utf8_encode = codecs.getencoder('utf8')
     _lstat = os.lstat
-    pending = []
     _directory = _directory_kind
     _listdir = os.listdir
-    _kind_from_mode = file_kind_from_stat_mode
+    _kind_from_mode = _formats.get
 
-    pending = [(safe_utf8(prefix), safe_unicode(top))]
+    pending = [(safe_utf8(prefix), None, None, None, safe_unicode(top))]
     while pending:
-        relroot, top = pending.pop()
-        if relroot == '':
-            rel_prefix = ''
+        relroot, _, _, _, top = pending.pop()
+        if relroot:
+            relprefix = relroot + '/'
         else:
-            rel_prefix = relroot + '/'
+            relprefix = ''
         top_slash = top + u'/'
-        # In plain for loop form
+
         dirblock = []
+        append = dirblock.append
         for name in sorted(_listdir(top)):
             name_utf8 = _utf8_encode(name)[0]
             abspath = top_slash + name
             statvalue = _lstat(abspath)
-            kind = _kind_from_mode(statvalue.st_mode)
-            dirblock.append((rel_prefix + name_utf8, name_utf8,
-                             kind, statvalue, abspath))
+            kind = _kind_from_mode(statvalue.st_mode & 0170000, 'unknown')
+            append((relprefix + name_utf8, name_utf8, kind, statvalue, abspath))
         yield (relroot, top), dirblock
+
         # push the user specified dirs from dirblock
-        pending.extend((d[0], d[4])
-                       for d in reversed(dirblock)
-                       if d[2] == _directory)
+        pending.extend(d for d in reversed(dirblock) if d[2] == _directory)
 
 
 def copy_tree(from_path, to_path, handlers={}):