Rev 1: Play around with a pyrex implementation of walkdirs. in http://bzr.arbash-meinel.com/plugins/alt_readdir

John Arbash Meinel john at arbash-meinel.com
Wed Sep 5 21:41:03 BST 2007


At http://bzr.arbash-meinel.com/plugins/alt_readdir

------------------------------------------------------------
revno: 1
revision-id: john at arbash-meinel.com-20070905204017-uz7oxwz1ks2ps3gg
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: alt_readdir
timestamp: Wed 2007-09-05 15:40:17 -0500
message:
  Play around with a pyrex implementation of walkdirs.
added:
  .bzrignore                     bzrignore-20070905203932-oh2i8bsfthna8xul-1
  Makefile                       makefile-20070905203935-juyid0ic3vgs0l9j-1
  __init__.py                    __init__.py-20070905203935-juyid0ic3vgs0l9j-2
  _alt_walkdirs.pyx              _alt_walkdirs.pyx-20070905203935-juyid0ic3vgs0l9j-3
  _walkdirs.py                   _walkdirs.py-20070905203935-juyid0ic3vgs0l9j-4
  setup.py                       setup.py-20070905203935-juyid0ic3vgs0l9j-6
-------------- next part --------------
=== added file '.bzrignore'
--- a/.bzrignore	1970-01-01 00:00:00 +0000
+++ b/.bzrignore	2007-09-05 20:40:17 +0000
@@ -0,0 +1,2 @@
+./_alt_walkdirs.c
+./build

=== added file 'Makefile'
--- a/Makefile	1970-01-01 00:00:00 +0000
+++ b/Makefile	2007-09-05 20:40:17 +0000
@@ -0,0 +1,55 @@
+# Copyright (C) 2007 Canonical Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307	USA
+
+# A relatively simple Makefile to assist in building parts of splib. Mostly for
+# building documentation, etc.
+
+.PHONY: all clean extensions pyflakes docs check-api-docs api-docs
+
+all: extensions
+
+check: extensions
+
+extensions: 
+	python setup.py build_ext -i
+
+# Run Python style checker (apt-get install pyflakes)
+pyflakes:
+	pyflakes bzrlib
+
+docfiles = splib
+api-docs:
+	mkdir -p doc/api
+	PYTHONPATH=$(PWD) epydoc --html -o doc/api --docformat 'restructuredtext en' $(docfiles)
+
+check-api-docs:
+	PYTHONPATH=$(PWD) epydoc --check --docformat 'restructuredtext en' $(docfiles)
+
+
+# translate rst docs to html
+doc_dir := doc 
+rst_files := $(wildcard $(addsuffix /*.rst, $(doc_dir)))
+html_files := $(patsubst %.rst, %.html, $(rst_files)) 
+
+pretty_files: $(patsubst doc/%.rst, $(PRETTYDIR)/%.htm, $(rst_files))
+
+docs: $(html_files)
+
+doc/%.html: doc/%.rst 
+	rst2html.py --link-stylesheet --stylesheet=default.css doc/$*.rst doc/$*.html
+
+
+# vim: noet ts=8 sw=8 sts=8 ai si

=== added file '__init__.py'
--- a/__init__.py	1970-01-01 00:00:00 +0000
+++ b/__init__.py	2007-09-05 20:40:17 +0000
@@ -0,0 +1,24 @@
+# Copyright (C) 2007 Canonical Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+"""A Pyrex implementation of _walkdirs_fs_utf8"""
+
+from _walkdirs import _walkdirs_fs_utf8
+
+
+from bzrlib import osutils
+
+osutils._walkdirs_fs_utf8 = _walkdirs_fs_utf8

=== added file '_alt_walkdirs.pyx'
--- a/_alt_walkdirs.pyx	1970-01-01 00:00:00 +0000
+++ b/_alt_walkdirs.pyx	2007-09-05 20:40:17 +0000
@@ -0,0 +1,152 @@
+# Copyright (C) 2007 Canonical Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+"""A Pyrex implementation of _walkdirs_fs_utf8"""
+
+cdef extern from *:
+    ctypedef unsigned long size_t
+
+cdef extern from "Python.h":
+    int PyList_Append(object lst, object item) except -1
+    int PyList_Append_object_void "PyList_Append" (object lst, void *item) except -1
+    # This is the #defined version
+    int PyList_GET_SIZE(object)
+    void *PyList_GET_ITEM_object_void "PyList_GET_ITEM" (object, int)
+    void *PyList_GET_ITEM_void_void "PyList_GET_ITEM" (void *, int)
+
+    object PyTuple_GET_ITEM "PyTuple_GET_ITEM" (object, int)
+    void *PyTuple_GET_ITEM_void_void "PyTuple_GET_ITEM" (void *, int)
+
+    char *PyString_AS_STRING_void "PyString_AS_STRING" (void *p)
+    int PyString_GET_SIZE_void "PyString_GET_SIZE" (void *p)
+
+cdef extern from "string.h":
+    int strncmp(char *s1, char *s2, int len)
+    void *memchr(void *s, int c, size_t len)
+    int memcmp(void *b1, void *b2, size_t len)
+
+
+import os
+
+from bzrlib import osutils
+
+
+# _formats = {
+#     stat.S_IFDIR:_directory_kind,
+#     stat.S_IFCHR:'chardev',
+#     stat.S_IFBLK:'block',
+#     stat.S_IFREG:'file',
+#     stat.S_IFIFO:'fifo',
+#     stat.S_IFLNK:'symlink',
+#     stat.S_IFSOCK:'socket',
+# }
+# kind = _kind_from_mode(statvalue.st_mode & 0170000, 'unknown')
+
+
+cdef class WalkdirsGenerator:
+    """Create a class which acts like walkdirs, having a .next, etc."""
+
+    cdef object _lstat
+    cdef object _listdir
+
+    cdef object _last_dirblock
+    cdef object _pending
+
+    def __init__(self, top, prefix=""):
+        self._lstat = os.lstat
+        self._listdir = os.listdir
+        self._pending = [(osutils.safe_utf8(prefix),
+                          None, None, None, osutils.safe_utf8(top))]
+        self._last_dirblock = []
+
+    # It seems that even though we have '.next()' defined, Python does't
+    # consider this object to be iterable.
+    def next(self):
+        """Return the next directory block."""
+        cdef int mode
+        cdef int i
+        cdef void *block_item
+        cdef void *item_kind
+
+        # We iterate in reversed order, so that we use self._pending as a stack
+        # rather than as a queue. Because List.pop() is a lot faster than
+        # List.pop(0). And deque() is quite a bit slower overall.
+        #for i from PyList_GET_SIZE(self._last_dirblock) > i >= 0:
+        #    block_item = PyList_GET_ITEM_object_void(self._last_dirblock, i)
+        #    item_kind = PyTuple_GET_ITEM_void_void(block_item, 2)
+        #    if (strncmp('directory', PyString_AS_STRING_void(item_kind),
+        #                PyString_GET_SIZE_void(item_kind)) == 0):
+        #        PyList_Append_object_void(self._pending, block_item)
+
+        for d in reversed(self._last_dirblock):
+            if d[2] == 'directory':
+                self._pending.append(d)
+
+        if not self._pending:
+            raise StopIteration
+
+        # relroot = PyTuple_GET_ITEM(obj, 0)
+        # top = PyTuple_GET_ITEM(obj, 4)
+        relroot, _, _, _, top = self._pending.pop()
+
+        if relroot:
+            relprefix = relroot + '/'
+        else:
+            relprefix = ''
+        top_slash = top + '/'
+
+        dirblock = []
+        append = dirblock.append
+        for name in sorted(self._listdir(top)):
+            abspath = top_slash + name
+            statvalue = self._lstat(abspath)
+            mode = statvalue.st_mode & 0170000
+            if mode == 0x8000:   # S_IFREG
+                kind = 'file'
+            elif mode == 0x4000: # S_IFDIR
+                kind = 'directory'
+            elif mode == 0xa000: # S_IFLNK
+                kind = 'symlink'
+            elif mode == 0x2000: # S_IFCHR
+                kind = 'chardev'
+            elif mode == 0x1000: # S_IFIFO
+                kind = 'fifo'
+            elif mode == 0xc000: # S_IFSOCK
+                kind = 'socket'
+            else:
+                kind = 'unknown'
+            #PyList_Append(dirblock,
+            #              (relprefix + name, name, kind, statvalue, abspath))
+            append((relprefix + name, name, kind, statvalue, abspath))
+
+        # push the user specified dirs from dirblock
+        self._last_dirblock = dirblock
+        return (relroot, top), dirblock
+
+
+class IteratingWrapper(object):
+    """A simple class which turns a pyrex object into an iterable."""
+
+    __slots__ = ['child']
+
+    def __init__(self, child):
+        self.child = child
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        return self.child.next()

=== added file '_walkdirs.py'
--- a/_walkdirs.py	1970-01-01 00:00:00 +0000
+++ b/_walkdirs.py	2007-09-05 20:40:17 +0000
@@ -0,0 +1,23 @@
+# Copyright (C) 2007 Canonical Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+"""A Pyrex implementation of _walkdirs_fs_utf8"""
+
+from _alt_walkdirs import WalkdirsGenerator, IteratingWrapper
+
+
+def _walkdirs_fs_utf8(top, prefix=""):
+    return IteratingWrapper(WalkdirsGenerator(top, prefix=prefix))

=== added file 'setup.py'
--- a/setup.py	1970-01-01 00:00:00 +0000
+++ b/setup.py	2007-09-05 20:40:17 +0000
@@ -0,0 +1,78 @@
+#! /usr/bin/env python
+# Copyright (C) 2007 Canonical Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+"""Installation script for splib.
+Run it with
+ './setup.py install', or
+ './setup.py --help' for more options
+"""
+
+import os
+import sys
+
+##
+# META INFORMATION FOR SETUP
+
+META_INFO = {'name':         'alt_readdir',
+#             'version':      bzrlib.__version__,
+             'author':       'Canonical Ltd',
+#             'author_email': 'bazaar at lists.canonical.com',
+#             'url':          'http://www.bazaar-vcs.org/',
+             'description':  'Implement walkdirs in pyrex',
+             'license':      'GNU GPL v2 or later',
+            }
+
+from distutils.core import setup
+from distutils.command.install_scripts import install_scripts
+from distutils.command.build import build
+
+########################
+## Setup
+########################
+
+command_classes = {}
+ext_modules = []
+try:
+    from Pyrex.Distutils import build_ext
+except ImportError:
+    # try to build the extension from the prior generated source.
+    print ("Pyrex not available, while sp2bzr will build, "
+           "you cannot modify the C extensions.")
+    from distutils.command.build_ext import build_ext
+    from distutils.extension import Extension
+    #ext_modules.append(
+    #    Extension("bzrlib.modulename", ["bzrlib/foo.c"], libraries = []))
+else:
+    from distutils.extension import Extension
+    #ext_modules.append(
+    #    Extension("bzrlib.modulename", ["bzrlib/foo.pyx"], libraries = []))
+    ext_modules.append(
+        Extension("_alt_walkdirs", ["_alt_walkdirs.pyx"],
+                  libraries=[],
+                  library_dirs=[],
+                  include_dirs=[],
+                  ))
+command_classes['build_ext'] = build_ext
+
+# std setup
+ARGS = {'cmdclass': command_classes,
+        'ext_modules': ext_modules,
+       }
+
+ARGS.update(META_INFO)
+
+setup(**ARGS)



More information about the bazaar-commits mailing list