Rev 5996: (mbp) cleanup export code (Martin Pool) in file:///home/pqm/archives/thelove/bzr/%2Btrunk/
Canonical.com Patch Queue Manager
pqm at pqm.ubuntu.com
Tue Jun 28 15:16:01 UTC 2011
At file:///home/pqm/archives/thelove/bzr/%2Btrunk/
------------------------------------------------------------
revno: 5996 [merge]
revision-id: pqm at pqm.ubuntu.com-20110628151559-dvc0trfnmfux4roq
parent: pqm at pqm.ubuntu.com-20110628121833-ax14ogkh8ks922nu
parent: mbp at canonical.com-20110628135539-6541falwx39fl46i
committer: Canonical.com Patch Queue Manager <pqm at pqm.ubuntu.com>
branch nick: +trunk
timestamp: Tue 2011-06-28 15:15:59 +0000
message:
(mbp) cleanup export code (Martin Pool)
modified:
bzrlib/export/__init__.py __init__.py-20051114235828-1ba62cb4062304e6
bzrlib/export/dir_exporter.py dir_exporter.py-20051114235828-b51397f56bc7b117
bzrlib/export/tar_exporter.py tar_exporter.py-20051114235828-1f6349a2f090a5d0
bzrlib/export/zip_exporter.py zip_exporter.py-20051114235828-8f57f954fba6497e
bzrlib/tests/test_export.py test_export.py-20090220201010-tpbxssdnezsvu9pk-1
doc/en/release-notes/bzr-2.4.txt bzr2.4.txt-20110114053217-k7ym9jfz243fddjm-1
=== modified file 'bzrlib/export/__init__.py'
--- a/bzrlib/export/__init__.py 2011-06-07 15:18:55 +0000
+++ b/bzrlib/export/__init__.py 2011-06-13 16:34:53 +0000
@@ -14,9 +14,7 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-"""Export functionality, which can take a Tree and create a different representation.
-
-Such as non-controlled directories, tarfiles, zipfiles, etc.
+"""Export trees to tarballs, non-controlled directories, zipfiles, etc.
"""
import os
@@ -32,6 +30,7 @@
# Maps filename extensions => export format name
_exporter_extensions = {}
+
def register_exporter(format, extensions, func, override=False):
"""Register an exporter.
@@ -61,15 +60,17 @@
"""
def _loader(tree, dest, root, subdir, filtered, force_mtime, fileobj):
func = pyutils.get_named_object(module, funcname)
- return func(tree, dest, root, subdir, filtered=filtered,
+ return func(tree, dest, root, subdir, filtered=filtered,
force_mtime=force_mtime, fileobj=fileobj)
+
register_exporter(scheme, extensions, _loader)
-
+
+
def get_export_generator(tree, dest=None, format=None, root=None, subdir=None,
filtered=False, per_file_timestamps=False,
fileobj=None):
"""Returns a generator that exports the given tree.
-
+
The generator is expected to yield None while exporting the tree while the
actual export is written to ``fileobj``.
@@ -123,17 +124,17 @@
try:
tree.lock_read()
-
+
for _ in _exporters[format](tree, dest, root, subdir,
- filtered=filtered,
+ filtered=filtered,
force_mtime=force_mtime, fileobj=fileobj):
-
+
yield
- finally:
+ finally:
tree.unlock()
-def export(tree, dest, format=None, root=None, subdir=None, filtered=False,
+def export(tree, dest, format=None, root=None, subdir=None, filtered=False,
per_file_timestamps=False, fileobj=None):
"""Export the given Tree to the specific destination.
@@ -153,16 +154,16 @@
a directory to start exporting from.
:param filtered: If True, content filtering is applied to the
files exported.
- :param per_file_timestamps: Whether to use the timestamp stored in the
- tree rather than now(). This will do a revision lookup
+ :param per_file_timestamps: Whether to use the timestamp stored in the
+ tree rather than now(). This will do a revision lookup
for every file so will be significantly slower.
:param fileobj: Optional file object to use
"""
- for _ in get_export_generator(tree, dest, format, root, subdir, filtered,
+ for _ in get_export_generator(tree, dest, format, root, subdir, filtered,
per_file_timestamps, fileobj):
-
pass
+
def get_root_name(dest):
"""Get just the root name for an export.
@@ -190,7 +191,7 @@
if subdir is not None:
subdir = subdir.rstrip('/')
entries = tree.iter_entries_by_dir()
- entries.next() # skip root
+ entries.next() # skip root
for path, entry in entries:
# The .bzr* namespace is reserved for "magic" files like
# .bzrignore and .bzrrules - do not export these
@@ -209,7 +210,7 @@
final_path = path
if not tree.has_filename(path):
continue
-
+
yield final_path, entry
@@ -219,7 +220,8 @@
'dir_exporter_generator')
register_lazy_exporter('tar', ['.tar'], 'bzrlib.export.tar_exporter',
'plain_tar_exporter_generator')
-register_lazy_exporter('tgz', ['.tar.gz', '.tgz'], 'bzrlib.export.tar_exporter',
+register_lazy_exporter('tgz', ['.tar.gz', '.tgz'],
+ 'bzrlib.export.tar_exporter',
'tgz_exporter_generator')
register_lazy_exporter('tbz2', ['.tar.bz2', '.tbz2'],
'bzrlib.export.tar_exporter', 'tbz_exporter_generator')
@@ -229,4 +231,3 @@
'tar_xz_exporter_generator')
register_lazy_exporter('zip', ['.zip'], 'bzrlib.export.zip_exporter',
'zip_exporter_generator')
-
=== modified file 'bzrlib/export/dir_exporter.py'
--- a/bzrlib/export/dir_exporter.py 2011-06-10 14:47:25 +0000
+++ b/bzrlib/export/dir_exporter.py 2011-06-28 13:55:39 +0000
@@ -1,4 +1,4 @@
-# Copyright (C) 2005, 2006, 2008, 2009, 2010 Canonical Ltd
+# Copyright (C) 2005-2011 Canonical Ltd
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -33,7 +33,7 @@
`dest` should either not exist or should be empty. If it does not exist it
will be created holding the contents of this tree.
-
+
:param fileobj: Is not used in this exporter
:note: If the export fails, the destination directory will be
@@ -66,7 +66,7 @@
try:
symlink_target = tree.get_symlink_target(ie.file_id)
os.symlink(symlink_target, fullpath)
- except OSError,e:
+ except OSError, e:
raise errors.BzrError(
"Failed to create symlink %r -> %r, error: %s"
% (fullpath, symlink_target, e))
@@ -104,10 +104,3 @@
os.utime(fullpath, (mtime, mtime))
yield
-
-def dir_exporter(tree, dest, root, subdir=None, filtered=False,
- force_mtime=None, fileobj=None):
-
- for _ in dir_exporter_generator(tree, dest, root, subdir, filtered,
- force_mtime, fileobj):
- pass
=== modified file 'bzrlib/export/tar_exporter.py'
--- a/bzrlib/export/tar_exporter.py 2011-06-09 15:13:54 +0000
+++ b/bzrlib/export/tar_exporter.py 2011-06-28 13:54:56 +0000
@@ -31,10 +31,11 @@
filtered_output_bytes,
)
+
def prepare_tarball_item(tree, root, final_path, entry, filtered=False,
force_mtime=None):
"""Prepare a tarball item for exporting
-
+
:param tree: Tree to export
:param final_path: Final path to place item
@@ -43,8 +44,9 @@
:param filtered: Whether to apply filters
- :param force_mtime: Option mtime to force, instead of using tree timestamps.
-
+ :param force_mtime: Option mtime to force, instead of using tree
+ timestamps.
+
Returns a (tarinfo, fileobj) tuple
"""
filename = osutils.pathjoin(root, final_path).encode('utf8')
@@ -87,13 +89,19 @@
% (entry.file_id, entry.kind))
return (item, fileobj)
+
def export_tarball_generator(tree, ball, root, subdir=None, filtered=False,
force_mtime=None):
- """Export tree contents to a tarball. This is a generator.
+ """Export tree contents to a tarball.
+
+ :returns: A generator that will repeatedly produce None as each file is
+ emitted. The entire generator must be consumed to complete writing
+ the file.
:param tree: Tree to export
- :param ball: Tarball to export to
+ :param ball: Tarball to export to; it will be closed when writing is
+ complete.
:param filtered: Whether to apply filters
@@ -102,21 +110,15 @@
:param force_mtime: Option mtime to force, instead of using tree
timestamps.
"""
- for final_path, entry in _export_iter_entries(tree, subdir):
-
- (item, fileobj) = prepare_tarball_item(tree, root, final_path,
- entry, filtered, force_mtime)
- ball.addfile(item, fileobj)
-
- yield
-
-
-def export_tarball(tree, ball, root, subdir=None, filtered=False,
- force_mtime=None):
-
- for _ in export_tarball_generator(tree, ball, root, subdir, filtered,
- force_mtime):
- pass
+ try:
+ for final_path, entry in _export_iter_entries(tree, subdir):
+ (item, fileobj) = prepare_tarball_item(
+ tree, root, final_path, entry, filtered, force_mtime)
+ ball.addfile(item, fileobj)
+ yield
+ finally:
+ ball.close()
+
def tgz_exporter_generator(tree, dest, root, subdir, filtered=False,
force_mtime=None, fileobj=None):
@@ -158,13 +160,9 @@
# Python < 2.7 doesn't support the mtime argument
zipstream = gzip.GzipFile(basename, 'w', fileobj=stream)
ball = tarfile.open(None, 'w|', fileobj=zipstream)
-
- for _ in export_tarball_generator(tree, ball, root, subdir, filtered,
- force_mtime):
-
+ for _ in export_tarball_generator(
+ tree, ball, root, subdir, filtered, force_mtime):
yield
- # Closing ball may trigger writes to zipstream
- ball.close()
# Closing zipstream may trigger writes to stream
zipstream.close()
if not is_stdout:
@@ -172,15 +170,6 @@
stream.close()
-
-def tgz_exporter(tree, dest, root, subdir, filtered=False, force_mtime=None,
- fileobj=None):
-
- for _ in tgz_exporter_generator(tree, dest, root, subdir, filtered,
- force_mtime, fileobj):
- pass
-
-
def tbz_exporter_generator(tree, dest, root, subdir, filtered=False,
force_mtime=None, fileobj=None):
"""Export this tree to a new tar file.
@@ -193,26 +182,14 @@
elif dest == '-':
ball = tarfile.open(None, 'w|bz2', sys.stdout)
else:
- # tarfile.open goes on to do 'os.getcwd() + dest' for opening
- # the tar file. With dest being unicode, this throws UnicodeDecodeError
+ # tarfile.open goes on to do 'os.getcwd() + dest' for opening the
+ # tar file. With dest being unicode, this throws UnicodeDecodeError
# unless we encode dest before passing it on. This works around
- # upstream python bug http://bugs.python.org/issue8396
- # (fixed in Python 2.6.5 and 2.7b1)
+ # upstream python bug http://bugs.python.org/issue8396 (fixed in
+ # Python 2.6.5 and 2.7b1)
ball = tarfile.open(dest.encode(osutils._fs_enc), 'w:bz2')
-
- for _ in export_tarball_generator(tree, ball, root, subdir, filtered,
- force_mtime):
- yield
-
- ball.close()
-
-
-def tbz_exporter(tree, dest, root, subdir, filtered=False, force_mtime=None,
- fileobj=None):
-
- for _ in tbz_exporter_generator(tree, dest, root, subdir, filtered,
- force_mtime, fileobj):
- pass
+ return export_tarball_generator(
+ tree, ball, root, subdir, filtered, force_mtime)
def plain_tar_exporter_generator(tree, dest, root, subdir, compression=None,
@@ -230,36 +207,16 @@
else:
stream = open(dest, 'wb')
ball = tarfile.open(None, 'w|', stream)
-
- for _ in export_tarball_generator(tree, ball, root, subdir, filtered,
- force_mtime):
-
- yield
-
- ball.close()
-
-def plain_tar_exporter(tree, dest, root, subdir, compression=None,
- filtered=False, force_mtime=None, fileobj=None):
-
- for _ in plain_tar_exporter_generator(
- tree, dest, root, subdir, compression, filtered, force_mtime, fileobj):
- pass
+ return export_tarball_generator(
+ tree, ball, root, subdir, filtered, force_mtime)
def tar_xz_exporter_generator(tree, dest, root, subdir, filtered=False,
force_mtime=None, fileobj=None):
-
return tar_lzma_exporter_generator(tree, dest, root, subdir, filtered,
force_mtime, fileobj, "xz")
-def tar_xz_exporter(tree, dest, root, subdir, filtered=False, force_mtime=None,
- fileobj=None):
- for _ in tar_xz_exporter_generator(tree, dest, root, subdir, filtered,
- force_mtime, fileobj):
- pass
-
-
def tar_lzma_exporter_generator(tree, dest, root, subdir, filtered=False,
force_mtime=None, fileobj=None,
compression_format="alone"):
@@ -280,20 +237,7 @@
raise errors.DependencyNotPresent('lzma', e)
stream = lzma.LZMAFile(dest.encode(osutils._fs_enc), 'w',
- options={"format": compression_format})
+ options={"format": compression_format})
ball = tarfile.open(None, 'w:', fileobj=stream)
-
- for _ in export_tarball_generator(
- tree, ball, root, subdir, filtered=filtered, force_mtime=force_mtime):
- yield
-
- ball.close()
-
-
-def tar_lzma_exporter(tree, dest, root, subdir, filtered=False,
- force_mtime=None, fileobj=None,
- compression_format="alone"):
- for _ in tar_lzma_exporter_generator(tree, dest, root, subdir, filtered,
- force_mtime, fileobj,
- compression_format):
- pass
+ return export_tarball_generator(
+ tree, ball, root, subdir, filtered=filtered, force_mtime=force_mtime)
=== modified file 'bzrlib/export/zip_exporter.py'
--- a/bzrlib/export/zip_exporter.py 2011-06-07 13:33:41 +0000
+++ b/bzrlib/export/zip_exporter.py 2011-06-13 16:34:53 +0000
@@ -14,7 +14,7 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-"""Export a Tree to a non-versioned directory.
+"""Export a Tree to a zip file.
"""
import os
@@ -34,8 +34,8 @@
from bzrlib.trace import mutter
-# Windows expects this bit to be set in the 'external_attr' section
-# Or it won't consider the entry a directory
+# Windows expects this bit to be set in the 'external_attr' section,
+# or it won't consider the entry a directory.
ZIP_DIRECTORY_BIT = (1 << 4)
FILE_PERMISSIONS = (0644 << 16)
DIR_PERMISSIONS = (0755 << 16)
@@ -44,8 +44,8 @@
_DIR_ATTR = stat.S_IFDIR | ZIP_DIRECTORY_BIT | DIR_PERMISSIONS
-def zip_exporter_generator(tree, dest, root, subdir=None, filtered=False,
- force_mtime=None, fileobj=None):
+def zip_exporter_generator(tree, dest, root, subdir=None, filtered=False,
+ force_mtime=None, fileobj=None):
""" Export this tree to a new zip file.
`dest` will be created holding the contents of this tree; if it
@@ -95,7 +95,7 @@
date_time=date_time)
zinfo.compress_type = compression
zinfo.external_attr = _DIR_ATTR
- zipf.writestr(zinfo,'')
+ zipf.writestr(zinfo, '')
elif ie.kind == "symlink":
zinfo = zipfile.ZipInfo(
filename=(filename + '.lnk'),
@@ -112,10 +112,3 @@
os.remove(dest)
from bzrlib.errors import BzrError
raise BzrError("Can't export non-ascii filenames to zip")
-
-
-def zip_exporter(tree, dest, root, subdir=None, filtered=False,
- force_mtime=None, fileobj=None):
- for _ in zip_exporter_generator(tree, dest, root, subdir, filtered,
- force_mtime, fileobj):
- pass
=== modified file 'bzrlib/tests/test_export.py'
--- a/bzrlib/tests/test_export.py 2011-06-10 14:47:25 +0000
+++ b/bzrlib/tests/test_export.py 2011-06-28 13:55:39 +0000
@@ -28,7 +28,7 @@
tests,
)
from bzrlib.export import get_root_name
-from bzrlib.export.tar_exporter import export_tarball
+from bzrlib.export.tar_exporter import export_tarball_generator
from bzrlib.tests import features
@@ -84,10 +84,12 @@
wt.add(['a', 'b', 'b/c'])
wt.commit('1')
self.build_tree(['target/', 'target/foo'])
- self.assertRaises(errors.BzrError, export.export, wt, 'target', format="dir")
+ self.assertRaises(errors.BzrError,
+ export.export, wt, 'target', format="dir")
def test_existing_single_file(self):
- self.build_tree(['dir1/', 'dir1/dir2/', 'dir1/first', 'dir1/dir2/second'])
+ self.build_tree([
+ 'dir1/', 'dir1/dir2/', 'dir1/first', 'dir1/dir2/second'])
wtree = self.make_branch_and_tree('dir1')
wtree.add(['dir2', 'first', 'dir2/second'])
wtree.commit('1')
@@ -110,6 +112,7 @@
self.addCleanup(b.unlock)
tree = b.basis_tree()
orig_iter_files_bytes = tree.iter_files_bytes
+
# Make iter_files_bytes slower, so we provoke mtime skew
def iter_files_bytes(to_fetch):
for thing in orig_iter_files_bytes(to_fetch):
@@ -239,7 +242,7 @@
self.assertRaises(errors.BzrError, export.export, wt, '-',
format="txz")
- def test_export_tarball(self):
+ def test_export_tarball_generator(self):
wt = self.make_branch_and_tree('.')
self.build_tree(['a'])
wt.add(["a"])
@@ -248,11 +251,15 @@
ball = tarfile.open(None, "w|", target)
wt.lock_read()
try:
- export_tarball(wt, ball, "bar")
+ for _ in export_tarball_generator(wt, ball, "bar"):
+ pass
finally:
wt.unlock()
- self.assertEquals(["bar/a"], ball.getnames())
- ball.close()
+ # Ball should now be closed.
+ target.seek(0)
+ ball2 = tarfile.open(None, "r", target)
+ self.addCleanup(ball2.close)
+ self.assertEquals(["bar/a"], ball2.getnames())
class ZipExporterTests(tests.TestCaseWithTransport):
=== modified file 'doc/en/release-notes/bzr-2.4.txt'
--- a/doc/en/release-notes/bzr-2.4.txt 2011-06-28 12:18:33 +0000
+++ b/doc/en/release-notes/bzr-2.4.txt 2011-06-28 13:55:39 +0000
@@ -198,6 +198,10 @@
.. Changes that may require updates in plugins or other code that uses
bzrlib.
+* Exporters are now all exposed as generators, rather than as single-call
+ functions, so that calling code can take stream the output.
+ (Xaav, Martin Pool)
+
* Information about held lockdir locks returned from eg `LockDir.peek` is
now represented as a `LockHeldInfo` object, rather than a plain
Python dict.
More information about the bazaar-commits
mailing list