Rev 4748: Merge in the latest bzr.dev in http://bazaar.launchpad.net/~jameinel/bzr/2.1-peak-mem-tweak
John Arbash Meinel
john at arbash-meinel.com
Sun Oct 18 18:53:54 BST 2009
At http://bazaar.launchpad.net/~jameinel/bzr/2.1-peak-mem-tweak
------------------------------------------------------------
revno: 4748 [merge]
revision-id: john at arbash-meinel.com-20091018175325-m7s57yrmm3bvj9b3
parent: john at arbash-meinel.com-20091017042950-bazb5obtctyq8333
parent: pqm at pqm.ubuntu.com-20091017222307-4x6xk4tkcp25q2ac
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: 2.1-peak-mem-tweak
timestamp: Sun 2009-10-18 12:53:25 -0500
message:
Merge in the latest bzr.dev
modified:
NEWS NEWS-20050323055033-4e00b5db738777ff
bzrlib/_readdir_pyx.pyx readdir.pyx-20060609152855-rm6v321vuaqyh9tu-1
bzrlib/_static_tuple_c.c _keys_type_c.c-20090908204220-aa346ccw4l37jzt7-1
bzrlib/_static_tuple_c.h _keys_type_c.h-20090913185138-f86v5xm1zlckguaj-1
bzrlib/_static_tuple_py.py _keys_type_py.py-20090908213415-o1ww98k9a8aqm0bm-1
bzrlib/inventory.py inventory.py-20050309040759-6648b84ca2005b37
bzrlib/repofmt/pack_repo.py pack_repo.py-20070813041115-gjv5ma7ktfqwsjgn-1
bzrlib/tests/test__static_tuple.py test__keys_type.py-20090908204220-aa346ccw4l37jzt7-2
bzrlib/tests/test_transform.py test_transaction.py-20060105172520-b3ffb3946550e6c4
bzrlib/tests/test_ui.py test_ui.py-20051130162854-458e667a7414af09
bzrlib/transform.py transform.py-20060105172343-dd99e54394d91687
bzrlib/ui/__init__.py ui.py-20050824083933-8cf663c763ba53a9
setup.py setup.py-20050314065409-02f8a0a6e3f9bc70
-------------- next part --------------
=== modified file 'NEWS'
--- a/NEWS 2009-10-15 18:18:44 +0000
+++ b/NEWS 2009-10-17 04:43:14 +0000
@@ -22,6 +22,9 @@
Bug Fixes
*********
+* TreeTransform.adjust_path updates the limbo paths of descendants of adjusted
+ files. (Aaron Bentley)
+
Improvements
************
@@ -30,12 +33,17 @@
memory, and can give a performance boost up to 40% on large projects.
(John Arbash Meinel)
+* Peak memory under certain operations has been reduced significantly.
+ (John Arbash Meinel)
+
Documentation
*************
API Changes
***********
+* Remove deprecated ``CLIUIFactory``. (Martin Pool)
+
* ``UIFactory`` now has new ``show_error``, ``show_message`` and
``show_warning`` methods, which can be hooked by non-text UIs.
(Martin Pool)
@@ -59,6 +67,15 @@
improve performance by removing objects from being inspected by the
garbage collector. (John Arbash Meinel)
+* ``GroupCompressBlock._ensure_content()`` will now release the
+ ``zlib.decompressobj()`` when the first request is for all of the
+ content. (Previously it would only be released if you made a request for
+ part of the content, and then all of it later.) This turns out to be a
+ significant memory savings, as a ``zstream`` carries around approx 260kB
+ of internal state and buffers. (For branching bzr.dev this drops peak
+ memory from 382MB => 345MB.) (John Arbash Meinel)
+
+
Testing
*******
@@ -78,6 +95,9 @@
Bug Fixes
*********
+* Avoid "NoneType has no attribute st_mode" error when files disappear
+ from a directory while it's being read. (Martin Pool, #446033)
+
Improvements
************
=== modified file 'bzrlib/_readdir_pyx.pyx'
--- a/bzrlib/_readdir_pyx.pyx 2009-07-27 04:24:36 +0000
+++ b/bzrlib/_readdir_pyx.pyx 2009-10-08 07:03:05 +0000
@@ -343,8 +343,10 @@
raise OSError(errno, "lstat: " + strerror(errno),
path + "/" + entry.d_name)
else:
- kind = _missing
- statvalue = None
+ # the file seems to have disappeared after being
+ # seen by readdir - perhaps a transient temporary
+ # file. there's no point returning it.
+ continue
# We append a 5-tuple that can be modified in-place by the C
# api:
# inode to sort on (to replace with top_path)
=== modified file 'bzrlib/_static_tuple_c.c'
--- a/bzrlib/_static_tuple_c.c 2009-10-15 18:18:44 +0000
+++ b/bzrlib/_static_tuple_c.c 2009-10-17 00:34:28 +0000
@@ -174,6 +174,52 @@
}
+static StaticTuple *
+StaticTuple_FromSequence(PyObject *sequence)
+{
+ StaticTuple *new;
+ PyObject *item;
+ Py_ssize_t i, size;
+
+ if (StaticTuple_CheckExact(sequence)) {
+ Py_INCREF(sequence);
+ return (StaticTuple *)sequence;
+ }
+ if (!PySequence_Check(sequence)) {
+ PyErr_Format(PyExc_TypeError, "Type %s is not a sequence type",
+ Py_TYPE(sequence)->tp_name);
+ return NULL;
+ }
+ size = PySequence_Size(sequence);
+ if (size == -1)
+ return NULL;
+ new = StaticTuple_New(size);
+ if (new == NULL) {
+ return NULL;
+ }
+ for (i = 0; i < size; ++i) {
+ // This returns a new reference, which we then 'steal' with
+ // StaticTuple_SET_ITEM
+ item = PySequence_GetItem(sequence, i);
+ if (item == NULL) {
+ Py_DECREF(new);
+ return NULL;
+ }
+ StaticTuple_SET_ITEM(new, i, item);
+ }
+ return (StaticTuple *)new;
+}
+
+static StaticTuple *
+StaticTuple_from_sequence(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+ PyObject *sequence;
+ if (!PyArg_ParseTuple(args, "O", &sequence))
+ return NULL;
+ return StaticTuple_FromSequence(sequence);
+}
+
+
static PyObject *
StaticTuple_new_constructor(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
@@ -571,6 +617,10 @@
{"intern", (PyCFunction)StaticTuple_Intern, METH_NOARGS, StaticTuple_Intern_doc},
{"_is_interned", (PyCFunction)StaticTuple__is_interned, METH_NOARGS,
StaticTuple__is_interned_doc},
+ {"from_sequence", (PyCFunction)StaticTuple_from_sequence,
+ METH_STATIC | METH_VARARGS,
+ "Create a StaticTuple from a given sequence. This functions"
+ " the same as the tuple() constructor."},
{NULL, NULL} /* sentinel */
};
@@ -690,6 +740,8 @@
"StaticTuple *(Py_ssize_t)");
_export_function(m, "StaticTuple_Intern", StaticTuple_Intern,
"StaticTuple *(StaticTuple *)");
+ _export_function(m, "StaticTuple_FromSequence", StaticTuple_FromSequence,
+ "StaticTuple *(PyObject *)");
_export_function(m, "_StaticTuple_CheckExact", _StaticTuple_CheckExact,
"int(PyObject *)");
}
=== modified file 'bzrlib/_static_tuple_c.h'
--- a/bzrlib/_static_tuple_c.h 2009-10-07 15:57:25 +0000
+++ b/bzrlib/_static_tuple_c.h 2009-10-13 18:00:16 +0000
@@ -74,6 +74,7 @@
static StaticTuple * StaticTuple_New(Py_ssize_t);
static StaticTuple * StaticTuple_Intern(StaticTuple *self);
+static StaticTuple * StaticTuple_FromSequence(PyObject *);
#define StaticTuple_CheckExact(op) (Py_TYPE(op) == &StaticTuple_Type)
#else
@@ -83,6 +84,7 @@
static StaticTuple *(*StaticTuple_New)(Py_ssize_t);
static StaticTuple *(*StaticTuple_Intern)(StaticTuple *);
+static StaticTuple *(*StaticTuple_FromSequence)(PyObject *);
static PyTypeObject *_p_StaticTuple_Type;
#define StaticTuple_CheckExact(op) (Py_TYPE(op) == _p_StaticTuple_Type)
@@ -98,6 +100,8 @@
"StaticTuple *(Py_ssize_t)"},
{"StaticTuple_Intern", (void **)&StaticTuple_Intern,
"StaticTuple *(StaticTuple *)"},
+ {"StaticTuple_FromSequence", (void **)&StaticTuple_FromSequence,
+ "StaticTuple *(PyObject *)"},
{"_StaticTuple_CheckExact", (void **)&_StaticTuple_CheckExact,
"int(PyObject *)"},
{NULL}};
=== modified file 'bzrlib/_static_tuple_py.py'
--- a/bzrlib/_static_tuple_py.py 2009-10-12 17:07:05 +0000
+++ b/bzrlib/_static_tuple_py.py 2009-10-13 18:00:16 +0000
@@ -51,6 +51,15 @@
def intern(self):
return _interned_tuples.setdefault(self, self)
+ @staticmethod
+ def from_sequence(seq):
+ """Convert a sequence object into a StaticTuple instance."""
+ if isinstance(seq, StaticTuple):
+ # it already is
+ return seq
+ return StaticTuple(*seq)
+
+
# Have to set it to None first, so that __new__ can determine whether
# the _empty_tuple singleton has been created yet or not.
=== modified file 'bzrlib/inventory.py'
--- a/bzrlib/inventory.py 2009-10-02 05:43:41 +0000
+++ b/bzrlib/inventory.py 2009-10-14 13:47:28 +0000
@@ -958,21 +958,21 @@
descend(self.root, u'')
return accum
- def path2id(self, name):
+ def path2id(self, relpath):
"""Walk down through directories to return entry of last component.
- names may be either a list of path components, or a single
- string, in which case it is automatically split.
+ :param relpath: may be either a list of path components, or a single
+ string, in which case it is automatically split.
This returns the entry of the last component in the path,
which may be either a file or a directory.
Returns None IFF the path is not found.
"""
- if isinstance(name, basestring):
- name = osutils.splitpath(name)
-
- # mutter("lookup path %r" % name)
+ if isinstance(relpath, basestring):
+ names = osutils.splitpath(relpath)
+ else:
+ names = relpath
try:
parent = self.root
@@ -981,7 +981,7 @@
return None
if parent is None:
return None
- for f in name:
+ for f in names:
try:
children = getattr(parent, 'children', None)
if children is None:
@@ -2170,35 +2170,41 @@
delta.append((old_path, new_path, file_id, entry))
return delta
- def path2id(self, name):
+ def path2id(self, relpath):
"""See CommonInventory.path2id()."""
# TODO: perhaps support negative hits?
- result = self._path_to_fileid_cache.get(name, None)
+ result = self._path_to_fileid_cache.get(relpath, None)
if result is not None:
return result
- if isinstance(name, basestring):
- names = osutils.splitpath(name)
+ if isinstance(relpath, basestring):
+ names = osutils.splitpath(relpath)
else:
- names = name
+ names = relpath
current_id = self.root_id
if current_id is None:
return None
parent_id_index = self.parent_id_basename_to_file_id
+ cur_path = None
for basename in names:
- # TODO: Cache each path we figure out in this function.
+ if cur_path is None:
+ cur_path = basename
+ else:
+ cur_path = cur_path + '/' + basename
basename_utf8 = basename.encode('utf8')
- key_filter = [(current_id, basename_utf8)]
- file_id = None
- for (parent_id, name_utf8), file_id in parent_id_index.iteritems(
- key_filter=key_filter):
- if parent_id != current_id or name_utf8 != basename_utf8:
- raise errors.BzrError("corrupt inventory lookup! "
- "%r %r %r %r" % (parent_id, current_id, name_utf8,
- basename_utf8))
+ file_id = self._path_to_fileid_cache.get(cur_path, None)
if file_id is None:
- return None
+ key_filter = [(current_id, basename_utf8)]
+ items = parent_id_index.iteritems(key_filter)
+ for (parent_id, name_utf8), file_id in items:
+ if parent_id != current_id or name_utf8 != basename_utf8:
+ raise errors.BzrError("corrupt inventory lookup! "
+ "%r %r %r %r" % (parent_id, current_id, name_utf8,
+ basename_utf8))
+ if file_id is None:
+ return None
+ else:
+ self._path_to_fileid_cache[cur_path] = file_id
current_id = file_id
- self._path_to_fileid_cache[name] = current_id
return current_id
def to_lines(self):
=== modified file 'bzrlib/repofmt/pack_repo.py'
--- a/bzrlib/repofmt/pack_repo.py 2009-10-15 04:01:26 +0000
+++ b/bzrlib/repofmt/pack_repo.py 2009-10-16 07:10:11 +0000
@@ -2089,7 +2089,7 @@
('signatures', self.repo.signatures),
):
missing = versioned_file.get_missing_compression_parent_keys()
- all_missing.update([(prefix,) + key for key in missing])
+ all_missing.update([(prefix,) + tuple(key) for key in missing])
if all_missing:
raise errors.BzrCheckError(
"Repository %s has missing compression parent(s) %r "
=== modified file 'bzrlib/tests/test__static_tuple.py'
--- a/bzrlib/tests/test__static_tuple.py 2009-10-14 14:03:56 +0000
+++ b/bzrlib/tests/test__static_tuple.py 2009-10-17 00:34:28 +0000
@@ -428,6 +428,37 @@
return
self.assertIsNot(None, self.module._C_API)
+ def test_from_sequence_tuple(self):
+ st = self.module.StaticTuple.from_sequence(('foo', 'bar'))
+ self.assertIsInstance(st, self.module.StaticTuple)
+ self.assertEqual(('foo', 'bar'), st)
+
+ def test_from_sequence_str(self):
+ st = self.module.StaticTuple.from_sequence('foo')
+ self.assertIsInstance(st, self.module.StaticTuple)
+ self.assertEqual(('f', 'o', 'o'), st)
+
+ def test_from_sequence_list(self):
+ st = self.module.StaticTuple.from_sequence(['foo', 'bar'])
+ self.assertIsInstance(st, self.module.StaticTuple)
+ self.assertEqual(('foo', 'bar'), st)
+
+ def test_from_sequence_static_tuple(self):
+ st = self.module.StaticTuple('foo', 'bar')
+ st2 = self.module.StaticTuple.from_sequence(st)
+ # If the source is a StaticTuple already, we return the exact object
+ self.assertIs(st, st2)
+
+ def test_from_sequence_not_sequence(self):
+ self.assertRaises(TypeError,
+ self.module.StaticTuple.from_sequence, object())
+
+ def test_from_sequence_incorrect_args(self):
+ self.assertRaises(TypeError,
+ self.module.StaticTuple.from_sequence, object(), 'a')
+ self.assertRaises(TypeError,
+ self.module.StaticTuple.from_sequence, foo='a')
+
def test_static_tuple_thunk(self):
# Make sure the right implementation is available from
# bzrlib.static_tuple.StaticTuple.
@@ -435,5 +466,5 @@
if CompiledStaticTuple.available():
# We will be using the C version
return
- self.assertIs(static_tuple.StaticTuple,
+ self.assertIs(static_tuple.StaticTuple,
self.module.StaticTuple)
=== modified file 'bzrlib/tests/test_transform.py'
--- a/bzrlib/tests/test_transform.py 2009-10-02 05:43:41 +0000
+++ b/bzrlib/tests/test_transform.py 2009-10-16 15:25:24 +0000
@@ -369,6 +369,18 @@
self.assertContainsRe(transform._limbo_name(first), 'new-1/file')
self.assertNotContainsRe(transform._limbo_name(second), 'new-1/FiLe')
+ def test_adjust_path_updates_child_limbo_names(self):
+ tree = self.make_branch_and_tree('tree')
+ transform = TreeTransform(tree)
+ self.addCleanup(transform.finalize)
+ foo_id = transform.new_directory('foo', transform.root)
+ bar_id = transform.new_directory('bar', foo_id)
+ baz_id = transform.new_directory('baz', bar_id)
+ qux_id = transform.new_directory('qux', baz_id)
+ transform.adjust_path('quxx', foo_id, bar_id)
+ self.assertStartsWith(transform._limbo_name(qux_id),
+ transform._limbo_name(bar_id))
+
def test_add_del(self):
start, root = self.get_transform()
start.new_directory('a', root, 'a')
=== modified file 'bzrlib/tests/test_ui.py'
--- a/bzrlib/tests/test_ui.py 2009-10-14 08:51:44 +0000
+++ b/bzrlib/tests/test_ui.py 2009-10-15 20:04:37 +0000
@@ -42,7 +42,6 @@
)
from bzrlib.ui import (
CannedInputUIFactory,
- CLIUIFactory,
SilentUIFactory,
UIFactory,
make_ui_for_terminal,
@@ -307,15 +306,6 @@
'TERM=%r' % (term_type,))
-class CLIUITests(TestCase):
-
- def test_cli_factory_deprecated(self):
- uif = self.applyDeprecated(deprecated_in((1, 18, 0)),
- CLIUIFactory,
- StringIO(), StringIO(), StringIO())
- self.assertIsInstance(uif, UIFactory)
-
-
class SilentUITests(TestCase):
def test_silent_factory_get_password(self):
=== modified file 'bzrlib/transform.py'
--- a/bzrlib/transform.py 2009-10-02 05:43:41 +0000
+++ b/bzrlib/transform.py 2009-10-16 15:29:50 +0000
@@ -1122,6 +1122,17 @@
continue
new_path = self._limbo_name(trans_id)
os.rename(old_path, new_path)
+ for descendant in self._limbo_descendants(trans_id):
+ desc_path = self._limbo_files[descendant]
+ desc_path = new_path + desc_path[len(old_path):]
+ self._limbo_files[descendant] = desc_path
+
+ def _limbo_descendants(self, trans_id):
+ """Return the set of trans_ids whose limbo paths descend from this."""
+ descendants = set(self._limbo_children.get(trans_id, []))
+ for descendant in list(descendants):
+ descendants.update(self._limbo_descendants(descendant))
+ return descendants
def create_file(self, contents, trans_id, mode_id=None):
"""Schedule creation of a new file.
=== modified file 'bzrlib/ui/__init__.py'
--- a/bzrlib/ui/__init__.py 2009-09-23 06:29:46 +0000
+++ b/bzrlib/ui/__init__.py 2009-10-15 20:04:37 +0000
@@ -226,95 +226,6 @@
-class CLIUIFactory(UIFactory):
- """Deprecated in favor of TextUIFactory."""
-
- @deprecated_method(deprecated_in((1, 18, 0)))
- def __init__(self, stdin=None, stdout=None, stderr=None):
- UIFactory.__init__(self)
- self.stdin = stdin or sys.stdin
- self.stdout = stdout or sys.stdout
- self.stderr = stderr or sys.stderr
-
- _accepted_boolean_strings = dict(y=True, n=False, yes=True, no=False)
-
- def get_boolean(self, prompt):
- while True:
- self.prompt(prompt + "? [y/n]: ")
- line = self.stdin.readline()
- line = line.rstrip('\n')
- val = bool_from_string(line, self._accepted_boolean_strings)
- if val is not None:
- return val
-
- def get_non_echoed_password(self):
- isatty = getattr(self.stdin, 'isatty', None)
- if isatty is not None and isatty():
- # getpass() ensure the password is not echoed and other
- # cross-platform niceties
- password = getpass.getpass('')
- else:
- # echo doesn't make sense without a terminal
- password = self.stdin.readline()
- if not password:
- password = None
- elif password[-1] == '\n':
- password = password[:-1]
- return password
-
- def get_password(self, prompt='', **kwargs):
- """Prompt the user for a password.
-
- :param prompt: The prompt to present the user
- :param kwargs: Arguments which will be expanded into the prompt.
- This lets front ends display different things if
- they so choose.
- :return: The password string, return None if the user
- canceled the request.
- """
- prompt += ': '
- self.prompt(prompt, **kwargs)
- # There's currently no way to say 'i decline to enter a password'
- # as opposed to 'my password is empty' -- does it matter?
- return self.get_non_echoed_password()
-
- def get_username(self, prompt, **kwargs):
- """Prompt the user for a username.
-
- :param prompt: The prompt to present the user
- :param kwargs: Arguments which will be expanded into the prompt.
- This lets front ends display different things if
- they so choose.
- :return: The username string, return None if the user
- canceled the request.
- """
- prompt += ': '
- self.prompt(prompt, **kwargs)
- username = self.stdin.readline()
- if not username:
- username = None
- elif username[-1] == '\n':
- username = username[:-1]
- return username
-
- def prompt(self, prompt, **kwargs):
- """Emit prompt on the CLI.
-
- :param kwargs: Dictionary of arguments to insert into the prompt,
- to allow UIs to reformat the prompt.
- """
- if kwargs:
- # See <https://launchpad.net/bugs/365891>
- prompt = prompt % kwargs
- prompt = prompt.encode(osutils.get_terminal_encoding(), 'replace')
- self.clear_term()
- self.stderr.write(prompt)
-
- def note(self, msg):
- """Write an already-formatted message."""
- self.stdout.write(msg + '\n')
-
-
class SilentUIFactory(UIFactory):
"""A UI Factory which never prints anything.
@@ -368,13 +279,6 @@
% (self,))
- at deprecated_function(deprecated_in((1, 18, 0)))
-def clear_decorator(func, *args, **kwargs):
- """Decorator that clears the term"""
- ui_factory.clear_term()
- func(*args, **kwargs)
-
-
ui_factory = SilentUIFactory()
# IMPORTANT: never import this symbol directly. ONLY ever access it as
# ui.ui_factory, so that you refer to the current value.
=== modified file 'setup.py'
--- a/setup.py 2009-10-12 20:12:32 +0000
+++ b/setup.py 2009-10-15 21:28:14 +0000
@@ -335,9 +335,6 @@
# Ensure tbzrlib itself is on sys.path
sys.path.append(tbzr_root)
- # Ensure our COM "entry-point" is on sys.path
- sys.path.append(os.path.join(tbzr_root, "shellext", "python"))
-
packages.append("tbzrlib")
# collect up our icons.
@@ -365,17 +362,6 @@
excludes.extend("""pywin pywin.dialogs pywin.dialogs.list
win32ui crawler.Crawler""".split())
- # NOTE: We still create a DLL version of the Python implemented shell
- # extension for testing purposes - but it is *not* registered by
- # default - our C++ one is instead. To discourage people thinking
- # this DLL is still necessary, its called 'tbzr_old.dll'
- tbzr = dict(
- modules=["tbzr"],
- create_exe = False, # we only want a .dll
- dest_base = 'tbzr_old',
- )
- com_targets.append(tbzr)
-
# tbzrcache executables - a "console" version for debugging and a
# GUI version that is generally used.
tbzrcache = dict(
@@ -406,8 +392,7 @@
console_targets.append(tracer)
# The C++ implemented shell extensions.
- dist_dir = os.path.join(tbzr_root, "shellext", "cpp", "tbzrshellext",
- "build", "dist")
+ dist_dir = os.path.join(tbzr_root, "shellext", "build")
data_files.append(('', [os.path.join(dist_dir, 'tbzrshellext_x86.dll')]))
data_files.append(('', [os.path.join(dist_dir, 'tbzrshellext_x64.dll')]))
@@ -644,7 +629,6 @@
'tools/win32/bzr_postinstall.py',
]
gui_targets = []
- com_targets = []
data_files = topics_files + plugins_files
if 'qbzr' in plugins:
@@ -695,7 +679,6 @@
setup(options=options_list,
console=console_targets,
windows=gui_targets,
- com_server=com_targets,
zipfile='lib/library.zip',
data_files=data_files,
cmdclass={'install_data': install_data_with_bytecompile},
More information about the bazaar-commits
mailing list