Rev 21: Handle compression parent fix even if the plugin has already been in file:///net/bigmamac/Volumes/home/vila/.bazaar/plugins/fix277537/
Vincent Ladeuil
v.ladeuil+lp at free.fr
Mon Feb 2 16:58:24 GMT 2009
At file:///net/bigmamac/Volumes/home/vila/.bazaar/plugins/fix277537/
------------------------------------------------------------
revno: 21
revision-id: v.ladeuil+lp at free.fr-20090202165822-z7odfgkrfyc3y80b
parent: john at arbash-meinel.com-20090130204404-n0blm8d9mogaade9
committer: Vincent Ladeuil <v.ladeuil+lp at free.fr>
branch nick: fix277537
timestamp: Mon 2009-02-02 17:58:22 +0100
message:
Handle compression parent fix even if the plugin has already been
applied.
* tests/test_fix277537.py:
(TestCommand.assertFixedParents): Update to take compression
parents fixed into account.
(TestCommand.test_no_parents_fixed,
TestCommand.test_parents_fixed_but_not_twice): Update
assertFixedParents uses.
* reconcile.py:
(InventoryAncestryReconcilePacker.__init__): Add
compression_parent_fixed attribute.
(InventoryAncestryReconcilePacker._copy_text_texts): Separate th
fixes themselves from the part that decide which shold be
applied. Fix compression parent even if it isn't mentioned in the
inconsistent parents description. Change fixed_nodes description
to include only the desired parents (the wrong ones weren't
strictly needed). Delete surprising reference to knit._counter.
(InventoryAncestryReconcilePacker._copy_fixed_nodes): Fix bug when
there is a single remaining node to start a new batch. Update
fixed_nodes uses.
(InventoryAncestryReconciler.__init__): Add
compression_parent_fixed attribute.
(InventoryAncestryReconciler._reconcile_steps): Forward
compression_parent_fixed.
* __init__.py:
(cmd_fix277537.run): Also report compression parents fixed.
-------------- next part --------------
=== modified file '__init__.py'
--- a/__init__.py 2008-11-27 07:31:30 +0000
+++ b/__init__.py 2009-02-02 16:58:22 +0000
@@ -54,6 +54,8 @@
reconciler.reconcile()
self.outf.write('Fixed %d wrong parents\n'
% reconciler.wrong_parents_fixed)
+ self.outf.write('Fixed %d compression parents\n'
+ % reconciler.compression_parent_fixed)
self.outf.write('%d wrong parents were already fixed\n'
% reconciler.wrong_parents_already_fixed)
if reconciler.poisoned_texts:
=== modified file 'reconcile.py'
--- a/reconcile.py 2009-01-30 20:44:04 +0000
+++ b/reconcile.py 2009-02-02 16:58:22 +0000
@@ -47,6 +47,7 @@
pack_collection, packs, suffix, revision_ids=revision_ids)
self._file_rev_ids = file_rev_ids
self.wrong_parents_fixed = 0
+ self.compression_parent_fixed = 0
self.wrong_parents_already_fixed = 0
self.poisoned_texts = 0
@@ -77,46 +78,63 @@
# Text for a poisoned rev_id, discard
self.poisoned_texts += 1
continue
+ actual_parents = node[3][0]
+ compression_parent = node[3][1]
+ # A delta must compressed against its left-hand (first) parent
+ delta_with_wrong_compression_parent = (
+ compression_parent
+ and compression_parent[0] != actual_parents[0])
+
ideal_parents = ideal_index.get(key, None)
+ fix_parents = False
+
if ideal_parents is None:
- # No parent fix needed.
- ok_nodes.append(node)
+ if delta_with_wrong_compression_parent:
+ # fixed_nodes needs some parents, give it some
+ ideal_parents = actual_parents
else:
ideal_parents = tuple(ideal_parents)
- actual_parents = node[3][0]
if ideal_parents == actual_parents:
- # that one has already been fixed
+ # that one has its parents already fixed, but may be that
+ # made its compression parent differs from its left-hand
+ # (first) parent.
self.wrong_parents_already_fixed += 1
- ok_nodes.append(node)
else:
wrong_parents = wrong_index[key]
- if actual_parents == wrong_parents:
- if not node[3][1] or node[3][1] == (ideal_parents[0],):
- # There is no delta compression, or it is already
- # pointing at the ideal_parent, so we can just copy
- # the data across as-is, and just update the index
- # appropriately.
- # Preserve the representation but fix the parents
- ok_nodes.append((node[0], node[1], node[2],
- (ideal_parents, node[3][1])))
- elif node[3][1] == (wrong_parents[0],):
- # We have a compression parent pointing at a
- # left-hand parent which we will be changing.
- # We will copy this as a fulltext
- # Reinsert this text completely
- fixed_nodes[node[1]] = (wrong_parents,
- ideal_parents)
- else:
- assert False, "Node layout does not conform to"\
- " expections, aborting"
- self._data_changed = True
- self.wrong_parents_fixed += 1
- else:
+ if actual_parents != wrong_parents:
# Bad input, we didn't find the parents we expected
raise errors.BzrError(
'Mismatched between expected and actual parents:\n'
'key: %r, expected: %r, got %r' %
(key, wrong_parents, actual_parents))
+ fix_parents = True
+ delta_with_wrong_compression_parent = (
+ compression_parent
+ and compression_parent[0] != ideal_parents[0])
+ self.wrong_parents_fixed += 1
+
+ if delta_with_wrong_compression_parent:
+ # We have a compression parent pointing at a left-hand parent
+ # This is either because we just changed the parents or we did
+ # it as part of a previous run of this plugin
+
+ # Reinsert completely as full text
+ fixed_nodes[key] = ideal_parents
+ self._data_changed = True
+ self.compression_parent_fixed += 1
+ elif fix_parents:
+ # There is no delta compression (i.e. it's a full text so it
+ # has no compression parent), or fixing the parents will make
+ # it point at the ideal_parent, so we can just copy the data
+ # across as-is, and just update the index appropriately.
+
+ # Preserve the representation but fix the parents
+ ok_nodes.append((node[0], node[1], node[2],
+ (ideal_parents, node[3][1])))
+ self._data_changed = True
+ else:
+ # All is well
+ ok_nodes.append(node)
# we're finished with some data.
del text_nodes
del wrong_index
@@ -127,7 +145,6 @@
self.new_pack.text_index, readv_group_iter, total_items))
# 4) Now copy the fixed nodes
self._copy_fixed_nodes(fixed_nodes)
- trace.note('Counter: %s', knit._counter)
self._log_copied_texts()
def _copy_fixed_nodes(self, fixed_nodes):
@@ -140,8 +157,8 @@
# This is the revision_id parent map, note that we use revision_ids,
# not keys in this map
parent_map = dict([(key[0], tuple(ref[0] for ref in refs[0])) for
- _1, key, _2, refs in
- self.new_pack.revision_index.iter_all_entries()])
+ _1, key, _2, refs in
+ self.new_pack.revision_index.iter_all_entries()])
# Copy the "fixed_nodes" by using get_record_stream() to extract
# them back to fulltexts, and insert them into becoming new deltas
repo = self._pack_collection.repo
@@ -165,6 +182,7 @@
def grouped_and_topo_order(key):
# key == (file_id, revision_id)
return (key[0], revid_to_offset[key[1]])
+
# We use reverse sorted order, because we will pop nodes off the
# end
remaining_nodes = sorted(fixed_nodes.iterkeys(),
@@ -182,18 +200,15 @@
or (len(next_batch) < 100 and file_id == next_key[0])):
next_batch.append(next_key)
file_id = next_key[0]
- continue
+ if remaining_nodes:
+ continue
this_batch = next_batch
next_batch = [next_key]
file_id = next_key[0]
chunks = {}
- trace.mutter('copying batch %s len=%s, counter=%s', count,
- len(this_batch), knit._counter)
for record in repo.texts.get_record_stream(this_batch,
'unordered', True):
chunks[record.key] = record.get_bytes_as('chunked')
- wrong_parents, ideal_parents = fixed_nodes[record.key]
- assert record.parents == wrong_parents
for key in this_batch:
if parent_file_id is None or parent_file_id != key[0]:
@@ -206,7 +221,7 @@
# the just-written data
self.new_pack.flush()
pb.update('Copying fulltext', count, num_texts)
- wrong_parents, ideal_parents = fixed_nodes[key]
+ ideal_parents = fixed_nodes[key]
lines = osutils.chunks_to_lines(chunks.pop(key))
try:
_, _, content = output_texts.add_lines(key,
@@ -214,6 +229,9 @@
check_content=False,
parent_texts=parent_texts)
except KeyError, e:
+ # This should never happen. But if it happens, we'd
+ # better call pdb to have some clue about what went
+ # wrong.
import pdb; pdb.set_trace()
parent_texts[key] = content
parent_file_id = key[0]
@@ -230,6 +248,7 @@
repo, other=other, thorough=thorough)
self._file_rev_ids = file_rev_ids
self.wrong_parents_fixed = 0
+ self.compression_parent_fixed = 0
self.wrong_parents_already_fixed = 0
self.poisoned_texts = 0
@@ -253,6 +272,8 @@
if new_pack is not None:
self._discard_and_save(packs)
self.wrong_parents_fixed = self._packer.wrong_parents_fixed
+ self.compression_parent_fixed = \
+ self._packer.compression_parent_fixed
self.wrong_parents_already_fixed = \
self._packer.wrong_parents_already_fixed
self.poisoned_texts = self._packer.poisoned_texts
=== modified file 'tests/test_fix277537.py'
--- a/tests/test_fix277537.py 2008-10-15 15:53:37 +0000
+++ b/tests/test_fix277537.py 2009-02-02 16:58:22 +0000
@@ -202,6 +202,7 @@
self.assertEquals(0, reconciler.wrong_parents_fixed)
self.assertEquals(1, reconciler.wrong_parents_already_fixed)
+
class TestFileParser(tests.TestCase):
def test_empty_file(self):
@@ -308,12 +309,13 @@
f.close()
return fname
- def assertFixedParents(self, out, tried, fixed, already_fixed=0):
+ def assertFixedParents(self, out, tried, fixed, comp, already_fixed=0):
self.assertContainsRe(out,
'Will try to fix %d wrong parents\n'
'Fixed %d wrong parents\n'
+ 'Fixed %d compression parents\n'
'%d wrong parents were already fixed\n'
- % (tried, fixed, already_fixed))
+ % (tried, fixed, comp, already_fixed))
def test_unknown_file(self):
self.run_bzr(['fix277537', 'I-do-not-exist'], retcode=3)
@@ -335,13 +337,13 @@
''')
out, err = self.run_bzr(['fix277537', fname,
self.repo_relpath])
- self.assertFixedParents(out, 1, 0, 0)
+ self.assertFixedParents(out, 1, 0, 0, 0)
def test_parents_fixed_but_not_twice(self):
repo = self.make_populated_repository()
fname = self.create_file_rev_ids_file()
out, err = self.run_bzr(['fix277537', fname, self.repo_relpath])
- self.assertFixedParents(out, 1, 1, 0)
+ self.assertFixedParents(out, 1, 1, 1, 0)
out, err = self.run_bzr(['fix277537', fname, self.repo_relpath])
- self.assertFixedParents(out, 1, 0, 1)
+ self.assertFixedParents(out, 1, 0, 0, 1)
More information about the bazaar-commits
mailing list