Rev 3028: Make sure we allow _coalesce_offsets to combine more ranges. in http://bzr.arbash-meinel.com/branches/bzr/0.93-dev/extra_range_collapse_165061
John Arbash Meinel
john at arbash-meinel.com
Tue Nov 27 12:37:01 GMT 2007
At http://bzr.arbash-meinel.com/branches/bzr/0.93-dev/extra_range_collapse_165061
------------------------------------------------------------
revno: 3028
revision-id:john at arbash-meinel.com-20071127123612-uyqqe8wwkg0va7d6
parent: john at arbash-meinel.com-20071126213640-2asaghs4qr1zs3vg
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: extra_range_collapse_165061
timestamp: Tue 2007-11-27 06:36:12 -0600
message:
Make sure we allow _coalesce_offsets to combine more ranges.
We had max_combine set that it was forcing us not to combine, which would
cause us to endlessly loop.
Change it so we do a maximum number of passes. As well as increasing the
number we allow to combine.
modified:
bzrlib/tests/test_http.py testhttp.py-20051018020158-b2eef6e867c514d9
bzrlib/transport/http/__init__.py http_transport.py-20050711212304-506c5fd1059ace96
-------------- next part --------------
=== modified file 'bzrlib/tests/test_http.py'
--- a/bzrlib/tests/test_http.py 2007-11-26 21:36:40 +0000
+++ b/bzrlib/tests/test_http.py 2007-11-27 12:36:12 +0000
@@ -670,6 +670,34 @@
_CoalescedOffset(29, 1, [(0, 1)]),
], t._get_coalesced_offsets(offsets))
+ def test_get_coalesced_offsets_max_combined(self):
+ server = self.get_readonly_server()
+ t = self._transport(server.get_url())
+ # Initially, these parameters should be somewhat incompatible, we want
+ # only 2 final readv ranges, but we want to limit how much we combine
+ # _get_coalesced_offsets should figure out how to change these so it
+ # can get its job done.
+ t._bytes_to_read_before_seek = 1
+ t._max_readv_ranges = 2
+ t._max_readv_combine = 2
+ offsets = ((0, 1), (1, 1), (5, 2), (29, 1), (30, 1), (35, 1))
+ self.assertEqual([_CoalescedOffset(0, 7, [(0, 1), (1, 1), (5, 2)]),
+ _CoalescedOffset(29, 7, [(0, 1), (1, 1), (6, 1)]),
+ ], t._get_coalesced_offsets(offsets))
+
+ def test_get_coalesced_impossible(self):
+ server = self.get_readonly_server()
+ t = self._transport(server.get_url())
+ # These cannot be satisfied, but the loop should stop rather than
+ # endlessly looping
+ t._bytes_to_read_before_seek = 0
+ t._max_readv_ranges = 1
+ t._max_readv_combine = 1
+ t._max_coalesce_attempts = 2
+ offsets = ((0, 1), (1, 1), (5, 2), (29, 1), (30, 1), (35, 1))
+ # We don't care what it returns, as long as it does finish
+ t._get_coalesced_offsets(offsets)
+
class TestSingleRangeRequestServer(TestRangeRequestServer):
"""Test readv against a server which accept only single range requests"""
=== modified file 'bzrlib/transport/http/__init__.py'
--- a/bzrlib/transport/http/__init__.py 2007-11-26 21:36:40 +0000
+++ b/bzrlib/transport/http/__init__.py 2007-11-27 12:36:12 +0000
@@ -257,20 +257,34 @@
# amount to be safe.
_max_readv_ranges = 200
+ # We double each time, so 2^32 = 4 Billion , or big enough that if we
+ # haven't gotten it yet, we won't get it.
+ _max_coalesce_attempts = 32
+
def _get_coalesced_offsets(self, offsets):
sorted_offsets = sorted(list(offsets))
fudge = self._bytes_to_read_before_seek
+ max_combine = self._max_readv_combine
coalesced = list(self._coalesce_offsets(sorted_offsets,
- limit=self._max_readv_combine,
+ limit=max_combine,
fudge_factor=fudge))
+ count = 0
while len(coalesced) > self._max_readv_ranges:
+ count += 1
+ if count > self._max_coalesce_attempts:
+ break
new_fudge = fudge * 2 + 1
+ # If it is at 0 it is already "max", so no need to +1
+ new_max_combine = max_combine * 2
mutter('http would request too many ranges (%d > %d),'
- 'increasing fudge factor from %d to %d',
- len(coalesced), self._max_readv_ranges, fudge, new_fudge)
+ 'increasing fudge factor from %d to %d'
+ 'and max combine from %d to %d',
+ len(coalesced), self._max_readv_ranges, fudge, new_fudge,
+ max_combine, new_max_combine)
fudge = new_fudge
+ max_combine = new_max_combine
coalesced = list(self._coalesce_offsets(sorted_offsets,
- limit=self._max_readv_combine,
+ limit=max_combine,
fudge_factor=fudge))
return coalesced
More information about the bazaar-commits
mailing list