Rev 3028: Make sure we allow _coalesce_offsets to combine more ranges. in http://bzr.arbash-meinel.com/branches/bzr/0.93-dev/extra_range_collapse_165061

John Arbash Meinel john at arbash-meinel.com
Tue Nov 27 12:37:01 GMT 2007


At http://bzr.arbash-meinel.com/branches/bzr/0.93-dev/extra_range_collapse_165061

------------------------------------------------------------
revno: 3028
revision-id:john at arbash-meinel.com-20071127123612-uyqqe8wwkg0va7d6
parent: john at arbash-meinel.com-20071126213640-2asaghs4qr1zs3vg
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: extra_range_collapse_165061
timestamp: Tue 2007-11-27 06:36:12 -0600
message:
  Make sure we allow _coalesce_offsets to combine more ranges.
  We had max_combine set that it was forcing us not to combine, which would
  cause us to endlessly loop.
  Change it so we do a maximum number of passes. As well as increasing the
  number we allow to combine.
modified:
  bzrlib/tests/test_http.py      testhttp.py-20051018020158-b2eef6e867c514d9
  bzrlib/transport/http/__init__.py http_transport.py-20050711212304-506c5fd1059ace96
-------------- next part --------------
=== modified file 'bzrlib/tests/test_http.py'
--- a/bzrlib/tests/test_http.py	2007-11-26 21:36:40 +0000
+++ b/bzrlib/tests/test_http.py	2007-11-27 12:36:12 +0000
@@ -670,6 +670,34 @@
                           _CoalescedOffset(29, 1, [(0, 1)]),
                          ], t._get_coalesced_offsets(offsets))
 
+    def test_get_coalesced_offsets_max_combined(self):
+        server = self.get_readonly_server()
+        t = self._transport(server.get_url())
+        # Initially, these parameters should be somewhat incompatible, we want
+        # only 2 final readv ranges, but we want to limit how much we combine
+        # _get_coalesced_offsets should figure out how to change these so it
+        # can get its job done.
+        t._bytes_to_read_before_seek = 1
+        t._max_readv_ranges = 2
+        t._max_readv_combine = 2
+        offsets = ((0, 1), (1, 1), (5, 2), (29, 1), (30, 1), (35, 1))
+        self.assertEqual([_CoalescedOffset(0, 7, [(0, 1), (1, 1), (5, 2)]),
+                          _CoalescedOffset(29, 7, [(0, 1), (1, 1), (6, 1)]),
+                         ], t._get_coalesced_offsets(offsets))
+
+    def test_get_coalesced_impossible(self):
+        server = self.get_readonly_server()
+        t = self._transport(server.get_url())
+        # These cannot be satisfied, but the loop should stop rather than
+        # endlessly looping
+        t._bytes_to_read_before_seek = 0
+        t._max_readv_ranges = 1
+        t._max_readv_combine = 1
+        t._max_coalesce_attempts = 2
+        offsets = ((0, 1), (1, 1), (5, 2), (29, 1), (30, 1), (35, 1))
+        # We don't care what it returns, as long as it does finish
+        t._get_coalesced_offsets(offsets)
+
 
 class TestSingleRangeRequestServer(TestRangeRequestServer):
     """Test readv against a server which accept only single range requests"""

=== modified file 'bzrlib/transport/http/__init__.py'
--- a/bzrlib/transport/http/__init__.py	2007-11-26 21:36:40 +0000
+++ b/bzrlib/transport/http/__init__.py	2007-11-27 12:36:12 +0000
@@ -257,20 +257,34 @@
     # amount to be safe.
     _max_readv_ranges = 200
 
+    # We double each time, so 2^32 = 4 Billion , or big enough that if we
+    # haven't gotten it yet, we won't get it.
+    _max_coalesce_attempts = 32
+
     def _get_coalesced_offsets(self, offsets):
         sorted_offsets = sorted(list(offsets))
         fudge = self._bytes_to_read_before_seek
+        max_combine = self._max_readv_combine
         coalesced = list(self._coalesce_offsets(sorted_offsets,
-                                                limit=self._max_readv_combine,
+                                                limit=max_combine,
                                                 fudge_factor=fudge))
+        count = 0
         while len(coalesced) > self._max_readv_ranges:
+            count += 1
+            if count > self._max_coalesce_attempts:
+                break
             new_fudge = fudge * 2 + 1
+            # If it is at 0 it is already "max", so no need to +1
+            new_max_combine = max_combine * 2
             mutter('http would request too many ranges (%d > %d),'
-                   'increasing fudge factor from %d to %d',
-                   len(coalesced), self._max_readv_ranges, fudge, new_fudge)
+                   'increasing fudge factor from %d to %d'
+                   'and max combine from %d to %d',
+                   len(coalesced), self._max_readv_ranges, fudge, new_fudge,
+                   max_combine, new_max_combine)
             fudge = new_fudge
+            max_combine = new_max_combine
             coalesced = list(self._coalesce_offsets(sorted_offsets,
-                            limit=self._max_readv_combine,
+                            limit=max_combine,
                             fudge_factor=fudge))
         return coalesced
 



More information about the bazaar-commits mailing list