Rev 3878: (jam) Fix bug #303538 by capping readv sections to 100MB in file:///home/pqm/archives/thelove/bzr/%2Btrunk/

[Canonical.com Patch Queue Manager]

At file:///home/pqm/archives/thelove/bzr/%2Btrunk/

------------------------------------------------------------
revno: 3878
revision-id: pqm at pqm.ubuntu.com-20081202015700-3mc9dola31w7h5h4
parent: pqm at pqm.ubuntu.com-20081201235052-tlg1kksgj0kuya56
parent: john at arbash-meinel.com-20081202012200-bihdf9fybjwk3ey0
committer: Canonical.com Patch Queue Manager <pqm at pqm.ubuntu.com>
branch nick: +trunk
timestamp: Tue 2008-12-02 01:57:00 +0000
message:
  (jam) Fix bug #303538 by capping readv sections to 100MB
modified:
  NEWS                           NEWS-20050323055033-4e00b5db738777ff
  bzrlib/tests/test_transport.py testtransport.py-20050718175618-e5cdb99f4555ddce
  bzrlib/transport/__init__.py   transport.py-20050711165921-4978aa7ce1285ad5
  bzrlib/transport/remote.py     ssh.py-20060608202016-c25gvf1ob7ypbus6-1
    ------------------------------------------------------------
    revno: 3876.1.2
    revision-id: john at arbash-meinel.com-20081202012200-bihdf9fybjwk3ey0
    parent: john at arbash-meinel.com-20081201232022-3vu5ekhfvnnzyujr
    committer: John Arbash Meinel <john at arbash-meinel.com>
    branch nick: 303538_max_readv
    timestamp: Mon 2008-12-01 19:22:00 -0600
    message:
      Add a test case that checks the 100MB limit.
    modified:
      bzrlib/tests/test_transport.py testtransport.py-20050718175618-e5cdb99f4555ddce
    ------------------------------------------------------------
    revno: 3876.1.1
    revision-id: john at arbash-meinel.com-20081201232022-3vu5ekhfvnnzyujr
    parent: pqm at pqm.ubuntu.com-20081201201721-zconkq0v7pow8nmw
    committer: John Arbash Meinel <john at arbash-meinel.com>
    branch nick: 303538_max_readv
    timestamp: Mon 2008-12-01 17:20:22 -0600
    message:
      Fix bug #303538 by capping the max per-hunk read size at 100MB.
    modified:
      NEWS                           NEWS-20050323055033-4e00b5db738777ff
      bzrlib/transport/__init__.py   transport.py-20050711165921-4978aa7ce1285ad5
      bzrlib/transport/remote.py     ssh.py-20060608202016-c25gvf1ob7ypbus6-1
=== modified file 'NEWS'
--- a/NEWS	2008-12-01 18:40:08 +0000
+++ b/NEWS	2008-12-01 23:20:22 +0000
@@ -23,6 +23,11 @@
     * Give proper error message for diff with non-existent dotted revno.
       (Marius Kruger, #301969)
 
+    * ``Transport.readv()`` defaults to not reading more than 100MB in a
+      single array. Further ``RemoteTransport.readv`` sets this to 5MB to
+      work better with how it splits its requests.
+      (John Arbash Meinel, #303538)
+
   DOCUMENTATION:
 
   API CHANGES:

=== modified file 'bzrlib/tests/test_transport.py'
--- a/bzrlib/tests/test_transport.py	2008-09-10 19:14:43 +0000
+++ b/bzrlib/tests/test_transport.py	2008-12-02 01:22:00 +0000
@@ -225,6 +225,16 @@
                    [(10, 10), (20, 10), (30, 50), (80, 100)],
                   )
 
+    def test_coalesce_default_limit(self):
+        # By default we use a 100MB max size.
+        ten_mb = 10*1024*1024
+        self.check([(0, 10*ten_mb, [(i*ten_mb, ten_mb) for i in range(10)]),
+                    (10*ten_mb, ten_mb, [(0, ten_mb)])],
+                   [(i*ten_mb, ten_mb) for i in range(11)])
+        self.check([(0, 11*ten_mb, [(i*ten_mb, ten_mb) for i in range(11)]),],
+                   [(i*ten_mb, ten_mb) for i in range(11)],
+                   max_size=1*1024*1024*1024)
+
 
 class TestMemoryTransport(TestCase):
 

=== modified file 'bzrlib/transport/__init__.py'
--- a/bzrlib/transport/__init__.py	2008-11-07 18:10:32 +0000
+++ b/bzrlib/transport/__init__.py	2008-12-01 23:20:22 +0000
@@ -797,6 +797,10 @@
         cur = _CoalescedOffset(None, None, [])
         coalesced_offsets = []
 
+        if max_size <= 0:
+            # 'unlimited', but we actually take this to mean 100MB buffer limit
+            max_size = 100*1024*1024
+
         for start, size in offsets:
             end = start + size
             if (last_end is not None

=== modified file 'bzrlib/transport/remote.py'
--- a/bzrlib/transport/remote.py	2008-11-21 00:23:54 +0000
+++ b/bzrlib/transport/remote.py	2008-12-01 23:20:22 +0000
@@ -323,7 +323,8 @@
         sorted_offsets = sorted(offsets)
         coalesced = list(self._coalesce_offsets(sorted_offsets,
                                limit=self._max_readv_combine,
-                               fudge_factor=self._bytes_to_read_before_seek))
+                               fudge_factor=self._bytes_to_read_before_seek,
+                               max_size=self._max_readv_bytes))
 
         # now that we've coallesced things, avoid making enormous requests
         requests = []