[MERGE] A hack to make urllib not call recv(1) lots and lots.

[Andrew Bennetts]

Andrew Bennetts wrote:
[...]
> I've used your recipe, and I've just repeated my experiments from last night.
> I also followed Robert's suggestion, and bumped the buffering up to 64kB
> (updated patch attached for completeness).

Actually attached this time.

-Andrew.

-------------- next part --------------
# Bazaar merge directive format 2 (Bazaar 0.90)
# revision_id: andrew.bennetts at canonical.com-20080317171649-\
#   n5kszez1vltj8rr1
# target_branch: http://bazaar-vcs.org/bzr/bzr.dev
# testament_sha1: 825984682f5e2189cbf32613f1d5ac82728bc7e2
# timestamp: 2008-03-17 12:17:04 -0500
# source_branch: http://people.ubuntu.com/~andrew/bzr/urllib-recv-1-\
#   hack
# base_revision_id: pqm at pqm.ubuntu.com-20080316165803-tisoc9mpob9z544o
# 
# Begin patch
=== modified file 'bzrlib/transport/http/_urllib2_wrappers.py'
--- bzrlib/transport/http/_urllib2_wrappers.py	2008-01-03 16:26:32 +0000
+++ bzrlib/transport/http/_urllib2_wrappers.py	2008-03-17 17:16:49 +0000
@@ -81,6 +81,18 @@
     # 8k chunks should be fine.
     _discarded_buf_size = 8192
 
+    def __init__(self, *args, **kwargs):
+        httplib.HTTPResponse.__init__(self, *args, **kwargs)
+        try:
+            # httplib creates a fileobject that doesn't do buffering, which
+            # makes fp.readline() very expensive because it only reads one byte
+            # at a time.  So we change the _rbufsize attribute here to fix
+            # this.
+            self.fp._rbufsize = 65536
+        except AttributeError:
+            # If self.fp is a cStringIO, it doesn't have a _rbufsize.
+            pass
+
     def begin(self):
         """Begin to read the response from the server.
 
@@ -534,7 +546,7 @@
             req = request
             r = response
             r.recv = r.read
-            fp = socket._fileobject(r)
+            fp = socket._fileobject(r, bufsize=65536)
             resp = urllib2.addinfourl(fp, r.msg, req.get_full_url())
             resp.code = r.status
             resp.msg = r.reason

# Begin bundle
IyBCYXphYXIgcmV2aXNpb24gYnVuZGxlIHY0CiMKQlpoOTFBWSZTWSGpWssABDzfgGRQWf///3//
/9C////wYAhvjrIAAAggUJJFAEhJKAQTaGp6mjamk/QTJjSaMpp6ZIZME0aGT0QcZGmTE0GTJhNM
gZDQGgNMmhgBNAYaoNJ+qNMgAZPUAAAAaDQAAAAcZGmTE0GTJhNMgZDQGgNMmhgBNAY4yNMmJoMm
TCaZAyGgNAaZNDACaAwVSEBMmgARqZDTTQU2U2pmTJqaaD1M0R6jaT1H4wTIX83TdGLdq6WRjcm8
/VRImIRidxu5vH2gOJm79mKv98kWkkVJO5wn2NuUqVQshCnbu3U3K1JL+232M8bpb5T5IQuRG7kx
Wdp1nwaUkSlKXy5v0iPk+di9VsbaVNu+itU+fTYlRfKyXHqfsr6mh3fYfgJjQED9leiJw7RIKmFM
4SIkmR1TQ+mChGHqUHAWxqOVvBN2VeXmsz3XMWjPjzrY2fOjZIksc37Ki6rTjtrKbZfYs2I7s+R7
qMBfJTESvV2fi5qHHXqtpCeewkg8kHQjIXycte7qmNpHm0l7NX2O0g6jP2efJAcJiOjfx6wQdwtS
PGcXnel3GHVBHRHlOLrThCt7npcTwb5WeKV7yN9l1vY0vXEQ6XcShDnFm+9Fu0758l+i7dwmnOf9
kN2w25kRuSk8TkjBJCJ+NROIhKQ86ril9S1F1jBXURE4iJ3lr+XDOYRj4LWOZ7coR6dL7Xb4YldB
VSU4lJKikeebqU4PM37ZL+1s4b9G/Mb2k92dYS328aJ8mZjxQfWLkSEyM25aq2zWQjazqqEzCsra
bTQnoY+Sz6CjXbvZp1bWkFjDPbrfS4q4LGtUy1FjNBf7tY5rF66V7Nqafmz46XKwpU0JwjYqQi4v
sYWknJczJFJVMnR9+d05vT2/HBUw0ZK69l6yxlCqj9dGzWs1tKu9S45ZSrW6JdT7SfiPqmviWsZH
OpJ01Zj97VNfQS1MI6Md6mbdbCNRobK68HgKorZ3Nc1NKmaUblhciiajJasasyTzCuboTvRWwhWl
CNRLTRXysZs6+4ZXy9tDY1PVfm464f+7e85L2ti7Fz1QjbjsylaGnp54KjJk6Z62hjF99/j0iwkx
VLWYz8jKK9CwnBGtwa0uFhcosYNmZdgwSYJkmauzOT/q/PpZZXH0EojOaIRMSrryc01RkxTWVWVQ
jQwmm0YKqFyzYZjgfj/D1YvC+5vYVSSJajKHIUNpqgrl9x4ntPEkR8Ejs+XQdD3NOYom3JtcbZJS
VUTiTM8T9Tg96itMuSTUTiIYe80/W/d0fZ+R/T8cEt3o1xoMSuJOLyDDlOuIsexE8VjUXjHGeQxb
jhMwdLWDZ7FOtGQ7n1O0sBhF2jwpT5f0nKOOUoOd8VsL9OONp/B3BkICPbmlCqEQxzMcPc/sPB8X
R3m4LZ7x5u34cuMs72kw8AobhWAhMtt2vX08QUrsR0ykIVealgK9I134NPBLY/BsMkrGc0oeGfHU
Dw+PFFGuIZjUZhigUQfTW0QiTFte9pSbF78FS46vD8IxWv52Rj8JNp0wcmMRDplxOBfY3fq4xPiT
+BKKUidl8RD1lyPfGpEb36PEpqxzTNl9WpjZ6Opthq16XOKke5PvMsHxObe7neUX2MmvsiIUqXWL
49epduJPOanxpoQ8jtdkI3MmtJUyOy/6+KtniIVfOIhx4vYlvbNl3UlfgmkydEQiqLIRhEW9O/od
Lq38NXBRfDWvHJQsgqp6SF0GsqBAnOyXkkxkl1xosU5DP13s8NLFBcgpM0zR30vByqmnrorVtnNz
dXVhqamok2Gq3akkl+k7lcH5OW6Nmj4xSPDLhbNxh5lsRDr1q3eRIjKM2rLDZqUz14I+iQws73Yd
L89JWkrcoIy1RCH7dlo1DNCL3TH9/PvhwnJ1O5Oj7Hh+7/3yXeRS5bdzRyMR6GY3lsGmZL1OdONo
iF0BAv9M4aEK9xIUIRmVCvB7mehZS044IuBIxxvM7ockZ3S5Cah7HUQoiYFtCbw+XT3RCTjEQ9M5
tTrQzHnNLj4YKoaumDHRizP9dbsVrmJCWn0IMSjU1rNEhlYPGoZEKYMOTpdKmPi1mEfNu+gl5QKg
dRZ5KSSMcoFDITwd7I0I+MSZhYKReQYAtnnmIQugXYTOkj3CpovVw0Yo/B41Dv99mwWvSfeeq6tX
wTO0SR0SGg9CyLYNScRCukmyCJ3JKqoiFFE+iCKoiF1KRCqRHE/VQebp8z2ebAt6zNGsU0BwtTJE
J/EhKsCLEpupIcq1BFdBYSoHnSLNvW/K2K0o7yrNHfd91D2WH450YjBUbKuhHP7j2YRnZyTn9vkR
6vf7s4+VuUvY40cxU3Sfn6vbVBnexHP7S7i11eKXnaLWhEcGRY4JIo7oIrMepdOeeDTETq3jb9g7
LhakSlCRbJ1zd1U5Y+QqqvRWmSSiIf8laW5+7m1+N4diRvbStNoRoSIv4FiTIcoRqWF70UxkaJGd
DqFWvjPf9J4alJ6mUYODyrixY8piflpoS2LdiLyUTlo0E4dZWJs0GCLkfxn+FF6hOIhfMZPqMeGu
CKyjeWnptF5L/tr4vGeCs/x5Yj1mXfHQlqm9tLwpPGErk6vxHnPFbhE/GINAEhx5ab6BVkLPbr3C
odkgo8WEtBe3PJCZBqvp8xbonU6MRwB1nKM8HUKoRb7Zv/i7kinChIENStZY