pycurl ready to land
Martin Pool
mbp at sourcefrog.net
Mon Mar 6 11:37:04 GMT 2006
I've updated the pycurl branch to integrate the previous review feedback
and to make it fall back to urllib if needed. Comments welcome.
--
Martin
-------------- next part --------------
=== added directory 'b/bzrlib/transport/http'
=== added file 'b/bzrlib/transport/http/_pycurl.py'
--- /dev/null
+++ b/bzrlib/transport/http/_pycurl.py
@@ -0,0 +1,126 @@
+# Copyright (C) 2006 Canonical Ltd
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+"""http/https transport using pycurl"""
+
+# TODO: test reporting of http errors
+
+from StringIO import StringIO
+
+import bzrlib
+from bzrlib.trace import mutter
+from bzrlib.errors import (TransportNotPossible, NoSuchFile,
+ TransportError, ConnectionError,
+ DependencyNotPresent)
+from bzrlib.transport import Transport
+from bzrlib.transport.http import HttpTransportBase, extract_auth, HttpServer
+
+try:
+ import pycurl
+except ImportError, e:
+ mutter("failed to import pycurl: %s", e)
+ raise DependencyNotPresent('pycurl', e)
+
+
+class PyCurlTransport(HttpTransportBase):
+ """http client transport using pycurl
+
+ PyCurl is a Python binding to the C "curl" multiprotocol client.
+
+ This transport can be significantly faster than the builtin Python client.
+ Advantages include: DNS caching, connection keepalive, and ability to
+ set headers to allow caching.
+ """
+
+ def __init__(self, base):
+ super(PyCurlTransport, self).__init__(base)
+ mutter('using pycurl %s' % pycurl.version)
+
+ def should_cache(self):
+ """Return True if the data pulled across should be cached locally.
+ """
+ return True
+
+ def has(self, relpath):
+ curl = pycurl.Curl()
+ abspath = self.abspath(relpath)
+ if isinstance(abspath, unicode):
+ abspath = abspath.encode('ascii', 'strict')
+ curl.setopt(pycurl.URL, abspath)
+ curl.setopt(pycurl.FOLLOWLOCATION, 1) # follow redirect responses
+ self._set_curl_options(curl)
+ # don't want the body - ie just do a HEAD request
+ curl.setopt(pycurl.NOBODY, 1)
+ self._curl_perform(curl)
+ code = curl.getinfo(pycurl.HTTP_CODE)
+ if code == 404: # not found
+ return False
+ elif code in (200, 302): # "ok", "found"
+ return True
+ else:
+ raise TransportError('http error %d probing for %s' %
+ (code, curl.getinfo(pycurl.EFFECTIVE_URL)))
+
+ def get(self, relpath):
+ curl = pycurl.Curl()
+ abspath = self.abspath(relpath)
+ sio = StringIO()
+ if isinstance(abspath, unicode):
+ abspath = abspath.encode('ascii')
+ curl.setopt(pycurl.URL, abspath)
+ self._set_curl_options(curl)
+ curl.setopt(pycurl.WRITEFUNCTION, sio.write)
+ curl.setopt(pycurl.NOBODY, 0)
+ self._curl_perform(curl)
+ code = curl.getinfo(pycurl.HTTP_CODE)
+ if code == 404:
+ raise NoSuchFile(abspath)
+ elif code == 200:
+ sio.seek(0)
+ return sio
+ else:
+ raise TransportError('http error %d acccessing %s' %
+ (code, curl.getinfo(pycurl.EFFECTIVE_URL)))
+
+ def _set_curl_options(self, curl):
+ """Set options for all requests"""
+ # There's no way in http/1.0 to say "must revalidate"; we don't want
+ # to force it to always retrieve. so just turn off the default Pragma
+ # provided by Curl.
+ headers = ['Cache-control: must-revalidate',
+ 'Pragma:']
+ ## curl.setopt(pycurl.VERBOSE, 1)
+ # TODO: maybe show a summary of the pycurl version
+ ua_str = 'bzr/%s (pycurl)' % (bzrlib.__version__)
+ curl.setopt(pycurl.USERAGENT, ua_str)
+ curl.setopt(pycurl.HTTPHEADER, headers)
+ curl.setopt(pycurl.FOLLOWLOCATION, 1) # follow redirect responses
+
+ def _curl_perform(self, curl):
+ """Perform curl operation and translate exceptions."""
+ try:
+ curl.perform()
+ except pycurl.error, e:
+ # XXX: There seem to be no symbolic constants for these values.
+ if e[0] == 6:
+ # couldn't resolve host
+ raise NoSuchFile(curl.getinfo(pycurl.EFFECTIVE_URL), e)
+
+
+def get_test_permutations():
+ """Return the permutations to be used in testing."""
+ return [(PyCurlTransport, HttpServer),
+ ]
=== added file 'b/bzrlib/transport/http/_urllib.py'
--- /dev/null
+++ b/bzrlib/transport/http/_urllib.py
@@ -0,0 +1,142 @@
+# Copyright (C) 2005, 2006 Canonical Ltd
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import urllib, urllib2
+
+import bzrlib # for the version
+from bzrlib.errors import BzrError
+from bzrlib.trace import mutter
+from bzrlib.transport.http import HttpTransportBase, extract_auth, HttpServer
+from bzrlib.errors import (TransportNotPossible, NoSuchFile,
+ TransportError, ConnectionError)
+
+
+class Request(urllib2.Request):
+ """Request object for urllib2 that allows the method to be overridden."""
+
+ method = None
+
+ def get_method(self):
+ if self.method is not None:
+ return self.method
+ else:
+ return urllib2.Request.get_method(self)
+
+
+class HttpTransport(HttpTransportBase):
+ """Python urllib transport for http and https.
+ """
+
+ # TODO: Implement pipelined versions of all of the *_multi() functions.
+
+ def __init__(self, base):
+ """Set the base path where files will be stored."""
+ super(HttpTransport, self).__init__(base)
+
+ def _get_url(self, url, method=None):
+ mutter("get_url %s" % url)
+ manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
+ url = extract_auth(url, manager)
+ auth_handler = urllib2.HTTPBasicAuthHandler(manager)
+ opener = urllib2.build_opener(auth_handler)
+ request = Request(url)
+ request.method = method
+ request.add_header('User-Agent', 'bzr/%s' % bzrlib.__version__)
+ response = opener.open(request)
+ return response
+
+ def should_cache(self):
+ """Return True if the data pulled across should be cached locally.
+ """
+ return True
+
+ def has(self, relpath):
+ """Does the target location exist?
+ """
+ path = relpath
+ try:
+ path = self.abspath(relpath)
+ f = self._get_url(path, 'HEAD')
+ # Without the read and then close()
+ # we tend to have busy sockets.
+ f.read()
+ f.close()
+ return True
+ except urllib2.URLError, e:
+ mutter('url error code: %s for has url: %r', e.code, path)
+ if e.code == 404:
+ return False
+ raise
+ except IOError, e:
+ mutter('io error: %s %s for has url: %r',
+ e.errno, errno.errorcode.get(e.errno), path)
+ if e.errno == errno.ENOENT:
+ return False
+ raise TransportError(orig_error=e)
+
+ def get(self, relpath):
+ """Get the file at the given relative path.
+
+ :param relpath: The relative path to the file
+ """
+ path = relpath
+ try:
+ path = self.abspath(relpath)
+ return self._get_url(path)
+ except urllib2.HTTPError, e:
+ mutter('url error code: %s for has url: %r', e.code, path)
+ if e.code == 404:
+ raise NoSuchFile(path, extra=e)
+ raise
+ except (BzrError, IOError), e:
+ if hasattr(e, 'errno'):
+ mutter('io error: %s %s for has url: %r',
+ e.errno, errno.errorcode.get(e.errno), path)
+ if e.errno == errno.ENOENT:
+ raise NoSuchFile(path, extra=e)
+ raise ConnectionError(msg = "Error retrieving %s: %s"
+ % (self.abspath(relpath), str(e)),
+ orig_error=e)
+
+ def copy_to(self, relpaths, other, mode=None, pb=None):
+ """Copy a set of entries from self into another Transport.
+
+ :param relpaths: A list/generator of entries to be copied.
+
+ TODO: if other is LocalTransport, is it possible to
+ do better than put(get())?
+ """
+ # At this point HttpTransport might be able to check and see if
+ # the remote location is the same, and rather than download, and
+ # then upload, it could just issue a remote copy_this command.
+ if isinstance(other, HttpTransport):
+ raise TransportNotPossible('http cannot be the target of copy_to()')
+ else:
+ return super(HttpTransport, self).copy_to(relpaths, other, mode=mode, pb=pb)
+
+ def move(self, rel_from, rel_to):
+ """Move the item at rel_from to the location at rel_to"""
+ raise TransportNotPossible('http does not support move()')
+
+ def delete(self, relpath):
+ """Delete the item at relpath"""
+ raise TransportNotPossible('http does not support delete()')
+
+def get_test_permutations():
+ """Return the permutations to be used in testing."""
+ # XXX: There are no HTTPS transport provider tests yet.
+ return [(HttpTransport, HttpServer),
+ ]
=== renamed file 'a/bzrlib/transport/http.py' => 'b/bzrlib/transport/http/__init__.py'
--- a/bzrlib/transport/http.py
+++ b/bzrlib/transport/http/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2005 Canonical Ltd
+# Copyright (C) 2005, 2006 Canonical Ltd
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -13,22 +13,28 @@
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-"""Implementation of Transport over http.
+
+"""Base implementation of Transport over http.
+
+There are separate implementation modules for each http client implementation.
"""
-import os, errno
+import errno
+import os
from cStringIO import StringIO
-import urllib, urllib2
import urlparse
+import urllib
from warnings import warn
-import bzrlib
-from bzrlib.transport import Transport, Server
-from bzrlib.errors import (TransportNotPossible, NoSuchFile,
+from bzrlib.transport import Transport, register_transport, Server
+from bzrlib.errors import (TransportNotPossible, NoSuchFile,
TransportError, ConnectionError)
from bzrlib.errors import BzrError, BzrCheckError
from bzrlib.branch import Branch
from bzrlib.trace import mutter
+# TODO: load these only when running http tests
+import BaseHTTPServer, SimpleHTTPServer, socket, time
+import threading
from bzrlib.ui import ui_factory
@@ -60,68 +66,24 @@
password_manager.add_password(None, host, username, password)
url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))
return url
-
-
-class Request(urllib2.Request):
- """Request object for urllib2 that allows the method to be overridden."""
-
- method = None
-
- def get_method(self):
- if self.method is not None:
- return self.method
- else:
- return urllib2.Request.get_method(self)
-
-
-def get_url(url, method=None):
- import urllib2
- mutter("get_url %s", url)
- manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
- url = extract_auth(url, manager)
- auth_handler = urllib2.HTTPBasicAuthHandler(manager)
- opener = urllib2.build_opener(auth_handler)
-
- request = Request(url)
- request.method = method
- request.add_header('User-Agent', 'bzr/%s' % bzrlib.__version__)
- response = opener.open(request)
- return response
-
-
-class HttpTransport(Transport):
- """This is the transport agent for http:// access.
- TODO: Implement pipelined versions of all of the *_multi() functions.
- """
-
+
+class HttpTransportBase(Transport):
+ """Base class for http implementations.
+
+ Does URL parsing, etc, but not any network IO."""
def __init__(self, base):
"""Set the base path where files will be stored."""
assert base.startswith('http://') or base.startswith('https://')
if base[-1] != '/':
base = base + '/'
- super(HttpTransport, self).__init__(base)
+ super(HttpTransportBase, self).__init__(base)
# In the future we might actually connect to the remote host
# rather than using get_url
# self._connection = None
(self._proto, self._host,
self._path, self._parameters,
self._query, self._fragment) = urlparse.urlparse(self.base)
-
- def should_cache(self):
- """Return True if the data pulled across should be cached locally.
- """
- return True
-
- def clone(self, offset=None):
- """Return a new HttpTransport with root at self.base + offset
- For now HttpTransport does not actually connect, so just return
- a new HttpTransport object.
- """
- if offset is None:
- return HttpTransport(self.base)
- else:
- return HttpTransport(self.abspath(offset))
def abspath(self, relpath):
"""Return the full url to the given relative path.
@@ -161,116 +123,11 @@
return urlparse.urlunparse((self._proto,
self._host, path, '', '', ''))
+ def get(self, relpath):
+ raise NotImplementedError("has() is abstract on %r" % self)
+
def has(self, relpath):
- """Does the target location exist?
-
- TODO: This should be changed so that we don't use
- urllib2 and get an exception, the code path would be
- cleaner if we just do an http HEAD request, and parse
- the return code.
- """
- path = relpath
- try:
- path = self.abspath(relpath)
- f = get_url(path, method='HEAD')
- # Without the read and then close()
- # we tend to have busy sockets.
- f.read()
- f.close()
- return True
- except urllib2.HTTPError, e:
- mutter('url error code: %s for has url: %r', e.code, path)
- if e.code == 404:
- return False
- raise
- except IOError, e:
- mutter('io error: %s %s for has url: %r',
- e.errno, errno.errorcode.get(e.errno), path)
- if e.errno == errno.ENOENT:
- return False
- raise TransportError(orig_error=e)
-
- def get(self, relpath, decode=False):
- """Get the file at the given relative path.
-
- :param relpath: The relative path to the file
- """
- path = relpath
- try:
- path = self.abspath(relpath)
- return get_url(path)
- except urllib2.HTTPError, e:
- mutter('url error code: %s for has url: %r', e.code, path)
- if e.code == 404:
- raise NoSuchFile(path, extra=e)
- raise
- except (BzrError, IOError), e:
- if hasattr(e, 'errno'):
- mutter('io error: %s %s for has url: %r',
- e.errno, errno.errorcode.get(e.errno), path)
- if e.errno == errno.ENOENT:
- raise NoSuchFile(path, extra=e)
- raise ConnectionError(msg = "Error retrieving %s: %s"
- % (self.abspath(relpath), str(e)),
- orig_error=e)
-
- def put(self, relpath, f, mode=None):
- """Copy the file-like or string object into the location.
-
- :param relpath: Location to put the contents, relative to base.
- :param f: File-like or string object.
- """
- raise TransportNotPossible('http PUT not supported')
-
- def mkdir(self, relpath, mode=None):
- """Create a directory at the given path."""
- raise TransportNotPossible('http does not support mkdir()')
-
- def rmdir(self, relpath):
- """See Transport.rmdir."""
- raise TransportNotPossible('http does not support rmdir()')
-
- def append(self, relpath, f):
- """Append the text in the file-like object into the final
- location.
- """
- raise TransportNotPossible('http does not support append()')
-
- def copy(self, rel_from, rel_to):
- """Copy the item at rel_from to the location at rel_to"""
- raise TransportNotPossible('http does not support copy()')
-
- def copy_to(self, relpaths, other, mode=None, pb=None):
- """Copy a set of entries from self into another Transport.
-
- :param relpaths: A list/generator of entries to be copied.
-
- TODO: if other is LocalTransport, is it possible to
- do better than put(get())?
- """
- # At this point HttpTransport might be able to check and see if
- # the remote location is the same, and rather than download, and
- # then upload, it could just issue a remote copy_this command.
- if isinstance(other, HttpTransport):
- raise TransportNotPossible('http cannot be the target of copy_to()')
- else:
- return super(HttpTransport, self).copy_to(relpaths, other, mode=mode, pb=pb)
-
- def move(self, rel_from, rel_to):
- """Move the item at rel_from to the location at rel_to"""
- raise TransportNotPossible('http does not support move()')
-
- def delete(self, relpath):
- """Delete the item at relpath"""
- raise TransportNotPossible('http does not support delete()')
-
- def is_readonly(self):
- """See Transport.is_readonly."""
- return True
-
- def listable(self):
- """See Transport.listable."""
- return False
+ raise NotImplementedError("has() is abstract on %r" % self)
def stat(self, relpath):
"""Return the stat information for a file.
@@ -298,11 +155,108 @@
"""
raise TransportNotPossible('http does not support lock_write()')
+ def clone(self, offset=None):
+ """Return a new HttpTransportBase with root at self.base + offset
+ For now HttpTransportBase does not actually connect, so just return
+ a new HttpTransportBase object.
+ """
+ if offset is None:
+ return self.__class__(self.base)
+ else:
+ return self.__class__(self.abspath(offset))
+
+ def listable(self):
+ """Returns false - http has no reliable way to list directories."""
+ # well, we could try DAV...
+ return False
+
+ def put(self, relpath, f, mode=None):
+ """Copy the file-like or string object into the location.
+
+ :param relpath: Location to put the contents, relative to base.
+ :param f: File-like or string object.
+ """
+ raise TransportNotPossible('http PUT not supported')
+
+ def mkdir(self, relpath, mode=None):
+ """Create a directory at the given path."""
+ raise TransportNotPossible('http does not support mkdir()')
+
+ def rmdir(self, relpath):
+ """See Transport.rmdir."""
+ raise TransportNotPossible('http does not support rmdir()')
+
+ def append(self, relpath, f):
+ """Append the text in the file-like object into the final
+ location.
+ """
+ raise TransportNotPossible('http does not support append()')
+
+ def copy(self, rel_from, rel_to):
+ """Copy the item at rel_from to the location at rel_to"""
+ raise TransportNotPossible('http does not support copy()')
+
+ def copy_to(self, relpaths, other, mode=None, pb=None):
+ """Copy a set of entries from self into another Transport.
+
+ :param relpaths: A list/generator of entries to be copied.
+
+ TODO: if other is LocalTransport, is it possible to
+ do better than put(get())?
+ """
+ # At this point HttpTransportBase might be able to check and see if
+ # the remote location is the same, and rather than download, and
+ # then upload, it could just issue a remote copy_this command.
+ if isinstance(other, HttpTransportBase):
+ raise TransportNotPossible('http cannot be the target of copy_to()')
+ else:
+ return super(HttpTransportBase, self).copy_to(relpaths, other, mode=mode, pb=pb)
+
+ def move(self, rel_from, rel_to):
+ """Move the item at rel_from to the location at rel_to"""
+ raise TransportNotPossible('http does not support move()')
+
+ def delete(self, relpath):
+ """Delete the item at relpath"""
+ raise TransportNotPossible('http does not support delete()')
+
+ def is_readonly(self):
+ """See Transport.is_readonly."""
+ return True
+
+ def listable(self):
+ """See Transport.listable."""
+ return False
+
+ def stat(self, relpath):
+ """Return the stat information for a file.
+ """
+ raise TransportNotPossible('http does not support stat()')
+
+ def lock_read(self, relpath):
+ """Lock the given file for shared (read) access.
+ :return: A lock object, which should be passed to Transport.unlock()
+ """
+ # The old RemoteBranch ignore lock for reading, so we will
+ # continue that tradition and return a bogus lock object.
+ class BogusLock(object):
+ def __init__(self, path):
+ self.path = path
+ def unlock(self):
+ pass
+ return BogusLock(relpath)
+
+ def lock_write(self, relpath):
+ """Lock the given file for exclusive (write) access.
+ WARNING: many transports do not support this, so trying avoid using it
+
+ :return: A lock object, which should be passed to Transport.unlock()
+ """
+ raise TransportNotPossible('http does not support lock_write()')
+
#---------------- test server facilities ----------------
-import BaseHTTPServer, SimpleHTTPServer, socket, time
-import threading
-
+# TODO: load these only when running tests
class WebserverNotAvailable(Exception):
pass
@@ -433,10 +387,3 @@
def get_bogus_url(self):
"""See bzrlib.transport.Server.get_bogus_url."""
return 'http://jasldkjsalkdjalksjdkljasd'
-
-
-def get_test_permutations():
- """Return the permutations to be used in testing."""
- warn("There are no HTTPS transport provider tests yet.")
- return [(HttpTransport, HttpServer),
- ]
=== modified file 'a/NEWS'
--- a/NEWS
+++ b/NEWS
@@ -62,6 +62,13 @@
been split into Branch, and Repository. The common locking and file
management routines are now in bzrlib.lockablefiles.
(Aaron Bentley, Robert Collins, Martin Pool)
+
+ * Transports can now raise DependencyNotPresent if they need a library
+ which is not installed, and then another implementation will be
+ tried. (Martin Pool)
+
+ * Remove obsolete (and no-op) `decode` parameter to `Transport.get`.
+ (Martin Pool)
* Using Tree Transform for merge, revert, tree-building
=== modified file 'a/bzrlib/errors.py'
--- a/bzrlib/errors.py
+++ b/bzrlib/errors.py
@@ -692,7 +692,7 @@
class DependencyNotPresent(BzrNewError):
- """Unable to import library: %(library)s, %(error)s"""
+ """Unable to import library "%(library)s": %(error)s"""
def __init__(self, library, error):
BzrNewError.__init__(self, library=library, error=error)
=== modified file 'a/bzrlib/tests/test_fetch.py'
--- a/bzrlib/tests/test_fetch.py
+++ b/bzrlib/tests/test_fetch.py
@@ -186,6 +186,18 @@
br_rem_a = Branch.open(self.get_readonly_url('branch1'))
fetch_steps(self, br_rem_a, br_b, br_a)
+ def _count_log_matches(self, target, logs):
+ """Count the number of times the target file pattern was fetched in an http log"""
+ log_pattern = '%s HTTP/1.1" 200 - "-" "bzr/%s' % \
+ (target, bzrlib.__version__)
+ c = 0
+ for line in logs:
+ # TODO: perhaps use a regexp instead so we can match more
+ # precisely?
+ if line.find(log_pattern) > -1:
+ c += 1
+ return c
+
def test_weaves_are_retrieved_once(self):
self.build_tree(("source/", "source/file", "target/"))
wt = self.make_branch_and_tree('source')
@@ -197,28 +209,27 @@
target = BzrDir.create_branch_and_repo("target/")
source = Branch.open(self.get_readonly_url("source/"))
self.assertEqual(target.fetch(source), (2, []))
- log_pattern = '%%s HTTP/1.1" 200 - "-" "bzr/%s"' % bzrlib.__version__
+ log_pattern = '%%s HTTP/1.1" 200 - "-" "bzr/%s' % bzrlib.__version__
# this is the path to the literal file. As format changes
# occur it needs to be updated. FIXME: ask the store for the
# path.
- weave_suffix = log_pattern % 'weaves/ce/id.weave'
- self.assertEqual(1,
- len([log for log in self.get_readonly_server().logs if log.endswith(weave_suffix)]))
- inventory_weave_suffix = log_pattern % 'inventory.weave'
- self.assertEqual(1,
- len([log for log in self.get_readonly_server().logs if log.endswith(
- inventory_weave_suffix)]))
+ self.log("web server logs are:")
+ http_logs = self.get_readonly_server().logs
+ self.log('\n'.join(http_logs))
+ self.assertEqual(1, self._count_log_matches('weaves/ce/id.weave', http_logs))
+ self.assertEqual(1, self._count_log_matches('inventory.weave', http_logs))
# this r-h check test will prevent regressions, but it currently already
# passes, before the patch to cache-rh is applied :[
- revision_history_suffix = log_pattern % 'revision-history'
- self.assertEqual(1,
- len([log for log in self.get_readonly_server().logs if log.endswith(
- revision_history_suffix)]))
+ self.assertEqual(1, self._count_log_matches('revision-history', http_logs))
# FIXME naughty poking in there.
self.get_readonly_server().logs = []
# check there is nothing more to fetch
source = Branch.open(self.get_readonly_url("source/"))
self.assertEqual(target.fetch(source), (0, []))
- self.failUnless(self.get_readonly_server().logs[0].endswith(log_pattern % 'branch-format'))
- self.failUnless(self.get_readonly_server().logs[1].endswith(log_pattern % 'revision-history'))
- self.assertEqual(2, len(self.get_readonly_server().logs))
+ # should make just two requests
+ http_logs = self.get_readonly_server().logs
+ self.log("web server logs are:")
+ self.log('\n'.join(http_logs))
+ self.assertEqual(1, self._count_log_matches('branch-format', http_logs[0:1]))
+ self.assertEqual(1, self._count_log_matches('revision-history', http_logs[1:2]))
+ self.assertEqual(2, len(http_logs))
=== modified file 'a/bzrlib/tests/test_http.py'
--- a/bzrlib/tests/test_http.py
+++ b/bzrlib/tests/test_http.py
@@ -1,9 +1,28 @@
-# (C) 2005 Canonical
+# Copyright (C) 2005, 2006 Canonical
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+# FIXME: This test should be repeated for each available http client
+# implementation; at the moment we have urllib and pycurl.
import bzrlib
from bzrlib.tests import TestCase
+from bzrlib.transport.http import extract_auth
+from bzrlib.transport.http._urllib import HttpTransport
+from bzrlib.transport.http._pycurl import PyCurlTransport
from bzrlib.tests.HTTPTestUtil import TestCaseWithWebserver
-from bzrlib.transport.http import HttpTransport, extract_auth
class FakeManager (object):
def __init__(self):
@@ -57,31 +76,42 @@
class TestHttpConnections(TestCaseWithWebserver):
+ _transport = HttpTransport
+
def setUp(self):
super(TestHttpConnections, self).setUp()
self.build_tree(['xxx', 'foo/', 'foo/bar'], line_endings='binary')
def test_http_has(self):
server = self.get_readonly_server()
- t = HttpTransport(server.get_url())
+ t = self._transport(server.get_url())
self.assertEqual(t.has('foo/bar'), True)
self.assertEqual(len(server.logs), 1)
- self.assertTrue(server.logs[0].endswith(
- '"HEAD /foo/bar HTTP/1.1" 200 - "-" "bzr/%s"'
- % bzrlib.__version__))
+ self.assertContainsRe(server.logs[0],
+ r'"HEAD /foo/bar HTTP/1.." (200|302) - "-" "bzr/')
+ def test_http_has_not_found(self):
+ server = self.get_readonly_server()
+ t = self._transport(server.get_url())
self.assertEqual(t.has('not-found'), False)
- self.assertTrue(server.logs[-1].endswith(
- '"HEAD /not-found HTTP/1.1" 404 - "-" "bzr/%s"'
- % bzrlib.__version__))
+ self.assertContainsRe(server.logs[1],
+ r'"HEAD /not-found HTTP/1.." 404 - "-" "bzr/')
def test_http_get(self):
server = self.get_readonly_server()
- t = HttpTransport(server.get_url())
+ t = self._transport(server.get_url())
fp = t.get('foo/bar')
self.assertEqualDiff(
fp.read(),
'contents of foo/bar\n')
self.assertEqual(len(server.logs), 1)
- self.assertTrue(server.logs[0].endswith(
- '"GET /foo/bar HTTP/1.1" 200 - "-" "bzr/%s"' % bzrlib.__version__))
+ self.assertTrue(server.logs[0].find(
+ '"GET /foo/bar HTTP/1.1" 200 - "-" "bzr/%s' % bzrlib.__version__) > -1)
+
+
+class TestHttpConnections_pycurl(TestHttpConnections):
+ _transport = PyCurlTransport
+
+ def setUp(self):
+ super(TestHttpConnections_pycurl, self).setUp()
+
=== modified file 'a/bzrlib/tests/test_selftest.py'
--- a/bzrlib/tests/test_selftest.py
+++ b/bzrlib/tests/test_selftest.py
@@ -153,7 +153,7 @@
has_paramiko = False
else:
has_paramiko = True
- from bzrlib.transport.http import (HttpTransport,
+ from bzrlib.transport.http import (HttpTransportBase,
HttpServer
)
from bzrlib.transport.ftp import FtpTransport
@@ -212,7 +212,7 @@
self.assertEqual(SFTPSiblingAbsoluteServer,
sftp_sibling_abs_test.transport_server)
- self.assertEqual(HttpTransport, http_test.transport_class)
+ self.assertTrue(issubclass(http_test.transport_class, HttpTransportBase))
self.assertEqual(HttpServer, http_test.transport_server)
# self.assertEqual(FtpTransport, ftp_test.transport_class)
@@ -382,16 +382,17 @@
def test_get_readonly_url_http(self):
from bzrlib.transport import get_transport
from bzrlib.transport.local import LocalRelpathServer
- from bzrlib.transport.http import HttpServer, HttpTransport
+ from bzrlib.transport.http import HttpServer, HttpTransportBase
self.transport_server = LocalRelpathServer
self.transport_readonly_server = HttpServer
# calling get_readonly_transport() gives us a HTTP server instance.
url = self.get_readonly_url()
url2 = self.get_readonly_url('foo/bar')
+ # the transport returned may be any HttpTransportBase subclass
t = get_transport(url)
t2 = get_transport(url2)
- self.failUnless(isinstance(t, HttpTransport))
- self.failUnless(isinstance(t2, HttpTransport))
+ self.failUnless(isinstance(t, HttpTransportBase))
+ self.failUnless(isinstance(t2, HttpTransportBase))
self.assertEqual(t2.base[:-1], t.abspath('foo/bar'))
=== modified file 'a/bzrlib/tests/test_transport.py'
--- a/bzrlib/tests/test_transport.py
+++ b/bzrlib/tests/test_transport.py
@@ -21,7 +21,10 @@
from cStringIO import StringIO
from bzrlib.errors import (NoSuchFile, FileExists,
- TransportNotPossible, ConnectionError)
+ TransportNotPossible,
+ ConnectionError,
+ DependencyNotPresent,
+ )
from bzrlib.tests import TestCase
from bzrlib.transport import (_get_protocol_handlers,
_get_transport_modules,
@@ -29,7 +32,10 @@
register_lazy_transport,
_set_protocol_handlers,
urlescape,
+ Transport,
)
+from bzrlib.transport.memory import MemoryTransport
+from bzrlib.transport.local import LocalTransport
class TestTransport(TestCase):
@@ -60,18 +66,126 @@
_get_transport_modules())
finally:
_set_protocol_handlers(handlers)
+
+ def test_transport_dependency(self):
+ """Transport with missing dependency causes no error"""
+ saved_handlers = _get_protocol_handlers()
+ try:
+ register_lazy_transport('foo', 'bzrlib.tests.test_transport',
+ 'BadTransportHandler')
+ t = get_transport('foo://fooserver/foo')
+ # because we failed to load the transport
+ self.assertTrue(isinstance(t, LocalTransport))
+ finally:
+ # restore original values
+ _set_protocol_handlers(saved_handlers)
-
-class MemoryTransportTest(TestCase):
- """Memory transport specific tests."""
+ def test_transport_fallback(self):
+ """Transport with missing dependency causes no error"""
+ saved_handlers = _get_protocol_handlers()
+ try:
+ _set_protocol_handlers({})
+ register_lazy_transport('foo', 'bzrlib.tests.test_transport',
+ 'BackupTransportHandler')
+ register_lazy_transport('foo', 'bzrlib.tests.test_transport',
+ 'BadTransportHandler')
+ t = get_transport('foo://fooserver/foo')
+ self.assertTrue(isinstance(t, BackupTransportHandler))
+ finally:
+ _set_protocol_handlers(saved_handlers)
+
+
+class TestMemoryTransport(TestCase):
+
+ def test_get_transport(self):
+ MemoryTransport()
+
+ def test_clone(self):
+ transport = MemoryTransport()
+ self.assertTrue(isinstance(transport, MemoryTransport))
+
+ def test_abspath(self):
+ transport = MemoryTransport()
+ self.assertEqual("memory:/relpath", transport.abspath('relpath'))
+
+ def test_relpath(self):
+ transport = MemoryTransport()
+
+ def test_append_and_get(self):
+ transport = MemoryTransport()
+ transport.append('path', StringIO('content'))
+ self.assertEqual(transport.get('path').read(), 'content')
+ transport.append('path', StringIO('content'))
+ self.assertEqual(transport.get('path').read(), 'contentcontent')
+
+ def test_put_and_get(self):
+ transport = MemoryTransport()
+ transport.put('path', StringIO('content'))
+ self.assertEqual(transport.get('path').read(), 'content')
+ transport.put('path', StringIO('content'))
+ self.assertEqual(transport.get('path').read(), 'content')
+
+ def test_append_without_dir_fails(self):
+ transport = MemoryTransport()
+ self.assertRaises(NoSuchFile,
+ transport.append, 'dir/path', StringIO('content'))
+
+ def test_put_without_dir_fails(self):
+ transport = MemoryTransport()
+ self.assertRaises(NoSuchFile,
+ transport.put, 'dir/path', StringIO('content'))
+
+ def test_get_missing(self):
+ transport = MemoryTransport()
+ self.assertRaises(NoSuchFile, transport.get, 'foo')
+
+ def test_has_missing(self):
+ transport = MemoryTransport()
+ self.assertEquals(False, transport.has('foo'))
+
+ def test_has_present(self):
+ transport = MemoryTransport()
+ transport.append('foo', StringIO('content'))
+ self.assertEquals(True, transport.has('foo'))
+
+ def test_mkdir(self):
+ transport = MemoryTransport()
+ transport.mkdir('dir')
+ transport.append('dir/path', StringIO('content'))
+ self.assertEqual(transport.get('dir/path').read(), 'content')
+
+ def test_mkdir_missing_parent(self):
+ transport = MemoryTransport()
+ self.assertRaises(NoSuchFile,
+ transport.mkdir, 'dir/dir')
+
+ def test_mkdir_twice(self):
+ transport = MemoryTransport()
+ transport.mkdir('dir')
+ self.assertRaises(FileExists, transport.mkdir, 'dir')
def test_parameters(self):
- import bzrlib.transport.memory as memory
- transport = memory.MemoryTransport()
+ transport = MemoryTransport()
self.assertEqual(True, transport.listable())
self.assertEqual(False, transport.should_cache())
self.assertEqual(False, transport.is_readonly())
-
+
+ def test_iter_files_recursive(self):
+ transport = MemoryTransport()
+ transport.mkdir('dir')
+ transport.put('dir/foo', StringIO('content'))
+ transport.put('dir/bar', StringIO('content'))
+ transport.put('bar', StringIO('content'))
+ paths = set(transport.iter_files_recursive())
+ self.assertEqual(set(['dir/foo', 'dir/bar', 'bar']), paths)
+
+ def test_stat(self):
+ transport = MemoryTransport()
+ transport.put('foo', StringIO('content'))
+ transport.put('bar', StringIO('phowar'))
+ self.assertEqual(7, transport.stat('foo').st_size)
+ self.assertEqual(6, transport.stat('bar').st_size)
+
class ReadonlyDecoratorTransportTest(TestCase):
"""Readonly decoration specific tests."""
@@ -99,3 +213,13 @@
self.assertEqual(True, transport.is_readonly())
finally:
server.tearDown()
+
+
+class BadTransportHandler(Transport):
+ def __init__(self, base_url):
+ raise DependencyNotPresent('some_lib', 'testing missing dependency')
+
+
+class BackupTransportHandler(Transport):
+ """Test transport that works as a backup for the BadTransportHandler"""
+ pass
=== modified file 'a/bzrlib/transport/__init__.py'
--- a/bzrlib/transport/__init__.py
+++ b/bzrlib/transport/__init__.py
@@ -13,10 +13,17 @@
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
"""Transport is an abstraction layer to handle file access.
The abstraction is to allow access from the local filesystem, as well
as remote (such as http or sftp).
+
+Transports are constructed from a string, being a URL or (as a degenerate
+case) a local filesystem path. This is typically the top directory of
+a bzrdir, repository, or similar object we are interested in working with.
+The Transport returned has methods to read, write and manipulate files within
+it.
"""
import errno
@@ -25,27 +32,58 @@
import sys
from unittest import TestSuite
-from bzrlib.trace import mutter
+from bzrlib.trace import mutter, warning
import bzrlib.errors as errors
-
+from bzrlib.errors import DependencyNotPresent
+from bzrlib.symbol_versioning import *
+
+# {prefix: [transport_classes]}
+# Transports are inserted onto the list LIFO and tried in order; as a result
+# transports provided by plugins are tried first, which is usually what we
+# want.
_protocol_handlers = {
}
-def register_transport(prefix, klass, override=True):
+def register_transport(prefix, klass, override=DEPRECATED_PARAMETER):
+ """Register a transport that can be used to open URLs
+
+ Normally you should use register_lazy_transport, which defers loading the
+ implementation until it's actually used, and so avoids pulling in possibly
+ large implementation libraries.
+ """
+ # Note that this code runs very early in library setup -- trace may not be
+ # working, etc.
global _protocol_handlers
- # trace messages commented out because they're typically
- # run during import before trace is set up
- if _protocol_handlers.has_key(prefix):
- if override:
- ## mutter('overriding transport: %s => %s' % (prefix, klass.__name__))
- _protocol_handlers[prefix] = klass
- else:
- ## mutter('registering transport: %s => %s' % (prefix, klass.__name__))
- _protocol_handlers[prefix] = klass
+ if deprecated_passed(override):
+ warn("register_transport(override) is deprecated")
+ _protocol_handlers.setdefault(prefix, []).insert(0, klass)
+
+
+def register_lazy_transport(scheme, module, classname):
+ """Register lazy-loaded transport class.
+
+ When opening a URL with the given scheme, load the module and then
+ instantiate the particular class.
+
+ If the module raises DependencyNotPresent when it's imported, it is
+ skipped and another implementation of the protocol is tried. This is
+ intended to be used when the implementation depends on an external
+ implementation that may not be present. If any other error is raised, it
+ propagates up and the attempt to open the url fails.
+ """
+ # TODO: If no implementation of a protocol is available because of missing
+ # dependencies, we should perhaps show the message about what dependency
+ # was missing.
+ def _loader(base):
+ mod = __import__(module, globals(), locals(), [classname])
+ klass = getattr(mod, classname)
+ return klass(base)
+ _loader.module = module
+ register_transport(scheme, _loader)
def _get_protocol_handlers():
- """Return a dictionary of prefix:transport-factories."""
+ """Return a dictionary of {urlprefix: [factory]}"""
return _protocol_handlers
@@ -58,16 +96,22 @@
_protocol_handlers = new_handlers
+def _clear_protocol_handlers():
+ global _protocol_handlers
+ _protocol_handlers = {}
+
+
def _get_transport_modules():
"""Return a list of the modules providing transports."""
modules = set()
- for prefix, factory in _protocol_handlers.items():
- if factory.__module__ == "bzrlib.transport":
- # this is a lazy load transport, because no real ones
- # are directlry in bzrlib.transport
- modules.add(factory.module)
- else:
- modules.add(factory.__module__)
+ for prefix, factory_list in _protocol_handlers.items():
+ for factory in factory_list:
+ if factory.__module__ == "bzrlib.transport":
+ # this is a lazy load transport, because no real ones
+ # are directlry in bzrlib.transport
+ modules.add(factory.module)
+ else:
+ modules.add(factory.__module__)
result = list(modules)
result.sort()
return result
@@ -117,7 +161,7 @@
using a subdirectory or parent directory. This allows connections
to be pooled, rather than a new one needed for each subdir.
"""
- raise NotImplementedError
+ raise NotImplementedError(self.clone)
def should_cache(self):
"""Return True if the data pulled across should be cached locally.
@@ -182,7 +226,7 @@
XXX: Robert Collins 20051016 - is this really needed in the public
interface ?
"""
- raise NotImplementedError
+ raise NotImplementedError(self.abspath)
def relpath(self, abspath):
"""Return the local path portion from a given absolute path.
@@ -205,7 +249,7 @@
Note that some transports MAY allow querying on directories, but this
is not part of the protocol.
"""
- raise NotImplementedError
+ raise NotImplementedError(self.has)
def has_multi(self, relpaths, pb=None):
"""Return True/False for each entry in relpaths"""
@@ -241,7 +285,7 @@
:param relpath: The relative path to the file
"""
- raise NotImplementedError
+ raise NotImplementedError(self.get)
def get_multi(self, relpaths, pb=None):
"""Get a list of file-like objects, one for each entry in relpaths.
@@ -268,7 +312,7 @@
:param mode: The mode for the newly created file,
None means just use the default
"""
- raise NotImplementedError
+ raise NotImplementedError(self.put)
def put_multi(self, files, mode=None, pb=None):
"""Put a set of files into the location.
@@ -284,7 +328,7 @@
def mkdir(self, relpath, mode=None):
"""Create a directory at the given path."""
- raise NotImplementedError
+ raise NotImplementedError(self.mkdir)
def mkdir_multi(self, relpaths, mode=None, pb=None):
"""Create a group of directories"""
@@ -296,7 +340,7 @@
"""Append the text in the file-like or string object to
the supplied location.
"""
- raise NotImplementedError
+ raise NotImplementedError(self.append)
def append_multi(self, files, pb=None):
"""Append the text in each file-like or string object to
@@ -414,11 +458,11 @@
"""
# This is not implemented, because you need to do special tricks to
# extract the basename, and add it to rel_to
- raise NotImplementedError
+ raise NotImplementedError(self.move_multi_to)
def delete(self, relpath):
"""Delete the item at relpath"""
- raise NotImplementedError
+ raise NotImplementedError(self.delete)
def delete_multi(self, relpaths, pb=None):
"""Queue up a bunch of deletes to be done.
@@ -461,7 +505,7 @@
ALSO NOTE: Stats of directories may not be supported on some
transports.
"""
- raise NotImplementedError
+ raise NotImplementedError(self.stat)
def rmdir(self, relpath):
"""Remove a directory at the given path."""
@@ -481,7 +525,7 @@
def listable(self):
"""Return True if this store supports listing."""
- raise NotImplementedError
+ raise NotImplementedError(self.listable)
def list_dir(self, relpath):
"""Return a list of all files at the given location.
@@ -499,7 +543,7 @@
:return: A lock object, which should contain an unlock() function.
"""
- raise NotImplementedError
+ raise NotImplementedError(self.lock_read)
def lock_write(self, relpath):
"""Lock the given file for exclusive (write) access.
@@ -507,7 +551,7 @@
:return: A lock object, which should contain an unlock() function.
"""
- raise NotImplementedError
+ raise NotImplementedError(self.lock_write)
def is_readonly(self):
"""Return true if this connection cannot be written to."""
@@ -519,31 +563,31 @@
base is either a URL or a directory name.
"""
+ # TODO: give a better error if base looks like a url but there's no
+ # handler for the scheme?
global _protocol_handlers
if base is None:
base = u'.'
else:
base = unicode(base)
- for proto, klass in _protocol_handlers.iteritems():
+ for proto, factory_list in _protocol_handlers.iteritems():
if proto is not None and base.startswith(proto):
- return klass(base)
- # The default handler is the filesystem handler
- # which has a lookup of None
- return _protocol_handlers[None](base)
-
-
-def register_lazy_transport(scheme, module, classname):
- """Register lazy-loaded transport class.
-
- When opening a URL with the given scheme, load the module and then
- instantiate the particular class.
- """
- def _loader(base):
- mod = __import__(module, globals(), locals(), [classname])
- klass = getattr(mod, classname)
- return klass(base)
- _loader.module = module
- register_transport(scheme, _loader)
+ t = _try_transport_factories(base, factory_list)
+ if t:
+ return t
+ # The default handler is the filesystem handler, stored as protocol None
+ return _try_transport_factories(base, _protocol_handlers[None])
+
+
+def _try_transport_factories(base, factory_list):
+ for factory in factory_list:
+ try:
+ return factory(base)
+ except DependencyNotPresent, e:
+ mutter("failed to instantiate transport %r for %r: %r" %
+ (factory, base, e))
+ continue
+ return None
def urlescape(relpath):
@@ -612,6 +656,10 @@
def get_transport_test_permutations(self, module):
"""Get the permutations module wants to have tested."""
+ if not hasattr(module, 'get_test_permutations'):
+ warning("transport module %s doesn't provide get_test_permutations()"
+ % module.__name__)
+ return []
return module.get_test_permutations()
def _test_permutations(self):
@@ -633,8 +681,10 @@
register_lazy_transport(None, 'bzrlib.transport.local', 'LocalTransport')
register_lazy_transport('file://', 'bzrlib.transport.local', 'LocalTransport')
register_lazy_transport('sftp://', 'bzrlib.transport.sftp', 'SFTPTransport')
-register_lazy_transport('http://', 'bzrlib.transport.http', 'HttpTransport')
-register_lazy_transport('https://', 'bzrlib.transport.http', 'HttpTransport')
+register_lazy_transport('http://', 'bzrlib.transport.http._urllib', 'HttpTransport')
+register_lazy_transport('https://', 'bzrlib.transport.http._urllib', 'HttpTransport')
+register_lazy_transport('http://', 'bzrlib.transport.http._pycurl', 'PyCurlTransport')
+register_lazy_transport('https://', 'bzrlib.transport.http._pycurl', 'PyCurlTransport')
register_lazy_transport('ftp://', 'bzrlib.transport.ftp', 'FtpTransport')
register_lazy_transport('aftp://', 'bzrlib.transport.ftp', 'FtpTransport')
register_lazy_transport('memory:/', 'bzrlib.transport.memory', 'MemoryTransport')
=== modified file 'a/bzrlib/transport/ftp.py'
--- a/bzrlib/transport/ftp.py
+++ b/bzrlib/transport/ftp.py
@@ -189,6 +189,7 @@
We're meant to return a file-like object which bzr will
then read from. For now we do this via the magic of StringIO
"""
+ # TODO: decode should be deprecated
try:
mutter("FTP get: %s" % self._abspath(relpath))
f = self._get_FTP()
=== modified file 'a/bzrlib/transport/sftp.py'
--- a/bzrlib/transport/sftp.py
+++ b/bzrlib/transport/sftp.py
@@ -356,7 +356,7 @@
except IOError:
return False
- def get(self, relpath, decode=False):
+ def get(self, relpath):
"""
Get the file at the given relative path.
=== modified file 'a/setup.py'
--- a/setup.py
+++ b/setup.py
@@ -96,6 +96,7 @@
'bzrlib.tests',
'bzrlib.tests.blackbox',
'bzrlib.transport',
+ 'bzrlib.transport.http',
'bzrlib.ui',
'bzrlib.util',
'bzrlib.util.elementtree',
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 191 bytes
Desc: Digital signature
Url : https://lists.ubuntu.com/archives/bazaar/attachments/20060306/7f9b4ba8/attachment.pgp
More information about the bazaar
mailing list