Rev 3518: Start supporting pyftpdlib as an ftp test server. in file:///net/bigmamac/Volumes/home/vila/src/bzr/experimental/more-ftp/
Vincent Ladeuil
v.ladeuil+lp at free.fr
Sun Mar 1 10:02:02 GMT 2009
At file:///net/bigmamac/Volumes/home/vila/src/bzr/experimental/more-ftp/
------------------------------------------------------------
revno: 3518
revision-id: v.ladeuil+lp at free.fr-20090301100200-1qdmf362uw5q35s0
parent: v.ladeuil+lp at free.fr-20090228214320-8wjuatnu074v8pny
committer: Vincent Ladeuil <v.ladeuil+lp at free.fr>
branch nick: pyftpdlib
timestamp: Sun 2009-03-01 11:02:00 +0100
message:
Start supporting pyftpdlib as an ftp test server.
* bzrlib/transport/ftp/__init__.py:
(FtpTransport._setmode): Report mode bits in octal.
* bzrlib/transport/__init__.py:
Don't register GSSAPI ftp transports if kerberos can't be
imported, this avoids useless repeated messages in log as the
import repeatedly fail.
* bzrlib/tests/test_transport_implementations.py:
(TransportTests.test_has): Fix too long lines.
(TransportTests.test_connect_twice_is_same_content): Fig bogus
test. transport.base is *not* usable, it miss the password for
authenticated urls. This was passing only because no test server
use authentication by default *except* medusa for which we had a
dummy authorizer with no password check.
* bzrlib/tests/ftp_server/pyftpdlib_based.py:
First shot at an ftp test server running is a separate thread.
* bzrlib/tests/ftp_server/__init__.py:
Add support for pyftpdlib based ftp test server.
(_FTPServerFeature._probe): One of medusa or pyftpdlib is needed.
-------------- next part --------------
=== modified file 'BRANCH.TODO'
--- a/BRANCH.TODO 2009-01-30 00:49:41 +0000
+++ b/BRANCH.TODO 2009-03-01 10:02:00 +0000
@@ -3,3 +3,8 @@
#
#
+- explicit test to forbid empty password for ftp ?
+- handle unicode or utf-8 encoded paths
+- try using _map to fix failing tests
+- call _ftp_server.close_all in the server thread and/or switch
+ to an implementation that doesn't use serve_forever
=== modified file 'bzrlib/tests/ftp_server/__init__.py'
--- a/bzrlib/tests/ftp_server/__init__.py 2009-02-28 21:43:00 +0000
+++ b/bzrlib/tests/ftp_server/__init__.py 2009-03-01 10:02:00 +0000
@@ -30,6 +30,13 @@
medusa_available = False
+try:
+ from bzrlib.tests.ftp_server import pyftpdlib_based
+ pyftpdlib_available = True
+except ImportError:
+ pyftpdlib_available = False
+
+
class _FTPServerFeature(tests.Feature):
"""Some tests want an FTP Server, check if one is available.
@@ -38,7 +45,7 @@
"""
def _probe(self):
- return medusa_available
+ return medusa_available or pyftpdlib_available
def feature_name(self):
return 'FTPServer'
@@ -50,7 +57,6 @@
class UnavailableFTPServer(object):
"""Dummy ftp test server.
-
This allows the test suite report the number of tests needing that
feature. We raise UnavailableFeature from methods before the test server is
being used. Doing so in the setUp method has bad side-effects (tearDown is
@@ -72,5 +78,7 @@
if medusa_available:
FTPServer = medusa_based.FTPServer
+elif pyftpdlib_available:
+ FTPServer = pyftpdlib_based.FTPServer
else:
FTPServer = UnavailableFTPServer
=== added file 'bzrlib/tests/ftp_server/pyftpdlib_based.py'
--- a/bzrlib/tests/ftp_server/pyftpdlib_based.py 1970-01-01 00:00:00 +0000
+++ b/bzrlib/tests/ftp_server/pyftpdlib_based.py 2009-03-01 10:02:00 +0000
@@ -0,0 +1,202 @@
+# Copyright (C) 2009 Canonical Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+"""
+FTP test server.
+
+Based on pyftpdlib: http://code.google.com/p/pyftpdlib/
+"""
+
+import errno
+import os
+from pyftpdlib import ftpserver
+import select
+import threading
+
+
+from bzrlib import (
+ trace,
+ transport,
+ )
+
+
+class AnonymousWithWriteAccessAuthorizer(ftpserver.DummyAuthorizer):
+
+ def _check_permissions(self, username, perm):
+ # Like base implementation but don't warn about write permissions
+ # assigned to anonynous, since that's exactly our purpose.
+ for p in perm:
+ if p not in self.read_perms + self.write_perms:
+ raise ftpserver.AuthorizerError('No such permission "%s"' %p)
+
+
+class BzrConformingFS(ftpserver.AbstractedFS):
+
+ def chmod(self, path, mode):
+ return os.chmod(path, mode)
+
+ def listdir(self, path):
+ """List the content of a directory."""
+ # XXX: Something just freaks out in asyncore if given unicode strings,
+ # that may need to be revisited once unicode or at least utf-8 encoded
+ # paths is better handled. -- vila 20090228
+ return [str(s) for s in os.listdir(path)]
+
+
+class BZRConformingFTPHandler(ftpserver.FTPHandler):
+
+ abstracted_fs = BzrConformingFS
+
+ def __init__(self, conn, server):
+ ftpserver.FTPHandler.__init__(self, conn, server)
+ self.authorizer = server.authorizer
+
+ def ftp_SIZE(self, path):
+ # bzr is overly picky here, but we want to make the test suite pass
+ # first. This may need to be revisited -- vila 20090226
+ line = self.fs.fs2ftp(path)
+ if self.fs.isdir(self.fs.realpath(path)):
+ why = "%s is a directory" % line
+ self.log('FAIL SIZE "%s". %s.' % (line, why))
+ self.respond("550 %s." %why)
+ else:
+ ftpserver.FTPHandler.ftp_SIZE(self, path)
+
+ def ftp_NLST(self, path):
+ # bzr is overly picky here, but we want to make the test suite pass
+ # first. This may need to be revisited -- vila 20090226
+ line = self.fs.fs2ftp(path)
+ if self.fs.isfile(self.fs.realpath(path)):
+ why = "Not a directory: %s" % line
+ self.log('FAIL SIZE "%s". %s.' % (line, why))
+ self.respond("550 %s." %why)
+ else:
+ ftpserver.FTPHandler.ftp_NLST(self, path)
+
+ def ftp_SITE_CHMOD(self, line):
+ try:
+ mode, path = line.split()
+ mode = int(mode, 8)
+ except ValueError:
+ # We catch both malformed line and malformed mode with the same
+ # ValueError.
+ self.respond("500 'SITE CHMOD %s': command not understood."
+ % line)
+ self.log('FAIL SITE CHMOD MKD ' % line)
+ return
+ ftp_path = self.fs.fs2ftp(path)
+ try:
+ self.run_as_current_user(self.fs.chmod, self.fs.ftp2fs(path), mode)
+ except OSError, err:
+ why = ftpserver._strerror(err)
+ self.log('FAIL SITE CHMOD 0%03o "%s". %s.' % (mode, ftp_path, why))
+ self.respond('550 %s.' % why)
+ else:
+ self.log('OK SITE CHMOD 0%03o "%s".' % (mode, ftp_path))
+ self.respond('200 SITE CHMOD succesful.')
+
+
+# pyftpdlib says to define SITE commands by declaring ftp_SITE_<CMD> methods,
+# but fails to recognize them.
+ftpserver.proto_cmds['SITE CHMOD'] = ftpserver._CommandProperty(
+ perm='w', # Best fit choice even if not exactly right (can be d, f or m too)
+ auth_needed=True, arg_needed=True, check_path=False,
+ help='Syntax: SITE CHMOD <SP> octal_mode_bits file-name (chmod file)',
+ )
+
+
+class ftp_server(ftpserver.FTPServer):
+
+ def __init__(self, address, handler, authorizer):
+ ftpserver.FTPServer.__init__(self, address, handler)
+ self.authorizer = authorizer
+
+
+class FTPServer(transport.Server):
+ """Common code for FTP server facilities."""
+
+ def __init__(self):
+ self._root = None
+ self._ftp_server = None
+ self._port = None
+ self._async_thread = None
+ # ftp server logs
+ self.logs = []
+
+ def get_url(self):
+ """Calculate an ftp url to this server."""
+ return 'ftp://foo:bar@localhost:%d/' % (self._port)
+
+ def get_bogus_url(self):
+ """Return a URL which cannot be connected to."""
+ return 'ftp://127.0.0.1:1/'
+
+ def log(self, message):
+ """This is used by medusa.ftp_server to log connections, etc."""
+ self.logs.append(message)
+
+ def setUp(self, vfs_server=None):
+ from bzrlib.transport.local import LocalURLServer
+ if not (vfs_server is None or isinstance(vfs_server, LocalURLServer)):
+ raise AssertionError(
+ "FTPServer currently assumes local transport, got %s"
+ % vfs_server)
+ self._root = os.getcwdu()
+
+ address = ('localhost', 0) # bind to a random port
+ authorizer = AnonymousWithWriteAccessAuthorizer()
+ authorizer.add_user('foo', 'bar', self._root, perm='elradfmw')
+ self._ftp_server = ftp_server(address, BZRConformingFTPHandler,
+ authorizer)
+ # This is hacky as hell, will not work if we need two servers working
+ # at the same time, but that's the best we can do so far...
+ # FIXME: At least log and logline could be overriden in the handler ?
+ # -- vila 20090227
+ ftpserver.log = self.log
+ ftpserver.logline = self.log
+ ftpserver.logerror = self.log
+
+ self._port = self._ftp_server.socket.getsockname()[1]
+ # Don't let it loop forever, or handle an infinite number of requests.
+ # In this case it will run for 1000s, or 10000 requests
+ self._async_thread = threading.Thread(
+ target=self._asyncore_loop_ignore_EBADF,
+ kwargs={'timeout':0.1,
+ 'count':100000,
+ })
+ self._async_thread.setDaemon(True)
+ self._async_thread.start()
+
+ def tearDown(self):
+ """See bzrlib.transport.Server.tearDown."""
+ self._ftp_server.close_all()
+ self._async_thread.join()
+
+ def _asyncore_loop_ignore_EBADF(self, *args, **kwargs):
+ """Ignore EBADF during server shutdown.
+
+ We close the socket to get the server to shutdown, but this causes
+ select.select() to raise EBADF.
+ """
+ try:
+ self._ftp_server.serve_forever(*args, **kwargs)
+ # FIXME: If we reach that point, we should raise an exception
+ # explaining that the 'count' parameter in setUp is too low or
+ # testers may wonder why their test just sits there waiting for a
+ # server that is already dead. Note that if the tester waits too
+ # long under pdb the server will also die.
+ except select.error, e:
+ if e.args[0] != errno.EBADF:
+ raise
=== modified file 'bzrlib/tests/test_transport_implementations.py'
--- a/bzrlib/tests/test_transport_implementations.py 2009-02-26 19:30:06 +0000
+++ b/bzrlib/tests/test_transport_implementations.py 2009-03-01 10:02:00 +0000
@@ -167,12 +167,17 @@
self.assertEqual(True, t.has('a'))
self.assertEqual(False, t.has('c'))
self.assertEqual(True, t.has(urlutils.escape('%')))
- self.assertEqual(list(t.has_multi(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'])),
- [True, True, False, False, True, False, True, False])
+ self.assertEqual(list(t.has_multi(['a', 'b', 'c', 'd',
+ 'e', 'f', 'g', 'h'])),
+ [True, True, False, False,
+ True, False, True, False])
self.assertEqual(True, t.has_any(['a', 'b', 'c']))
- self.assertEqual(False, t.has_any(['c', 'd', 'f', urlutils.escape('%%')]))
- self.assertEqual(list(t.has_multi(iter(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']))),
- [True, True, False, False, True, False, True, False])
+ self.assertEqual(False, t.has_any(['c', 'd', 'f',
+ urlutils.escape('%%')]))
+ self.assertEqual(list(t.has_multi(iter(['a', 'b', 'c', 'd',
+ 'e', 'f', 'g', 'h']))),
+ [True, True, False, False,
+ True, False, True, False])
self.assertEqual(False, t.has_any(['c', 'c', 'c']))
self.assertEqual(True, t.has_any(['b', 'b', 'b']))
@@ -1507,16 +1512,10 @@
transport.put_bytes('foo', 'bar')
transport3 = self.get_transport()
self.check_transport_contents('bar', transport3, 'foo')
- # its base should be usable. XXX: This is true only if we don't use
- # auhentication, otherwise 'base' doesn't mention the password and we
- # can't access it anymore since the password is lost (it *could* be
- # mentioned in the url given by the test server) --vila 090226
- transport4 = get_transport(transport.base)
- self.check_transport_contents('bar', transport4, 'foo')
# now opening at a relative url should give use a sane result:
transport.mkdir('newdir')
- transport5 = get_transport(transport.base + "newdir")
+ transport5 = get_transport("newdir")
transport6 = transport5.clone('..')
self.check_transport_contents('bar', transport6, 'foo')
=== modified file 'bzrlib/transport/__init__.py'
--- a/bzrlib/transport/__init__.py 2009-02-23 15:42:47 +0000
+++ b/bzrlib/transport/__init__.py 2009-03-01 10:02:00 +0000
@@ -1787,24 +1787,32 @@
register_transport_proto('aftp://', help="Access using active FTP.")
register_lazy_transport('aftp://', 'bzrlib.transport.ftp', 'FtpTransport')
-# Default to trying GSSAPI authentication (if the kerberos module is available)
-register_transport_proto('ftp+gssapi://', register_netloc=True)
-register_lazy_transport('ftp+gssapi://', 'bzrlib.transport.ftp._gssapi',
- 'GSSAPIFtpTransport')
-register_transport_proto('aftp+gssapi://', register_netloc=True)
-register_lazy_transport('aftp+gssapi://', 'bzrlib.transport.ftp._gssapi',
- 'GSSAPIFtpTransport')
-register_transport_proto('ftp+nogssapi://', register_netloc=True)
-register_transport_proto('aftp+nogssapi://', register_netloc=True)
-
-register_lazy_transport('ftp://', 'bzrlib.transport.ftp._gssapi',
- 'GSSAPIFtpTransport')
-register_lazy_transport('aftp://', 'bzrlib.transport.ftp._gssapi',
- 'GSSAPIFtpTransport')
-register_lazy_transport('ftp+nogssapi://', 'bzrlib.transport.ftp',
- 'FtpTransport')
-register_lazy_transport('aftp+nogssapi://', 'bzrlib.transport.ftp',
- 'FtpTransport')
+try:
+ import kerberos
+ kerberos_available = True
+except ImportError:
+ kerberos_available = False
+
+if kerberos_available:
+ # Default to trying GSSAPI authentication (if the kerberos module is
+ # available)
+ register_transport_proto('ftp+gssapi://', register_netloc=True)
+ register_lazy_transport('ftp+gssapi://', 'bzrlib.transport.ftp._gssapi',
+ 'GSSAPIFtpTransport')
+ register_transport_proto('aftp+gssapi://', register_netloc=True)
+ register_lazy_transport('aftp+gssapi://', 'bzrlib.transport.ftp._gssapi',
+ 'GSSAPIFtpTransport')
+ register_transport_proto('ftp+nogssapi://', register_netloc=True)
+ register_transport_proto('aftp+nogssapi://', register_netloc=True)
+
+ register_lazy_transport('ftp://', 'bzrlib.transport.ftp._gssapi',
+ 'GSSAPIFtpTransport')
+ register_lazy_transport('aftp://', 'bzrlib.transport.ftp._gssapi',
+ 'GSSAPIFtpTransport')
+ register_lazy_transport('ftp+nogssapi://', 'bzrlib.transport.ftp',
+ 'FtpTransport')
+ register_lazy_transport('aftp+nogssapi://', 'bzrlib.transport.ftp',
+ 'FtpTransport')
register_transport_proto('memory://')
register_lazy_transport('memory://', 'bzrlib.transport.memory',
=== modified file 'bzrlib/transport/ftp/__init__.py'
--- a/bzrlib/transport/ftp/__init__.py 2009-02-27 13:26:13 +0000
+++ b/bzrlib/transport/ftp/__init__.py 2009-03-01 10:02:00 +0000
@@ -439,7 +439,7 @@
if mode:
try:
mutter("FTP site chmod: setting permissions to %s on %s",
- str(mode), self._remote_path(relpath))
+ oct(mode), self._remote_path(relpath))
ftp = self._get_FTP()
cmd = "SITE CHMOD %s %s" % (oct(mode),
self._remote_path(relpath))
@@ -447,7 +447,7 @@
except ftplib.error_perm, e:
# Command probably not available on this server
warning("FTP Could not set permissions to %s on %s. %s",
- str(mode), self._remote_path(relpath), str(e))
+ oct(mode), self._remote_path(relpath), str(e))
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
# to copy something to another machine. And you may be able
More information about the bazaar-commits
mailing list