Rev 4809: Stop requiring testtools for sftp use. in file:///home/vila/src/bzr/bugs/516183-test-server/

Vincent Ladeuil v.ladeuil+lp at free.fr
Wed Feb 3 14:08:56 GMT 2010


At file:///home/vila/src/bzr/bugs/516183-test-server/

------------------------------------------------------------
revno: 4809
revision-id: v.ladeuil+lp at free.fr-20100203140855-yuwq660bb3jayr1d
parent: v.ladeuil+lp at free.fr-20100203140252-gx0y05pyn21gg3mg
committer: Vincent Ladeuil <v.ladeuil+lp at free.fr>
branch nick: 516183-test-server
timestamp: Wed 2010-02-03 15:08:55 +0100
message:
  Stop requiring testtools for sftp use.
  
  * bzrlib/transport/sftp.py: 
  Move the test servers code to tests.stub_sftp.
  (get_test_permutations): Import the test servers only when the
  tests are run.
  
  * bzrlib/tests/test_transport.py:
  (TestSSHConnections.test_bzr_connect_to_bzr_ssh): Get the tests
  servers from tests.stub_sftp.
  
  * bzrlib/tests/test_sftp_transport.py: 
  Get the tests servers  from tests.stub_sftp.
  
  * bzrlib/tests/stub_sftp.py: 
  Fix imports, move the test servers code from transport.sftp.
  
  * bzrlib/tests/blackbox/test_selftest.py: 
  Fix imports.
  
  * bzrlib/builtins.py:
  (cmd_selftest.get_transport_type): Fix the test sftp server
  import.
-------------- next part --------------
=== modified file 'NEWS'
--- a/NEWS	2010-02-02 08:36:12 +0000
+++ b/NEWS	2010-02-03 14:08:55 +0000
@@ -14,6 +14,9 @@
 Bug Fixes
 *********
 
+* Don't require testtools to use sftp.
+  (Vincent Ladeuil, #516183)
+
 * Fix "AttributeError in Inter1and2Helper" during fetch.
   (Martin Pool, #513432)
 

=== modified file 'bzrlib/builtins.py'
--- a/bzrlib/builtins.py	2010-01-21 17:54:58 +0000
+++ b/bzrlib/builtins.py	2010-02-03 14:08:55 +0000
@@ -3444,8 +3444,8 @@
     def get_transport_type(typestring):
         """Parse and return a transport specifier."""
         if typestring == "sftp":
-            from bzrlib.transport.sftp import SFTPAbsoluteServer
-            return SFTPAbsoluteServer
+            from bzrlib.tests import stub_sftp
+            return stub_sftp.SFTPAbsoluteServer
         if typestring == "memory":
             from bzrlib.transport.memory import MemoryServer
             return MemoryServer

=== modified file 'bzrlib/tests/blackbox/test_selftest.py'
--- a/bzrlib/tests/blackbox/test_selftest.py	2009-12-22 15:37:23 +0000
+++ b/bzrlib/tests/blackbox/test_selftest.py	2010-02-03 14:08:55 +0000
@@ -16,18 +16,14 @@
 
 """UI tests for the test framework."""
 
-import bzrlib.transport
 from bzrlib import (
     benchmarks,
     tests,
     )
-from bzrlib.errors import ParamikoNotPresent
 from bzrlib.tests import (
-                          features,
-                          TestCase,
-                          TestCaseInTempDir,
-                          TestSkipped,
-                          )
+    features,
+    stub_sftp,
+    )
 
 
 class SelfTestPatch:
@@ -51,7 +47,7 @@
             tests.selftest = original_selftest
 
 
-class TestOptionsWritingToDisk(TestCaseInTempDir, SelfTestPatch):
+class TestOptionsWritingToDisk(tests.TestCaseInTempDir, SelfTestPatch):
 
     def test_benchmark_runs_benchmark_tests(self):
         """selftest --benchmark should change the suite factory."""
@@ -69,7 +65,7 @@
         self.assertEqual(0, len(lines))
 
 
-class TestOptions(TestCase, SelfTestPatch):
+class TestOptions(tests.TestCase, SelfTestPatch):
 
     def test_load_list(self):
         params = self.get_params_passed_to_core('selftest --load-list foo')
@@ -80,7 +76,7 @@
         # version.
         self.requireFeature(features.paramiko)
         params = self.get_params_passed_to_core('selftest --transport=sftp')
-        self.assertEqual(bzrlib.transport.sftp.SFTPAbsoluteServer,
+        self.assertEqual(stub_sftp.SFTPAbsoluteServer,
             params[1]["transport"])
 
     def test_transport_set_to_memory(self):

=== modified file 'bzrlib/tests/stub_sftp.py'
--- a/bzrlib/tests/stub_sftp.py	2010-02-03 14:02:52 +0000
+++ b/bzrlib/tests/stub_sftp.py	2010-02-03 14:08:55 +0000
@@ -21,13 +21,22 @@
 
 import os
 import paramiko
+import select
+import socket
 import sys
+import threading
+import time
 
 from bzrlib import (
     osutils,
     trace,
-    )
-
+    urlutils,
+    )
+from bzrlib.transport import (
+    local,
+    Server,
+    ssh,
+    )
 
 class StubServer (paramiko.ServerInterface):
 
@@ -231,3 +240,323 @@
 
     # removed: chattr, symlink, readlink
     # (nothing in bzr's sftp transport uses those)
+
+# ------------- server test implementation --------------
+
+STUB_SERVER_KEY = """
+-----BEGIN RSA PRIVATE KEY-----
+MIICWgIBAAKBgQDTj1bqB4WmayWNPB+8jVSYpZYk80Ujvj680pOTh2bORBjbIAyz
+oWGW+GUjzKxTiiPvVmxFgx5wdsFvF03v34lEVVhMpouqPAYQ15N37K/ir5XY+9m/
+d8ufMCkjeXsQkKqFbAlQcnWMCRnOoPHS3I4vi6hmnDDeeYTSRvfLbW0fhwIBIwKB
+gBIiOqZYaoqbeD9OS9z2K9KR2atlTxGxOJPXiP4ESqP3NVScWNwyZ3NXHpyrJLa0
+EbVtzsQhLn6rF+TzXnOlcipFvjsem3iYzCpuChfGQ6SovTcOjHV9z+hnpXvQ/fon
+soVRZY65wKnF7IAoUwTmJS9opqgrN6kRgCd3DASAMd1bAkEA96SBVWFt/fJBNJ9H
+tYnBKZGw0VeHOYmVYbvMSstssn8un+pQpUm9vlG/bp7Oxd/m+b9KWEh2xPfv6zqU
+avNwHwJBANqzGZa/EpzF4J8pGti7oIAPUIDGMtfIcmqNXVMckrmzQ2vTfqtkEZsA
+4rE1IERRyiJQx6EJsz21wJmGV9WJQ5kCQQDwkS0uXqVdFzgHO6S++tjmjYcxwr3g
+H0CoFYSgbddOT6miqRskOQF3DZVkJT3kyuBgU2zKygz52ukQZMqxCb1fAkASvuTv
+qfpH87Qq5kQhNKdbbwbmd2NxlNabazPijWuphGTdW0VfJdWfklyS2Kr+iqrs/5wV
+HhathJt636Eg7oIjAkA8ht3MQ+XSl9yIJIS8gVpbPxSw5OMfw0PjVE7tBdQruiSc
+nvuQES5C9BMHjF39LZiGH1iLQy7FgdHyoP+eodI7
+-----END RSA PRIVATE KEY-----
+"""
+
+
+class SocketListener(threading.Thread):
+
+    def __init__(self, callback):
+        threading.Thread.__init__(self)
+        self._callback = callback
+        self._socket = socket.socket()
+        self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        self._socket.bind(('localhost', 0))
+        self._socket.listen(1)
+        self.host, self.port = self._socket.getsockname()[:2]
+        self._stop_event = threading.Event()
+
+    def stop(self):
+        # called from outside this thread
+        self._stop_event.set()
+        # use a timeout here, because if the test fails, the server thread may
+        # never notice the stop_event.
+        self.join(5.0)
+        self._socket.close()
+
+    def run(self):
+        while True:
+            readable, writable_unused, exception_unused = \
+                select.select([self._socket], [], [], 0.1)
+            if self._stop_event.isSet():
+                return
+            if len(readable) == 0:
+                continue
+            try:
+                s, addr_unused = self._socket.accept()
+                # because the loopback socket is inline, and transports are
+                # never explicitly closed, best to launch a new thread.
+                threading.Thread(target=self._callback, args=(s,)).start()
+            except socket.error, x:
+                sys.excepthook(*sys.exc_info())
+                warning('Socket error during accept() within unit test server'
+                        ' thread: %r' % x)
+            except Exception, x:
+                # probably a failed test; unit test thread will log the
+                # failure/error
+                sys.excepthook(*sys.exc_info())
+                warning('Exception from within unit test server thread: %r' %
+                        x)
+
+
+class SocketDelay(object):
+    """A socket decorator to make TCP appear slower.
+
+    This changes recv, send, and sendall to add a fixed latency to each python
+    call if a new roundtrip is detected. That is, when a recv is called and the
+    flag new_roundtrip is set, latency is charged. Every send and send_all
+    sets this flag.
+
+    In addition every send, sendall and recv sleeps a bit per character send to
+    simulate bandwidth.
+
+    Not all methods are implemented, this is deliberate as this class is not a
+    replacement for the builtin sockets layer. fileno is not implemented to
+    prevent the proxy being bypassed.
+    """
+
+    simulated_time = 0
+    _proxied_arguments = dict.fromkeys([
+        "close", "getpeername", "getsockname", "getsockopt", "gettimeout",
+        "setblocking", "setsockopt", "settimeout", "shutdown"])
+
+    def __init__(self, sock, latency, bandwidth=1.0,
+                 really_sleep=True):
+        """
+        :param bandwith: simulated bandwith (MegaBit)
+        :param really_sleep: If set to false, the SocketDelay will just
+        increase a counter, instead of calling time.sleep. This is useful for
+        unittesting the SocketDelay.
+        """
+        self.sock = sock
+        self.latency = latency
+        self.really_sleep = really_sleep
+        self.time_per_byte = 1 / (bandwidth / 8.0 * 1024 * 1024)
+        self.new_roundtrip = False
+
+    def sleep(self, s):
+        if self.really_sleep:
+            time.sleep(s)
+        else:
+            SocketDelay.simulated_time += s
+
+    def __getattr__(self, attr):
+        if attr in SocketDelay._proxied_arguments:
+            return getattr(self.sock, attr)
+        raise AttributeError("'SocketDelay' object has no attribute %r" %
+                             attr)
+
+    def dup(self):
+        return SocketDelay(self.sock.dup(), self.latency, self.time_per_byte,
+                           self._sleep)
+
+    def recv(self, *args):
+        data = self.sock.recv(*args)
+        if data and self.new_roundtrip:
+            self.new_roundtrip = False
+            self.sleep(self.latency)
+        self.sleep(len(data) * self.time_per_byte)
+        return data
+
+    def sendall(self, data, flags=0):
+        if not self.new_roundtrip:
+            self.new_roundtrip = True
+            self.sleep(self.latency)
+        self.sleep(len(data) * self.time_per_byte)
+        return self.sock.sendall(data, flags)
+
+    def send(self, data, flags=0):
+        if not self.new_roundtrip:
+            self.new_roundtrip = True
+            self.sleep(self.latency)
+        bytes_sent = self.sock.send(data, flags)
+        self.sleep(bytes_sent * self.time_per_byte)
+        return bytes_sent
+
+
+class SFTPServer(Server):
+    """Common code for SFTP server facilities."""
+
+    def __init__(self, server_interface=StubServer):
+        self._original_vendor = None
+        self._homedir = None
+        self._server_homedir = None
+        self._listener = None
+        self._root = None
+        self._vendor = ssh.ParamikoVendor()
+        self._server_interface = server_interface
+        # sftp server logs
+        self.logs = []
+        self.add_latency = 0
+
+    def _get_sftp_url(self, path):
+        """Calculate an sftp url to this server for path."""
+        return 'sftp://foo:bar@%s:%d/%s' % (self._listener.host,
+                                            self._listener.port, path)
+
+    def log(self, message):
+        """StubServer uses this to log when a new server is created."""
+        self.logs.append(message)
+
+    def _run_server_entry(self, sock):
+        """Entry point for all implementations of _run_server.
+
+        If self.add_latency is > 0.000001 then sock is given a latency adding
+        decorator.
+        """
+        if self.add_latency > 0.000001:
+            sock = SocketDelay(sock, self.add_latency)
+        return self._run_server(sock)
+
+    def _run_server(self, s):
+        ssh_server = paramiko.Transport(s)
+        key_file = osutils.pathjoin(self._homedir, 'test_rsa.key')
+        f = open(key_file, 'w')
+        f.write(STUB_SERVER_KEY)
+        f.close()
+        host_key = paramiko.RSAKey.from_private_key_file(key_file)
+        ssh_server.add_server_key(host_key)
+        server = self._server_interface(self)
+        ssh_server.set_subsystem_handler('sftp', paramiko.SFTPServer,
+                                         StubSFTPServer, root=self._root,
+                                         home=self._server_homedir)
+        event = threading.Event()
+        ssh_server.start_server(event, server)
+        event.wait(5.0)
+
+    def start_server(self, backing_server=None):
+        # XXX: TODO: make sftpserver back onto backing_server rather than local
+        # disk.
+        if not (backing_server is None or
+                isinstance(backing_server, local.LocalURLServer)):
+            raise AssertionError(
+                "backing_server should not be %r, because this can only serve the "
+                "local current working directory." % (backing_server,))
+        self._original_vendor = ssh._ssh_vendor_manager._cached_ssh_vendor
+        ssh._ssh_vendor_manager._cached_ssh_vendor = self._vendor
+        if sys.platform == 'win32':
+            # Win32 needs to use the UNICODE api
+            self._homedir = getcwd()
+        else:
+            # But Linux SFTP servers should just deal in bytestreams
+            self._homedir = os.getcwd()
+        if self._server_homedir is None:
+            self._server_homedir = self._homedir
+        self._root = '/'
+        if sys.platform == 'win32':
+            self._root = ''
+        self._listener = SocketListener(self._run_server_entry)
+        self._listener.setDaemon(True)
+        self._listener.start()
+
+    def stop_server(self):
+        self._listener.stop()
+        ssh._ssh_vendor_manager._cached_ssh_vendor = self._original_vendor
+
+    def get_bogus_url(self):
+        """See bzrlib.transport.Server.get_bogus_url."""
+        # this is chosen to try to prevent trouble with proxies, wierd dns, etc
+        # we bind a random socket, so that we get a guaranteed unused port
+        # we just never listen on that port
+        s = socket.socket()
+        s.bind(('localhost', 0))
+        return 'sftp://%s:%s/' % s.getsockname()
+
+
+class SFTPFullAbsoluteServer(SFTPServer):
+    """A test server for sftp transports, using absolute urls and ssh."""
+
+    def get_url(self):
+        """See bzrlib.transport.Server.get_url."""
+        homedir = self._homedir
+        if sys.platform != 'win32':
+            # Remove the initial '/' on all platforms but win32
+            homedir = homedir[1:]
+        return self._get_sftp_url(urlutils.escape(homedir))
+
+
+class SFTPServerWithoutSSH(SFTPServer):
+    """An SFTP server that uses a simple TCP socket pair rather than SSH."""
+
+    def __init__(self):
+        super(SFTPServerWithoutSSH, self).__init__()
+        self._vendor = ssh.LoopbackVendor()
+
+    def _run_server(self, sock):
+        # Re-import these as locals, so that they're still accessible during
+        # interpreter shutdown (when all module globals get set to None, leading
+        # to confusing errors like "'NoneType' object has no attribute 'error'".
+        class FakeChannel(object):
+            def get_transport(self):
+                return self
+            def get_log_channel(self):
+                return 'paramiko'
+            def get_name(self):
+                return '1'
+            def get_hexdump(self):
+                return False
+            def close(self):
+                pass
+
+        server = paramiko.SFTPServer(
+            FakeChannel(), 'sftp', StubServer(self), StubSFTPServer,
+            root=self._root, home=self._server_homedir)
+        try:
+            server.start_subsystem(
+                'sftp', None, ssh.SocketAsChannelAdapter(sock))
+        except socket.error, e:
+            if (len(e.args) > 0) and (e.args[0] == errno.EPIPE):
+                # it's okay for the client to disconnect abruptly
+                # (bug in paramiko 1.6: it should absorb this exception)
+                pass
+            else:
+                raise
+        except Exception, e:
+            # This typically seems to happen during interpreter shutdown, so
+            # most of the useful ways to report this error are won't work.
+            # Writing the exception type, and then the text of the exception,
+            # seems to be the best we can do.
+            import sys
+            sys.stderr.write('\nEXCEPTION %r: ' % (e.__class__,))
+            sys.stderr.write('%s\n\n' % (e,))
+        server.finish_subsystem()
+
+
+class SFTPAbsoluteServer(SFTPServerWithoutSSH):
+    """A test server for sftp transports, using absolute urls."""
+
+    def get_url(self):
+        """See bzrlib.transport.Server.get_url."""
+        homedir = self._homedir
+        if sys.platform != 'win32':
+            # Remove the initial '/' on all platforms but win32
+            homedir = homedir[1:]
+        return self._get_sftp_url(urlutils.escape(homedir))
+
+
+class SFTPHomeDirServer(SFTPServerWithoutSSH):
+    """A test server for sftp transports, using homedir relative urls."""
+
+    def get_url(self):
+        """See bzrlib.transport.Server.get_url."""
+        return self._get_sftp_url("~/")
+
+
+class SFTPSiblingAbsoluteServer(SFTPAbsoluteServer):
+    """A test server for sftp transports where only absolute paths will work.
+
+    It does this by serving from a deeply-nested directory that doesn't exist.
+    """
+
+    def start_server(self, backing_server=None):
+        self._server_homedir = '/dev/noone/runs/tests/here'
+        super(SFTPSiblingAbsoluteServer, self).start_server(backing_server)
+

=== modified file 'bzrlib/tests/test_selftest.py'
--- a/bzrlib/tests/test_selftest.py	2010-01-17 21:48:39 +0000
+++ b/bzrlib/tests/test_selftest.py	2010-02-03 14:08:55 +0000
@@ -62,6 +62,7 @@
     )
 from bzrlib.tests import (
     features,
+    stub_sftp,
     test_lsprof,
     test_sftp_transport,
     TestUtil,
@@ -1927,7 +1928,7 @@
 
     def test_transport_sftp(self):
         self.requireFeature(features.paramiko)
-        self.check_transport_set(bzrlib.transport.sftp.SFTPAbsoluteServer)
+        self.check_transport_set(stub_sftp.SFTPAbsoluteServer)
 
     def test_transport_memory(self):
         self.check_transport_set(bzrlib.transport.memory.MemoryServer)

=== modified file 'bzrlib/tests/test_sftp_transport.py'
--- a/bzrlib/tests/test_sftp_transport.py	2010-01-07 03:03:01 +0000
+++ b/bzrlib/tests/test_sftp_transport.py	2010-02-03 14:08:55 +0000
@@ -46,11 +46,7 @@
 
 if features.paramiko.available():
     from bzrlib.transport import sftp as _mod_sftp
-    from bzrlib.transport.sftp import (
-        SFTPAbsoluteServer,
-        SFTPHomeDirServer,
-        SFTPTransport,
-        )
+    from bzrlib.tests import stub_sftp
 
 from bzrlib.workingtree import WorkingTree
 
@@ -60,9 +56,9 @@
     if getattr(testcase, '_get_remote_is_absolute', None) is None:
         testcase._get_remote_is_absolute = True
     if testcase._get_remote_is_absolute:
-        testcase.transport_server = SFTPAbsoluteServer
+        testcase.transport_server = stub_sftp.SFTPAbsoluteServer
     else:
-        testcase.transport_server = SFTPHomeDirServer
+        testcase.transport_server = stub_sftp.SFTPHomeDirServer
     testcase.transport_readonly_server = HttpServer
 
 
@@ -162,7 +158,8 @@
         self.requireFeature(features.paramiko)
 
     def test_parse_url_with_home_dir(self):
-        s = SFTPTransport('sftp://ro%62ey:h%40t@example.com:2222/~/relative')
+        s = _mod_sftp.SFTPTransport(
+            'sftp://ro%62ey:h%40t@example.com:2222/~/relative')
         self.assertEquals(s._host, 'example.com')
         self.assertEquals(s._port, 2222)
         self.assertEquals(s._user, 'robey')
@@ -170,7 +167,7 @@
         self.assertEquals(s._path, '/~/relative/')
 
     def test_relpath(self):
-        s = SFTPTransport('sftp://user@host.com/abs/path')
+        s = _mod_sftp.SFTPTransport('sftp://user@host.com/abs/path')
         self.assertRaises(errors.PathNotChild, s.relpath,
                           'sftp://user@host.com/~/rel/path/sub')
 
@@ -190,8 +187,7 @@
             ssh._ssh_vendor_manager._cached_ssh_vendor = orig_vendor
 
     def test_abspath_root_sibling_server(self):
-        from bzrlib.transport.sftp import SFTPSiblingAbsoluteServer
-        server = SFTPSiblingAbsoluteServer()
+        server = stub_sftp.SFTPSiblingAbsoluteServer()
         server.start_server()
         try:
             transport = get_transport(server.get_url())
@@ -255,14 +251,13 @@
 
     def setUp(self):
         super(SSHVendorConnection, self).setUp()
-        from bzrlib.transport.sftp import SFTPFullAbsoluteServer
 
         def create_server():
             """Just a wrapper so that when created, it will set _vendor"""
             # SFTPFullAbsoluteServer can handle any vendor,
             # it just needs to be set between the time it is instantiated
             # and the time .setUp() is called
-            server = SFTPFullAbsoluteServer()
+            server = stub_sftp.SFTPFullAbsoluteServer()
             server._vendor = self._test_vendor
             return server
         self._test_vendor = 'loopback'
@@ -415,21 +410,20 @@
         self.requireFeature(features.paramiko)
 
     def test_delay(self):
-        from bzrlib.transport.sftp import SocketDelay
         sending = FakeSocket()
-        receiving = SocketDelay(sending, 0.1, bandwidth=1000000,
-                                really_sleep=False)
+        receiving = stub_sftp.SocketDelay(sending, 0.1, bandwidth=1000000,
+                                          really_sleep=False)
         # check that simulated time is charged only per round-trip:
-        t1 = SocketDelay.simulated_time
+        t1 = stub_sftp.SocketDelay.simulated_time
         receiving.send("connect1")
         self.assertEqual(sending.recv(1024), "connect1")
-        t2 = SocketDelay.simulated_time
+        t2 = stub_sftp.SocketDelay.simulated_time
         self.assertAlmostEqual(t2 - t1, 0.1)
         receiving.send("connect2")
         self.assertEqual(sending.recv(1024), "connect2")
         sending.send("hello")
         self.assertEqual(receiving.recv(1024), "hello")
-        t3 = SocketDelay.simulated_time
+        t3 = stub_sftp.SocketDelay.simulated_time
         self.assertAlmostEqual(t3 - t2, 0.1)
         sending.send("hello")
         self.assertEqual(receiving.recv(1024), "hello")
@@ -437,21 +431,20 @@
         self.assertEqual(receiving.recv(1024), "hello")
         sending.send("hello")
         self.assertEqual(receiving.recv(1024), "hello")
-        t4 = SocketDelay.simulated_time
+        t4 = stub_sftp.SocketDelay.simulated_time
         self.assertAlmostEqual(t4, t3)
 
     def test_bandwidth(self):
-        from bzrlib.transport.sftp import SocketDelay
         sending = FakeSocket()
-        receiving = SocketDelay(sending, 0, bandwidth=8.0/(1024*1024),
-                                really_sleep=False)
+        receiving = stub_sftp.SocketDelay(sending, 0, bandwidth=8.0/(1024*1024),
+                                          really_sleep=False)
         # check that simulated time is charged only per round-trip:
-        t1 = SocketDelay.simulated_time
+        t1 = stub_sftp.SocketDelay.simulated_time
         receiving.send("connect")
         self.assertEqual(sending.recv(1024), "connect")
         sending.send("a" * 100)
         self.assertEqual(receiving.recv(1024), "a" * 100)
-        t2 = SocketDelay.simulated_time
+        t2 = stub_sftp.SocketDelay.simulated_time
         self.assertAlmostEqual(t2 - t1, 100 + 7)
 
 

=== modified file 'bzrlib/tests/test_transport.py'
--- a/bzrlib/tests/test_transport.py	2010-01-08 20:03:04 +0000
+++ b/bzrlib/tests/test_transport.py	2010-02-03 14:08:55 +0000
@@ -914,8 +914,7 @@
         # SFTPFullAbsoluteServer has a get_url method, and doesn't
         # override the interface (doesn't change self._vendor).
         # Note that this does encryption, so can be slow.
-        from bzrlib.transport.sftp import SFTPFullAbsoluteServer
-        from bzrlib.tests.stub_sftp import StubServer
+        from bzrlib.tests import stub_sftp
 
         # Start an SSH server
         self.command_executed = []
@@ -924,7 +923,7 @@
         # SSH channel ourselves.  Surely this has already been implemented
         # elsewhere?
         started = []
-        class StubSSHServer(StubServer):
+        class StubSSHServer(stub_sftp.StubServer):
 
             test = self
 
@@ -958,7 +957,7 @@
 
                 return True
 
-        ssh_server = SFTPFullAbsoluteServer(StubSSHServer)
+        ssh_server = stub_sftp.SFTPFullAbsoluteServer(StubSSHServer)
         # We *don't* want to override the default SSH vendor: the detected one
         # is the one to use.
         self.start_server(ssh_server)

=== modified file 'bzrlib/transport/sftp.py'
--- a/bzrlib/transport/sftp.py	2010-01-07 03:03:01 +0000
+++ b/bzrlib/transport/sftp.py	2010-02-03 14:08:55 +0000
@@ -28,8 +28,6 @@
 import itertools
 import os
 import random
-import select
-import socket
 import stat
 import sys
 import time
@@ -884,332 +882,11 @@
         else:
             return True
 
-# ------------- server test implementation --------------
-import threading
-
-from bzrlib.tests.stub_sftp import StubServer, StubSFTPServer
-
-STUB_SERVER_KEY = """
------BEGIN RSA PRIVATE KEY-----
-MIICWgIBAAKBgQDTj1bqB4WmayWNPB+8jVSYpZYk80Ujvj680pOTh2bORBjbIAyz
-oWGW+GUjzKxTiiPvVmxFgx5wdsFvF03v34lEVVhMpouqPAYQ15N37K/ir5XY+9m/
-d8ufMCkjeXsQkKqFbAlQcnWMCRnOoPHS3I4vi6hmnDDeeYTSRvfLbW0fhwIBIwKB
-gBIiOqZYaoqbeD9OS9z2K9KR2atlTxGxOJPXiP4ESqP3NVScWNwyZ3NXHpyrJLa0
-EbVtzsQhLn6rF+TzXnOlcipFvjsem3iYzCpuChfGQ6SovTcOjHV9z+hnpXvQ/fon
-soVRZY65wKnF7IAoUwTmJS9opqgrN6kRgCd3DASAMd1bAkEA96SBVWFt/fJBNJ9H
-tYnBKZGw0VeHOYmVYbvMSstssn8un+pQpUm9vlG/bp7Oxd/m+b9KWEh2xPfv6zqU
-avNwHwJBANqzGZa/EpzF4J8pGti7oIAPUIDGMtfIcmqNXVMckrmzQ2vTfqtkEZsA
-4rE1IERRyiJQx6EJsz21wJmGV9WJQ5kCQQDwkS0uXqVdFzgHO6S++tjmjYcxwr3g
-H0CoFYSgbddOT6miqRskOQF3DZVkJT3kyuBgU2zKygz52ukQZMqxCb1fAkASvuTv
-qfpH87Qq5kQhNKdbbwbmd2NxlNabazPijWuphGTdW0VfJdWfklyS2Kr+iqrs/5wV
-HhathJt636Eg7oIjAkA8ht3MQ+XSl9yIJIS8gVpbPxSw5OMfw0PjVE7tBdQruiSc
-nvuQES5C9BMHjF39LZiGH1iLQy7FgdHyoP+eodI7
------END RSA PRIVATE KEY-----
-"""
-
-
-class SocketListener(threading.Thread):
-
-    def __init__(self, callback):
-        threading.Thread.__init__(self)
-        self._callback = callback
-        self._socket = socket.socket()
-        self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        self._socket.bind(('localhost', 0))
-        self._socket.listen(1)
-        self.host, self.port = self._socket.getsockname()[:2]
-        self._stop_event = threading.Event()
-
-    def stop(self):
-        # called from outside this thread
-        self._stop_event.set()
-        # use a timeout here, because if the test fails, the server thread may
-        # never notice the stop_event.
-        self.join(5.0)
-        self._socket.close()
-
-    def run(self):
-        while True:
-            readable, writable_unused, exception_unused = \
-                select.select([self._socket], [], [], 0.1)
-            if self._stop_event.isSet():
-                return
-            if len(readable) == 0:
-                continue
-            try:
-                s, addr_unused = self._socket.accept()
-                # because the loopback socket is inline, and transports are
-                # never explicitly closed, best to launch a new thread.
-                threading.Thread(target=self._callback, args=(s,)).start()
-            except socket.error, x:
-                sys.excepthook(*sys.exc_info())
-                warning('Socket error during accept() within unit test server'
-                        ' thread: %r' % x)
-            except Exception, x:
-                # probably a failed test; unit test thread will log the
-                # failure/error
-                sys.excepthook(*sys.exc_info())
-                warning('Exception from within unit test server thread: %r' %
-                        x)
-
-
-class SocketDelay(object):
-    """A socket decorator to make TCP appear slower.
-
-    This changes recv, send, and sendall to add a fixed latency to each python
-    call if a new roundtrip is detected. That is, when a recv is called and the
-    flag new_roundtrip is set, latency is charged. Every send and send_all
-    sets this flag.
-
-    In addition every send, sendall and recv sleeps a bit per character send to
-    simulate bandwidth.
-
-    Not all methods are implemented, this is deliberate as this class is not a
-    replacement for the builtin sockets layer. fileno is not implemented to
-    prevent the proxy being bypassed.
-    """
-
-    simulated_time = 0
-    _proxied_arguments = dict.fromkeys([
-        "close", "getpeername", "getsockname", "getsockopt", "gettimeout",
-        "setblocking", "setsockopt", "settimeout", "shutdown"])
-
-    def __init__(self, sock, latency, bandwidth=1.0,
-                 really_sleep=True):
-        """
-        :param bandwith: simulated bandwith (MegaBit)
-        :param really_sleep: If set to false, the SocketDelay will just
-        increase a counter, instead of calling time.sleep. This is useful for
-        unittesting the SocketDelay.
-        """
-        self.sock = sock
-        self.latency = latency
-        self.really_sleep = really_sleep
-        self.time_per_byte = 1 / (bandwidth / 8.0 * 1024 * 1024)
-        self.new_roundtrip = False
-
-    def sleep(self, s):
-        if self.really_sleep:
-            time.sleep(s)
-        else:
-            SocketDelay.simulated_time += s
-
-    def __getattr__(self, attr):
-        if attr in SocketDelay._proxied_arguments:
-            return getattr(self.sock, attr)
-        raise AttributeError("'SocketDelay' object has no attribute %r" %
-                             attr)
-
-    def dup(self):
-        return SocketDelay(self.sock.dup(), self.latency, self.time_per_byte,
-                           self._sleep)
-
-    def recv(self, *args):
-        data = self.sock.recv(*args)
-        if data and self.new_roundtrip:
-            self.new_roundtrip = False
-            self.sleep(self.latency)
-        self.sleep(len(data) * self.time_per_byte)
-        return data
-
-    def sendall(self, data, flags=0):
-        if not self.new_roundtrip:
-            self.new_roundtrip = True
-            self.sleep(self.latency)
-        self.sleep(len(data) * self.time_per_byte)
-        return self.sock.sendall(data, flags)
-
-    def send(self, data, flags=0):
-        if not self.new_roundtrip:
-            self.new_roundtrip = True
-            self.sleep(self.latency)
-        bytes_sent = self.sock.send(data, flags)
-        self.sleep(bytes_sent * self.time_per_byte)
-        return bytes_sent
-
-
-class SFTPServer(Server):
-    """Common code for SFTP server facilities."""
-
-    def __init__(self, server_interface=StubServer):
-        self._original_vendor = None
-        self._homedir = None
-        self._server_homedir = None
-        self._listener = None
-        self._root = None
-        self._vendor = ssh.ParamikoVendor()
-        self._server_interface = server_interface
-        # sftp server logs
-        self.logs = []
-        self.add_latency = 0
-
-    def _get_sftp_url(self, path):
-        """Calculate an sftp url to this server for path."""
-        return 'sftp://foo:bar@%s:%d/%s' % (self._listener.host,
-                                            self._listener.port, path)
-
-    def log(self, message):
-        """StubServer uses this to log when a new server is created."""
-        self.logs.append(message)
-
-    def _run_server_entry(self, sock):
-        """Entry point for all implementations of _run_server.
-
-        If self.add_latency is > 0.000001 then sock is given a latency adding
-        decorator.
-        """
-        if self.add_latency > 0.000001:
-            sock = SocketDelay(sock, self.add_latency)
-        return self._run_server(sock)
-
-    def _run_server(self, s):
-        ssh_server = paramiko.Transport(s)
-        key_file = pathjoin(self._homedir, 'test_rsa.key')
-        f = open(key_file, 'w')
-        f.write(STUB_SERVER_KEY)
-        f.close()
-        host_key = paramiko.RSAKey.from_private_key_file(key_file)
-        ssh_server.add_server_key(host_key)
-        server = self._server_interface(self)
-        ssh_server.set_subsystem_handler('sftp', paramiko.SFTPServer,
-                                         StubSFTPServer, root=self._root,
-                                         home=self._server_homedir)
-        event = threading.Event()
-        ssh_server.start_server(event, server)
-        event.wait(5.0)
-
-    def start_server(self, backing_server=None):
-        # XXX: TODO: make sftpserver back onto backing_server rather than local
-        # disk.
-        if not (backing_server is None or
-                isinstance(backing_server, local.LocalURLServer)):
-            raise AssertionError(
-                "backing_server should not be %r, because this can only serve the "
-                "local current working directory." % (backing_server,))
-        self._original_vendor = ssh._ssh_vendor_manager._cached_ssh_vendor
-        ssh._ssh_vendor_manager._cached_ssh_vendor = self._vendor
-        if sys.platform == 'win32':
-            # Win32 needs to use the UNICODE api
-            self._homedir = getcwd()
-        else:
-            # But Linux SFTP servers should just deal in bytestreams
-            self._homedir = os.getcwd()
-        if self._server_homedir is None:
-            self._server_homedir = self._homedir
-        self._root = '/'
-        if sys.platform == 'win32':
-            self._root = ''
-        self._listener = SocketListener(self._run_server_entry)
-        self._listener.setDaemon(True)
-        self._listener.start()
-
-    def stop_server(self):
-        self._listener.stop()
-        ssh._ssh_vendor_manager._cached_ssh_vendor = self._original_vendor
-
-    def get_bogus_url(self):
-        """See bzrlib.transport.Server.get_bogus_url."""
-        # this is chosen to try to prevent trouble with proxies, wierd dns, etc
-        # we bind a random socket, so that we get a guaranteed unused port
-        # we just never listen on that port
-        s = socket.socket()
-        s.bind(('localhost', 0))
-        return 'sftp://%s:%s/' % s.getsockname()
-
-
-class SFTPFullAbsoluteServer(SFTPServer):
-    """A test server for sftp transports, using absolute urls and ssh."""
-
-    def get_url(self):
-        """See bzrlib.transport.Server.get_url."""
-        homedir = self._homedir
-        if sys.platform != 'win32':
-            # Remove the initial '/' on all platforms but win32
-            homedir = homedir[1:]
-        return self._get_sftp_url(urlutils.escape(homedir))
-
-
-class SFTPServerWithoutSSH(SFTPServer):
-    """An SFTP server that uses a simple TCP socket pair rather than SSH."""
-
-    def __init__(self):
-        super(SFTPServerWithoutSSH, self).__init__()
-        self._vendor = ssh.LoopbackVendor()
-
-    def _run_server(self, sock):
-        # Re-import these as locals, so that they're still accessible during
-        # interpreter shutdown (when all module globals get set to None, leading
-        # to confusing errors like "'NoneType' object has no attribute 'error'".
-        class FakeChannel(object):
-            def get_transport(self):
-                return self
-            def get_log_channel(self):
-                return 'paramiko'
-            def get_name(self):
-                return '1'
-            def get_hexdump(self):
-                return False
-            def close(self):
-                pass
-
-        server = paramiko.SFTPServer(
-            FakeChannel(), 'sftp', StubServer(self), StubSFTPServer,
-            root=self._root, home=self._server_homedir)
-        try:
-            server.start_subsystem(
-                'sftp', None, ssh.SocketAsChannelAdapter(sock))
-        except socket.error, e:
-            if (len(e.args) > 0) and (e.args[0] == errno.EPIPE):
-                # it's okay for the client to disconnect abruptly
-                # (bug in paramiko 1.6: it should absorb this exception)
-                pass
-            else:
-                raise
-        except Exception, e:
-            # This typically seems to happen during interpreter shutdown, so
-            # most of the useful ways to report this error are won't work.
-            # Writing the exception type, and then the text of the exception,
-            # seems to be the best we can do.
-            import sys
-            sys.stderr.write('\nEXCEPTION %r: ' % (e.__class__,))
-            sys.stderr.write('%s\n\n' % (e,))
-        server.finish_subsystem()
-
-
-class SFTPAbsoluteServer(SFTPServerWithoutSSH):
-    """A test server for sftp transports, using absolute urls."""
-
-    def get_url(self):
-        """See bzrlib.transport.Server.get_url."""
-        homedir = self._homedir
-        if sys.platform != 'win32':
-            # Remove the initial '/' on all platforms but win32
-            homedir = homedir[1:]
-        return self._get_sftp_url(urlutils.escape(homedir))
-
-
-class SFTPHomeDirServer(SFTPServerWithoutSSH):
-    """A test server for sftp transports, using homedir relative urls."""
-
-    def get_url(self):
-        """See bzrlib.transport.Server.get_url."""
-        return self._get_sftp_url("~/")
-
-
-class SFTPSiblingAbsoluteServer(SFTPAbsoluteServer):
-    """A test server for sftp transports where only absolute paths will work.
-
-    It does this by serving from a deeply-nested directory that doesn't exist.
-    """
-
-    def start_server(self, backing_server=None):
-        self._server_homedir = '/dev/noone/runs/tests/here'
-        super(SFTPSiblingAbsoluteServer, self).start_server(backing_server)
-
 
 def get_test_permutations():
     """Return the permutations to be used in testing."""
-    return [(SFTPTransport, SFTPAbsoluteServer),
-            (SFTPTransport, SFTPHomeDirServer),
-            (SFTPTransport, SFTPSiblingAbsoluteServer),
+    from bzrlib.tests import stub_sftp
+    return [(SFTPTransport, stub_sftp.SFTPAbsoluteServer),
+            (SFTPTransport, stub_sftp.SFTPHomeDirServer),
+            (SFTPTransport, stub_sftp.SFTPSiblingAbsoluteServer),
             ]



More information about the bazaar-commits mailing list