Rev 2673: Start open_file_stream logic. in http://people.ubuntu.com/~robertc/baz2.0/transport
Robert Collins
robertc at robertcollins.net
Sun Aug 5 03:57:48 BST 2007
At http://people.ubuntu.com/~robertc/baz2.0/transport
------------------------------------------------------------
revno: 2673
revision-id: robertc at robertcollins.net-20070805025745-eg2qmr8jzsky39y2
parent: robertc at robertcollins.net-20070805014730-qjx8zkquv3pagglo
committer: Robert Collins <robertc at robertcollins.net>
branch nick: transport-get-file
timestamp: Sun 2007-08-05 12:57:45 +1000
message:
Start open_file_stream logic.
modified:
bzrlib/tests/test_transport_implementations.py test_transport_implementations.py-20051227111451-f97c5c7d5c49fce7
bzrlib/transport/__init__.py transport.py-20050711165921-4978aa7ce1285ad5
bzrlib/transport/chroot.py chroot.py-20061011104729-0us9mgm97z378vnt-1
bzrlib/transport/decorator.py decorator.py-20060402223305-e913a0f25319ab42
bzrlib/transport/ftp.py ftp.py-20051116161804-58dc9506548c2a53
bzrlib/transport/local.py local_transport.py-20050711165921-9b1f142bfe480c24
bzrlib/transport/memory.py memory.py-20051016101338-cd008dbdf69f04fc
bzrlib/transport/remote.py ssh.py-20060608202016-c25gvf1ob7ypbus6-1
bzrlib/transport/sftp.py sftp.py-20051019050329-ab48ce71b7e32dfe
=== modified file 'bzrlib/tests/test_transport_implementations.py'
--- a/bzrlib/tests/test_transport_implementations.py 2007-08-05 01:47:30 +0000
+++ b/bzrlib/tests/test_transport_implementations.py 2007-08-05 02:57:45 +0000
@@ -633,6 +633,16 @@
t.mkdir('dnomode', mode=None)
self.assertTransportMode(t, 'dnomode', 0777 & ~umask)
+ def test_opening_a_file_stream_creates_file(self):
+ t = self.get_transport()
+ if t.is_readonly():
+ return
+ handle = t.open_file_stream('foo')
+ try:
+ self.assertEqual('', t.get_bytes('foo'))
+ finally:
+ t.close_file_stream('foo')
+
def test_copy_to(self):
# FIXME: test: same server to same server (partly done)
# same protocol two servers
=== modified file 'bzrlib/transport/__init__.py'
--- a/bzrlib/transport/__init__.py 2007-08-05 01:47:30 +0000
+++ b/bzrlib/transport/__init__.py 2007-08-05 02:57:45 +0000
@@ -66,6 +66,11 @@
from bzrlib import registry
+# a dictionary of open file streams. Keys are absolute paths, values are
+# transport defined.
+_file_streams = {}
+
+
def _get_protocol_handlers():
"""Return a dictionary of {urlprefix: [factory]}"""
return transport_list_registry
@@ -313,6 +318,15 @@
"""
raise NotImplementedError(self.clone)
+ def close_file_stream(self, relpath):
+ """Close a file stream at relpath.
+
+ :raises: NoSuchFile if there is no open file stream for relpath.
+ :seealso: open_file_stream.
+ :return: None
+ """
+ raise NotImplementedError(self.close_file_stream)
+
def ensure_base(self):
"""Ensure that the directory this transport references exists.
@@ -832,6 +846,20 @@
self.mkdir(path, mode=mode)
return len(self._iterate_over(relpaths, mkdir, pb, 'mkdir', expand=False))
+ def open_file_stream(self, relpath):
+ """Open a file stream at relpath.
+
+ A file stream is a callback which adds data to the file. Buffering
+ may occur internally until the stream is closed with close_file_stream.
+ Calls to readv or the get_* methods will be synchronised with any
+ internal buffering that may be present.
+
+ :seealso: close_file_stream.
+ :param relpath: The relative path to the file.
+ :return: A write callback to add data to the file.
+ """
+ raise NotImplementedError(self.open_file_stream)
+
@deprecated_method(zero_eleven)
def append(self, relpath, f, mode=None):
"""Append the text in the file-like object to the supplied location.
=== modified file 'bzrlib/transport/chroot.py'
--- a/bzrlib/transport/chroot.py 2007-07-20 03:20:20 +0000
+++ b/bzrlib/transport/chroot.py 2007-08-05 02:57:45 +0000
@@ -92,6 +92,9 @@
def clone(self, relpath):
return ChrootTransport(self.server, self.abspath(relpath))
+ def close_file_stream(self, relpath):
+ return self._call('close_file_stream', relpath)
+
def delete(self, relpath):
return self._call('delete', relpath)
@@ -133,6 +136,9 @@
def mkdir(self, relpath, mode=None):
return self._call('mkdir', relpath, mode)
+ def open_file_stream(self, relpath):
+ return self._call('open_file_stream', relpath)
+
def put_file(self, relpath, f, mode=None):
return self._call('put_file', relpath, f, mode)
=== modified file 'bzrlib/transport/decorator.py'
--- a/bzrlib/transport/decorator.py 2007-08-05 01:47:30 +0000
+++ b/bzrlib/transport/decorator.py 2007-08-05 02:57:45 +0000
@@ -71,6 +71,10 @@
return self.__class__(
self._get_url_prefix() + decorated_clone.base, decorated_clone)
+ def close_file_stream(self, relpath):
+ """See Transport.close_file_stream."""
+ return self._decorated.close_file_stream(relpath)
+
def delete(self, relpath):
"""See Transport.delete()."""
return self._decorated.delete(relpath)
@@ -110,6 +114,10 @@
"""See Transport.mkdir()."""
return self._decorated.mkdir(relpath, mode)
+ def open_file_stream(self, relpath):
+ """See Transport.open_file_stream."""
+ return self._decorated.open_file_stream(relpath)
+
def put_file(self, relpath, f, mode=None):
"""See Transport.put_file()."""
return self._decorated.put_file(relpath, f, mode)
=== modified file 'bzrlib/transport/ftp.py'
--- a/bzrlib/transport/ftp.py 2007-08-05 01:47:30 +0000
+++ b/bzrlib/transport/ftp.py 2007-08-05 02:57:45 +0000
@@ -46,6 +46,7 @@
)
from bzrlib.trace import mutter, warning
from bzrlib.transport import (
+ _file_streams,
Server,
ConnectedTransport,
)
@@ -105,6 +106,10 @@
self._set_connection(connection, credentials)
return connection
+ def close_file_stream(self, relpath):
+ """See Transport.close_file_stream."""
+ del _file_streams[self.abspath(relpath)]
+
def _create_connection(self, credentials=None):
"""Create a new connection with the provided credentials.
@@ -323,6 +328,14 @@
self._translate_perm_error(e, abspath,
unknown_exc=errors.FileExists)
+ def open_file_stream(self, relpath):
+ """See Transport.open_file_stream."""
+ def append_data(bytes):
+ self.append_bytes(relpath, bytes)
+ self.put_bytes(relpath, "")
+ _file_streams[self.abspath(relpath)] = append_data
+ return append_data
+
def recommended_page_size(self):
"""See Transport.recommended_page_size().
=== modified file 'bzrlib/transport/local.py'
--- a/bzrlib/transport/local.py 2007-08-02 03:17:46 +0000
+++ b/bzrlib/transport/local.py 2007-08-05 02:57:45 +0000
@@ -33,6 +33,7 @@
osutils,
urlutils,
symbol_versioning,
+ transport,
)
from bzrlib.trace import mutter
from bzrlib.transport import LateReadError
@@ -84,6 +85,11 @@
abspath = self.base
return LocalTransport(abspath)
+ def close_file_stream(self, relpath):
+ """See Transport.close_file_stream."""
+ handle = transport._file_streams.pop(self.abspath(relpath))
+ handle.close()
+
def _abspath(self, relative_reference):
"""Return a path for use in os calls.
@@ -301,6 +307,14 @@
"""Create a directory at the given path."""
self._mkdir(self._abspath(relpath), mode=mode)
+ def open_file_stream(self, relpath):
+ """See Transport.open_file_stream."""
+ # initialise the file
+ self.put_bytes_non_atomic(relpath, "")
+ handle = open(self._abspath(relpath), 'wb')
+ transport._file_streams[self.abspath(relpath)] = handle
+ return handle.write
+
def _get_append_file(self, relpath, mode=None):
"""Call os.open() for the given relpath"""
file_abspath = self._abspath(relpath)
=== modified file 'bzrlib/transport/memory.py'
--- a/bzrlib/transport/memory.py 2007-07-20 03:20:20 +0000
+++ b/bzrlib/transport/memory.py 2007-08-05 02:57:45 +0000
@@ -36,6 +36,7 @@
)
from bzrlib.trace import mutter
from bzrlib.transport import (
+ _file_streams,
LateReadError,
register_transport,
Server,
@@ -89,6 +90,10 @@
result._locks = self._locks
return result
+ def close_file_stream(self, relpath):
+ """See Transport.close_file_stream."""
+ del _file_streams[self.abspath(relpath)]
+
def abspath(self, relpath):
"""See Transport.abspath()."""
# while a little slow, this is sufficiently fast to not matter in our
@@ -165,6 +170,14 @@
raise FileExists(relpath)
self._dirs[_abspath]=mode
+ def open_file_stream(self, relpath):
+ """See Transport.open_file_stream."""
+ def append_data(bytes):
+ self.append_bytes(relpath, bytes)
+ self.put_bytes(relpath, "")
+ _file_streams[self.abspath(relpath)] = append_data
+ return append_data
+
def listable(self):
"""See Transport.listable."""
return True
=== modified file 'bzrlib/transport/remote.py'
--- a/bzrlib/transport/remote.py 2007-07-30 14:36:04 +0000
+++ b/bzrlib/transport/remote.py 2007-08-05 02:57:45 +0000
@@ -127,6 +127,10 @@
# No credentials
return None, None
+ def close_file_stream(self, relpath):
+ """See Transport.close_file_stream."""
+ del transport._file_streams[self.abspath(relpath)]
+
def is_readonly(self):
"""Smart server transport can do read/write file operations."""
resp = self._call2('Transport.is_readonly')
@@ -213,6 +217,14 @@
self._serialise_optional_mode(mode))
self._translate_error(resp)
+ def open_file_stream(self, relpath):
+ """See Transport.open_file_stream."""
+ def append_data(bytes):
+ self.append_bytes(relpath, bytes)
+ self.put_bytes(relpath, "")
+ transport._file_streams[self.abspath(relpath)] = append_data
+ return append_data
+
def put_bytes(self, relpath, upload_contents, mode=None):
# FIXME: upload_file is probably not safe for non-ascii characters -
# should probably just pass all parameters as length-delimited
=== modified file 'bzrlib/transport/sftp.py'
--- a/bzrlib/transport/sftp.py 2007-08-05 01:47:30 +0000
+++ b/bzrlib/transport/sftp.py 2007-08-05 02:57:45 +0000
@@ -53,6 +53,7 @@
)
from bzrlib.trace import mutter, warning
from bzrlib.transport import (
+ _file_streams,
local,
register_urlparse_netloc_protocol,
Server,
@@ -156,6 +157,11 @@
super(SFTPTransport, self).__init__(base,
_from_transport=_from_transport)
+ def close_file_stream(self, relpath):
+ """See Transport.close_file_stream."""
+ handle = _file_streams.pop(self.abspath(relpath))
+ handle.close()
+
def _remote_path(self, relpath):
"""Return the path to be passed along the sftp protocol for relpath.
@@ -533,6 +539,28 @@
"""Create a directory at the given path."""
self._mkdir(self._remote_path(relpath), mode=mode)
+ def open_file_stream(self, relpath):
+ """See Transport.open_file_stream."""
+ # initialise the file to zero-length
+ # this is three round trips, but we don't use this
+ # api more than once per write_group at the moment so
+ # it is a tolerable overhead. Better would be to truncate
+ # the file after opening. RBC 20070805
+ self.put_bytes_non_atomic(relpath, "")
+ abspath = self._remote_path(relpath)
+ # TODO: jam 20060816 paramiko doesn't publicly expose a way to
+ # set the file mode at create time. If it does, use it.
+ # But for now, we just chmod later anyway.
+ handle = None
+ try:
+ handle = self._get_sftp().file(abspath, mode='wb')
+ handle.set_pipelined(True)
+ except (paramiko.SSHException, IOError), e:
+ self._translate_io_exception(e, abspath,
+ ': unable to open')
+ _file_streams[self.abspath(relpath)] = handle
+ return handle.write
+
def _translate_io_exception(self, e, path, more_info='',
failure_exc=PathError):
"""Translate a paramiko or IOError into a friendlier exception.
More information about the bazaar-commits
mailing list