Rev 30: More precision for latency. in http://code.launchpad.net/%7Ev-ladeuil/bzr/transportstats

Vincent Ladeuil v.ladeuil+lp at free.fr
Fri Dec 7 13:51:16 GMT 2007


At http://code.launchpad.net/%7Ev-ladeuil/bzr/transportstats

------------------------------------------------------------
revno: 30
revision-id:v.ladeuil+lp at free.fr-20071207135112-wuaf24l9ovhcypz6
parent: v.ladeuil+lp at free.fr-20071207110923-g1fzl339sb4665h8
committer: Vincent Ladeuil <v.ladeuil+lp at free.fr>
branch nick: transportstats
timestamp: Fri 2007-12-07 14:51:12 +0100
message:
  More precision for latency.
  
  * stats.py:
  (HTTPTransportReadv): readv should not be counted as requests
  since we count the _get.
  
  * decorator.py:
  (HTTPStatsCollector): readv and get should not be instrumented
  since we instrument _get directly.
  (HTTPStatsCollector.__init__): Rename the original and injected
  _get attributes with better names.
  (HTTPStatsCollector._get_nb_bytes_streamed): Take into account the
  latency of the reads on the socket. Clean up dead code.
modified:
  decorator.py                   decorator.py-20070926152401-52kuiex1mu755ajk-1
  stats.py                       stats.py-20070928061304-7i3r2h4gg6rbi03e-1
-------------- next part --------------
=== modified file 'decorator.py'
--- a/decorator.py	2007-12-07 11:09:23 +0000
+++ b/decorator.py	2007-12-07 13:51:12 +0000
@@ -295,8 +295,10 @@
                                                  _from_transport)
         # Install our hooks. Note that the methods are already bound so they
         # get the right 'self' when executing.
-        self._get_of_decorated = self._decorated._get
-        self._decorated._get = self._get_for_decorated
+        self._original_get = self._decorated._get
+        self._decorated._get = self._injected_get
+        # Different http implementation needs different ways of measuring the
+        # bytes
         if http_response_streamed:
             self._get_nb_bytes = self._get_nb_bytes_streamed
         else:
@@ -307,27 +309,36 @@
 
         Since the whole response is buffered we just have to peak under the
         covers of the two possible implementations.
+
+        :return: a tuple (response, nb_bytes, latency)
         """
         if isinstance(resp, response.RangeFile):
             nb_bytes = len(resp._data)
         else:
             nb_bytes = len(resp.getvalue())
-        return nb_bytes, resp
+        # the get latency has already been measured, so no additional latency
+        return resp, nb_bytes, 0
 
     def _get_nb_bytes_streamed(self, resp):
         """Return the number of bytes in the response body.
 
         Since the whole response is streamed, we have to create a temp file
         containing an equivalent content.
+
+        :return: a tuple (response, nb_bytes, latency)
         """
         if not isinstance(resp, response.RangeFile):
             nb_bytes = len(resp.getvalue())
-            return nb_bytes, resp
+            # the get latency has already been measured, so no additional
+            # latency
+            return resp, nb_bytes, 0
 
         temp = tempfile.TemporaryFile()
         if resp._size == -1 or resp._boundary is None:
             # The content is raw bytes
+            self._start()
             data = resp.read()
+            latency = self._latency()
             nb_bytes = len(data)
             temp.write(data)
             temp.seek(0)
@@ -343,37 +354,22 @@
             temp.write(''.join(resp._headers.headers) + '\r\n')
             # Copy the rest of the body bypassing RangeFile.read() to get the
             # raw content
+            self._start()
             data = resp._file.read()
+            latency = self._latency()
             while data:
                 nb_bytes = len(data)
                 temp.write(data)
+                self._start()
                 data = resp._file.read()
+                latency += self._latency()
             temp.seek(0)
             new_resp = response.RangeFile(resp._path, temp)
             new_resp.set_boundary(resp._boundary)
 
-        return nb_bytes, new_resp
-
-        data = response.read()
-        nb_bytes = len(data)
-        temp_file.write(data)
-        again = True
-        while again:
-            try:
-                # We try to seek when at the end of the range (but staying at
-                # the same offset), that should position us to the beginning of
-                # the next range if it exist.
-                response.seek(0, 1)
-                data = response.read()
-                nb_bytes += len(data)
-                temp_file.write(data)
-                again = True
-            except (errors.InvalidRange, errors.InvalidHttpResponse):
-                again = False
-        temp_file.seek(0)
-        return nb_bytes
-
-    def _get_for_decorated(self, relpath, ranges, tail_amount=0):
+        return new_resp, nb_bytes, latency
+
+    def _injected_get(self, relpath, ranges, tail_amount=0):
         """See HttpTransportBase._get().
 
         This method is defined here but will be executed with self being the
@@ -381,18 +377,29 @@
         decorated transport.
         """
         self._start()
-        code, f = self._get_of_decorated(relpath, ranges, tail_amount)
+        code, f = self._original_get(relpath, ranges, tail_amount)
         latency = self._latency()
 
         if code in (200, 206):
             # We collect data on valid responses only (nb_bytes is still an
             # approximation since it doesn't include the HTTP headers.)
-            nb_bytes, f = self._get_nb_bytes(f)
+            f, nb_bytes, added_latency = self._get_nb_bytes(f)
+            latency += added_latency
         else:
             nb_bytes = 0
         self._collect('HttpTransportBase._get', relpath, nb_bytes, latency)
         return code, f
 
+    def get(self, relpath):
+        """See Transport.get()."""
+        # Nothing to do since we instrumented _get
+        return super(DefaultStatsCollector, self).get(relpath)
+
+    def get_bytes(self, relpath):
+        """See Transport.get_bytes()."""
+        # Nothing to do since we instrumented _get
+        return super(DefaultStatsCollector, self).get_bytes(relpath)
+
     def readv(self, relpath, offsets, *args, **kwargs):
         """See Transport.readv().
 

=== modified file 'stats.py'
--- a/stats.py	2007-12-07 11:09:23 +0000
+++ b/stats.py	2007-12-07 13:51:12 +0000
@@ -58,7 +58,7 @@
 class TransportStat(StatsPart):
 
     def __init__(self, *args):
-        StatsPart.__init__(self, *args)
+        super(TransportStat, self).__init__(*args)
         # Default values for most of the stats
         self.requests = 1
 
@@ -66,11 +66,19 @@
 class TransportReadvOffset(TransportStat):
 
     def __init__(self, *args):
-        super(self.__class__, self).__init__(*args)
+        super(TransportReadvOffset, self).__init__(*args)
         # Special case, this stat is part of a request, not a request itself
         self.requests = 0
 
 
+class HTTPTransportReadv(TransportStat):
+
+    def __init__(self, *args):
+        super(TransportStat, self).__init__(*args)
+        # Special case, the _get requests are counted, not the readv
+        self.requests = 0
+
+
 class StatsRegistry(registry.Registry):
     """Statitics registry."""
 
@@ -142,7 +150,7 @@
               '%(base)us%(relpath)us%(latency)L')
 
 
-register_stat('HttpTransportBase.readv', TransportStat,
+register_stat('HttpTransportBase.readv', HTTPTransportReadv,
               '%(base)us%(relpath)us')
 register_stat('HttpTransportBase.readv/offset', TransportReadvOffset,
               '%(base)us%(relpath)us%(offset)L%(bytes_requested)L')



More information about the bazaar-commits mailing list