Rev 2764: gzip usage tuning. in http://people.ubuntu.com/~robertc/baz2.0/repository

[Robert Collins]

At http://people.ubuntu.com/~robertc/baz2.0/repository

------------------------------------------------------------
revno: 2764
revision-id: robertc at robertcollins.net-20070913032853-mgpq1d9vpqm1sxsc
parent: robertc at robertcollins.net-20070912234007-9j5fd6isl84t7tw1
committer: Robert Collins <robertc at robertcollins.net>
branch nick: repository
timestamp: Thu 2007-09-13 13:28:53 +1000
message:
  gzip usage tuning.
modified:
  bzrlib/knit.py                 knit.py-20051212171256-f056ac8f0fbe1bd9
  bzrlib/tuned_gzip.py           tuned_gzip.py-20060407014720-5aadc518e928e8d2
=== modified file 'bzrlib/knit.py'
--- a/bzrlib/knit.py	2007-09-12 23:30:46 +0000
+++ b/bzrlib/knit.py	2007-09-13 03:28:53 +0000
@@ -100,7 +100,7 @@
     RevisionNotPresent,
     RevisionAlreadyPresent,
     )
-from bzrlib.tuned_gzip import GzipFile
+from bzrlib.tuned_gzip import GzipFile, bytes_to_gzip
 from bzrlib.osutils import (
     contains_whitespace,
     contains_linebreaks,
@@ -1981,22 +1981,15 @@
         
         :return: (len, a StringIO instance with the raw data ready to read.)
         """
-        sio = StringIO()
-        data_file = GzipFile(None, mode='wb', fileobj=sio,
-            compresslevel=Z_DEFAULT_COMPRESSION)
-
-        assert isinstance(version_id, str)
-        data_file.writelines(chain(
+        bytes = (''.join(chain(
             ["version %s %d %s\n" % (version_id,
                                      len(lines),
                                      digest)],
             lines,
-            ["end %s\n" % version_id]))
-        data_file.close()
-        length= sio.tell()
-
-        sio.seek(0)
-        return length, sio
+            ["end %s\n" % version_id])))
+        assert bytes.__class__ == str
+        compressed_bytes = bytes_to_gzip(bytes)
+        return len(compressed_bytes), compressed_bytes
 
     def add_raw_records(self, sizes, raw_data):
         """Append a prepared record to the data file.
@@ -2014,10 +2007,10 @@
         
         Returns index data for retrieving it later, as per add_raw_records.
         """
-        size, sio = self._record_to_data(version_id, digest, lines)
-        result = self.add_raw_records([size], sio.getvalue())
+        size, bytes = self._record_to_data(version_id, digest, lines)
+        result = self.add_raw_records([size], bytes)
         if self._do_cache:
-            self._cache[version_id] = sio.getvalue()
+            self._cache[version_id] = bytes
         return result[0]
 
     def _parse_record_header(self, version_id, raw_data):

=== modified file 'bzrlib/tuned_gzip.py'
--- a/bzrlib/tuned_gzip.py	2006-10-11 23:08:27 +0000
+++ b/bzrlib/tuned_gzip.py	2007-09-13 03:28:53 +0000
@@ -29,7 +29,37 @@
 # we want a \n preserved, break on \n only splitlines.
 import bzrlib
 
-__all__ = ["GzipFile"]
+__all__ = ["GzipFile", "bytes_to_gzip"]
+
+
+def bytes_to_gzip(bytes, factory=zlib.compressobj,
+    level=zlib.Z_DEFAULT_COMPRESSION, method=zlib.DEFLATED,
+    width=-zlib.MAX_WBITS, mem=zlib.DEF_MEM_LEVEL,
+    crc32=zlib.crc32):
+    """Create a gzip file containing bytes and return its content."""
+    result = [
+        '\037\213'  # self.fileobj.write('\037\213')  # magic header
+        '\010'      # self.fileobj.write('\010')      # compression method
+                    # fname = self.filename[:-3]
+                    # flags = 0
+                    # if fname:
+                    #     flags = FNAME
+        '\x00'      # self.fileobj.write(chr(flags))
+        '\0\0\0\0'  # write32u(self.fileobj, long(time.time()))
+        '\002'      # self.fileobj.write('\002')
+        '\377'      # self.fileobj.write('\377')
+                    # if fname:
+        ''          #     self.fileobj.write(fname + '\000')
+        ]
+    # using a compressobj avoids a small header and trailer that the compress()
+    # utility function adds.
+    compress = factory(level, method, width, mem, 0)
+    result.append(compress.compress(bytes))
+    result.append(compress.flush())
+    result.append(struct.pack("<L", LOWU32(crc32(bytes))))
+    # size may exceed 2GB, or even 4GB
+    result.append(struct.pack("<L", LOWU32(len(bytes))))
+    return ''.join(result)
 
 
 class GzipFile(gzip.GzipFile):