Rev 2674: Implement the inner loop to retry for more data. in http://bzr.arbash-meinel.com/branches/bzr/0.19-dev/pyrex_knit_extract

John Arbash Meinel john at arbash-meinel.com
Thu Aug 2 23:12:35 BST 2007


At http://bzr.arbash-meinel.com/branches/bzr/0.19-dev/pyrex_knit_extract

------------------------------------------------------------
revno: 2674
revision-id: john at arbash-meinel.com-20070802221201-fq0ze6hinmbs5j9q
parent: john at arbash-meinel.com-20070802215505-hahicpf9014hhe3o
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: pyrex_knit_extract
timestamp: Thu 2007-08-02 17:12:01 -0500
message:
  Implement the inner loop to retry for more data.
  Now we can handle any data that fits inside our 1MB extraction buffer.
  But if a single line is >1MB, we will fail.
  Need to figure out how to do that in a better manner.
modified:
  bzrlib/_knit_helpers_c.pyx     knit_c.pyx-20070509143944-u42gy8w387a10m0j-1
-------------- next part --------------
=== modified file 'bzrlib/_knit_helpers_c.pyx'
--- a/bzrlib/_knit_helpers_c.pyx	2007-08-02 21:55:05 +0000
+++ b/bzrlib/_knit_helpers_c.pyx	2007-08-02 22:12:01 +0000
@@ -121,6 +121,7 @@
 cdef extern from "string.h":
     void *memchr(void *s, int c, size_t n)
     int memcmp(void *b1, void *b2, size_t len)
+    void *memmove(void *dest, void *src, size_t len)
 
 
 cdef int string_to_int_safe(char *s, char *end, int *out) except -1:
@@ -540,6 +541,17 @@
         # to the start of the buffer, and do another decompression from the
         # tail.
         avail_out = self.strm.avail_out
+        if avail_out == 0: # Reset the buffer
+            # Move the current data to the front of the decompress buffer
+            # and open up more data at the tail.
+            memmove(self.decompress_buffer, self.cur_available,
+                    self.bytes_available)
+            self.cur_available = self.decompress_buffer
+            self.strm.next_out = <Bytef*>(self.cur_available
+                                          + self.bytes_available)
+            self.strm.avail_out = (self.decompress_buf_size
+                                   - self.bytes_available)
+            avail_out = self.strm.avail_out
         retval = inflate(&self.strm, Z_NO_FLUSH)
         if retval == Z_STREAM_END:
             self.stream_finished = 1 # True
@@ -580,7 +592,9 @@
         # self.strm.next_in should point to the complete gzip'd hunk.
 
         cdef char *end_of_line
+        cdef int bytes_processed
 
+        bytes_processed = 0
         # TODO: jam 2007-07-24 Figure out how to make this code support lines
         #       of "unlimited" length. Such as a versioned ISO where a single
         #       line is unlikely to fit in the decompression buffer
@@ -595,7 +609,20 @@
             if end_of_line == NULL:
                 # We reached the end of the buffer without finding a newline
                 # we need to extract more bytes, and try again
-                assert False, 'not implemented yet'
+                if self.stream_finished:
+                    raise errors.KnitCorrupt(self.knit_data_name,
+                        "Missing a trailing newline")
+                # Extract a bit more data out of the stream
+                bytes_processed = self.bytes_available
+                self._extract_from_stream()
+                assert self.bytes_available > bytes_processed
+                end_of_line = <char *>memchr(
+                        self.cur_available+bytes_processed,
+                        c'\n', self.bytes_available-bytes_processed)
+                if end_of_line == NULL:
+                    # This seems like we have a single line which
+                    # is longer than our buffer :(
+                    raise AssertionError('Not implemented')
             self.cur_line = self.cur_available
             self.cur_available = end_of_line + 1
             self.cur_line_size = self.cur_available - self.cur_line



More information about the bazaar-commits mailing list