Rev 4814: (jam) Release the gil during some of the core groupcompress code in file:///home/pqm/archives/thelove/bzr/%2Btrunk/
Canonical.com Patch Queue Manager
pqm at pqm.ubuntu.com
Thu Nov 19 16:41:37 GMT 2009
At file:///home/pqm/archives/thelove/bzr/%2Btrunk/
------------------------------------------------------------
revno: 4814 [merge]
revision-id: pqm at pqm.ubuntu.com-20091119164131-pg4ky6zrxe6kpzhq
parent: pqm at pqm.ubuntu.com-20091119154726-sd7fhzjnshehq9i5
parent: john at arbash-meinel.com-20091106180503-q3hcg1nxmut5b2cb
committer: Canonical.com Patch Queue Manager <pqm at pqm.ubuntu.com>
branch nick: +trunk
timestamp: Thu 2009-11-19 16:41:31 +0000
message:
(jam) Release the gil during some of the core groupcompress code
paths.
modified:
NEWS NEWS-20050323055033-4e00b5db738777ff
bzrlib/_groupcompress_pyx.pyx _groupcompress_c.pyx-20080724041824-yelg6ii7c7zxt4z0-1
=== modified file 'NEWS'
--- a/NEWS 2009-11-19 06:30:25 +0000
+++ b/NEWS 2009-11-19 16:41:31 +0000
@@ -149,6 +149,10 @@
* Include Japanese translations for documentation (Inada Naoki)
+* Some of the core groupcompress functionality now releases the GIL before
+ operation. Similar to how zlib and bz2 operate without the GIL in the
+ core compression and decompression routines. (John Arbash Meinel)
+
Testing
*******
=== modified file 'bzrlib/_groupcompress_pyx.pyx'
--- a/bzrlib/_groupcompress_pyx.pyx 2009-06-10 03:56:49 +0000
+++ b/bzrlib/_groupcompress_pyx.pyx 2009-11-06 18:05:03 +0000
@@ -31,10 +31,10 @@
cdef extern from *:
ctypedef unsigned long size_t
- void * malloc(size_t)
- void * realloc(void *, size_t)
- void free(void *)
- void memcpy(void *, void *, size_t)
+ void * malloc(size_t) nogil
+ void * realloc(void *, size_t) nogil
+ void free(void *) nogil
+ void memcpy(void *, void *, size_t) nogil
cdef extern from "delta.h":
@@ -44,19 +44,16 @@
unsigned long agg_offset
struct delta_index:
pass
- delta_index * create_delta_index(source_info *src, delta_index *old)
+ delta_index * create_delta_index(source_info *src, delta_index *old) nogil
delta_index * create_delta_index_from_delta(source_info *delta,
- delta_index *old)
- void free_delta_index(delta_index *index)
+ delta_index *old) nogil
+ void free_delta_index(delta_index *index) nogil
void *create_delta(delta_index *indexes,
void *buf, unsigned long bufsize,
- unsigned long *delta_size, unsigned long max_delta_size)
+ unsigned long *delta_size, unsigned long max_delta_size) nogil
unsigned long get_delta_hdr_size(unsigned char **datap,
- unsigned char *top)
+ unsigned char *top) nogil
Py_ssize_t DELTA_SIZE_MIN
- void *patch_delta(void *src_buf, unsigned long src_size,
- void *delta_buf, unsigned long delta_size,
- unsigned long *dst_size)
cdef void *safe_malloc(size_t count) except NULL:
@@ -148,7 +145,8 @@
src.buf = c_delta
src.size = c_delta_size
src.agg_offset = self._source_offset + unadded_bytes
- index = create_delta_index_from_delta(src, self._index)
+ with nogil:
+ index = create_delta_index_from_delta(src, self._index)
self._source_offset = src.agg_offset + src.size
if index != NULL:
free_delta_index(self._index)
@@ -188,7 +186,8 @@
self._source_offset = src.agg_offset + src.size
# We delay creating the index on the first insert
if source_location != 0:
- index = create_delta_index(src, self._index)
+ with nogil:
+ index = create_delta_index(src, self._index)
if index != NULL:
free_delta_index(self._index)
self._index = index
@@ -201,7 +200,8 @@
# We know that self._index is already NULL, so whatever
# create_delta_index returns is fine
- self._index = create_delta_index(&self._source_infos[0], NULL)
+ with nogil:
+ self._index = create_delta_index(&self._source_infos[0], NULL)
assert self._index != NULL
cdef _expand_sources(self):
@@ -218,6 +218,7 @@
cdef Py_ssize_t target_size
cdef void * delta
cdef unsigned long delta_size
+ cdef unsigned long c_max_delta_size
if self._index == NULL:
if len(self._sources) == 0:
@@ -234,9 +235,11 @@
# TODO: inline some of create_delta so we at least don't have to double
# malloc, and can instead use PyString_FromStringAndSize, to
# allocate the bytes into the final string
- delta = create_delta(self._index,
- target, target_size,
- &delta_size, max_delta_size)
+ c_max_delta_size = max_delta_size
+ with nogil:
+ delta = create_delta(self._index,
+ target, target_size,
+ &delta_size, c_max_delta_size)
result = None
if delta:
result = PyString_FromStringAndSize(<char *>delta, delta_size)
@@ -276,7 +279,7 @@
cdef unsigned char *_decode_copy_instruction(unsigned char *bytes,
- unsigned char cmd, unsigned int *offset, unsigned int *length):
+ unsigned char cmd, unsigned int *offset, unsigned int *length) nogil:
"""Decode a copy instruction from the next few bytes.
A copy instruction is a variable number of bytes, so we will parse the
@@ -326,6 +329,7 @@
cdef unsigned char *dst_buf, *out, cmd
cdef Py_ssize_t size
cdef unsigned int cp_off, cp_size
+ cdef int failed
data = <unsigned char *>delta
top = data + delta_size
@@ -335,37 +339,49 @@
result = PyString_FromStringAndSize(NULL, size)
dst_buf = <unsigned char*>PyString_AS_STRING(result)
- out = dst_buf
- while (data < top):
- cmd = data[0]
- data = data + 1
- if (cmd & 0x80):
- # Copy instruction
- data = _decode_copy_instruction(data, cmd, &cp_off, &cp_size)
- if (cp_off + cp_size < cp_size or
- cp_off + cp_size > source_size or
- cp_size > size):
- raise RuntimeError('Something wrong with:'
- ' cp_off = %s, cp_size = %s'
- ' source_size = %s, size = %s'
- % (cp_off, cp_size, source_size, size))
- memcpy(out, source + cp_off, cp_size)
- out = out + cp_size
- size = size - cp_size
- else:
- # Insert instruction
- if cmd == 0:
- # cmd == 0 is reserved for future encoding
- # extensions. In the mean time we must fail when
- # encountering them (might be data corruption).
- raise RuntimeError('Got delta opcode: 0, not supported')
- if (cmd > size):
- raise RuntimeError('Insert instruction longer than remaining'
- ' bytes: %d > %d' % (cmd, size))
- memcpy(out, data, cmd)
- out = out + cmd
- data = data + cmd
- size = size - cmd
+ failed = 0
+ with nogil:
+ out = dst_buf
+ while (data < top):
+ cmd = data[0]
+ data = data + 1
+ if (cmd & 0x80):
+ # Copy instruction
+ data = _decode_copy_instruction(data, cmd, &cp_off, &cp_size)
+ if (cp_off + cp_size < cp_size or
+ cp_off + cp_size > source_size or
+ cp_size > size):
+ failed = 1
+ break
+ memcpy(out, source + cp_off, cp_size)
+ out = out + cp_size
+ size = size - cp_size
+ else:
+ # Insert instruction
+ if cmd == 0:
+ # cmd == 0 is reserved for future encoding
+ # extensions. In the mean time we must fail when
+ # encountering them (might be data corruption).
+ failed = 2
+ break
+ if cmd > size:
+ failed = 3
+ break
+ memcpy(out, data, cmd)
+ out = out + cmd
+ data = data + cmd
+ size = size - cmd
+ if failed:
+ if failed == 1:
+ raise ValueError('Something wrong with:'
+ ' cp_off = %s, cp_size = %s'
+ ' source_size = %s, size = %s'
+ % (cp_off, cp_size, source_size, size))
+ elif failed == 2:
+ raise ValueError('Got delta opcode: 0, not supported')
+ elif failed == 3:
+ raise ValueError('Insert instruction longer than remaining'
+ ' bytes: %d > %d' % (cmd, size))
# sanity check
if (data != top or size != 0):
More information about the bazaar-commits
mailing list