Rev 4703: Implement Key_New like Tuple_New to create a key from a 3rd-party source. in http://bazaar.launchpad.net/~jameinel/bzr/2.1-memory-consumption
John Arbash Meinel
john at arbash-meinel.com
Sun Sep 13 21:28:02 BST 2009
At http://bazaar.launchpad.net/~jameinel/bzr/2.1-memory-consumption
------------------------------------------------------------
revno: 4703
revision-id: john at arbash-meinel.com-20090913202746-2uu54no9eh659glh
parent: john at arbash-meinel.com-20090912051620-5u3buw5euy3h3q26
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: 2.1-memory-consumption
timestamp: Sun 2009-09-13 15:27:46 -0500
message:
Implement Key_New like Tuple_New to create a key from a 3rd-party source.
Have Keys_item() return Key types rather than tuples.
Drops the time to 738ms, probably because we then mostly compare Key <=> Key,
rather than comparing Key <=> Tuple.
-------------- next part --------------
=== modified file 'bzrlib/_keys_type_c.c'
--- a/bzrlib/_keys_type_c.c 2009-09-12 05:16:20 +0000
+++ b/bzrlib/_keys_type_c.c 2009-09-13 20:27:46 +0000
@@ -15,7 +15,7 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
-#include <Python.h>
+#include "_keys_type_c.h"
#include "python-compat.h"
@@ -28,41 +28,10 @@
#endif
-/* This defines a single variable-width key.
- * It is basically the same as a tuple, but
- * 1) Lighter weight in memory
- * 2) Only supports strings.
- * It is mostly used as a helper. Note that Keys() is a similar structure for
- * lists of Key objects. Its main advantage, though, is that it inlines all of
- * the Key objects so that you have 1 python object overhead for N Keys, rather
- * than N objects.
- */
-typedef struct {
- PyObject_VAR_HEAD
- long hash;
- PyStringObject *key_bits[1];
-} Key;
-extern PyTypeObject KeyType;
-
-/* Because of object alignment, it seems that using unsigned char doesn't make
- * things any smaller than using an 'int'... :(
- * Perhaps we should use the high bits for extra flags?
- */
-typedef struct {
- PyObject_HEAD
- // unsigned char key_width;
- // unsigned char num_keys;
- // unsigned char flags; /* not used yet */
- unsigned int info; /* Broken down into 4 1-byte fields */
- PyStringObject *key_bits[1]; /* key_width * num_keys entries */
-} Keys;
-
-/* Forward declaration */
-extern PyTypeObject KeysType;
static PyObject *Keys_item(Keys *self, Py_ssize_t offset);
-#define Key_CheckExact(op) (Py_TYPE(op) == &KeyType)
+#define Key_CheckExact(op) (Py_TYPE(op) == &Key_Type)
static PyObject *
Key_as_tuple(Key *self)
@@ -97,6 +66,26 @@
}
+/* Similar to PyTuple_New() */
+PyObject *
+Key_New(Py_ssize_t size)
+{
+ Key *key;
+ if (size < 0) {
+ PyErr_BadInternalCall();
+ return NULL;
+ }
+
+ key = PyObject_NewVar(Key, &Key_Type, size);
+ if (key == NULL) {
+ return NULL;
+ }
+ memset(key->key_bits, 0, sizeof(PyStringObject *) * size);
+ key->hash = -1;
+ return (PyObject *)key;
+}
+
+
static PyObject *
Key_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
@@ -104,7 +93,7 @@
PyObject *obj = NULL;
Py_ssize_t i, len = 0;
- if (type != &KeyType) {
+ if (type != &Key_Type) {
PyErr_SetString(PyExc_TypeError, "we only support creating Key");
return NULL;
}
@@ -187,6 +176,11 @@
x += 97531L;
if (x == -1)
x = -2;
+ if (self->hash != -1) {
+ if (self->hash != x) {
+ fprintf(stderr, "hash changed: %d => %d\n", self->hash, x);
+ }
+ }
self->hash = x;
return x;
}
@@ -401,7 +395,7 @@
0, /* sq_contains */
};
-static PyTypeObject KeyType = {
+static PyTypeObject Key_Type = {
PyObject_HEAD_INIT(NULL)
0, /* ob_size */
"Key", /* tp_name */
@@ -478,7 +472,7 @@
return (int)((self->info >> 24) & 0xFF);
}
-#define Keys_CheckExact(op) (Py_TYPE(op) == &KeysType)
+#define Keys_CheckExact(op) (Py_TYPE(op) == &Keys_Type)
static void
Keys_dealloc(Keys *self)
@@ -511,7 +505,7 @@
PyObject *obj= NULL;
Keys *self;
- if (type != &KeysType) {
+ if (type != &Keys_Type) {
PyErr_SetString(PyExc_TypeError, "we only support creating Keys");
return NULL;
}
@@ -708,15 +702,16 @@
{
long start, i;
int key_width;
- PyObject *tpl, *obj;
+ PyObject *obj;
+ Key *key;
if (offset < 0 || offset >= Keys_get_num_keys(self)) {
PyErr_SetString(PyExc_IndexError, "Keys index out of range");
return NULL;
}
key_width = Keys_get_key_width(self);
- tpl = PyTuple_New(key_width);
- if (!tpl) {
+ key = (Key *)Key_New(key_width);
+ if (!key) {
/* Malloc failure */
return NULL;
}
@@ -724,9 +719,9 @@
for (i = 0; i < key_width; ++i) {
obj = (PyObject *)self->key_bits[start + i];
Py_INCREF(obj);
- PyTuple_SET_ITEM(tpl, i, obj);
+ key->key_bits[i] = (PyStringObject *)obj;
}
- return tpl;
+ return (PyObject *)key;
}
@@ -748,7 +743,7 @@
0, /* sq_contains */
};
-static PyTypeObject KeysType = {
+static PyTypeObject Keys_Type = {
PyObject_HEAD_INIT(NULL)
0, /* ob_size */
"Keys", /* tp_name */
@@ -807,9 +802,9 @@
{
PyObject* m;
- if (PyType_Ready(&KeyType) < 0)
+ if (PyType_Ready(&Key_Type) < 0)
return;
- if (PyType_Ready(&KeysType) < 0)
+ if (PyType_Ready(&Keys_Type) < 0)
return;
m = Py_InitModule3("_keys_type_c", keys_type_c_methods,
@@ -817,8 +812,8 @@
if (m == NULL)
return;
- Py_INCREF(&KeyType);
- PyModule_AddObject(m, "Key", (PyObject *)&KeyType);
- Py_INCREF(&KeysType);
- PyModule_AddObject(m, "Keys", (PyObject *)&KeysType);
+ Py_INCREF(&Key_Type);
+ PyModule_AddObject(m, "Key", (PyObject *)&Key_Type);
+ Py_INCREF(&Keys_Type);
+ PyModule_AddObject(m, "Keys", (PyObject *)&Keys_Type);
}
=== added file 'bzrlib/_keys_type_c.h'
--- a/bzrlib/_keys_type_c.h 1970-01-01 00:00:00 +0000
+++ b/bzrlib/_keys_type_c.h 2009-09-13 20:27:46 +0000
@@ -0,0 +1,54 @@
+/* Copyright (C) 2009 Canonical Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <Python.h>
+
+
+/* This defines a single variable-width key.
+ * It is basically the same as a tuple, but
+ * 1) Lighter weight in memory
+ * 2) Only supports strings.
+ * It is mostly used as a helper. Note that Keys() is a similar structure for
+ * lists of Key objects. Its main advantage, though, is that it inlines all of
+ * the Key objects so that you have 1 python object overhead for N Keys, rather
+ * than N objects.
+ */
+typedef struct {
+ PyObject_VAR_HEAD
+ long hash;
+ PyStringObject *key_bits[1];
+} Key;
+extern PyTypeObject Key_Type;
+
+/* Do we need a PyAPI_FUNC sort of wrapper? */
+PyObject * Key_New(Py_ssize_t size);
+
+/* Because of object alignment, it seems that using unsigned char doesn't make
+ * things any smaller than using an 'int'... :(
+ * Perhaps we should use the high bits for extra flags?
+ */
+typedef struct {
+ PyObject_HEAD
+ // unsigned char key_width;
+ // unsigned char num_keys;
+ // unsigned char flags; /* not used yet */
+ unsigned int info; /* Broken down into 4 1-byte fields */
+ PyStringObject *key_bits[1]; /* key_width * num_keys entries */
+} Keys;
+
+/* Forward declaration */
+extern PyTypeObject Keys_Type;
More information about the bazaar-commits
mailing list