Rev 4680: Start working on a Keys type. in http://bazaar.launchpad.net/~jameinel/bzr/2.1-memory-consumption
John Arbash Meinel
john at arbash-meinel.com
Tue Sep 8 21:43:20 BST 2009
At http://bazaar.launchpad.net/~jameinel/bzr/2.1-memory-consumption
------------------------------------------------------------
revno: 4680
revision-id: john at arbash-meinel.com-20090908204304-bvrvpt10o8ux0e6z
parent: pqm at pqm.ubuntu.com-20090908065926-at5dgy2j2spzl8u5
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: 2.1-memory-consumption
timestamp: Tue 2009-09-08 15:43:04 -0500
message:
Start working on a Keys type.
This is a pure C extension type, because we want a lot of control
over the memory representation.
-------------- next part --------------
=== added file 'bzrlib/_keys_type_c.c'
--- a/bzrlib/_keys_type_c.c 1970-01-01 00:00:00 +0000
+++ b/bzrlib/_keys_type_c.c 2009-09-08 20:43:04 +0000
@@ -0,0 +1,189 @@
+/* Copyright (C) 2009 Canonical Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+*/
+
+#include <Python.h>
+
+#include "python-compat.h"
+
+#if defined(__GNUC__)
+# define inline __inline__
+#elif defined(_MSC_VER)
+# define inline __inline
+#else
+# define inline
+#endif
+
+
+typedef struct {
+ PyObject_HEAD
+ unsigned char key_width;
+ unsigned char num_keys; /* Not a Py_ssize_t like most containers */
+ PyStringObject *key_strings[1];
+} Keys;
+
+/* Forward declaration */
+extern PyTypeObject KeysType;
+
+static PyObject *
+Keys_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+ Py_ssize_t num_args;
+ Py_ssize_t i;
+ long key_width;
+ long num_keys;
+ long num_key_bits;
+ PyObject *obj= NULL;
+ Keys *self;
+
+ if (type != &KeysType) {
+ PyErr_BadInternalCall();
+ return NULL;
+ }
+ if (!PyTuple_CheckExact(args)) {
+ PyErr_BadInternalCall();
+ return NULL;
+ }
+ num_args = PyTuple_GET_SIZE(args);
+ if (num_args < 1) {
+ PyErr_SetString(PyExc_TypeError, "Keys.__init__(width, ...) takes at"
+ " least one argument.");
+ return NULL;
+ }
+ key_width = PyInt_AsLong(PyTuple_GET_ITEM(args, 0));
+ if (key_width == -1 && PyErr_Occurred()) {
+ return NULL;
+ }
+ if (key_width <= 0) {
+ PyErr_SetString(PyExc_ValueError, "Keys.__init__(width, ...), width"
+ " should be a positive integer.");
+ return NULL;
+ }
+ if (key_width > 256) {
+ PyErr_SetString(PyExc_ValueError, "Keys.__init__(width, ...), width"
+ " must be <= 256");
+ return NULL;
+ }
+ /* First arg is the key width, the rest are the actual key items */
+ num_key_bits = num_args - 1;
+ num_keys = num_key_bits / key_width;
+ if (num_keys * key_width != num_key_bits) {
+ PyErr_SetString(PyExc_ValueError, "Keys.__init__(width, ...), was"
+ " supplied a number of key bits that was not an even multiple"
+ " of the key width.");
+ return NULL;
+ }
+ if (num_keys > 256) {
+ PyErr_SetString(PyExc_ValueError, "Keys.__init__(width, ...), was"
+ " supplied more than 256 keys");
+ return NULL;
+ }
+ self = (Keys *)(type->tp_alloc(type, num_key_bits));
+ self->key_width = (unsigned char)key_width;
+ self->num_keys = (unsigned char)num_keys;
+ for (i = 0; i < num_key_bits; i++) {
+ obj = PyTuple_GET_ITEM(args, i + 1);
+ if (!PyString_CheckExact(obj)) {
+ PyErr_SetString(PyExc_TypeError, "Keys.__init__(width, ...)"
+ " requires that all key bits are strings.");
+ /* TODO: What is the proper way to dealloc ? */
+ type->tp_dealloc((PyObject *)self);
+ return NULL;
+ }
+ Py_INCREF(obj);
+ self->key_strings[i] = obj;
+ }
+ return (PyObject *)self;
+}
+
+static char Keys_doc[] =
+ "C implementation of a Keys structure";
+
+static PyMethodDef Keys_methods[] = {
+ {NULL, NULL} /* sentinel */
+};
+
+static PyTypeObject KeysType = {
+ PyObject_HEAD_INIT(NULL)
+ 0, /* ob_size */
+ "Keys", /* tp_name */
+ sizeof(Keys) - sizeof(PyStringObject *), /* tp_basicsize */
+ sizeof(PyObject *), /* tp_itemsize */
+ // We probably need a custom dealloc so that it decrefs the referenced
+ // PyString objects
+ 0, // (destructor)PatienceSequenceMatcher_dealloc, /* tp_dealloc */
+ 0, /* tp_print */
+ 0, /* tp_getattr */
+ 0, /* tp_setattr */
+ 0, /* tp_compare */
+ 0, /* tp_repr */
+ 0, /* tp_as_number */
+ 0, /* tp_as_sequence */
+ 0, /* tp_as_mapping */
+ 0, /* tp_hash */
+ 0, /* tp_call */
+ 0, /* tp_str */
+ PyObject_GenericGetAttr, /* tp_getattro */
+ 0, /* tp_setattro */
+ 0, /* tp_as_buffer */
+ Py_TPFLAGS_DEFAULT, /* tp_flags*/
+ Keys_doc, /* tp_doc */
+ // might want to set this, except we don't participate in gc, so it might
+ // confuse things
+ 0, /* tp_traverse */
+ 0, /* tp_clear */
+ // Probably worth implementing
+ 0, /* tp_richcompare */
+ 0, /* tp_weaklistoffset */
+ // We could implement this as returning tuples of keys...
+ 0, /* tp_iter */
+ 0, /* tp_iternext */
+ Keys_methods, /* tp_methods */
+ 0, /* tp_members */
+ 0, /* tp_getset */
+ 0, /* tp_base */
+ 0, /* tp_dict */
+ 0, /* tp_descr_get */
+ 0, /* tp_descr_set */
+ 0, /* tp_dictoffset */
+ 0, /* tp_init */
+ 0, /* tp_alloc */
+ Keys_new, /* tp_new */
+};
+
+static PyMethodDef keys_type_c_methods[] = {
+// {"unique_lcs_c", py_unique_lcs, METH_VARARGS},
+// {"recurse_matches_c", py_recurse_matches, METH_VARARGS},
+ {NULL, NULL}
+};
+
+
+PyMODINIT_FUNC
+init_keys_type_c(void)
+{
+ PyObject* m;
+
+ if (PyType_Ready(&KeysType) < 0)
+ return;
+
+ m = Py_InitModule3("_keys_type_c", keys_type_c_methods,
+ "C implementation of a Keys structure");
+ if (m == NULL)
+ return;
+
+ Py_INCREF(&KeysType);
+ PyModule_AddObject(m, "Keys", (PyObject *)&KeysType);
+}
=== modified file 'bzrlib/tests/__init__.py'
--- a/bzrlib/tests/__init__.py 2009-08-28 21:05:31 +0000
+++ b/bzrlib/tests/__init__.py 2009-09-08 20:43:04 +0000
@@ -3511,6 +3511,7 @@
'bzrlib.tests.test__chk_map',
'bzrlib.tests.test__dirstate_helpers',
'bzrlib.tests.test__groupcompress',
+ 'bzrlib.tests.test__keys_type',
'bzrlib.tests.test__known_graph',
'bzrlib.tests.test__rio',
'bzrlib.tests.test__walkdirs_win32',
=== added file 'bzrlib/tests/test__keys_type.py'
--- a/bzrlib/tests/test__keys_type.py 1970-01-01 00:00:00 +0000
+++ b/bzrlib/tests/test__keys_type.py 2009-09-08 20:43:04 +0000
@@ -0,0 +1,76 @@
+# Copyright (C) 2009 Canonical Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+"""Tests for the Keys type."""
+
+from bzrlib import (
+ # _keys_py,
+ errors,
+ tests,
+ )
+
+
+def load_tests(standard_tests, module, loader):
+ """Parameterize tests for all versions of groupcompress."""
+ scenarios = [
+ # ('python', {'module': _keys_py}),
+ ]
+ suite = loader.suiteClass()
+ if CompiledKeysType.available():
+ from bzrlib import _keys_type_c
+ scenarios.append(('C', {'module': _keys_type_c}))
+ else:
+ # the compiled module isn't available, so we add a failing test
+ class FailWithoutFeature(tests.TestCase):
+ def test_fail(self):
+ self.requireFeature(CompiledKeysType)
+ suite.addTest(loader.loadTestsFromTestCase(FailWithoutFeature))
+ result = tests.multiply_tests(standard_tests, scenarios, suite)
+ return result
+
+
+class _CompiledKeysType(tests.Feature):
+
+ def _probe(self):
+ try:
+ import bzrlib._keys_type_c
+ except ImportError:
+ return False
+ return True
+
+ def feature_name(self):
+ return 'bzrlib._keys_type_c'
+
+CompiledKeysType = _CompiledKeysType()
+
+
+class TestKeysType(tests.TestCase):
+
+ def test_create(self):
+ t = self.module.Keys(1, 'foo', 'bar')
+
+ def test_create_bad_args(self):
+ self.assertRaises(TypeError, self.module.Keys)
+ self.assertRaises(TypeError, self.module.Keys, 'foo')
+ self.assertRaises(ValueError, self.module.Keys, 0)
+ self.assertRaises(ValueError, self.module.Keys, -1)
+ self.assertRaises(ValueError, self.module.Keys, -200)
+ self.assertRaises(ValueError, self.module.Keys, 2, 'foo')
+ self.assertRaises(ValueError, self.module.Keys, 257)
+ lots_of_args = ['a']*300
+ # too many args
+ self.assertRaises(ValueError, self.module.Keys, 1, *lots_of_args)
+ self.assertRaises(TypeError, self.module.Keys, 1, 'foo', 10)
=== modified file 'setup.py'
--- a/setup.py 2009-08-14 14:34:35 +0000
+++ b/setup.py 2009-09-08 20:43:04 +0000
@@ -295,6 +295,8 @@
add_pyrex_extension('bzrlib._chk_map_pyx', libraries=[z_lib])
ext_modules.append(Extension('bzrlib._patiencediff_c',
['bzrlib/_patiencediff_c.c']))
+ext_modules.append(Extension('bzrlib._keys_type_c',
+ ['bzrlib/_keys_type_c.c']))
if unavailable_files:
More information about the bazaar-commits
mailing list