[PATCH 3.16.y-ckt 140/183] libceph: fix double __remove_osd() problem

Luis Henriques luis.henriques at canonical.com
Fri Mar 6 09:57:11 UTC 2015

3.16.7-ckt8 -stable review patch.  If anyone has any objections, please let me know.


From: Ilya Dryomov <idryomov at gmail.com>

commit 7eb71e0351fbb1b242ae70abb7bb17107fe2f792 upstream.

It turns out it's possible to get __remove_osd() called twice on the
same OSD.  That doesn't sit well with rb_erase() - depending on the
shape of the tree we can get a NULL dereference, a soft lockup or
a random crash at some point in the future as we end up touching freed
memory.  One scenario that I was able to reproduce is as follows:

            <osd3 is idle, on the osd lru list>
<con reset - osd3>
                              <osdmap - osd3 down>
                                <takes map_sem>
                                  <takes request_mutex>
                                  <releases request_mutex>
                                <releases map_sem>
    <takes map_sem>
    <takes request_mutex>
        __remove_osd() <-- !!!

A case can be made that osd refcounting is imperfect and reworking it
would be a proper resolution, but for now Sage and I decided to fix
this by adding a safe guard around __remove_osd().

Fixes: http://tracker.ceph.com/issues/8087

Cc: Sage Weil <sage at redhat.com>
Signed-off-by: Ilya Dryomov <idryomov at gmail.com>
Reviewed-by: Sage Weil <sage at redhat.com>
Reviewed-by: Alex Elder <elder at linaro.org>
Signed-off-by: Luis Henriques <luis.henriques at canonical.com>
 net/ceph/osd_client.c | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
index 18a59c60a2b7..8cbd41f5c424 100644
--- a/net/ceph/osd_client.c
+++ b/net/ceph/osd_client.c
@@ -1006,14 +1006,24 @@ static void put_osd(struct ceph_osd *osd)
 static void __remove_osd(struct ceph_osd_client *osdc, struct ceph_osd *osd)
-	dout("__remove_osd %p\n", osd);
+	dout("%s %p osd%d\n", __func__, osd, osd->o_osd);
-	rb_erase(&osd->o_node, &osdc->osds);
-	ceph_con_close(&osd->o_con);
-	put_osd(osd);
+	rb_erase(&osd->o_node, &osdc->osds);
+	RB_CLEAR_NODE(&osd->o_node);
+static void remove_osd(struct ceph_osd_client *osdc, struct ceph_osd *osd)
+	dout("%s %p osd%d\n", __func__, osd, osd->o_osd);
+	if (!RB_EMPTY_NODE(&osd->o_node)) {
+		ceph_con_close(&osd->o_con);
+		__remove_osd(osdc, osd);
+		put_osd(osd);
+	}
 static void remove_all_osds(struct ceph_osd_client *osdc)
@@ -1023,7 +1033,7 @@ static void remove_all_osds(struct ceph_osd_client *osdc)
 	while (!RB_EMPTY_ROOT(&osdc->osds)) {
 		struct ceph_osd *osd = rb_entry(rb_first(&osdc->osds),
 						struct ceph_osd, o_node);
-		__remove_osd(osdc, osd);
+		remove_osd(osdc, osd);
@@ -1053,7 +1063,7 @@ static void remove_old_osds(struct ceph_osd_client *osdc)
 	list_for_each_entry_safe(osd, nosd, &osdc->osd_lru, o_osd_lru) {
 		if (time_before(jiffies, osd->lru_ttl))
-		__remove_osd(osdc, osd);
+		remove_osd(osdc, osd);
@@ -1068,8 +1078,7 @@ static int __reset_osd(struct ceph_osd_client *osdc, struct ceph_osd *osd)
 	dout("__reset_osd %p osd%d\n", osd, osd->o_osd);
 	if (list_empty(&osd->o_requests) &&
 	    list_empty(&osd->o_linger_requests)) {
-		__remove_osd(osdc, osd);
+		remove_osd(osdc, osd);
 		return -ENODEV;
@@ -1869,6 +1878,7 @@ static void reset_changed_osds(struct ceph_osd_client *osdc)
 	struct rb_node *p, *n;
+	dout("%s %p\n", __func__, osdc);
 	for (p = rb_first(&osdc->osds); p; p = n) {
 		struct ceph_osd *osd = rb_entry(p, struct ceph_osd, o_node);

More information about the kernel-team mailing list