From e1dc3fd300999dd35530ce619fd433094365d6cd Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 29 May 2013 11:27:38 -0700 Subject: osd: augment osd heartbeat peers with neighbors and randoms, to up some min - always include our neighbors to ensure we have a fully-connected graph - include some random neighbors to get at least some min number of peers. Signed-off-by: Sage Weil --- src/common/config_opts.h | 1 + src/osd/OSD.cc | 75 +++++++++++++++++++++++++++++++++++++----------- src/osd/OSDMap.h | 24 ++++++++++++++++ 3 files changed, 84 insertions(+), 16 deletions(-) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 56a46aec0c0..01e8b5a5a15 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -408,6 +408,7 @@ OPTION(osd_age_time, OPT_INT, 0) OPTION(osd_heartbeat_addr, OPT_ADDR, entity_addr_t()) OPTION(osd_heartbeat_interval, OPT_INT, 6) // (seconds) how often we ping peers OPTION(osd_heartbeat_grace, OPT_INT, 20) // (seconds) how long before we decide a peer has failed +OPTION(osd_heartbeat_min_peers, OPT_INT, 10) // minimum number of peers OPTION(osd_mon_heartbeat_interval, OPT_INT, 30) // (seconds) how often to ping monitor if no peers OPTION(osd_mon_report_interval_max, OPT_INT, 120) OPTION(osd_mon_report_interval_min, OPT_INT, 5) // pg stats, failures, up_thru, boot. diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 4bbcd06243e..cf08418262b 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -2292,6 +2292,8 @@ void OSD::maybe_update_heartbeat_peers() return; heartbeat_need_update = false; + dout(10) << "maybe_update_heartbeat_peers updating" << dendl; + heartbeat_epoch = osdmap->get_epoch(); // build heartbeat from set @@ -2314,25 +2316,66 @@ void OSD::maybe_update_heartbeat_peers() pg->heartbeat_peer_lock.Unlock(); } - map::iterator p = heartbeat_peers.begin(); - while (p != heartbeat_peers.end()) { + // include next and previous up osds to ensure we have a fully-connected set + set want, extras; + int next = osdmap->get_next_up_osd_after(whoami); + if (next >= 0) + want.insert(next); + int prev = osdmap->get_previous_up_osd_before(whoami); + if (prev >= 0) + want.insert(prev); + + for (set::iterator p = want.begin(); p != want.end(); ++p) { + dout(10) << " adding neighbor peer osd." << *p << dendl; + extras.insert(*p); + _add_heartbeat_peer(*p); + } + + // identify extras + for (map::iterator p = heartbeat_peers.begin(); + p != heartbeat_peers.end(); + ++p) { if (p->second.epoch < osdmap->get_epoch()) { - dout(20) << " removing heartbeat peer osd." << p->first - << " " << p->second.con_back->get_peer_addr() - << " " << (p->second.con_front ? p->second.con_front->get_peer_addr() : entity_addr_t()) - << dendl; - hbclient_messenger->mark_down(p->second.con_back); - p->second.con_back->put(); - if (p->second.con_front) { - hbclient_messenger->mark_down(p->second.con_front); - p->second.con_front->put(); - } - heartbeat_peers.erase(p++); - } else { - ++p; + extras.insert(p->first); + } + } + + // too few? + int start = osdmap->get_next_up_osd_after(whoami); + for (int n = start; n >= 0; ) { + if ((int)heartbeat_peers.size() >= g_conf->osd_heartbeat_min_peers) + break; + if (!extras.count(n) && !want.count(n) && n != whoami) { + dout(10) << " adding random peer osd." << n << dendl; + extras.insert(n); + _add_heartbeat_peer(n); + } + n = osdmap->get_next_up_osd_after(n); + if (n == start) + break; // came full circle; stop + } + + // too many? + for (set::iterator p = extras.begin(); + (int)heartbeat_peers.size() > g_conf->osd_heartbeat_min_peers && p != extras.end(); + ++p) { + if (want.count(*p)) + continue; + map::iterator q = heartbeat_peers.find(*p); + dout(20) << " removing heartbeat peer osd." << q->first + << " " << q->second.con_back->get_peer_addr() + << " " << (q->second.con_front ? q->second.con_front->get_peer_addr() : entity_addr_t()) + << dendl; + hbclient_messenger->mark_down(q->second.con_back); + q->second.con_back->put(); + if (q->second.con_front) { + hbclient_messenger->mark_down(q->second.con_front); + q->second.con_front->put(); } + heartbeat_peers.erase(q); } - dout(10) << "maybe_update_heartbeat_peers " << heartbeat_peers.size() << " peers" << dendl; + + dout(10) << "maybe_update_heartbeat_peers " << heartbeat_peers.size() << " peers, extras " << extras << dendl; } void OSD::reset_heartbeat_peers() diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h index deebc376a91..0a00c40e23b 100644 --- a/src/osd/OSDMap.h +++ b/src/osd/OSDMap.h @@ -404,6 +404,30 @@ private: return -1; } + int get_next_up_osd_after(int n) const { + for (int i = n + 1; i != n; ++i) { + if (i >= get_max_osd()) + i = 0; + if (i == n) + break; + if (is_up(i)) + return i; + } + return -1; + } + + int get_previous_up_osd_before(int n) const { + for (int i = n - 1; i != n; --i) { + if (i < 0) + i = get_max_osd() - 1; + if (i == n) + break; + if (is_up(i)) + return i; + } + return -1; + } + /** * get feature bits required by the current structure * -- cgit v1.2.1