mon: MDSMonitor: tighter leash on cross-proposals to the osdmon

Fixes: #3495

Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>

2 files changed, 47 insertions, 6 deletions

diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc
index 3ea6d860039..547e9b806ef 100644
--- a/src/mon/MDSMonitor.cc
+++ b/src/mon/MDSMonitor.cc
@@ -359,8 +359,18 @@ bool MDSMonitor::prepare_beacon(MMDSBeacon *m)
   if (state == MDSMap::STATE_BOOT) {
     // zap previous instance of this name?
     if (g_conf->mds_enforce_unique_name) {
+      bool failed_mds = false;
       while (uint64_t existing = pending_mdsmap.find_mds_gid_by_name(m->get_name())) {
+        if (!mon->osdmon()->is_writeable()) {
+          mon->osdmon()->wait_for_writeable(new C_RetryMessage(this, m));
+          return false;
+        }
 	fail_mds_gid(existing);
+        failed_mds = true;
+      }
+      if (failed_mds) {
+        assert(mon->osdmon()->is_writeable());
+        request_proposal(mon->osdmon());
       }
     }
 
@@ -696,7 +706,6 @@ void MDSMonitor::fail_mds_gid(uint64_t gid)
   until += g_conf->mds_blacklist_interval;
 
   pending_mdsmap.last_failure_osd_epoch = mon->osdmon()->blacklist(info.addr, until);
-  mon->osdmon()->propose_pending();
 
   if (info.rank >= 0) {
     pending_mdsmap.up.erase(info.rank);
@@ -720,16 +729,22 @@ int MDSMonitor::fail_mds(std::ostream &ss, const std::string &arg)
     w = mds_info->rank;
   }
 
+  bool failed_mds_gid = false;
   if (pending_mdsmap.up.count(w)) {
     uint64_t gid = pending_mdsmap.up[w];
-    if (pending_mdsmap.mds_info.count(gid))
+    if (pending_mdsmap.mds_info.count(gid)) {
       fail_mds_gid(gid);
+      failed_mds_gid = true;
+    }
     ss << "failed mds." << w;
   } else if (pending_mdsmap.mds_info.count(w)) {
     fail_mds_gid(w);
+    failed_mds_gid = true;
     ss << "failed mds gid " << w;
   }
-  return 0;
+  // fyi, no particular reason why we return -EAGAIN besides being
+  // an errno error that this function wouldn't return in the first place.
+  return (failed_mds_gid ? -EAGAIN : 0);
 }
 
 int MDSMonitor::cluster_fail(std::ostream &ss)
@@ -757,7 +772,7 @@ int MDSMonitor::cluster_fail(std::ostream &ss)
       dout(10) << " blacklisting gid " << p->second << " " << info.addr << dendl;
       pending_mdsmap.last_failure_osd_epoch = mon->osdmon()->blacklist(info.addr, until);
     }
-    mon->osdmon()->propose_pending();
+    request_proposal(mon->osdmon());
   }
   pending_mdsmap.up.clear();
   pending_mdsmap.failed.insert(pending_mdsmap.in.begin(),
@@ -857,7 +872,16 @@ bool MDSMonitor::prepare_command(MMonCommand *m)
       }
     }
     else if (m->cmd[1] == "fail" && m->cmd.size() == 3) {
+      if (!mon->osdmon()->is_writeable()) {
+        mon->osdmon()->wait_for_writeable(new C_RetryMessage(this, m));
+        return false;
+      }
       r = fail_mds(ss, m->cmd[2]);
+      if (r < 0 && r == -EAGAIN) {
+        assert(mon->osdmon()->is_writeable());
+        request_proposal(mon->osdmon());
+        r = 0;
+      }
     }
     else if (m->cmd[1] == "rm" && m->cmd.size() == 3) {
       int64_t gid = parse_pos_long(m->cmd[2].c_str(), &ss);
@@ -1177,7 +1201,7 @@ void MDSMonitor::tick()
     }
 
     if (propose_osdmap)
-      mon->osdmon()->propose_pending();
+      request_proposal(mon->osdmon());
 
   }
 
@@ -1284,6 +1308,14 @@ void MDSMonitor::do_stop()
   if (!mon->is_leader() ||
       !is_active()) {
     dout(0) << "do_stop can't stop right now, mdsmap not writeable" << dendl;
+    if (mon->is_leader())
+      wait_for_active(new C_Stop(this));
+    return;
+  }
+
+  if (!mon->osdmon()->is_writeable()) {
+    dout(0) << __func__ << " osdmon not writeable -- waiting to stop" << dendl;
+    mon->osdmon()->wait_for_writeable(new C_Stop(this));
     return;
   }
 
@@ -1331,5 +1363,5 @@ void MDSMonitor::do_stop()
 
   propose_pending();
   if (propose_osdmap)
-    mon->osdmon()->propose_pending();
+    request_proposal(mon->osdmon());
 }
diff --git a/src/mon/MDSMonitor.h b/src/mon/MDSMonitor.h
index 3b1fbf92e2d..76e6da5e7e2 100644
--- a/src/mon/MDSMonitor.h
+++ b/src/mon/MDSMonitor.h
@@ -63,6 +63,15 @@ class MDSMonitor : public PaxosService {
     }
   };
 
+  class C_Stop : public Context {
+    MDSMonitor *mm;
+  public:
+    C_Stop(MDSMonitor *m) : mm(m) { }
+    void finish(int r) {
+      if (r >= 0)
+        mm->do_stop();
+    }
+  };
 
   void create_new_fs(MDSMap &m, int metadata_pool, int data_pool);