mds: inject mds failure during mdstable operations

4 files changed, 23 insertions, 0 deletions

diff --git a/src/config.cc b/src/config.cc
index 53a439d0ef6..6e6fb95fafd 100644
--- a/src/config.cc
+++ b/src/config.cc
@@ -470,6 +470,7 @@ static struct config_option config_optionsp[] = {
 	OPTION(mds_dump_cache_on_map, 0, OPT_BOOL, false),
 	OPTION(mds_dump_cache_after_rejoin, 0, OPT_BOOL, true),
 	OPTION(mds_hack_log_expire_for_better_stats, 0, OPT_BOOL, false),
+	OPTION(mds_kill_mdstable_at, 0, OPT_INT, 0),
 	OPTION(osd_data, 0, OPT_STR, ""),
 	OPTION(osd_journal, 0, OPT_STR, ""),
 	OPTION(osd_journal_size, 0, OPT_INT, 0),         // in mb
diff --git a/src/config.h b/src/config.h
index 3c5327d52b0..800964ab9f9 100644
--- a/src/config.h
+++ b/src/config.h
@@ -273,6 +273,8 @@ struct md_config_t {
 
   bool mds_hack_log_expire_for_better_stats;
 
+  int mds_kill_mdstable_at;   // non-zero to specify kill point
+
   // osd
   const char *osd_data;
   const char *osd_journal;
diff --git a/src/mds/MDSTableClient.cc b/src/mds/MDSTableClient.cc
index 1cfea7616b1..5885eb9a513 100644
--- a/src/mds/MDSTableClient.cc
+++ b/src/mds/MDSTableClient.cc
@@ -53,6 +53,9 @@ void MDSTableClient::handle_request(class MMDSTableRequest *m)
   case TABLESERVER_OP_AGREE:
     if (pending_prepare.count(reqid)) {
       dout(10) << "got agree on " << reqid << " atid " << tid << dendl;
+
+      assert(g_conf.mds_kill_mdstable_at != 3);
+
       Context *onfinish = pending_prepare[reqid].onfinish;
       *pending_prepare[reqid].ptid = tid;
       if (pending_prepare[reqid].pbl)
@@ -84,6 +87,8 @@ void MDSTableClient::handle_request(class MMDSTableRequest *m)
   case TABLESERVER_OP_ACK:
     dout(10) << "got ack on tid " << tid << ", logging" << dendl;
 
+    assert(g_conf.mds_kill_mdstable_at != 7);
+
     // remove from committing list
     assert(pending_commit.count(tid));
     assert(pending_commit[tid]->pending_commit_tids[table].count(tid));
@@ -105,6 +110,8 @@ void MDSTableClient::_logged_ack(version_t tid)
 {
   dout(10) << "_logged_ack " << tid << dendl;
 
+  assert(g_conf.mds_kill_mdstable_at != 8);
+
   assert(pending_commit.count(tid));
   assert(pending_commit[tid]->pending_commit_tids[table].count(tid));
   
@@ -145,6 +152,8 @@ void MDSTableClient::commit(version_t tid, LogSegment *ls)
   pending_commit[tid] = ls;
   ls->pending_commit_tids[table].insert(tid);
 
+  assert(g_conf.mds_kill_mdstable_at != 4);
+
   // send message
   MMDSTableRequest *req = new MMDSTableRequest(table, TABLESERVER_OP_COMMIT, 0, tid);
   mds->send_message_mds(req, mds->mdsmap->get_tableserver());
diff --git a/src/mds/MDSTableServer.cc b/src/mds/MDSTableServer.cc
index dc864dbc547..6f3fb599436 100644
--- a/src/mds/MDSTableServer.cc
+++ b/src/mds/MDSTableServer.cc
@@ -48,6 +48,8 @@ void MDSTableServer::handle_prepare(MMDSTableRequest *req)
   _prepare(req->bl, req->reqid, from);
   _note_prepare(from, req->reqid);
 
+  assert(g_conf.mds_kill_mdstable_at != 1);
+
   ETableServer *le = new ETableServer(table, TABLESERVER_OP_PREPARE, req->reqid, from, version, version);
   le->mutation = bl;  // original request, NOT modified return value coming out of _prepare!
   mds->mdlog->submit_entry(le, new C_Prepare(this, req, version));
@@ -57,6 +59,9 @@ void MDSTableServer::handle_prepare(MMDSTableRequest *req)
 void MDSTableServer::_prepare_logged(MMDSTableRequest *req, version_t tid)
 {
   dout(7) << "_create_logged " << *req << " tid " << tid << dendl;
+
+  assert(g_conf.mds_kill_mdstable_at != 2);
+
   MMDSTableRequest *reply = new MMDSTableRequest(table, TABLESERVER_OP_AGREE, req->reqid, tid);
   reply->bl = req->bl;
   mds->send_message_mds(reply, req->get_source().num());
@@ -73,6 +78,9 @@ void MDSTableServer::handle_commit(MMDSTableRequest *req)
   version_t tid = req->tid;
 
   if (pending_for_mds.count(tid)) {
+
+    assert(g_conf.mds_kill_mdstable_at != 5);
+
     _commit(tid);
     _note_commit(tid);
     mds->mdlog->submit_entry(new ETableServer(table, TABLESERVER_OP_COMMIT, 0, -1, tid, version));
@@ -94,6 +102,9 @@ void MDSTableServer::handle_commit(MMDSTableRequest *req)
 void MDSTableServer::_commit_logged(MMDSTableRequest *req)
 {
   dout(7) << "_commit_logged, sending ACK" << dendl;
+
+  assert(g_conf.mds_kill_mdstable_at != 6);
+
   MMDSTableRequest *reply = new MMDSTableRequest(table, TABLESERVER_OP_ACK, req->reqid, req->tid);
   mds->send_message_mds(reply, req->get_source().num());
   delete req;