OSD: add pg deletion cancelation

DeletingState now allows _create_lock_pg() to attempt to cancel
pg deletion.

PG::init() must mark the PG as backfill iff we stopped a deletion.

Signed-off-by: Samuel Just <sam.just@inktank.com>

4 files changed, 143 insertions, 12 deletions

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index bf630ec0ee3..14f88d5e7fc 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1644,9 +1644,24 @@ PG *OSD::_create_lock_pg(
 
   PG *pg = _open_lock_pg(createmap, pgid, true, hold_map_lock);
 
-  t.create_collection(coll_t(pgid));
+  DeletingStateRef df = service.deleting_pgs.lookup(pgid);
+  bool backfill = false;
 
-  pg->init(role, up, acting, history, pi, &t);
+  if (df && df->try_stop_deletion()) {
+    dout(10) << __func__ << ": halted deletion on pg " << pgid << dendl;
+    backfill = true;
+    service.deleting_pgs.remove(pgid); // PG is no longer being removed!
+  } else {
+    if (df) {
+      // raced, ensure we don't see DeletingStateRef when we try to
+      // delete this pg
+      service.deleting_pgs.remove(pgid);
+    }
+    // either it's not deleting, or we failed to get to it in time
+    t.create_collection(coll_t(pgid));
+  }
+
+  pg->init(role, up, acting, history, pi, backfill, &t);
 
   dout(7) << "_create_lock_pg " << *pg << dendl;
   return pg;
@@ -2780,7 +2795,7 @@ void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store,
 }
 
 // =========================================
-void remove_dir(
+bool remove_dir(
   ObjectStore *store, SnapMapper *mapper,
   OSDriver *osdriver,
   ObjectStore::Sequencer *osr,
@@ -2801,12 +2816,17 @@ void remove_dir(
     if (num >= g_conf->osd_target_transaction_size) {
       store->apply_transaction(osr, *t);
       delete t;
+      if (!dstate->check_canceled()) {
+	// canceled!
+	return false;
+      }
       t = new ObjectStore::Transaction;
       num = 0;
     }
   }
   store->apply_transaction(*t);
   delete t;
+  return true;
 }
 
 void OSD::RemoveWQ::_process(pair<PGRef, DeletingStateRef> item)
@@ -2817,10 +2837,22 @@ void OSD::RemoveWQ::_process(pair<PGRef, DeletingStateRef> item)
   coll_t coll = coll_t(pg->info.pgid);
   pg->osr->flush();
 
-  if (pg->have_temp_coll())
-    remove_dir(
+  if (!item.second->start_clearing())
+    return;
+
+  if (pg->have_temp_coll()) {
+    bool cont = remove_dir(
       store, &mapper, &driver, pg->osr.get(), pg->get_temp_coll(), item.second);
-  remove_dir(store, &mapper, &driver, pg->osr.get(), coll, item.second);
+    if (!cont)
+      return;
+  }
+  bool cont = remove_dir(
+    store, &mapper, &driver, pg->osr.get(), coll, item.second);
+  if (!cont)
+    return;
+
+  if (!item.second->start_deleting())
+    return;
 
   ObjectStore::Transaction *t = new ObjectStore::Transaction;
   PG::clear_info_log(
@@ -2831,11 +2863,19 @@ void OSD::RemoveWQ::_process(pair<PGRef, DeletingStateRef> item)
   if (pg->have_temp_coll())
     t->remove_collection(pg->get_temp_coll());
   t->remove_collection(coll);
+
+  // We need the sequencer to stick around until the op is complete
   store->queue_transaction(
     pg->osr.get(),
     t,
-    new ObjectStore::C_DeleteTransactionHolder<pair<PGRef, DeletingStateRef> >(
-      t, item));
+    0, // onapplied
+    0, // oncommit
+    0, // onreadable sync
+    new ObjectStore::C_DeleteTransactionHolder<PGRef>(
+      t, pg), // oncomplete
+    TrackedOpRef());
+
+  item.second->finish_deleting();
 }
 // =========================================
 
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index e5886d959af..2a9f2827488 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -142,9 +142,19 @@ typedef std::tr1::shared_ptr<ObjectStore::Sequencer> SequencerRef;
 
 class DeletingState {
   Mutex lock;
+  Cond cond;
   list<Context *> on_deletion_complete;
+  enum {
+    QUEUED,
+    CLEARING_DIR,
+    DELETING_DIR,
+    DELETED_DIR,
+    CANCELED,
+  } status;
+  bool stop_deleting;
 public:
-  DeletingState() : lock("DeletingState::lock") {}
+  DeletingState() :
+    lock("DeletingState::lock"), status(QUEUED), stop_deleting(false) {}
   void register_on_delete(Context *completion) {
     Mutex::Locker l(lock);
     on_deletion_complete.push_front(completion);
@@ -156,6 +166,72 @@ public:
       (*i)->complete(0);
     }
   }
+
+  /// check whether removal was canceled
+  bool check_canceled() {
+    Mutex::Locker l(lock);
+    assert(status == CLEARING_DIR);
+    if (stop_deleting) {
+      status = CANCELED;
+      cond.Signal();
+      return false;
+    }
+    return true;
+  } ///< @return false if canceled, true if we should continue
+
+  /// transition status to clearing
+  bool start_clearing() {
+    Mutex::Locker l(lock);
+    assert(
+      status == QUEUED ||
+      status == DELETED_DIR);
+    if (stop_deleting) {
+      status = CANCELED;
+      cond.Signal();
+      return false;
+    }
+    status = CLEARING_DIR;
+    return true;
+  } ///< @return false if we should cancel deletion
+
+  /// transition status to deleting
+  bool start_deleting() {
+    Mutex::Locker l(lock);
+    assert(status == CLEARING_DIR);
+    if (stop_deleting) {
+      status = CANCELED;
+      cond.Signal();
+      return false;
+    }
+    status = DELETING_DIR;
+    return true;
+  } ///< @return false if we should cancel deletion
+
+  /// signal collection removal queued
+  void finish_deleting() {
+    Mutex::Locker l(lock);
+    assert(status == DELETING_DIR);
+    status = DELETED_DIR;
+    cond.Signal();
+  }
+
+  /// try to halt the deletion
+  bool try_stop_deletion() {
+    Mutex::Locker l(lock);
+    stop_deleting = true;
+    /**
+     * If we are in DELETING_DIR or DELETED_DIR, there are in progress
+     * operations we have to wait for before continuing on.  States
+     * DELETED_DIR, QUEUED, and CANCELED either check for stop_deleting
+     * prior to performing any operations or signify the end of the
+     * deleting process.  We don't want to wait to leave the QUEUED
+     * state, because this might block the caller behind entire pg
+     * removals.
+     */
+    while (status == DELETING_DIR || status == DELETING_DIR)
+      cond.Wait(lock);
+    return status != DELETED_DIR;
+  } ///< @return true if we don't need to recreate the collection
 };
 typedef std::tr1::shared_ptr<DeletingState> DeletingStateRef;
 
diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index ae88be652da..31aaae36a1f 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -2407,10 +2407,13 @@ void PG::clear_publish_stats()
  * @param newacting acting set
  * @param history pg history
  * @param pi past_intervals
+ * @param backfill true if info should be marked as backfill
  * @param t transaction to write out our new state in
  */
-void PG::init(int role, vector<int>& newup, vector<int>& newacting, pg_history_t& history,
+void PG::init(int role, vector<int>& newup, vector<int>& newacting,
+	      pg_history_t& history,
 	      pg_interval_map_t& pi,
+	      bool backfill,
 	      ObjectStore::Transaction *t)
 {
   dout(10) << "init role " << role << " up " << newup << " acting " << newacting
@@ -2429,6 +2432,12 @@ void PG::init(int role, vector<int>& newup, vector<int>& newacting, pg_history_t
   info.stats.acting = acting;
   info.stats.mapping_epoch = info.history.same_interval_since;
 
+  if (backfill) {
+    dout(10) << __func__ << ": Setting backfill" << dendl;
+    info.last_backfill = hobject_t();
+    info.last_complete = info.last_update;
+  }
+
   reg_next_scrub();
 
   dirty_info = true;
diff --git a/src/osd/PG.h b/src/osd/PG.h
index 720fcb58772..982710b339b 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -1846,8 +1846,14 @@ public:
 
   bool  is_empty() const { return info.last_update == eversion_t(0,0); }
 
-  void init(int role, vector<int>& up, vector<int>& acting, pg_history_t& history,
-	    pg_interval_map_t& pim, ObjectStore::Transaction *t);
+  void init(
+    int role,
+    vector<int>& up,
+    vector<int>& acting,
+    pg_history_t& history,
+    pg_interval_map_t& pim,
+    bool backfill,
+    ObjectStore::Transaction *t);
 
   // pg on-disk state
   void do_pending_flush();