osd: pull requeued requests off one at a time

Pull items off the finished queue on at a time.  In certain cases, an
event may result in new items betting added to the finished queue that
will be put at the *front* instead of the back.  See latest incarnation
of #2947.

Note that this is a significant changed in behavior in that we can
theoretically starve if an event keeps resulting in new events getting
generated.  Beware!

Signed-off-by: Sage Weil <sage@inktank.com>
Reviewed-by: Samuel Just <sam.just@inktank.com>

1 files changed, 8 insertions, 13 deletions

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index a1546bc606d..d5f2b2299a4 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -3545,22 +3545,17 @@ void OSD::do_waiters()
 {
   assert(osd_lock.is_locked());
   
+  dout(10) << "do_waiters -- start" << dendl;
   finished_lock.Lock();
-  if (finished.empty()) {
+  while (!finished.empty()) {
+    OpRequestRef next = finished.front();
+    finished.pop_front();
     finished_lock.Unlock();
-  } else {
-    list<OpRequestRef> waiting;
-    waiting.splice(waiting.begin(), finished);
-
-    finished_lock.Unlock();
-    
-    dout(10) << "do_waiters -- start" << dendl;
-    for (list<OpRequestRef>::iterator it = waiting.begin();
-         it != waiting.end();
-         it++)
-      dispatch_op(*it);
-    dout(10) << "do_waiters -- finish" << dendl;
+    dispatch_op(next);
+    finished_lock.Lock();
   }
+  finished_lock.Unlock();
+  dout(10) << "do_waiters -- finish" << dendl;
 }
 
 void OSD::dispatch_op(OpRequestRef op)