osd: requeue pg waiters at the front of the finished queue

We could have a sequence like:

- op1
- notify
- op2

in the finished queue.  Op1 gets put on waiting_for_pg, the notify
creates the pg and requeues op1 (and the end), op2 is handled, and
finally op1 is handled.  That breaks ordering; see #2947.

Instead, when we wake up a pg, queue the waiting messages at the front
of the dispatch queue.

Signed-off-by: Sage Weil <sage@inktank.com>
Reviewed-by: Samuel Just <sam.just@inktank.com>

1 files changed, 7 insertions, 2 deletions

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 5680acca178..1837195d339 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -618,6 +618,11 @@ private:
     finished.splice(finished.end(), ls);
     finished_lock.Unlock();
   }
+  void take_waiters_front(list<OpRequestRef>& ls) {
+    finished_lock.Lock();
+    finished.splice(finished.begin(), ls);
+    finished_lock.Unlock();
+  }
   void take_waiter(OpRequestRef op) {
     finished_lock.Lock();
     finished.push_back(op);
@@ -884,7 +889,7 @@ protected:
 
   void wake_pg_waiters(pg_t pgid) {
     if (waiting_for_pg.count(pgid)) {
-      take_waiters(waiting_for_pg[pgid]);
+      take_waiters_front(waiting_for_pg[pgid]);
       waiting_for_pg.erase(pgid);
     }
   }
@@ -892,7 +897,7 @@ protected:
     for (map<pg_t, list<OpRequestRef> >::iterator p = waiting_for_pg.begin();
 	 p != waiting_for_pg.end();
 	 p++)
-      take_waiters(p->second);
+      take_waiters_front(p->second);
     waiting_for_pg.clear();
   }