msgr: clear_pipe+queue reset when replacing lossy connections

We already handle the lossless replacement and lossy fault paths, but
not the lossy replacement.  This fixes an assert(!cleared) in the
reaper.  Adjust comments appropriately.

Signed-off-by: Sage Weil <sage@inktank.com>

2 files changed, 10 insertions, 3 deletions

diff --git a/src/msg/Pipe.cc b/src/msg/Pipe.cc
index e425bc1bb2d..b581e367332 100644
--- a/src/msg/Pipe.cc
+++ b/src/msg/Pipe.cc
@@ -555,7 +555,12 @@ int Pipe::accept()
   existing->unregister_pipe();
   replaced = true;
 
-  if (!existing->policy.lossy) {
+  if (existing->policy.lossy) {
+    // disconnect from the Connection
+    assert(existing->connection_state);
+    if (existing->connection_state->clear_pipe(existing))
+      msgr->dispatch_queue.queue_reset(existing->connection_state.get());
+  } else {
     // queue a reset on the old connection
     msgr->dispatch_queue.queue_reset(connection_state.get());
 
@@ -565,7 +570,7 @@ int Pipe::accept()
     connection_state = existing->connection_state;
 
     // make existing Connection reference us
-    existing->connection_state->reset_pipe(this);
+    connection_state->reset_pipe(this);
 
     // flush/queue any existing delayed messages
     if (existing->delay_thread)
diff --git a/src/msg/SimpleMessenger.cc b/src/msg/SimpleMessenger.cc
index 0b0fa662eba..48e37d87098 100644
--- a/src/msg/SimpleMessenger.cc
+++ b/src/msg/SimpleMessenger.cc
@@ -223,7 +223,9 @@ void SimpleMessenger::reaper()
     p->pipe_lock.Lock();
     p->discard_out_queue();
     if (p->connection_state) {
-      // mark_down, mark_down_all, or fault() should have done this, but make sure!
+      // mark_down, mark_down_all, or fault() should have done this,
+      // or accept() may have switch the Connection to a different
+      // Pipe... but make sure!
       bool cleared = p->connection_state->clear_pipe(p);
       assert(!cleared);
     }