objectcacher: wait directly from writex()

This gives us access to the original ObjectExtent (useful later), and
simplifies the callers.

Signed-off-by: Sage Weil <sage@newdream.net>

4 files changed, 17 insertions, 14 deletions

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 7f220093912..8512e8ac86a 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -5313,10 +5313,8 @@ int Client::_write(Fh *f, int64_t offset, uint64_t size, const char *buf)
 
     // async, caching, non-blocking.
     objectcacher->file_write(&in->oset, &in->layout, in->snaprealm->get_snap_context(),
-			     offset, size, bl, ceph_clock_now(cct), 0);
-
-    // wait? (this may block!)
-    objectcacher->wait_for_write(size, client_lock);
+			     offset, size, bl, ceph_clock_now(cct), 0,
+			     client_lock);
 
     put_cap_ref(in, CEPH_CAP_FILE_BUFFER);
   } else {
diff --git a/src/librbd.cc b/src/librbd.cc
index 57a4eb8cfc9..d37ef1312c5 100644
--- a/src/librbd.cc
+++ b/src/librbd.cc
@@ -288,8 +288,7 @@ namespace librbd {
       wr->extents.push_back(extent);
       {
 	Mutex::Locker l(cache_lock);
-	object_cacher->writex(wr, object_set);
-	object_cacher->wait_for_write(len, cache_lock);
+	object_cacher->writex(wr, object_set, cache_lock);
       }
     }
 
diff --git a/src/osdc/ObjectCacher.cc b/src/osdc/ObjectCacher.cc
index 621655f79dc..ea0205add71 100644
--- a/src/osdc/ObjectCacher.cc
+++ b/src/osdc/ObjectCacher.cc
@@ -1048,7 +1048,7 @@ int ObjectCacher::readx(OSDRead *rd, ObjectSet *oset, Context *onfinish)
 }
 
 
-int ObjectCacher::writex(OSDWrite *wr, ObjectSet *oset)
+int ObjectCacher::writex(OSDWrite *wr, ObjectSet *oset, Mutex& wait_on_lock)
 {
   assert(lock.is_locked());
   utime_t now = ceph_clock_now(cct);
@@ -1115,20 +1115,23 @@ int ObjectCacher::writex(OSDWrite *wr, ObjectSet *oset)
     }
   }
 
+  int r = _wait_for_write(wr, bytes_written, oset, wait_on_lock);
+
   delete wr;
 
   //verify_stats();
   trim();
-  return 0;
+  return r;
 }
  
 
 // blocking wait for write.
-bool ObjectCacher::wait_for_write(uint64_t len, Mutex& lock)
+int ObjectCacher::_wait_for_write(OSDWrite *wr, uint64_t len, ObjectSet *oset, Mutex& lock)
 {
   int blocked = 0;
   const md_config_t *conf = cct->_conf;
   utime_t start = ceph_clock_now(cct);
+  int ret = 0;
 
   // wait for writeback?
   //  - wait for dirty and tx bytes (relative to the max_dirty threshold)
@@ -1160,7 +1163,7 @@ bool ObjectCacher::wait_for_write(uint64_t len, Mutex& lock)
     utime_t blocked = ceph_clock_now(cct) - start;
     perfcounter->finc(l_objectcacher_write_time_blocked, (double) blocked);
   }
-  return blocked;
+  return ret;
 }
 
 void ObjectCacher::flusher_entry()
diff --git a/src/osdc/ObjectCacher.h b/src/osdc/ObjectCacher.h
index 85a8cead4b9..07ed0342736 100644
--- a/src/osdc/ObjectCacher.h
+++ b/src/osdc/ObjectCacher.h
@@ -469,12 +469,14 @@ class ObjectCacher {
 
   // non-blocking.  async.
   int readx(OSDRead *rd, ObjectSet *oset, Context *onfinish);
-  int writex(OSDWrite *wr, ObjectSet *oset);
+  int writex(OSDWrite *wr, ObjectSet *oset, Mutex& wait_on_lock);
   bool is_cached(ObjectSet *oset, vector<ObjectExtent>& extents, snapid_t snapid);
 
+private:
   // write blocking
-  bool wait_for_write(uint64_t len, Mutex& lock);
+  int _wait_for_write(OSDWrite *wr, uint64_t len, ObjectSet *oset, Mutex& lock);
   
+public:
   bool set_is_cached(ObjectSet *oset);
   bool set_is_dirty_or_committing(ObjectSet *oset);
 
@@ -512,10 +514,11 @@ class ObjectCacher {
 
   int file_write(ObjectSet *oset, ceph_file_layout *layout, const SnapContext& snapc,
                  loff_t offset, uint64_t len, 
-                 bufferlist& bl, utime_t mtime, int flags) {
+                 bufferlist& bl, utime_t mtime, int flags,
+		 Mutex& wait_on_lock) {
     OSDWrite *wr = prepare_write(snapc, bl, mtime, flags);
     Filer::file_to_extents(cct, oset->ino, layout, offset, len, wr->extents);
-    return writex(wr, oset);
+    return writex(wr, oset, wait_on_lock);
   }
 };