objectcacher: make cache sizes explicit

Make ObjectCacher users specify the cache size for each ObjectCacher
instances.  This avoids the confusing config namespace for the object
cache (client_oc_*), and also will make it possible to eventually have
cache sizes that vary between (say) RBD images.

- drop unused client_oc_max_sync_write
- add rbd_cache_max_size, max_dirty, target_dirty config values (these are
  the defaults for each image)

We probably want to add librbd calls to specify the cache size on a
per-image basis?  Alternatively, we should make it possible to share a
cache pool between multiple images in some explicit way.

Signed-off-by: Sage Weil <sage@newdream.net>

5 files changed, 40 insertions, 21 deletions

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 8512e8ac86a..86d3c7605b9 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -153,6 +153,9 @@ Client::Client(Messenger *m, MonClient *mc)
   objectcacher = new ObjectCacher(cct, "libcephfs", *writeback_handler, client_lock,
 				  client_flush_set_callback,    // all commit callback
 				  (void*)this);
+  objectcacher->set_max_size(cct->_conf->client_oc_size);
+  objectcacher->set_max_dirty(cct->_conf->client_oc_max_dirty);
+  objectcacher->set_target_dirty(cct->_conf->client_oc_target_dirty);
   filer = new Filer(objecter);
 }
 
diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 6ae487bf002..b454eff01c1 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -154,7 +154,6 @@ OPTION(client_oc_size, OPT_INT, 1024*1024* 200)    // MB * n
 OPTION(client_oc_max_dirty, OPT_INT, 1024*1024* 100)    // MB * n  (dirty OR tx.. bigish)
 OPTION(client_oc_target_dirty, OPT_INT, 1024*1024* 8) // target dirty (keep this smallish)
 // note: the max amount of "in flight" dirty data is roughly (max - target)
-OPTION(client_oc_max_sync_write, OPT_U64, 128*1024)   // sync writes >= this use wrlock
 OPTION(fuse_use_invalidate_cb, OPT_BOOL, false) // use fuse 2.8+ invalidate callback to keep page cache consistent
 OPTION(fuse_big_writes, OPT_BOOL, true)
 OPTION(objecter_tick_interval, OPT_DOUBLE, 5.0)
@@ -376,6 +375,9 @@ OPTION(journal_align_min_size, OPT_INT, 64 << 10)  // align data payloads >= thi
 OPTION(journal_replay_from, OPT_INT, 0)
 OPTION(journal_zero_on_create, OPT_BOOL, false)
 OPTION(rbd_cache, OPT_BOOL, false) // whether to enable writeback caching
+OPTION(rbd_cache_size, OPT_LONGLONG, 8<<20)         // cache size
+OPTION(rbd_cache_max_dirty, OPT_LONGLONG, 6<<20)    // dirty limit
+OPTION(rbd_cache_target_dirty, OPT_LONGLONG, 4<<20) // target dirty limit
 OPTION(rgw_cache_enabled, OPT_BOOL, true)   // rgw cache enabled
 OPTION(rgw_cache_lru_size, OPT_INT, 10000)   // num of entries in rgw cache
 OPTION(rgw_socket_path, OPT_STR, "")   // path to unix domain socket, if not specified, rgw will not run as external fcgi
diff --git a/src/librbd.cc b/src/librbd.cc
index d37ef1312c5..bd43dd2c132 100644
--- a/src/librbd.cc
+++ b/src/librbd.cc
@@ -148,6 +148,9 @@ namespace librbd {
 	writeback_handler = new LibrbdWriteback(data_ctx, cache_lock);
 	object_cacher = new ObjectCacher(cct, pname, *writeback_handler, cache_lock,
 					 NULL, NULL);
+	object_cacher->set_max_size(cct->_conf->rbd_cache_size);
+	object_cacher->set_max_dirty(cct->_conf->rbd_cache_max_dirty);
+	object_cacher->set_target_dirty(cct->_conf->rbd_cache_target_dirty);
 	object_set = new ObjectCacher::ObjectSet(NULL, data_ctx.get_id(), 0);
 	object_cacher->start();
       }
diff --git a/src/osdc/ObjectCacher.cc b/src/osdc/ObjectCacher.cc
index 41ed4dd712d..ee62de404e4 100644
--- a/src/osdc/ObjectCacher.cc
+++ b/src/osdc/ObjectCacher.cc
@@ -414,6 +414,7 @@ ObjectCacher::ObjectCacher(CephContext *cct_, string name, WritebackHandler& wb,
 			   void *flush_callback_arg)
   : perfcounter(NULL),
     cct(cct_), writeback_handler(wb), name(name), lock(l),
+    max_dirty(0), target_dirty(0), max_size(0),
     flush_set_callback(flush_callback), flush_set_callback_arg(flush_callback_arg),
     flusher_stop(false), flusher_thread(this),
     stat_clean(0), stat_dirty(0), stat_rx(0), stat_tx(0), stat_missing(0), stat_dirty_waiting(0)
@@ -764,7 +765,6 @@ void ObjectCacher::bh_write_commit(int64_t poolid, sobject_t oid, loff_t start,
 void ObjectCacher::flush(loff_t amount)
 {
   utime_t cutoff = ceph_clock_now(cct);
-  //cutoff.sec_ref() -= cct->_conf->client_oc_max_dirty_age;
 
   ldout(cct, 10) << "flush " << amount << dendl;
   
@@ -788,7 +788,7 @@ void ObjectCacher::flush(loff_t amount)
 void ObjectCacher::trim(loff_t max)
 {
   if (max < 0) 
-    max = cct->_conf->client_oc_size;
+    max = max_size;
   
   ldout(cct, 10) << "trim  start: max " << max 
            << "  clean " << get_stat_clean()
@@ -1088,20 +1088,19 @@ int ObjectCacher::writex(OSDWrite *wr, ObjectSet *oset, Mutex& wait_on_lock)
 int ObjectCacher::_wait_for_write(OSDWrite *wr, uint64_t len, ObjectSet *oset, Mutex& lock)
 {
   int blocked = 0;
-  const md_config_t *conf = cct->_conf;
   utime_t start = ceph_clock_now(cct);
   int ret = 0;
 
-  if (conf->client_oc_max_dirty > 0) {
+  if (max_dirty > 0) {
     // wait for writeback?
     //  - wait for dirty and tx bytes (relative to the max_dirty threshold)
     //  - do not wait for bytes other waiters are waiting on.  this means that
     //    threads do not wait for each other.  this effectively allows the cache size
     //    to balloon proportional to the data that is in flight.
-    while (get_stat_dirty() + get_stat_tx() >= conf->client_oc_max_dirty + get_stat_dirty_waiting()) {
+    while (get_stat_dirty() + get_stat_tx() >= max_dirty + get_stat_dirty_waiting()) {
       ldout(cct, 10) << "wait_for_write waiting on " << len << ", dirty|tx " 
 		     << (get_stat_dirty() + get_stat_tx()) 
-		     << " >= max " << conf->client_oc_max_dirty << " + dirty_waiting " << get_stat_dirty_waiting()
+		     << " >= max " << max_dirty << " + dirty_waiting " << get_stat_dirty_waiting()
 		     << dendl;
       flusher_cond.Signal();
       stat_dirty_waiting += len;
@@ -1124,9 +1123,9 @@ int ObjectCacher::_wait_for_write(OSDWrite *wr, uint64_t len, ObjectSet *oset, M
   }
 
   // start writeback anyway?
-  if (get_stat_dirty() > conf->client_oc_target_dirty) {
+  if (get_stat_dirty() > target_dirty) {
     ldout(cct, 10) << "wait_for_write " << get_stat_dirty() << " > target "
-		   << conf->client_oc_target_dirty << ", nudging flusher" << dendl;
+		   << target_dirty << ", nudging flusher" << dendl;
     flusher_cond.Signal();
   }
   if (blocked && perfcounter) {
@@ -1140,29 +1139,28 @@ int ObjectCacher::_wait_for_write(OSDWrite *wr, uint64_t len, ObjectSet *oset, M
 
 void ObjectCacher::flusher_entry()
 {
-  const md_config_t *conf = cct->_conf;
   ldout(cct, 10) << "flusher start" << dendl;
   lock.Lock();
   while (!flusher_stop) {
     while (!flusher_stop) {
       loff_t all = get_stat_tx() + get_stat_rx() + get_stat_clean() + get_stat_dirty();
       ldout(cct, 11) << "flusher "
-               << all << " / " << conf->client_oc_size << ":  "
-               << get_stat_tx() << " tx, "
-               << get_stat_rx() << " rx, "
-               << get_stat_clean() << " clean, "
-               << get_stat_dirty() << " dirty ("
-	       << conf->client_oc_target_dirty << " target, "
-	       << conf->client_oc_max_dirty << " max)"
-               << dendl;
-      if (get_stat_dirty() + get_stat_dirty_waiting() > conf->client_oc_target_dirty) {
+		     << all << " / " << max_size << ":  "
+		     << get_stat_tx() << " tx, "
+		     << get_stat_rx() << " rx, "
+		     << get_stat_clean() << " clean, "
+		     << get_stat_dirty() << " dirty ("
+		     << target_dirty << " target, "
+		     << max_dirty << " max)"
+		     << dendl;
+      if (get_stat_dirty() + get_stat_dirty_waiting() > target_dirty) {
         // flush some dirty pages
         ldout(cct, 10) << "flusher " 
 		       << get_stat_dirty() << " dirty + " << get_stat_dirty_waiting()
 		       << " dirty_waiting > target "
-		       << conf->client_oc_target_dirty
+		       << target_dirty
 		       << ", flushing some dirty bhs" << dendl;
-        flush(get_stat_dirty() + get_stat_dirty_waiting() - conf->client_oc_target_dirty);
+        flush(get_stat_dirty() + get_stat_dirty_waiting() - target_dirty);
       }
       else {
         // check tail of lru for old dirty items
diff --git a/src/osdc/ObjectCacher.h b/src/osdc/ObjectCacher.h
index 83738a2ec16..4e055e298b9 100644
--- a/src/osdc/ObjectCacher.h
+++ b/src/osdc/ObjectCacher.h
@@ -295,6 +295,8 @@ class ObjectCacher {
   string name;
   Mutex& lock;
   
+  int64_t max_dirty, target_dirty, max_size;
+
   flush_set_callback_t flush_set_callback;
   void *flush_set_callback_arg;
 
@@ -521,6 +523,17 @@ public:
 
   void discard_set(ObjectSet *oset, vector<ObjectExtent>& ex);
 
+  // cache sizes
+  void set_max_dirty(int64_t v) {
+    max_dirty = v;
+  }
+  void set_target_dirty(int64_t v) {
+    target_dirty = v;
+  }
+  void set_max_size(int64_t v) {
+    max_size = v;
+  }
+
   // file functions
 
   /*** async+caching (non-blocking) file interface ***/