config: provide settings for the LevelDB stores we use

Now that we can set up the LevelDB options internally, provide
config options on the OSD and the Monitor. We leave the OSD values
at the defaults for now as they're performance-sensitive, but we
set new values on the Monitor so that it can scale to large PGMaps.
(Previously there were issues with large PGMaps taking forever to write;
these changes to the use of compression and the default block and
write buffers counteract them.)

Since we pass these variables through, users who are interested in
doing so now can test and tune them more appropriately.

Reported-by: Jim Schutt <jaschut@sandia.gov>
Signed-off-by: Greg Farnum <greg@inktank.com>

3 files changed, 26 insertions, 0 deletions

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 19f3082ec4b..3a9cfd70827 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -180,6 +180,12 @@ OPTION(mon_sync_debug_provider_fallback, OPT_STR, "") // monitor to be used as f
 OPTION(mon_sync_leader_kill_at, OPT_INT, 0) // kill the sync leader at a specifc point in the work flow
 OPTION(mon_sync_provider_kill_at, OPT_INT, 0) // kill the sync provider at a specific point in the work flow
 OPTION(mon_sync_requester_kill_at, OPT_INT, 0) // kill the sync requester at a specific point in the work flow
+OPTION(mon_leveldb_write_buffer_size, OPT_U64, 32*1024*1024) // monitor's leveldb write buffer size
+OPTION(mon_leveldb_cache_size, OPT_U64, 0) // monitor's leveldb cache size
+OPTION(mon_leveldb_block_size, OPT_U64, 4*1024*1024) // monitor's leveldb block size
+OPTION(mon_leveldb_bloom_size, OPT_INT, 0) // monitor's leveldb bloom bits per entry
+OPTION(mon_leveldb_max_open_files, OPT_INT, 0) // monitor's leveldb max open files
+OPTION(mon_leveldb_compression, OPT_BOOL, false) // monitor's leveldb uses compression
 OPTION(paxos_max_join_drift, OPT_INT, 10) // max paxos iterations before we must first sync the monitor stores
 OPTION(paxos_propose_interval, OPT_DOUBLE, 1.0)  // gather updates for this long before proposing a map update
 OPTION(paxos_min_wait, OPT_DOUBLE, 0.05)  // min time to gather updates for after period of inactivity
@@ -429,6 +435,12 @@ OPTION(osd_op_history_duration, OPT_U32, 600) // Oldest completed op to track
 OPTION(osd_target_transaction_size, OPT_INT, 30)     // to adjust various transactions that batch smaller items
 OPTION(osd_failsafe_full_ratio, OPT_FLOAT, .97) // what % full makes an OSD "full" (failsafe)
 OPTION(osd_failsafe_nearfull_ratio, OPT_FLOAT, .90) // what % full makes an OSD near full (failsafe)
+OPTION(osd_leveldb_write_buffer_size, OPT_U64, 0) // OSD's leveldb write buffer size
+OPTION(osd_leveldb_cache_size, OPT_U64, 0) // OSD's leveldb cache size
+OPTION(osd_leveldb_block_size, OPT_U64, 0) // OSD's leveldb block size
+OPTION(osd_leveldb_bloom_size, OPT_INT, 0) // OSD's leveldb bloom bits per entry
+OPTION(osd_leveldb_max_open_files, OPT_INT, 0) // OSD's leveldb max open files
+OPTION(osd_leveldb_compression, OPT_BOOL, true) // OSD's leveldb uses compression
 
 /**
  * osd_client_op_priority and osd_recovery_op_priority adjust the relative
diff --git a/src/mon/MonitorDBStore.h b/src/mon/MonitorDBStore.h
index 99c9b4fe9f5..ac2703ec5e6 100644
--- a/src/mon/MonitorDBStore.h
+++ b/src/mon/MonitorDBStore.h
@@ -474,6 +474,12 @@ class MonitorDBStore
       assert(0 != "MonitorDBStore: error initializing level db back storage");
     }
     db.reset(db_ptr);
+    db->options.write_buffer_size = g_conf->mon_leveldb_write_buffer_size;
+    db->options.cache_size = g_conf->mon_leveldb_cache_size;
+    db->options.block_size = g_conf->mon_leveldb_block_size;
+    db->options.bloom_size = g_conf->mon_leveldb_bloom_size;
+    db->options.compression_enabled = g_conf->mon_leveldb_compression;
+    db->options.max_open_files = g_conf->mon_leveldb_max_open_files;
   }
   MonitorDBStore(LevelDBStore *db_ptr) {
     db.reset(db_ptr);
diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc
index 9d393022f14..5170412183e 100644
--- a/src/os/FileStore.cc
+++ b/src/os/FileStore.cc
@@ -1647,6 +1647,14 @@ int FileStore::mount()
 
   {
     LevelDBStore *omap_store = new LevelDBStore(omap_dir);
+
+    omap_store->options.write_buffer_size = g_conf->osd_leveldb_write_buffer_size;
+    omap_store->options.cache_size = g_conf->osd_leveldb_cache_size;
+    omap_store->options.block_size = g_conf->osd_leveldb_block_size;
+    omap_store->options.bloom_size = g_conf->osd_leveldb_bloom_size;
+    omap_store->options.compression_enabled = g_conf->osd_leveldb_compression;
+    omap_store->options.max_open_files = g_conf->osd_leveldb_max_open_files;
+
     stringstream err;
     if (omap_store->create_and_open(err)) {
       delete omap_store;