rgw: bucket quota threshold

Add bucket quota threshold so that when we're passed that value we
reread the bucket stats before every write and not rely on cached value.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

3 files changed, 41 insertions, 5 deletions

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index c2784ce287e..b0dead48763 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -716,6 +716,7 @@ OPTION(rgw_data_log_obj_prefix, OPT_STR, "data_log") //
 OPTION(rgw_replica_log_obj_prefix, OPT_STR, "replica_log") // 
 
 OPTION(rgw_bucket_quota_ttl, OPT_INT, 600) // time for cached bucket stats to be cached within rgw instance
+OPTION(rgw_bucket_quota_soft_threshold, OPT_DOUBLE, 0.95) // threshold from which we don't rely on cached info for quota decisions
 OPTION(rgw_bucket_quota_cache_size, OPT_INT, 10000) // number of entries in bucket quota cache
 
 OPTION(mutex_perf_counter, OPT_BOOL, false) // enable/disable mutex perf counter
diff --git a/src/rgw/rgw_quota.cc b/src/rgw/rgw_quota.cc
index 9380403c7bc..d6bad572e35 100644
--- a/src/rgw/rgw_quota.cc
+++ b/src/rgw/rgw_quota.cc
@@ -31,14 +31,45 @@ public:
     async_refcount->put_wait(); /* wait for all pending async requests to complete */
   }
 
-  int get_bucket_stats(rgw_bucket& bucket, RGWBucketStats& stats);
+  int get_bucket_stats(rgw_bucket& bucket, RGWBucketStats& stats, RGWQuotaInfo& quota);
   void adjust_bucket_stats(rgw_bucket& bucket, int objs_delta, uint64_t added_bytes, uint64_t removed_bytes);
 
+  bool can_use_cached_stats(RGWQuotaInfo& quota, RGWBucketStats& stats);
+
   void set_stats(rgw_bucket& bucket, RGWQuotaBucketStats& qs, RGWBucketStats& stats);
   int async_refresh(rgw_bucket& bucket, RGWQuotaBucketStats& qs);
   void async_refresh_response(rgw_bucket& bucket, RGWBucketStats& stats);
 };
 
+bool RGWBucketStatsCache::can_use_cached_stats(RGWQuotaInfo& quota, RGWBucketStats& cached_stats)
+{
+  if (quota.max_size_kb >= 0) {
+    if (quota.max_size_soft_threshold < 0) {
+      quota.max_size_soft_threshold = quota.max_size_kb * store->ctx()->_conf->rgw_bucket_quota_soft_threshold;
+    }
+
+    if (cached_stats.num_kb_rounded >= (uint64_t)quota.max_size_soft_threshold) {
+      ldout(store->ctx(), 20) << "quota: can't use cached stats, exceeded soft threshold (size): "
+        << cached_stats.num_kb_rounded << " >= " << quota.max_size_soft_threshold << dendl;
+      return false;
+    }
+  }
+
+  if (quota.max_objects >= 0) {
+    if (quota.max_objs_soft_threshold < 0) {
+      quota.max_objs_soft_threshold = quota.max_objects * store->ctx()->_conf->rgw_bucket_quota_soft_threshold;
+    }
+
+    if (cached_stats.num_objects >= (uint64_t)quota.max_objs_soft_threshold) {
+      ldout(store->ctx(), 20) << "quota: can't use cached stats, exceeded soft threshold (num objs): "
+        << cached_stats.num_objects << " >= " << quota.max_objs_soft_threshold << dendl;
+      return false;
+    }
+  }
+
+  return true;
+}
+
 int RGWBucketStatsCache::fetch_bucket_totals(rgw_bucket& bucket, RGWBucketStats& stats)
 {
   RGWBucketInfo bucket_info;
@@ -176,7 +207,7 @@ void RGWBucketStatsCache::set_stats(rgw_bucket& bucket, RGWQuotaBucketStats& qs,
   stats_map.add(bucket, qs);
 }
 
-int RGWBucketStatsCache::get_bucket_stats(rgw_bucket& bucket, RGWBucketStats& stats) {
+int RGWBucketStatsCache::get_bucket_stats(rgw_bucket& bucket, RGWBucketStats& stats, RGWQuotaInfo& quota) {
   RGWQuotaBucketStats qs;
   utime_t now = ceph_clock_now(store->ctx());
   if (stats_map.find(bucket, qs)) {
@@ -188,7 +219,8 @@ int RGWBucketStatsCache::get_bucket_stats(rgw_bucket& bucket, RGWBucketStats& st
         /* continue processing, might be a transient error, async refresh is just optimization */
       }
     }
-    if (qs.expiration > ceph_clock_now(store->ctx())) {
+
+    if (can_use_cached_stats(quota, qs.stats) && qs.expiration > ceph_clock_now(store->ctx())) {
       stats = qs.stats;
       return 0;
     }
@@ -245,7 +277,7 @@ public:
 
     RGWBucketStats stats;
 
-    int ret = stats_cache.get_bucket_stats(bucket, stats);
+    int ret = stats_cache.get_bucket_stats(bucket, stats, bucket_quota);
     if (ret < 0)
       return ret;
 
diff --git a/src/rgw/rgw_quota.h b/src/rgw/rgw_quota.h
index 9af91e3986d..55e2738a0ca 100644
--- a/src/rgw/rgw_quota.h
+++ b/src/rgw/rgw_quota.h
@@ -13,8 +13,11 @@ struct RGWQuotaInfo {
   int64_t max_size_kb;
   int64_t max_objects;
   bool enabled;
+  int64_t max_size_soft_threshold;
+  int64_t max_objs_soft_threshold;
 
-  RGWQuotaInfo() : max_size_kb(-1), max_objects(-1), enabled(false) {}
+  RGWQuotaInfo() : max_size_kb(-1), max_objects(-1), enabled(false),
+                   max_size_soft_threshold(-1), max_objs_soft_threshold(-1) {}
 
   void encode(bufferlist& bl) const {
     ENCODE_START(1, 1, bl);