rgw: stream list buckets (containers) request

Fixes: #4760
Instead of retrieving the entire list of buckets in one
chunk, streamline it. This makes it so that if the request
takes too long, client isn't going to timeout before getting
any data.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

12 files changed, 340 insertions, 277 deletions

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 2658c2d8785..c1bb236ac9b 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -587,6 +587,7 @@ OPTION(rgw_exit_timeout_secs, OPT_INT, 120) // how many seconds to wait for proc
 OPTION(rgw_get_obj_window_size, OPT_INT, 16 << 20) // window size in bytes for single get obj request
 OPTION(rgw_get_obj_max_req_size, OPT_INT, 4 << 20) // max length of a single get obj rados op
 OPTION(rgw_relaxed_s3_bucket_names, OPT_BOOL, false) // enable relaxed bucket name rules for US region buckets
+OPTION(rgw_list_buckets_max_chunk, OPT_INT, 1000) // max buckets to retrieve in a single op when listing user buckets
 
 OPTION(mutex_perf_counter, OPT_BOOL, false) // enable/disable mutex perf counter
 
diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index 22cab20914f..2f05264778e 100644
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -29,64 +29,34 @@ void rgw_get_buckets_obj(string& user_id, string& buckets_obj_id)
   buckets_obj_id += RGW_BUCKETS_OBJ_PREFIX;
 }
 
-static int rgw_read_buckets_from_attr(RGWRados *store, string& user_id, RGWUserBuckets& buckets)
-{
-  bufferlist bl;
-  rgw_obj obj(store->zone.user_uid_pool, user_id);
-  int ret = store->get_attr(NULL, obj, RGW_ATTR_BUCKETS, bl);
-  if (ret)
-    return ret;
-
-  bufferlist::iterator iter = bl.begin();
-  try {
-    buckets.decode(iter);
-  } catch (buffer::error& err) {
-    ldout(store->ctx(), 0) << "ERROR: failed to decode buckets info, caught buffer::error" << dendl;
-    return -EIO;
-  }
-  return 0;
-}
-
 /**
  * Get all the buckets owned by a user and fill up an RGWUserBuckets with them.
  * Returns: 0 on success, -ERR# on failure.
  */
-int rgw_read_user_buckets(RGWRados *store, string user_id, RGWUserBuckets& buckets, bool need_stats)
+int rgw_read_user_buckets(RGWRados *store, string user_id, RGWUserBuckets& buckets,
+                          const string& marker, uint64_t max, bool need_stats)
 {
   int ret;
   buckets.clear();
-  if (store->supports_omap()) {
-    string buckets_obj_id;
-    rgw_get_buckets_obj(user_id, buckets_obj_id);
-    bufferlist bl;
-    rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
-    bufferlist header;
-    map<string,bufferlist> m;
-
-    ret = store->omap_get_all(obj, header, m);
-    if (ret == -ENOENT)
-      ret = 0;
+  string buckets_obj_id;
+  rgw_get_buckets_obj(user_id, buckets_obj_id);
+  bufferlist bl;
+  rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
+  bufferlist header;
+  map<string,bufferlist> m;
 
-    if (ret < 0)
-      return ret;
+  ret = store->omap_get_vals(obj, header, marker, max, m);
+  if (ret == -ENOENT)
+    ret = 0;
 
-    for (map<string,bufferlist>::iterator q = m.begin(); q != m.end(); ++q) {
-      bufferlist::iterator iter = q->second.begin();
-      RGWBucketEnt bucket;
-      ::decode(bucket, iter);
-      buckets.add(bucket);
-    }
-  } else {
-    ret = rgw_read_buckets_from_attr(store, user_id, buckets);
-    switch (ret) {
-    case 0:
-      break;
-    case -ENODATA:
-      ret = 0;
-      return 0;
-    default:
-      return ret;
-    }
+  if (ret < 0)
+    return ret;
+
+  for (map<string,bufferlist>::iterator q = m.begin(); q != m.end(); ++q) {
+    bufferlist::iterator iter = q->second.begin();
+    RGWBucketEnt bucket;
+    ::decode(bucket, iter);
+    buckets.add(bucket);
   }
 
   if (need_stats) {
@@ -122,44 +92,22 @@ int rgw_add_bucket(RGWRados *store, string user_id, rgw_bucket& bucket)
   int ret;
   string& bucket_name = bucket.name;
 
-  if (store->supports_omap()) {
-    bufferlist bl;
-
-    RGWBucketEnt new_bucket;
-    new_bucket.bucket = bucket;
-    new_bucket.size = 0;
-    time(&new_bucket.mtime);
-    ::encode(new_bucket, bl);
+  bufferlist bl;
 
-    string buckets_obj_id;
-    rgw_get_buckets_obj(user_id, buckets_obj_id);
+  RGWBucketEnt new_bucket;
+  new_bucket.bucket = bucket;
+  new_bucket.size = 0;
+  time(&new_bucket.mtime);
+  ::encode(new_bucket, bl);
 
-    rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
-    ret = store->omap_set(obj, bucket_name, bl);
-    if (ret < 0) {
-      ldout(store->ctx(), 0) << "ERROR: error adding bucket to directory: "
-          << cpp_strerror(-ret)<< dendl;
-    }
-  } else {
-    RGWUserBuckets buckets;
+  string buckets_obj_id;
+  rgw_get_buckets_obj(user_id, buckets_obj_id);
 
-    ret = rgw_read_user_buckets(store, user_id, buckets, false);
-    RGWBucketEnt new_bucket;
-
-    switch (ret) {
-    case 0:
-    case -ENOENT:
-    case -ENODATA:
-      new_bucket.bucket = bucket;
-      new_bucket.size = 0;
-      time(&new_bucket.mtime);
-      buckets.add(new_bucket);
-      ret = rgw_write_buckets_attr(store, user_id, buckets);
-      break;
-    default:
-      ldout(store->ctx(), 10) << "rgw_write_buckets_attr returned " << ret << dendl;
-      break;
-    }
+  rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
+  ret = store->omap_set(obj, bucket_name, bl);
+  if (ret < 0) {
+    ldout(store->ctx(), 0) << "ERROR: error adding bucket to directory: "
+        << cpp_strerror(-ret)<< dendl;
   }
 
   return ret;
@@ -169,27 +117,16 @@ int rgw_remove_user_bucket_info(RGWRados *store, string user_id, rgw_bucket& buc
 {
   int ret;
 
-  if (store->supports_omap()) {
-    bufferlist bl;
-
-    string buckets_obj_id;
-    rgw_get_buckets_obj(user_id, buckets_obj_id);
-
-    rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
-    ret = store->omap_del(obj, bucket.name);
-    if (ret < 0) {
-      ldout(store->ctx(), 0) << "ERROR: error removing bucket from directory: "
-          << cpp_strerror(-ret)<< dendl;
-    }
-  } else {
-    RGWUserBuckets buckets;
+  bufferlist bl;
 
-    ret = rgw_read_user_buckets(store, user_id, buckets, false);
+  string buckets_obj_id;
+  rgw_get_buckets_obj(user_id, buckets_obj_id);
 
-    if (ret == 0 || ret == -ENOENT) {
-      buckets.remove(bucket.name);
-      ret = rgw_write_buckets_attr(store, user_id, buckets);
-    }
+  rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
+  ret = store->omap_del(obj, bucket.name);
+  if (ret < 0) {
+    ldout(store->ctx(), 0) << "ERROR: error removing bucket from directory: "
+        << cpp_strerror(-ret)<< dendl;
   }
 
   return ret;
@@ -254,42 +191,54 @@ static void dump_mulipart_index_results(list<std::string>& objs_to_unlink,
 void check_bad_user_bucket_mapping(RGWRados *store, const string& user_id, bool fix)
 {
   RGWUserBuckets user_buckets;
-  int ret = rgw_read_user_buckets(store, user_id, user_buckets, false);
-  if (ret < 0) {
-    ldout(store->ctx(), 0) << "failed to read user buckets: " << cpp_strerror(-ret) << dendl;
-    return;
-  }
+  bool done;
+  string marker;
 
-  map<string, RGWBucketEnt>& buckets = user_buckets.get_buckets();
-  for (map<string, RGWBucketEnt>::iterator i = buckets.begin();
-       i != buckets.end();
-       ++i) {
-    RGWBucketEnt& bucket_ent = i->second;
-    rgw_bucket& bucket = bucket_ent.bucket;
+  CephContext *cct = store->ctx();
 
-    RGWBucketInfo bucket_info;
-    int r = store->get_bucket_info(NULL, bucket.name, bucket_info);
-    if (r < 0) {
-      ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket << dendl;
-      continue;
+  size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk;
+
+  do {
+    int ret = rgw_read_user_buckets(store, user_id, user_buckets, marker, max_entries, false);
+    if (ret < 0) {
+      ldout(store->ctx(), 0) << "failed to read user buckets: " << cpp_strerror(-ret) << dendl;
+      return;
     }
 
-    rgw_bucket& actual_bucket = bucket_info.bucket;
-
-    if (actual_bucket.name.compare(bucket.name) != 0 ||
-        actual_bucket.pool.compare(bucket.pool) != 0 ||
-        actual_bucket.marker.compare(bucket.marker) != 0 ||
-        actual_bucket.bucket_id.compare(bucket.bucket_id) != 0) {
-      cout << "bucket info mismatch: expected " << actual_bucket << " got " << bucket << std::endl;
-      if (fix) {
-        cout << "fixing" << std::endl;
-        r = rgw_add_bucket(store, user_id, actual_bucket);
-        if (r < 0) {
-          cerr << "failed to fix bucket: " << cpp_strerror(-r) << std::endl;
+    map<string, RGWBucketEnt>& buckets = user_buckets.get_buckets();
+    for (map<string, RGWBucketEnt>::iterator i = buckets.begin();
+         i != buckets.end();
+         ++i) {
+      marker = i->first;
+
+      RGWBucketEnt& bucket_ent = i->second;
+      rgw_bucket& bucket = bucket_ent.bucket;
+
+      RGWBucketInfo bucket_info;
+      int r = store->get_bucket_info(NULL, bucket.name, bucket_info);
+      if (r < 0) {
+        ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket << dendl;
+        continue;
+      }
+
+      rgw_bucket& actual_bucket = bucket_info.bucket;
+
+      if (actual_bucket.name.compare(bucket.name) != 0 ||
+          actual_bucket.pool.compare(bucket.pool) != 0 ||
+          actual_bucket.marker.compare(bucket.marker) != 0 ||
+          actual_bucket.bucket_id.compare(bucket.bucket_id) != 0) {
+        cout << "bucket info mismatch: expected " << actual_bucket << " got " << bucket << std::endl;
+        if (fix) {
+          cout << "fixing" << std::endl;
+          r = rgw_add_bucket(store, user_id, actual_bucket);
+          if (r < 0) {
+            cerr << "failed to fix bucket: " << cpp_strerror(-r) << std::endl;
+          }
         }
       }
     }
-  }
+    done = (buckets.size() < max_entries);
+  } while (!done);
 }
 
 static bool bucket_object_check_filter(const string& name)
@@ -415,11 +364,6 @@ int RGWBucket::init(RGWRados *storage, RGWBucketAdminOpState& op_state)
     if (r < 0)
       return r;
 
-    r = rgw_read_user_buckets(store, user_id, user_buckets, true);
-    if (r < 0)
-      return r;
-
-    op_state.set_user_buckets(user_buckets);
     op_state.display_name = info.display_name;
   }
 
@@ -927,21 +871,39 @@ int RGWBucketAdminOp::info(RGWRados *store, RGWBucketAdminOpState& op_state,
   Formatter *formatter = flusher.get_formatter();
   flusher.start(0);
 
+  CephContext *cct = store->ctx();
+
+  size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk;
+
   bool show_stats = op_state.will_fetch_stats();
   if (op_state.is_user_op()) {
     formatter->open_array_section("buckets");
 
-    RGWUserBuckets buckets = op_state.get_user_buckets();
-    map<string, RGWBucketEnt>& m = buckets.get_buckets();
-    map<string, RGWBucketEnt>::iterator iter;
-
-    for (iter = m.begin(); iter != m.end(); ++iter) {
-      std::string  obj_name = iter->first;
-      if (show_stats)
-        bucket_stats(store, obj_name, formatter);
-      else
-        formatter->dump_string("bucket", obj_name);
-    }
+    RGWUserBuckets buckets;
+    string marker;
+    bool done;
+
+    do {
+      ret = rgw_read_user_buckets(store, op_state.get_user_id(), buckets, marker, max_entries, false);
+      if (ret < 0)
+        return ret;
+
+      map<string, RGWBucketEnt>& m = buckets.get_buckets();
+      map<string, RGWBucketEnt>::iterator iter;
+
+      for (iter = m.begin(); iter != m.end(); ++iter) {
+        std::string  obj_name = iter->first;
+        if (show_stats)
+          bucket_stats(store, obj_name, formatter);
+        else
+          formatter->dump_string("bucket", obj_name);
+
+        marker = obj_name;
+      }
+
+      flusher.flush();
+      done = (m.size() < max_entries);
+    } while (!done);
 
     formatter->close_section();
   } else if (!bucket_name.empty()) {
diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h
index c1a8f60deea..9eb16cf3707 100644
--- a/src/rgw/rgw_bucket.h
+++ b/src/rgw/rgw_bucket.h
@@ -81,7 +81,8 @@ WRITE_CLASS_ENCODER(RGWUserBuckets)
  * Get all the buckets owned by a user and fill up an RGWUserBuckets with them.
  * Returns: 0 on success, -ERR# on failure.
  */
-extern int rgw_read_user_buckets(RGWRados *store, string user_id, RGWUserBuckets& buckets, bool need_stats);
+extern int rgw_read_user_buckets(RGWRados *store, string user_id, RGWUserBuckets& buckets,
+                                 const string& marker, uint64_t max, bool need_stats);
 
 /**
  * Store the set of buckets associated with a user.
@@ -113,7 +114,6 @@ struct RGWBucketAdminOpState {
   bool bucket_stored;
 
   rgw_bucket bucket;
-  RGWUserBuckets buckets;
 
   void set_fetch_stats(bool value) { stat_buckets = value; }
   void set_check_objects(bool value) { check_objects = value; }
@@ -142,9 +142,6 @@ struct RGWBucketAdminOpState {
     bucket_stored = true;
   }
 
-  RGWUserBuckets& get_user_buckets() { return buckets; };
-  void set_user_buckets(RGWUserBuckets& _buckets) { buckets = _buckets; };
-
   bool will_fetch_stats() { return stat_buckets; };
   bool will_fix_index() { return fix_index; };
   bool will_delete_children() { return delete_child_objects; };
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index d2fbeeb14d7..4e03b053e5c 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -653,21 +653,54 @@ void RGWListBuckets::execute()
   if (ret < 0)
     return;
 
-  ret = rgw_read_user_buckets(store, s->user.user_id, buckets, !!(s->prot_flags & RGW_REST_SWIFT));
-  if (ret < 0) {
-    /* hmm.. something wrong here.. the user was authenticated, so it
-       should exist, just try to recreate */
-    ldout(s->cct, 10) << "WARNING: failed on rgw_get_user_buckets uid=" << s->user.user_id << dendl;
+  bool done;
 
-    /*
+  bool started = false;
+  uint64_t total_count = 0;
 
-    on a second thought, this is probably a bug and we should fail
+  size_t max_buckets = s->cct->_conf->rgw_list_buckets_max_chunk;
 
-    rgw_put_user_buckets(s->user.user_id, buckets);
-    ret = 0;
+  do {
+    RGWUserBuckets buckets;
+    uint64_t read_count = min(limit - total_count, max_buckets);
+    ret = rgw_read_user_buckets(store, s->user.user_id, buckets,
+                                marker, read_count, !!(s->prot_flags & RGW_REST_SWIFT));
 
-    */
-  }
+    if (!started) {
+      send_response_begin();
+      started = true;
+    }
+
+    if (ret < 0) {
+      /* hmm.. something wrong here.. the user was authenticated, so it
+         should exist, just try to recreate */
+      ldout(s->cct, 10) << "WARNING: failed on rgw_get_user_buckets uid=" << s->user.user_id << dendl;
+
+      /*
+
+      on a second thought, this is probably a bug and we should fail
+
+      rgw_put_user_buckets(s->user.user_id, buckets);
+      ret = 0;
+
+      */
+      break;
+    }
+    map<string, RGWBucketEnt>& m = buckets.get_buckets();
+
+    total_count += m.size();
+
+    done = (m.size() < read_count);
+
+    if (m.size()) {
+      send_response_data(buckets);
+
+      map<string, RGWBucketEnt>::reverse_iterator riter = m.rbegin();
+      marker = riter->first;
+    }
+  } while (!done);
+
+  send_response_end();
 }
 
 int RGWStatAccount::verify_permission()
@@ -677,33 +710,43 @@ int RGWStatAccount::verify_permission()
 
 void RGWStatAccount::execute()
 {
-  RGWUserBuckets buckets;
+  string marker;
+  bool done;
+  size_t max_buckets = s->cct->_conf->rgw_list_buckets_max_chunk;
 
-  ret = rgw_read_user_buckets(store, s->user.user_id, buckets, true);
-  if (ret < 0) {
-    /* hmm.. something wrong here.. the user was authenticated, so it
-       should exist, just try to recreate */
-    ldout(s->cct, 10) << "WARNING: failed on rgw_get_user_buckets uid=" << s->user.user_id << dendl;
+  do {
+    RGWUserBuckets buckets;
 
-    /*
+    ret = rgw_read_user_buckets(store, s->user.user_id, buckets, marker, max_buckets, true);
+    if (ret < 0) {
+      /* hmm.. something wrong here.. the user was authenticated, so it
+         should exist, just try to recreate */
+      ldout(s->cct, 10) << "WARNING: failed on rgw_get_user_buckets uid=" << s->user.user_id << dendl;
 
-    on a second thought, this is probably a bug and we should fail
+      /*
 
-    rgw_put_user_buckets(s->user.user_id, buckets);
-    ret = 0;
+      on a second thought, this is probably a bug and we should fail
 
-    */
-  } else {
-    map<string, RGWBucketEnt>& m = buckets.get_buckets();
-    map<string, RGWBucketEnt>::iterator iter;
-    for (iter = m.begin(); iter != m.end(); ++iter) {
-      RGWBucketEnt& bucket = iter->second;
-      buckets_size += bucket.size;
-      buckets_size_rounded += bucket.size_rounded;
-      buckets_objcount += bucket.count;
+      rgw_put_user_buckets(s->user.user_id, buckets);
+      ret = 0;
+
+      */
+
+      break;
+    } else {
+      map<string, RGWBucketEnt>& m = buckets.get_buckets();
+      map<string, RGWBucketEnt>::iterator iter;
+      for (iter = m.begin(); iter != m.end(); ++iter) {
+        RGWBucketEnt& bucket = iter->second;
+        buckets_size += bucket.size;
+        buckets_size_rounded += bucket.size_rounded;
+        buckets_objcount += bucket.count;
+      }
+      buckets_count = m.size();
+
+      done = (m.size() < max_buckets);
     }
-    buckets_count = m.size();
-  }
+  } while (!done);
 }
 
 int RGWStatBucket::verify_permission()
@@ -788,11 +831,13 @@ int RGWCreateBucket::verify_permission()
 
   if (s->user.max_buckets) {
     RGWUserBuckets buckets;
-    int ret = rgw_read_user_buckets(store, s->user.user_id, buckets, false);
+    string marker;
+    int ret = rgw_read_user_buckets(store, s->user.user_id, buckets, marker, s->user.max_buckets, false);
     if (ret < 0)
       return ret;
 
-    if (buckets.count() >= s->user.max_buckets) {
+    map<string, RGWBucketEnt>& m = buckets.get_buckets();
+    if (m.size() >= s->user.max_buckets) {
       return -ERR_TOO_MANY_BUCKETS;
     }
   }
diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index dab1404c6aa..334ecd89abb 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -124,16 +124,24 @@ public:
 class RGWListBuckets : public RGWOp {
 protected:
   int ret;
-  RGWUserBuckets buckets;
+  bool sent_data;
+  string marker;
+  uint64_t limit;
+  uint64_t limit_max;
 
 public:
-  RGWListBuckets() : ret(0) {}
+  RGWListBuckets() : ret(0), sent_data(false) {
+    limit = limit_max = 10000;
+  }
 
   int verify_permission();
   void execute();
 
   virtual int get_params() = 0;
-  virtual void send_response() = 0;
+  virtual void send_response_begin() = 0;
+  virtual void send_response_data(RGWUserBuckets& buckets) = 0;
+  virtual void send_response_end() = 0;
+  virtual void send_response() {}
 
   virtual const char *name() { return "list_buckets"; }
 };
diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index e47e2e90e8e..89daf7651d3 100644
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -3293,7 +3293,7 @@ int RGWRados::put_bucket_info(string& bucket_name, RGWBucketInfo& info, bool exc
   return ret;
 }
 
-int RGWRados::omap_get_all(rgw_obj& obj, bufferlist& header, std::map<string, bufferlist>& m)
+int RGWRados::omap_get_vals(rgw_obj& obj, bufferlist& header, const string& marker, uint64_t count, std::map<string, bufferlist>& m)
 {
   bufferlist bl;
   librados::IoCtx io_ctx;
@@ -3306,8 +3306,7 @@ int RGWRados::omap_get_all(rgw_obj& obj, bufferlist& header, std::map<string, bu
 
   io_ctx.locator_set_key(key);
 
-  string start_after;
-  r = io_ctx.omap_get_vals(oid, start_after, -1, &m);
+  r = io_ctx.omap_get_vals(oid, marker, count, &m);
   if (r < 0)
     return r;
 
@@ -3315,6 +3314,13 @@ int RGWRados::omap_get_all(rgw_obj& obj, bufferlist& header, std::map<string, bu
  
 }
 
+int RGWRados::omap_get_all(rgw_obj& obj, bufferlist& header, std::map<string, bufferlist>& m)
+{
+  string start_after;
+
+  return omap_get_vals(obj, header, start_after, (uint64_t)-1, m);
+}
+
 int RGWRados::omap_set(rgw_obj& obj, std::string& key, bufferlist& bl)
 {
   rgw_bucket bucket;
diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index bed481573bb..9df662c9b68 100644
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -661,6 +661,7 @@ public:
                        uint64_t *epoch, map<string, bufferlist> *attrs, bufferlist *first_chunk);
 
   virtual bool supports_omap() { return true; }
+  int omap_get_vals(rgw_obj& obj, bufferlist& header, const std::string& marker, uint64_t count, std::map<string, bufferlist>& m);
   virtual int omap_get_all(rgw_obj& obj, bufferlist& header, std::map<string, bufferlist>& m);
   virtual int omap_set(rgw_obj& obj, std::string& key, bufferlist& bl);
   virtual int omap_set(rgw_obj& obj, map<std::string, bufferlist>& m);
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 7a5e941b613..45ded2c5f17 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -155,28 +155,40 @@ send_data:
   return 0;
 }
 
-void RGWListBuckets_ObjStore_S3::send_response()
+void RGWListBuckets_ObjStore_S3::send_response_begin()
 {
   if (ret)
     set_req_state_err(s, ret);
   dump_errno(s);
   dump_start(s);
+  end_header(s, "application/xml");
 
-  list_all_buckets_start(s);
-  dump_owner(s, s->user.user_id, s->user.display_name);
+  if (!ret) {
+    list_all_buckets_start(s);
+    dump_owner(s, s->user.user_id, s->user.display_name);
+    s->formatter->open_array_section("Buckets");
+    sent_data = true;
+  }
+}
 
+void RGWListBuckets_ObjStore_S3::send_response_data(RGWUserBuckets& buckets)
+{
   map<string, RGWBucketEnt>& m = buckets.get_buckets();
   map<string, RGWBucketEnt>::iterator iter;
 
-  s->formatter->open_array_section("Buckets");
   for (iter = m.begin(); iter != m.end(); ++iter) {
     RGWBucketEnt obj = iter->second;
     dump_bucket(s, obj);
   }
-  s->formatter->close_section();
-  list_all_buckets_end(s);
-  dump_content_length(s, s->formatter->get_len());
-  end_header(s, "application/xml");
+  rgw_flush_formatter(s, s->formatter);
+}
+
+void RGWListBuckets_ObjStore_S3::send_response_end()
+{
+  if (sent_data) {
+    s->formatter->close_section();
+    list_all_buckets_end(s);
+  }
   rgw_flush_formatter_and_reset(s, s->formatter);
 }
 
diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h
index 204bee84456..16325978a7a 100644
--- a/src/rgw/rgw_rest_s3.h
+++ b/src/rgw/rgw_rest_s3.h
@@ -26,7 +26,9 @@ public:
   ~RGWListBuckets_ObjStore_S3() {}
 
   int get_params() { return 0; }
-  void send_response();
+  virtual void send_response_begin();
+  virtual void send_response_data(RGWUserBuckets& buckets);
+  virtual void send_response_end();
 };
 
 class RGWListBucket_ObjStore_S3 : public RGWListBucket_ObjStore {
diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index 576eb99cec6..bbe2349d27d 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -17,52 +17,53 @@ int RGWListBuckets_ObjStore_SWIFT::get_params()
   marker = s->args.get("marker");
   string limit_str;
   limit_str = s->args.get("limit");
-  limit = strtol(limit_str.c_str(), NULL, 10);
-  if (limit > limit_max || limit < 0)
+  long l = strtol(limit_str.c_str(), NULL, 10);
+  if (l > (long)limit_max || l < 0)
     return -ERR_PRECONDITION_FAILED;
 
+  limit = (uint64_t)l;
+
   if (limit == 0)
     limit = limit_max;
 
   return 0;
 }
 
-void RGWListBuckets_ObjStore_SWIFT::send_response()
+void RGWListBuckets_ObjStore_SWIFT::send_response_begin()
 {
-  map<string, RGWBucketEnt>& m = buckets.get_buckets();
-  map<string, RGWBucketEnt>::iterator iter;
-
-  if (ret < 0)
-    goto done;
+  if (ret)
+    set_req_state_err(s, ret);
+  dump_errno(s);
+  end_header(s);
 
-  dump_start(s);
+  if (!ret) {
+    dump_start(s);
+    s->formatter->open_array_section("account");
 
-  s->formatter->open_array_section("account");
+    sent_data = true;
+  }
+}
 
-  if (marker.empty())
-    iter = m.begin();
-  else
-    iter = m.upper_bound(marker);
+void RGWListBuckets_ObjStore_SWIFT::send_response_data(RGWUserBuckets& buckets)
+{
+  map<string, RGWBucketEnt>& m = buckets.get_buckets();
+  map<string, RGWBucketEnt>::iterator iter;
 
-  for (int i = 0; i < limit && iter != m.end(); ++iter, ++i) {
+  for (iter = m.begin(); iter != m.end(); ++iter) {
     RGWBucketEnt obj = iter->second;
     s->formatter->open_object_section("container");
     s->formatter->dump_string("name", obj.bucket.name);
     s->formatter->dump_int("count", obj.count);
     s->formatter->dump_int("bytes", obj.size);
     s->formatter->close_section();
+    rgw_flush_formatter(s, s->formatter);
   }
-  s->formatter->close_section();
-
-  if (!ret && s->formatter->get_len() == 0)
-    ret = STATUS_NO_CONTENT;
-done:
-  set_req_state_err(s, ret);
-  dump_errno(s);
-  end_header(s);
+}
 
-  if (ret < 0) {
-    return;
+void RGWListBuckets_ObjStore_SWIFT::send_response_end()
+{
+  if (sent_data) {
+    s->formatter->close_section();
   }
 
   rgw_flush_formatter_and_reset(s, s->formatter);
diff --git a/src/rgw/rgw_rest_swift.h b/src/rgw/rgw_rest_swift.h
index 420d1d86873..fef1b4df528 100644
--- a/src/rgw/rgw_rest_swift.h
+++ b/src/rgw/rgw_rest_swift.h
@@ -14,18 +14,14 @@ public:
 };
 
 class RGWListBuckets_ObjStore_SWIFT : public RGWListBuckets_ObjStore {
-  int limit_max;
-  int limit;
-  string marker;
 public:
-  RGWListBuckets_ObjStore_SWIFT() {
-    limit_max = 10000;
-    limit = limit_max;
-  }
+  RGWListBuckets_ObjStore_SWIFT() {}
   ~RGWListBuckets_ObjStore_SWIFT() {}
 
   int get_params();
-  void send_response();
+  void send_response_begin();
+  void send_response_data(RGWUserBuckets& buckets);
+  void send_response_end();
 };
 
 class RGWListBucket_ObjStore_SWIFT : public RGWListBucket_ObjStore {
diff --git a/src/rgw/rgw_user.cc b/src/rgw/rgw_user.cc
index d094ff915f6..9a75ba69b9e 100644
--- a/src/rgw/rgw_user.cc
+++ b/src/rgw/rgw_user.cc
@@ -240,19 +240,33 @@ int rgw_remove_swift_name_index(RGWRados *store, string& swift_name)
  * themselves alone, as well as any ACLs embedded in object xattrs.
  */
 int rgw_delete_user(RGWRados *store, RGWUserInfo& info) {
-  RGWUserBuckets user_buckets;
-  int ret = rgw_read_user_buckets(store, info.user_id, user_buckets, false);
-  if (ret < 0)
-    return ret;
-
-  map<string, RGWBucketEnt>& buckets = user_buckets.get_buckets();
+  string marker;
   vector<rgw_bucket> buckets_vec;
-  for (map<string, RGWBucketEnt>::iterator i = buckets.begin();
-      i != buckets.end();
-      ++i) {
-    RGWBucketEnt& bucket = i->second;
-    buckets_vec.push_back(bucket.bucket);
-  }
+
+  bool done;
+  int ret;
+  CephContext *cct = store->ctx();
+  size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk;
+
+  do {
+    RGWUserBuckets user_buckets;
+    ret = rgw_read_user_buckets(store, info.user_id, user_buckets, marker, max_buckets, false);
+    if (ret < 0)
+      return ret;
+
+    map<string, RGWBucketEnt>& buckets = user_buckets.get_buckets();
+    for (map<string, RGWBucketEnt>::iterator i = buckets.begin();
+        i != buckets.end();
+        ++i) {
+      RGWBucketEnt& bucket = i->second;
+      buckets_vec.push_back(bucket.bucket);
+
+      marker = i->first;
+    }
+
+    done = (buckets.size() < max_buckets);
+  } while (!done);
+
   map<string, RGWAccessKey>::iterator kiter = info.access_keys.begin();
   for (; kiter != info.access_keys.end(); ++kiter) {
     ldout(store->ctx(), 10) << "removing key index: " << kiter->first << dendl;
@@ -1694,20 +1708,24 @@ int RGWUser::execute_remove(RGWUserAdminOpState& op_state, std::string *err_msg)
     return -EINVAL;
   }
 
-  RGWUserBuckets buckets;
-  ret = rgw_read_user_buckets(store, uid, buckets, false);
-  if (ret < 0) {
-    set_err_msg(err_msg, "unable to read user bucket info");
-    return ret;
-  }
+  bool done;
+  string marker;
+  CephContext *cct = store->ctx();
+  size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk;
+  do {
+    RGWUserBuckets buckets;
+    ret = rgw_read_user_buckets(store, uid, buckets, marker, max_buckets, false);
+    if (ret < 0) {
+      set_err_msg(err_msg, "unable to read user bucket info");
+      return ret;
+    }
 
-  map<std::string, RGWBucketEnt>& m = buckets.get_buckets();
-  if (!m.empty() && !purge_data) {
-    set_err_msg(err_msg, "must specify purge data to remove user with buckets");
-    return -EEXIST; // change to code that maps to 409: conflict
-  }
+    map<std::string, RGWBucketEnt>& m = buckets.get_buckets();
+    if (!m.empty() && !purge_data) {
+      set_err_msg(err_msg, "must specify purge data to remove user with buckets");
+      return -EEXIST; // change to code that maps to 409: conflict
+    }
 
-  if (!m.empty()) {
     std::map<std::string, RGWBucketEnt>::iterator it;
     for (it = m.begin(); it != m.end(); ++it) {
       ret = rgw_remove_bucket(store, ((*it).second).bucket, true);
@@ -1715,8 +1733,12 @@ int RGWUser::execute_remove(RGWUserAdminOpState& op_state, std::string *err_msg)
         set_err_msg(err_msg, "unable to delete user data");
         return ret;
       }
+
+      marker = it->first;
     }
-  }
+
+    done = (m.size() < max_buckets);
+  } while (!done);
 
   ret = rgw_delete_user(store, user_info);
   if (ret < 0) {
@@ -1824,26 +1846,36 @@ int RGWUser::execute_modify(RGWUserAdminOpState& op_state, std::string *err_msg)
       return -EINVAL;
     }
 
-    ret = rgw_read_user_buckets(store, user_id, buckets, false);
-    if (ret < 0) {
-      set_err_msg(err_msg, "could not get buckets for uid:  " + user_id);
-      return ret;
-    }
+    bool done;
+    string marker;
+    CephContext *cct = store->ctx();
+    size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk;
+    do {
+      ret = rgw_read_user_buckets(store, user_id, buckets, marker, max_buckets, false);
+      if (ret < 0) {
+        set_err_msg(err_msg, "could not get buckets for uid:  " + user_id);
+        return ret;
+      }
 
-    map<string, RGWBucketEnt>& m = buckets.get_buckets();
-    map<string, RGWBucketEnt>::iterator iter;
+      map<string, RGWBucketEnt>& m = buckets.get_buckets();
+      map<string, RGWBucketEnt>::iterator iter;
 
-    vector<rgw_bucket> bucket_names;
-    for (iter = m.begin(); iter != m.end(); ++iter) {
-      RGWBucketEnt obj = iter->second;
-      bucket_names.push_back(obj.bucket);
-    }
+      vector<rgw_bucket> bucket_names;
+      for (iter = m.begin(); iter != m.end(); ++iter) {
+        RGWBucketEnt obj = iter->second;
+        bucket_names.push_back(obj.bucket);
 
-    ret = store->set_buckets_enabled(bucket_names, !suspended);
-    if (ret < 0) {
-      set_err_msg(err_msg, "failed to change pool");
-      return ret;
-    }
+        marker = iter->first;
+      }
+
+      ret = store->set_buckets_enabled(bucket_names, !suspended);
+      if (ret < 0) {
+        set_err_msg(err_msg, "failed to modify bucket");
+        return ret;
+      }
+
+      done = (m.size() < max_buckets);
+    } while (!done);
   }
   op_state.set_user_info(user_info);