rgw: a tool to fix buckets with leaked multipart references

Checks specified bucket for the #4011 symptoms, optionally fix
the issue.

sytax:
  radosgw-admin bucket check --bucket=<bucket> [--fix]

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
(cherry picked from commit 2d8faf8e5f15e833e6b556b0f3c4ac92e4a4151e)

Conflicts:
	src/rgw/rgw_admin.cc
	src/rgw/rgw_rados.h

3 files changed, 102 insertions, 10 deletions

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index a3b1271d9ce..75af8c6a139 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -143,8 +143,8 @@ enum {
   OPT_BUCKET_LINK,
   OPT_BUCKET_UNLINK,
   OPT_BUCKET_STATS,
-  OPT_BUCKET_RM,
   OPT_BUCKET_CHECK,
+  OPT_BUCKET_RM,
   OPT_POLICY,
   OPT_POOL_ADD,
   OPT_POOL_RM,
@@ -579,6 +579,88 @@ enum ObjectKeyType {
   KEY_TYPE_S3,
 };
 
+static void check_bad_index_multipart(RGWRados *store, rgw_bucket& bucket, bool fix)
+{
+  int max = 1000;
+  string prefix;
+  string marker;
+  string delim;
+
+  map<string, bool> common_prefixes;
+  string ns = "multipart";
+
+  bool is_truncated;
+  list<string> objs_to_unlink;
+  map<string, bool> meta_objs;
+  map<string, string> all_objs;
+
+  do {
+    vector<RGWObjEnt> result;
+    int r = store->list_objects(bucket, max, prefix, delim, marker,
+				result, common_prefixes, false, ns,
+				&is_truncated, NULL);
+
+    if (r < 0) {
+      cerr << "failed to list objects in bucket=" << bucket << " err=" << cpp_strerror(-r) << std::endl;
+      return;
+    }
+
+    vector<RGWObjEnt>::iterator iter;
+    for (iter = result.begin(); iter != result.end(); ++iter) {
+      RGWObjEnt& ent = *iter;
+
+      rgw_obj obj(bucket, ent.name);
+      obj.set_ns(ns);
+
+      string& oid = obj.object;
+      marker = oid;
+
+      int pos = oid.find_last_of('.');
+      if (pos < 0)
+	continue;
+
+      string name = oid.substr(0, pos);
+      string suffix = oid.substr(pos + 1);
+
+      if (suffix.compare("meta") == 0) {
+	meta_objs[name] = true;
+      } else {
+	all_objs[oid] = name;
+      }
+    }
+
+  } while (is_truncated);
+
+  map<string, string>::iterator aiter;
+  for (aiter = all_objs.begin(); aiter != all_objs.end(); ++aiter) {
+    string& name = aiter->second;
+
+    if (meta_objs.find(name) == meta_objs.end()) {
+      objs_to_unlink.push_back(aiter->first);
+    }
+  }
+
+  if (objs_to_unlink.empty())
+    return;
+
+  if (!fix) {
+    cout << "Need to unlink the following objects from bucket=" << bucket << std::endl;
+  } else {
+    cout << "Unlinking the following objects from bucket=" << bucket << std::endl;
+  }
+  for (list<string>::iterator oiter = objs_to_unlink.begin(); oiter != objs_to_unlink.end(); ++oiter) {
+    cout << *oiter << std::endl;
+  }
+
+  if (fix) {
+    int r = store->remove_objs_from_index(bucket, objs_to_unlink);
+    if (r < 0) {
+      cerr << "ERROR: remove_obj_from_index() returned error: " << cpp_strerror(-r) << std::endl;
+    }
+  }
+
+}
+
 static int remove_object(RGWRados *store, rgw_bucket& bucket, std::string& object)
 {
   int ret = -EINVAL;
@@ -1633,7 +1715,9 @@ next:
   }
 
   if (opt_cmd == OPT_OBJECT_UNLINK) {
-    int ret = store->remove_obj_from_index(bucket, object);
+    list<string> oid_list;
+    oid_list.push_back(object);
+    int ret = store->remove_objs_from_index(bucket, oid_list);
     if (ret < 0) {
       cerr << "ERROR: remove_obj_from_index() returned error: " << cpp_strerror(-ret) << std::endl;
       return 1;
@@ -1641,6 +1725,8 @@ next:
   }
 
   if (opt_cmd == OPT_BUCKET_CHECK) {
+    check_bad_index_multipart(store, bucket, fix);
+
     map<RGWObjCategory, RGWBucketStats> existing_stats;
     map<RGWObjCategory, RGWBucketStats> calculated_stats;
 
diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 895dd7001e1..dfec5387fea 100644
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -3228,7 +3228,7 @@ int RGWRados::cls_obj_usage_log_trim(string& oid, string& user, uint64_t start_e
   return r;
 }
 
-int RGWRados::remove_obj_from_index(rgw_bucket& bucket, const string& oid)
+int RGWRados::remove_objs_from_index(rgw_bucket& bucket, list<string>& oid_list)
 {
   librados::IoCtx io_ctx;
 
@@ -3239,13 +3239,19 @@ int RGWRados::remove_obj_from_index(rgw_bucket& bucket, const string& oid)
   string dir_oid = dir_oid_prefix;
   dir_oid.append(bucket.marker);
 
-  rgw_bucket_dir_entry entry;
-  entry.epoch = (uint64_t)-1; // ULLONG_MAX, needed to that objclass doesn't skip out request
-  entry.name = oid;
-
   bufferlist updates;
-  updates.append(CEPH_RGW_REMOVE);
-  ::encode(entry, updates);
+
+  list<string>::iterator iter;
+
+  for (iter = oid_list.begin(); iter != oid_list.end(); ++iter) {
+    string& oid = *iter;
+    dout(2) << "RGWRados::remove_objs_from_index bucket=" << bucket << " oid=" << oid << dendl;
+    rgw_bucket_dir_entry entry;
+    entry.epoch = (uint64_t)-1; // ULLONG_MAX, needed to that objclass doesn't skip out request
+    entry.name = oid;
+    updates.append(CEPH_RGW_REMOVE);
+    ::encode(entry, updates);
+  }
 
   bufferlist out;
 
diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index e50ef5d4490..f86ef8cd833 100644
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -720,7 +720,7 @@ public:
                          map<RGWObjCategory, RGWBucketStats> *existing_stats,
                          map<RGWObjCategory, RGWBucketStats> *calculated_stats);
   int bucket_rebuild_index(rgw_bucket& bucket);
-  int remove_obj_from_index(rgw_bucket& bucket, const string& oid);
+  int remove_objs_from_index(rgw_bucket& bucket, list<string>& oid_list);
 
  private:
   int process_intent_log(rgw_bucket& bucket, string& oid,