Merge remote-tracking branch 'gh/next'

15 files changed, 231 insertions, 15 deletions

diff --git a/src/common/ceph_context.cc b/src/common/ceph_context.cc
index decf86db70a..0be1c8cea34 100644
--- a/src/common/ceph_context.cc
+++ b/src/common/ceph_context.cc
@@ -183,7 +183,7 @@ void CephContext::do_command(std::string command, std::string args, bufferlist *
       std::string var = args;
       size_t pos = var.find(' ');
       if (pos == string::npos) {
-	jf.dump_string("error", "set_config syntax is 'set_config <var> <value>'");
+	jf.dump_string("error", "syntax error: 'config set <var> <value>'");
       } else {
 	std::string val = var.substr(pos+1);
 	var.resize(pos);
@@ -253,7 +253,7 @@ CephContext::CephContext(uint32_t module_type_)
   _admin_socket->register_command("2", _admin_hook, "");
   _admin_socket->register_command("perf schema", _admin_hook, "dump perfcounters schema");
   _admin_socket->register_command("config show", _admin_hook, "dump current config settings");
-  _admin_socket->register_command("config set", _admin_hook, "set_config <field> <val>: set a config settings");
+  _admin_socket->register_command("config set", _admin_hook, "config set <field> <val>: set a config variable");
   _admin_socket->register_command("log flush", _admin_hook, "flush log entries to log file");
   _admin_socket->register_command("log dump", _admin_hook, "dump recent log entries to log file");
   _admin_socket->register_command("log reopen", _admin_hook, "reopen log file");
diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc
index 0dd3f1b95d9..0a3480ebced 100644
--- a/src/mds/MDS.cc
+++ b/src/mds/MDS.cc
@@ -780,10 +780,10 @@ void MDS::handle_command(MMonCommand *m)
 	  CDir *dir = in->get_dirfrag(frag_t());
 	  if (dir && dir->is_auth()) {
 	    mdcache->migrator->export_dir(dir, target);
-	  } else dout(0) << "bad migrate_dir path dirfrag frag_t() or dir not auth" << dendl;
-	} else dout(0) << "bad migrate_dir path" << dendl;
-      } else dout(0) << "bad migrate_dir target syntax" << dendl;
-    } else dout(0) << "bad migrate_dir syntax" << dendl;
+	  } else dout(0) << "bad export_dir path dirfrag frag_t() or dir not auth" << dendl;
+	} else dout(0) << "bad export_dir path" << dendl;
+      } else dout(0) << "bad export_dir target syntax" << dendl;
+    } else dout(0) << "bad export_dir syntax" << dendl;
   } 
   else if (m->cmd[0] == "cpu_profiler") {
     ostringstream ss;
diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 209bb982d86..f889f25166f 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -3041,7 +3041,7 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
 	      paxos->wait_for_commit(new Monitor::C_Command(mon, m, 0, rs, paxos->get_version()));
 	      return true;
 	    } else if (m->cmd[4] == "pg_num") {
-	      if (m->cmd.size() < 6 ||
+	      if (m->cmd.size() < 7 ||
 		  m->cmd[6] != "--allow-experimental-feature") {
 		ss << "increasing pg_num is currently experimental, add "
 		   << "--allow-experimental-feature as the last argument "
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index f09055e5bc8..669729f05ff 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1000,7 +1000,7 @@ int OSD::init()
   assert(r == 0);
   test_ops_hook = new TestOpsSocketHook(&(this->service), this->store);
   r = admin_socket->register_command("setomapval", test_ops_hook,
-                              "setomap <pool-id> <obj-name> <key> <val>");
+                              "setomapval <pool-id> <obj-name> <key> <val>");
   assert(r == 0);
   r = admin_socket->register_command("rmomapkey", test_ops_hook,
                                "rmomapkey <pool-id> <obj-name> <key>");
@@ -1032,6 +1032,11 @@ int OSD::init()
 
   osd_lock.Lock();
 
+  dout(10) << "ensuring pgs have consumed prior maps" << dendl;
+  consume_map();
+  peering_wq.drain();
+
+  dout(10) << "done with init, starting boot process" << dendl;
   state = STATE_BOOTING;
   start_boot();
 
diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 595cd3fdf7b..523904e1f96 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -5015,6 +5015,7 @@ void PG::handle_activate_map(RecoveryCtx *rctx)
   dout(10) << "handle_activate_map " << dendl;
   ActMap evt;
   recovery_state.handle_event(evt, rctx);
+  dirty_info = true;
 }
 
 void PG::handle_loaded(RecoveryCtx *rctx)
diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index 5e543d0a18e..fba6af90f23 100644
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -3490,6 +3490,11 @@ void ReplicatedPG::do_osd_op_effects(OpContext *ctx)
   }
 }
 
+bool ReplicatedPG::have_temp_coll()
+{
+  return temp_created || osd->store->collection_exists(temp_coll);
+}
+
 coll_t ReplicatedPG::get_temp_coll(ObjectStore::Transaction *t)
 {
   if (temp_created)
diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h
index 2b5019f3374..ae506e55074 100644
--- a/src/osd/ReplicatedPG.h
+++ b/src/osd/ReplicatedPG.h
@@ -1011,9 +1011,7 @@ private:
   coll_t temp_coll;
   coll_t get_temp_coll(ObjectStore::Transaction *t);
 public:
-  bool have_temp_coll() {
-    return temp_created;
-  }
+  bool have_temp_coll();
   coll_t get_temp_coll() {
     return temp_coll;
   }
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 860a4d19eba..bcb52c4420b 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -41,6 +41,7 @@ void _usage()
   cerr << "  user rm                    remove user\n";
   cerr << "  user suspend               suspend a user\n";
   cerr << "  user enable                reenable user after suspension\n";
+  cerr << "  user check                 check user info\n";
   cerr << "  caps add                   add user capabilities\n";
   cerr << "  caps rm                    remove user capabilities\n";
   cerr << "  subuser create             create a new subuser\n" ;
@@ -55,6 +56,7 @@ void _usage()
   cerr << "  bucket rm                  remove bucket\n";
   cerr << "  bucket check               check bucket index\n";
   cerr << "  object rm                  remove object\n";
+  cerr << "  object unlink              unlink object from bucket index\n";
   cerr << "  cluster info               show cluster params info\n";
   cerr << "  pool add                   add an existing pool for data placement\n";
   cerr << "  pool rm                    remove an existing pool from data placement set\n";
@@ -133,6 +135,7 @@ enum {
   OPT_USER_RM,
   OPT_USER_SUSPEND,
   OPT_USER_ENABLE,
+  OPT_USER_CHECK,
   OPT_SUBUSER_CREATE,
   OPT_SUBUSER_MODIFY,
   OPT_SUBUSER_RM,
@@ -142,8 +145,8 @@ enum {
   OPT_BUCKET_LINK,
   OPT_BUCKET_UNLINK,
   OPT_BUCKET_STATS,
-  OPT_BUCKET_RM,
   OPT_BUCKET_CHECK,
+  OPT_BUCKET_RM,
   OPT_POLICY,
   OPT_POOL_ADD,
   OPT_POOL_RM,
@@ -155,6 +158,7 @@ enum {
   OPT_USAGE_TRIM,
   OPT_TEMP_REMOVE,
   OPT_OBJECT_RM,
+  OPT_OBJECT_UNLINK,
   OPT_GC_LIST,
   OPT_GC_PROCESS,
   OPT_CLUSTER_INFO,
@@ -227,6 +231,7 @@ static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more)
       strcmp(cmd, "key") == 0 ||
       strcmp(cmd, "buckets") == 0 ||
       strcmp(cmd, "bucket") == 0 ||
+      strcmp(cmd, "object") == 0 ||
       strcmp(cmd, "pool") == 0 ||
       strcmp(cmd, "pools") == 0 ||
       strcmp(cmd, "log") == 0 ||
@@ -259,6 +264,8 @@ static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more)
       return OPT_USER_SUSPEND;
     if (strcmp(cmd, "enable") == 0)
       return OPT_USER_ENABLE;
+    if (strcmp(cmd, "check") == 0)
+      return OPT_USER_CHECK;
   } else if (strcmp(prev_cmd, "subuser") == 0) {
     if (strcmp(cmd, "create") == 0)
       return OPT_SUBUSER_CREATE;
@@ -318,6 +325,8 @@ static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more)
   } else if (strcmp(prev_cmd, "object") == 0) {
     if (strcmp(cmd, "rm") == 0)
       return OPT_OBJECT_RM;
+    if (strcmp(cmd, "unlink") == 0)
+      return OPT_OBJECT_UNLINK;
   } else if (strcmp(prev_cmd, "cluster") == 0) {
     if (strcmp(cmd, "info") == 0)
       return OPT_CLUSTER_INFO;
@@ -574,6 +583,128 @@ enum ObjectKeyType {
   KEY_TYPE_S3,
 };
 
+static void check_bad_index_multipart(RGWRados *store, rgw_bucket& bucket, bool fix)
+{
+  int max = 1000;
+  string prefix;
+  string marker;
+  string delim;
+
+  map<string, bool> common_prefixes;
+  string ns = "multipart";
+
+  bool is_truncated;
+  list<string> objs_to_unlink;
+  map<string, bool> meta_objs;
+  map<string, string> all_objs;
+
+  do {
+    vector<RGWObjEnt> result;
+    int r = store->list_objects(bucket, max, prefix, delim, marker,
+				result, common_prefixes, false, ns,
+				&is_truncated, NULL);
+
+    if (r < 0) {
+      cerr << "failed to list objects in bucket=" << bucket << " err=" << cpp_strerror(-r) << std::endl;
+      return;
+    }
+
+    vector<RGWObjEnt>::iterator iter;
+    for (iter = result.begin(); iter != result.end(); ++iter) {
+      RGWObjEnt& ent = *iter;
+
+      rgw_obj obj(bucket, ent.name);
+      obj.set_ns(ns);
+
+      string& oid = obj.object;
+      marker = oid;
+
+      int pos = oid.find_last_of('.');
+      if (pos < 0)
+	continue;
+
+      string name = oid.substr(0, pos);
+      string suffix = oid.substr(pos + 1);
+
+      if (suffix.compare("meta") == 0) {
+	meta_objs[name] = true;
+      } else {
+	all_objs[oid] = name;
+      }
+    }
+
+  } while (is_truncated);
+
+  map<string, string>::iterator aiter;
+  for (aiter = all_objs.begin(); aiter != all_objs.end(); ++aiter) {
+    string& name = aiter->second;
+
+    if (meta_objs.find(name) == meta_objs.end()) {
+      objs_to_unlink.push_back(aiter->first);
+    }
+  }
+
+  if (objs_to_unlink.empty())
+    return;
+
+  if (!fix) {
+    cout << "Need to unlink the following objects from bucket=" << bucket << std::endl;
+  } else {
+    cout << "Unlinking the following objects from bucket=" << bucket << std::endl;
+  }
+  for (list<string>::iterator oiter = objs_to_unlink.begin(); oiter != objs_to_unlink.end(); ++oiter) {
+    cout << *oiter << std::endl;
+  }
+
+  if (fix) {
+    int r = store->remove_objs_from_index(bucket, objs_to_unlink);
+    if (r < 0) {
+      cerr << "ERROR: remove_obj_from_index() returned error: " << cpp_strerror(-r) << std::endl;
+    }
+  }
+}
+
+static void check_bad_user_bucket_mapping(RGWRados *store, const string& user_id, bool fix)
+{
+  RGWUserBuckets user_buckets;
+  int ret = rgw_read_user_buckets(store, user_id, user_buckets, false);
+  if (ret < 0) {
+    cerr << "failed to read user buckets: " << cpp_strerror(-ret) << std::endl;
+    return;
+  }
+
+  map<string, RGWBucketEnt>& buckets = user_buckets.get_buckets();
+  for (map<string, RGWBucketEnt>::iterator i = buckets.begin();
+       i != buckets.end();
+       ++i) {
+    RGWBucketEnt& bucket_ent = i->second;
+    rgw_bucket& bucket = bucket_ent.bucket;
+
+    RGWBucketInfo bucket_info;
+    int r = store->get_bucket_info(NULL, bucket.name, bucket_info);
+    if (r < 0) {
+      cerr << "could not get bucket info for bucket=" << bucket << std::endl;
+      continue;
+    }
+
+    rgw_bucket& actual_bucket = bucket_info.bucket;
+
+    if (actual_bucket.name.compare(bucket.name) != 0 ||
+        actual_bucket.pool.compare(bucket.pool) != 0 ||
+        actual_bucket.marker.compare(bucket.marker) != 0 ||
+        actual_bucket.bucket_id.compare(bucket.bucket_id) != 0) {
+      cout << "bucket info mismatch: expected " << actual_bucket << " got " << bucket << std::endl;
+      if (fix) {
+	cout << "fixing" << std::endl;
+	r = rgw_add_bucket(store, user_id, actual_bucket);
+	if (r < 0) {
+	  cerr << "failed to fix bucket: " << cpp_strerror(-r) << std::endl;
+	}
+      }
+    }
+  }
+}
+
 static int remove_object(RGWRados *store, rgw_bucket& bucket, std::string& object)
 {
   int ret = -EINVAL;
@@ -1627,7 +1758,19 @@ next:
     }
   }
 
+  if (opt_cmd == OPT_OBJECT_UNLINK) {
+    list<string> oid_list;
+    oid_list.push_back(object);
+    int ret = store->remove_objs_from_index(bucket, oid_list);
+    if (ret < 0) {
+      cerr << "ERROR: remove_obj_from_index() returned error: " << cpp_strerror(-ret) << std::endl;
+      return 1;
+    }
+  }
+
   if (opt_cmd == OPT_BUCKET_CHECK) {
+    check_bad_index_multipart(store, bucket, fix);
+
     map<RGWObjCategory, RGWBucketStats> existing_stats;
     map<RGWObjCategory, RGWBucketStats> calculated_stats;
 
@@ -1757,5 +1900,10 @@ next:
     store->params.dump(formatter);
     formatter->flush(cout);
   }
+
+  if (opt_cmd == OPT_USER_CHECK) {
+    check_bad_user_bucket_mapping(store, user_id, fix);
+  }
+
   return 0;
 }
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index ee4be18c320..ea1cad1e925 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -836,6 +836,27 @@ void RGWCreateBucket::execute()
 
   existed = (ret == -EEXIST);
 
+  if (existed) {
+    /* bucket already existed, might have raced with another bucket creation, or
+     * might be partial bucket creation that never completed. Read existing bucket
+     * info, verify that the reported bucket owner is the current user.
+     * If all is ok then update the user's list of buckets
+     */
+    RGWBucketInfo info;
+    map<string, bufferlist> attrs;
+    int r = store->get_bucket_info(NULL, s->bucket.name, info, &attrs);
+    if (r < 0) {
+      ldout(s->cct, 0) << "ERROR: get_bucket_info on bucket=" << s->bucket.name << " returned err=" << r << " after create_bucket returned -EEXIST" << dendl;
+      ret = r;
+      return;
+    }
+    if (info.owner.compare(s->user.user_id) != 0) {
+      ret = -ERR_BUCKET_EXISTS;
+      return;
+    }
+    s->bucket = info.bucket;
+  }
+
   ret = rgw_add_bucket(store, s->user.user_id, s->bucket);
   if (ret && !existed && ret != -EEXIST)   /* if it exists (or previously existed), don't remove it! */
     rgw_remove_user_bucket_info(store, s->user.user_id, s->bucket);
diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index eb117ca33d4..a58ff371b7c 100644
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -806,6 +806,9 @@ int RGWRados::create_bucket(string& owner, rgw_bucket& bucket,
   info.bucket = bucket;
   info.owner = owner;
   ret = store_bucket_info(info, &attrs, exclusive);
+  if (ret == -EEXIST) {
+    io_ctx.remove(dir_oid);
+  }
 
   return ret;
 }
@@ -3634,6 +3637,38 @@ int RGWRados::cls_obj_usage_log_trim(string& oid, string& user, uint64_t start_e
   return r;
 }
 
+int RGWRados::remove_objs_from_index(rgw_bucket& bucket, list<string>& oid_list)
+{
+  librados::IoCtx io_ctx;
+
+  int r = open_bucket_ctx(bucket, io_ctx);
+  if (r < 0)
+    return r;
+
+  string dir_oid = dir_oid_prefix;
+  dir_oid.append(bucket.marker);
+
+  bufferlist updates;
+
+  list<string>::iterator iter;
+
+  for (iter = oid_list.begin(); iter != oid_list.end(); ++iter) {
+    string& oid = *iter;
+    dout(2) << "RGWRados::remove_objs_from_index bucket=" << bucket << " oid=" << oid << dendl;
+    rgw_bucket_dir_entry entry;
+    entry.epoch = (uint64_t)-1; // ULLONG_MAX, needed to that objclass doesn't skip out request
+    entry.name = oid;
+    updates.append(CEPH_RGW_REMOVE);
+    ::encode(entry, updates);
+  }
+
+  bufferlist out;
+
+  r = io_ctx.exec(dir_oid, "rgw", "dir_suggest_changes", updates, out);
+
+  return r;
+}
+
 int RGWRados::check_disk_state(librados::IoCtx io_ctx,
                                rgw_bucket& bucket,
                                rgw_bucket_dir_entry& list_state,
diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 67bbec98fc7..3ae13c8524c 100644
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -744,6 +744,8 @@ public:
                          map<RGWObjCategory, RGWBucketStats> *existing_stats,
                          map<RGWObjCategory, RGWBucketStats> *calculated_stats);
   int bucket_rebuild_index(rgw_bucket& bucket);
+  int remove_objs_from_index(rgw_bucket& bucket, list<string>& oid_list);
+
  private:
   int process_intent_log(rgw_bucket& bucket, string& oid,
 			 time_t epoch, int flags, bool purge);
diff --git a/src/test/cli/ceph/help.t b/src/test/cli/ceph/help.t
index 9a035c6e7e4..eac011dc814 100644
--- a/src/test/cli/ceph/help.t
+++ b/src/test/cli/ceph/help.t
@@ -23,7 +23,7 @@
   
   MONITOR (MON) COMMANDS
     ceph mon add <name> <ip>[:<port>]
-    ceph mon delete <name>
+    ceph mon remove <name>
     ceph mon stat
     ceph mon tell <mon-id or *> injectargs '--<switch> <value> [--<switch> <value>...]'
   
diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t
index 0ab4eace52b..1927023a2a7 100644
--- a/src/test/cli/radosgw-admin/help.t
+++ b/src/test/cli/radosgw-admin/help.t
@@ -21,6 +21,7 @@
     bucket rm                  remove bucket
     bucket check               check bucket index
     object rm                  remove object
+    object unlink              unlink object from bucket index
     cluster info               show cluster params info
     pool add                   add an existing pool for data placement
     pool rm                    remove an existing pool from data placement set
diff --git a/src/tools/ceph.cc b/src/tools/ceph.cc
index e791a0a748d..a1f8024cfe3 100644
--- a/src/tools/ceph.cc
+++ b/src/tools/ceph.cc
@@ -66,7 +66,7 @@ static void usage()
   cout << "\n";
   cout << "MONITOR (MON) COMMANDS\n";
   cout << "  ceph mon add <name> <ip>[:<port>]\n";
-  cout << "  ceph mon delete <name>\n";
+  cout << "  ceph mon remove <name>\n";
   cout << "  ceph mon stat\n";
   cout << "  ceph mon tell <mon-id or *> injectargs '--<switch> <value> [--<switch> <value>...]'\n";
   cout << "\n";
diff --git a/src/tools/common.cc b/src/tools/common.cc
index 9a3763a34ae..13aff67ac6e 100644
--- a/src/tools/common.cc
+++ b/src/tools/common.cc
@@ -274,7 +274,7 @@ int do_command(CephToolCtx *ctx,
   }
   if (cmd.size() > 0 && cmd[0] == "pg") {
     if (cmd.size() == 1) {
-      cerr << "no pgid specified" << std::endl;
+      cerr << "pg requires at least one argument" << std::endl;
       return -EINVAL;
     }
     if (pending_target_pgid.parse(cmd[1].c_str())) {