rgw: region creation

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

6 files changed, 291 insertions, 17 deletions

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 1783ca3de52..ca618d28878 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -481,7 +481,11 @@ OPTION(rgw_op_thread_suicide_timeout, OPT_INT, 0)
 OPTION(rgw_thread_pool_size, OPT_INT, 100)
 OPTION(rgw_num_control_oids, OPT_INT, 8)
 
-OPTION(rgw_zone_root_pool, OPT_STR, ".rgw.root")
+OPTION(rgw_zone, OPT_STR, "") // zone name
+OPTION(rgw_zone_root_pool, OPT_STR, ".rgw.root")    // pool where zone specific info is stored
+OPTION(rgw_region, OPT_STR, "") // region name
+OPTION(rgw_region_root_pool, OPT_STR, ".rgw.root")  // pool where all region info is stored
+OPTION(rgw_default_region_info_oid, OPT_STR, "default.region")  // oid where default region info is stored
 OPTION(rgw_log_nonexistent_bucket, OPT_BOOL, false)
 OPTION(rgw_log_object_name, OPT_STR, "%Y-%m-%d-%H-%i-%n")      // man date to see codes (a subset are supported)
 OPTION(rgw_log_object_name_utc, OPT_BOOL, false)
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 5c3b4ad8580..e55613f3e21 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -58,6 +58,7 @@ void _usage()
   cerr << "  bucket rm                  remove bucket\n";
   cerr << "  bucket check               check bucket index\n";
   cerr << "  object rm                  remove object\n";
+  cerr << "  region info                show region info\n";
   cerr << "  zone info                  show zone params info\n";
   cerr << "  pool add                   add an existing pool for data placement\n";
   cerr << "  pool rm                    remove an existing pool from data placement set\n";
@@ -160,6 +161,7 @@ enum {
   OPT_OBJECT_RM,
   OPT_GC_LIST,
   OPT_GC_PROCESS,
+  OPT_REGION_INFO,
   OPT_ZONE_INFO,
   OPT_ZONE_SET,
   OPT_CAPS_ADD,
@@ -194,6 +196,7 @@ static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more)
       strcmp(cmd, "log") == 0 ||
       strcmp(cmd, "usage") == 0 ||
       strcmp(cmd, "object") == 0 ||
+      strcmp(cmd, "region") == 0 ||
       strcmp(cmd, "zone") == 0 ||
       strcmp(cmd, "temp") == 0 ||
       strcmp(cmd, "caps") == 0 ||
@@ -280,6 +283,9 @@ static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more)
   } else if (strcmp(prev_cmd, "object") == 0) {
     if (strcmp(cmd, "rm") == 0)
       return OPT_OBJECT_RM;
+  } else if (strcmp(prev_cmd, "region") == 0) {
+    if (strcmp(cmd, "info") == 0)
+      return OPT_REGION_INFO;
   } else if (strcmp(prev_cmd, "zone") == 0) {
     if (strcmp(cmd, "info") == 0)
       return OPT_ZONE_INFO;
diff --git a/src/rgw/rgw_cache.h b/src/rgw/rgw_cache.h
index 4de16384f1a..2448c7769ca 100644
--- a/src/rgw/rgw_cache.h
+++ b/src/rgw/rgw_cache.h
@@ -165,10 +165,10 @@ class RGWCache  : public T
     return normal_name(obj.bucket, obj.object);
   }
 
-  int initialize() {
+  int init_rados() {
     int ret;
     cache.set_ctx(T::cct);
-    ret = T::initialize();
+    ret = T::init_rados();
     if (ret < 0)
       return ret;
 
diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc
index 77e7a83cf72..c9874a05170 100644
--- a/src/rgw/rgw_json_enc.cc
+++ b/src/rgw/rgw_json_enc.cc
@@ -470,12 +470,20 @@ void RGWRegion::dump(Formatter *f) const
   encode_json("zones", zones, f);
 }
 
+static void decode_zones(map<string, RGWZone>& zones, JSONObj *o)
+{
+  RGWZone z;
+  z.decode_json(o);
+  zones[z.name] = z;
+}
+
+
 void RGWRegion::decode_json(JSONObj *obj)
 {
   JSONDecoder::decode_json("name", name, obj);
   JSONDecoder::decode_json("endpoints", endpoints, obj);
   JSONDecoder::decode_json("master_zone", master_zone, obj);
-  JSONDecoder::decode_json("zones", zones, obj);
+  JSONDecoder::decode_json("zones", zones, decode_zones, obj);
 }
 
 
diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 7f6b0b29da5..c2b9c21c62e 100644
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -48,6 +48,9 @@ static string default_storage_pool = ".rgw.buckets";
 static string avail_pools = ".pools.avail";
 
 static string zone_info_oid = "zone_info";
+static string region_info_oid_prefix = "region_info";
+
+static string default_region_info_oid = "default.region";
 
 
 static RGWObjCategory shadow_category = RGW_OBJ_CATEGORY_SHADOW;
@@ -56,10 +59,183 @@ static RGWObjCategory main_category = RGW_OBJ_CATEGORY_MAIN;
 #define RGW_USAGE_OBJ_PREFIX "usage."
 
 #define RGW_DEFAULT_ZONE_ROOT_POOL ".rgw.root"
+#define RGW_DEFAULT_REGION_ROOT_POOL ".rgw.root"
 
 
 #define dout_subsys ceph_subsys_rgw
 
+struct RGWDefaultRegionInfo {
+  string default_region;
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(default_region, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(1, bl);
+    ::decode(default_region, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const {
+    encode_json("default_region", default_region, f);
+  }
+  void decode_json(JSONObj *obj) {
+    JSONDecoder::decode_json("default_region", default_region, obj);
+  }
+};
+WRITE_CLASS_ENCODER(RGWDefaultRegionInfo);
+
+string RGWRegion::get_pool_name(CephContext *cct)
+{
+  string pool_name = cct->_conf->rgw_region_root_pool;
+  if (pool_name.empty()) {
+    pool_name = RGW_DEFAULT_REGION_ROOT_POOL;
+  }
+  return pool_name;
+}
+
+int RGWRegion::read_default()
+{
+  string pool_name = get_pool_name(cct);
+
+  string oid = cct->_conf->rgw_default_region_info_oid;
+  if (oid.empty()) {
+    oid = default_region_info_oid;
+  }
+
+  rgw_bucket pool(pool_name.c_str());
+  bufferlist bl;
+  int ret = rgw_get_obj(store, NULL, pool, oid, bl);
+  if (ret < 0)
+    return ret;
+
+  RGWDefaultRegionInfo default_info;
+  try {
+    bufferlist::iterator iter = bl.begin();
+    ::decode(default_info, iter);
+  } catch (buffer::error& err) {
+    derr << "error decoding data from " << pool << ":" << oid << dendl;
+    return -EIO;
+  }
+
+  name = default_info.default_region;
+
+  return 0;
+}
+
+int RGWRegion::set_as_default()
+{
+  string pool_name = get_pool_name(cct);
+
+  string oid = cct->_conf->rgw_default_region_info_oid;
+  if (oid.empty()) {
+    oid = default_region_info_oid;
+  }
+
+  rgw_bucket pool(pool_name.c_str());
+  bufferlist bl;
+
+  RGWDefaultRegionInfo default_info;
+  default_info.default_region = name;
+
+  ::encode(default_info, bl);
+
+  int ret = rgw_put_system_obj(store, pool, oid, bl.c_str(), bl.length(), false, NULL);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWRegion::init(CephContext *_cct, RGWRados *_store, bool create_region)
+{
+  cct = _cct;
+  store = _store;
+
+  string pool_name = get_pool_name(cct);
+
+  name = cct->_conf->rgw_region;
+
+  if (name.empty()) {
+    int r = read_default();
+    if (r == -ENOENT) {
+      r = init_default();
+      if (r < 0)
+	return r;
+      r = set_as_default();
+      if (r < 0)
+	return r;
+    } else if (r < 0) {
+      derr << "failed reading default region info: " << cpp_strerror(-r) << dendl;
+      return r;
+    }
+  }
+
+  rgw_bucket pool(pool_name.c_str());
+  bufferlist bl;
+
+  string oid = region_info_oid_prefix + "." + name;
+
+  int ret = rgw_get_obj(store, NULL, pool, oid, bl);
+  if (ret == -ENOENT && create_region) {
+    return init_default();
+  }
+  if (ret < 0)
+    return ret;
+
+  try {
+    bufferlist::iterator iter = bl.begin();
+    ::decode(*this, iter);
+  } catch (buffer::error& err) {
+    ldout(cct, 0) << "ERROR: failed to decode region from " << pool << ":" << oid << dendl;
+    return -EIO;
+  }
+
+  return 0;
+}
+
+int RGWRegion::init_default()
+{
+  name = "default";
+  string zone_name = "default";
+
+  RGWZone default_zone;
+  default_zone.name = zone_name;
+
+  RGWZoneParams zone_params;
+  zone_params.name = zone_name;
+  zone_params.init_default();
+  int r = zone_params.store_info(cct, store);
+  if (r < 0) {
+    derr << "error storing zone params: " << cpp_strerror(-r) << dendl;
+    return r;
+  }
+
+  r = store_info(true);
+  if (r < 0) {
+    derr << "error storing region info: " << cpp_strerror(-r) << dendl;
+  }
+
+  return 0;
+}
+
+int RGWRegion::store_info(bool exclusive)
+{
+  string pool_name = get_pool_name(cct);
+
+  rgw_bucket pool(pool_name.c_str());
+
+  string oid = region_info_oid_prefix + "." + name;
+
+  bufferlist bl;
+  ::encode(*this, bl);
+  int ret = rgw_put_system_obj(store, pool, oid, bl.c_str(), bl.length(), exclusive, NULL);
+
+  return ret;
+}
+
 void RGWZoneParams::init_default()
 {
   domain_root = ".rgw";
@@ -74,17 +250,28 @@ void RGWZoneParams::init_default()
   user_uid_pool = ".users.uid";
 }
 
-int RGWZoneParams::init(CephContext *cct, RGWRados *store)
+string RGWZoneParams::get_pool_name(CephContext *cct, const string& zone_name)
 {
   string pool_name = cct->_conf->rgw_zone_root_pool;
-  if (pool_name.empty())
+  if (pool_name.empty()) {
     pool_name = RGW_DEFAULT_ZONE_ROOT_POOL;
+    if (!zone_name.empty()) {
+      pool_name.append("." + zone_name);
+    }
+  }
+  return pool_name;
+}
+
+int RGWZoneParams::init(CephContext *cct, RGWRados *store, bool create_zone)
+{
+  name = cct->_conf->rgw_zone;
+  string pool_name = get_pool_name(cct, name);
 
   rgw_bucket pool(pool_name.c_str());
   bufferlist bl;
 
   int ret = rgw_get_obj(store, NULL, pool, zone_info_oid, bl);
-  if (ret == -ENOENT) {
+  if (ret == -ENOENT && create_zone) {
     init_default();
     return 0; // don't try to store obj, we're not fully initialized yet
   }
@@ -104,9 +291,7 @@ int RGWZoneParams::init(CephContext *cct, RGWRados *store)
 
 int RGWZoneParams::store_info(CephContext *cct, RGWRados *store)
 {
-  string pool_name = cct->_conf->rgw_zone_root_pool;
-  if (pool_name.empty())
-    pool_name = RGW_DEFAULT_ZONE_ROOT_POOL;
+  string pool_name = get_pool_name(cct, name);
 
   rgw_bucket pool(pool_name.c_str());
 
@@ -178,7 +363,7 @@ void RGWRados::finalize()
  * Initialize the RADOS instance and prepare to do other ops
  * Returns 0 on success, -ERR# on failure.
  */
-int RGWRados::initialize()
+int RGWRados::init_rados()
 {
   int ret;
 
@@ -194,7 +379,24 @@ int RGWRados::initialize()
   if (ret < 0)
    return ret;
 
-  zone.init(cct, this);
+  return ret;
+}
+
+/** 
+ * Initialize the RADOS instance and prepare to do other ops
+ * Returns 0 on success, -ERR# on failure.
+ */
+int RGWRados::init_complete()
+{
+  int ret;
+
+  ret = region.init(cct, this, create_region);
+  if (ret < 0)
+    return ret;
+
+  ret = zone.init(cct, this, create_zone);
+  if (ret < 0)
+    return ret;
 
   ret = open_root_pool_ctx();
   if (ret < 0)
@@ -215,6 +417,23 @@ int RGWRados::initialize()
   return ret;
 }
 
+/** 
+ * Initialize the RADOS instance and prepare to do other ops
+ * Returns 0 on success, -ERR# on failure.
+ */
+int RGWRados::initialize()
+{
+  int ret;
+
+  ret = init_rados();
+  if (ret < 0)
+    return ret;
+
+  ret = init_complete();
+
+  return ret;
+}
+
 void RGWRados::finalize_watch()
 {
   for (int i = 0; i < num_watchers; i++) {
diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index dec71bf98d3..4b9460beb17 100644
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -244,12 +244,15 @@ struct RGWZoneParams {
   rgw_bucket user_swift_pool;
   rgw_bucket user_uid_pool;
 
-  int init(CephContext *cct, RGWRados *store);
+  string name;
+
+  string get_pool_name(CephContext *cct, const string& zone_name);
+  int init(CephContext *cct, RGWRados *store, bool create_zone);
   void init_default();
   int store_info(CephContext *cct, RGWRados *store);
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 1, bl);
     ::encode(domain_root, bl);
     ::encode(control_pool, bl);
     ::encode(gc_pool, bl);
@@ -260,11 +263,12 @@ struct RGWZoneParams {
     ::encode(user_email_pool, bl);
     ::encode(user_swift_pool, bl);
     ::encode(user_uid_pool, bl);
+    ::encode(name, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::iterator& bl) {
-     DECODE_START(1, bl);
+     DECODE_START(2, bl);
     ::decode(domain_root, bl);
     ::decode(control_pool, bl);
     ::decode(gc_pool, bl);
@@ -275,6 +279,8 @@ struct RGWZoneParams {
     ::decode(user_email_pool, bl);
     ::decode(user_swift_pool, bl);
     ::decode(user_uid_pool, bl);
+    if (struct_v >= 2)
+      ::decode(name, bl);
     DECODE_FINISH(bl);
   }
   void dump(Formatter *f) const;
@@ -309,7 +315,10 @@ struct RGWRegion {
   list<string> endpoints;
 
   string master_zone;
-  list<RGWZone> zones;
+  map<string, RGWZone> zones;
+
+  CephContext *cct;
+  RGWRados *store;
 
   void encode(bufferlist& bl) const {
     ENCODE_START(1, 1, bl);
@@ -326,6 +335,14 @@ struct RGWRegion {
     ::decode(zones, bl);
     DECODE_FINISH(bl);
   }
+
+  string get_pool_name(CephContext *cct);
+  int init(CephContext *_cct, RGWRados *_store, bool create_zone);
+  int init_default();
+  int store_info(bool exclusive);
+  int read_default();
+  int set_as_default();
+
   void dump(Formatter *f) const;
   void decode_json(JSONObj *obj);
 };
@@ -450,14 +467,32 @@ protected:
 
   bool pools_initialized;
 
+  string region_name;
+  string zone_name;
+
+  bool create_region;
+  bool create_zone;
+
 public:
   RGWRados() : lock("rados_timer_lock"), timer(NULL),
                gc(NULL), use_gc_thread(false),
                num_watchers(0), watchers(NULL), watch_handles(NULL),
                bucket_id_lock("rados_bucket_id"), max_bucket_id(0),
                cct(NULL), rados(NULL),
-               pools_initialized(false) {}
+               pools_initialized(false),
+	       create_region(false), create_zone(false) {}
+
+  void set_region(const string& name, bool create) {
+    region_name = name;
+    create_region = create;
+  }
+
+  void set_zone(const string& name, bool create) {
+    zone_name = name;
+    create_zone = create;
+  }
 
+  RGWRegion region;
   RGWZoneParams zone;
 
   virtual ~RGWRados() {
@@ -477,6 +512,8 @@ public:
     return initialize();
   }
   /** Initialize the RADOS instance and prepare to do other ops */
+  virtual int init_rados();
+  int init_complete();
   virtual int initialize();
   virtual void finalize();