rgw: handle bucket creation with specified placement pool

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

9 files changed, 171 insertions, 31 deletions

diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index 7b758b08fd1..0325a60840c 100644
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -126,7 +126,8 @@ int rgw_bucket_store_info(RGWRados *store, string& bucket_name, bufferlist& bl,
   return store->meta_mgr->put_entry(bucket_meta_handler, bucket_name, bl, exclusive, objv_tracker, mtime, pattrs);
 }
 
-
+#warning removed RGWBucket::create_bucket(), clean this up when ready
+#if 0
 int RGWBucket::create_bucket(string bucket_str, string& user_id, string& region_name, string& display_name)
 {
   RGWAccessControlPolicy policy, old_policy;
@@ -152,8 +153,9 @@ int RGWBucket::create_bucket(string bucket_str, string& user_id, string& region_
   rgw_bucket& bucket = bucket_info.bucket;
 
   RGWBucketInfo new_info;
+  string placement_rule;
 
-  ret = store->create_bucket(user_id, bucket, region_name, attrs, objv_tracker,
+  ret = store->create_bucket(user_info, bucket, region_name, placement_rule, attrs, objv_tracker,
                              NULL, bucket_info.creation_time, NULL, &new_info);
   if (ret && ret != -EEXIST)
     goto done;
@@ -173,6 +175,7 @@ int RGWBucket::create_bucket(string bucket_str, string& user_id, string& region_
 done:
   return ret;
 }
+#endif
 
 int rgw_bucket_set_attrs(RGWRados *store, rgw_obj& obj,
                          map<string, bufferlist>& attrs,
@@ -360,10 +363,9 @@ int RGWBucket::init(RGWRados *storage, RGWBucketAdminOpState& op_state)
 
   store = storage;
 
-  RGWUserInfo info;
   RGWBucketInfo bucket_info;
 
-  user_id = op_state.get_user_id();
+  string user_id = op_state.get_user_id();
   bucket_name = op_state.get_bucket_name();
   RGWUserBuckets user_buckets;
 
@@ -382,11 +384,11 @@ int RGWBucket::init(RGWRados *storage, RGWBucketAdminOpState& op_state)
   }
 
   if (!user_id.empty()) {
-    int r = rgw_get_user_info_by_uid(store, user_id, info);
+    int r = rgw_get_user_info_by_uid(store, user_id, user_info);
     if (r < 0)
       return r;
 
-    op_state.display_name = info.display_name;
+    op_state.display_name = user_info.display_name;
   }
 
   clear_failure();
@@ -405,7 +407,7 @@ int RGWBucket::link(RGWBucketAdminOpState& op_state, std::string *err_msg)
   std::string display_name = op_state.get_user_display_name();
   rgw_bucket bucket = op_state.get_bucket();
 
-  string uid_str(user_id);
+  string uid_str(user_info.user_id);
   bufferlist aclbl;
   rgw_obj obj(bucket, no_oid);
   RGWObjVersionTracker objv_tracker;
@@ -431,9 +433,9 @@ int RGWBucket::link(RGWBucketAdminOpState& op_state, std::string *err_msg)
 
     // now update the user for the bucket...
     if (display_name.empty()) {
-      ldout(store->ctx(), 0) << "WARNING: user " << user_id << " has no display name set" << dendl;
+      ldout(store->ctx(), 0) << "WARNING: user " << user_info.user_id << " has no display name set" << dendl;
     }
-    policy.create_default(user_id, display_name);
+    policy.create_default(user_info.user_id, display_name);
 
     owner = policy.get_owner();
     r = store->set_bucket_owner(bucket, owner);
@@ -450,10 +452,13 @@ int RGWBucket::link(RGWBucketAdminOpState& op_state, std::string *err_msg)
     if (r < 0)
       return r;
 
-    r = rgw_add_bucket(store, user_id, bucket, 0);
+    r = rgw_add_bucket(store, user_info.user_id, bucket, 0);
     if (r < 0)
       return r;
-  } else {
+  }
+#warning not creating bucket on bucket link, clean this up later
+#if 0
+  else {
     // the bucket seems not to exist, so we should probably create it...
     r = create_bucket(bucket_name.c_str(), uid_str, store->region.name, display_name);
     if (r < 0) {
@@ -462,6 +467,7 @@ int RGWBucket::link(RGWBucketAdminOpState& op_state, std::string *err_msg)
 
     return r;
   }
+#endif
 
   return 0;
 }
@@ -475,7 +481,7 @@ int RGWBucket::unlink(RGWBucketAdminOpState& op_state, std::string *err_msg)
     return -EINVAL;
   }
 
-  int r = rgw_remove_user_bucket_info(store, user_id, bucket);
+  int r = rgw_remove_user_bucket_info(store, user_info.user_id, bucket);
   if (r < 0) {
     set_err_msg(err_msg, "error unlinking bucket" + cpp_strerror(-r));
   }
@@ -1344,7 +1350,7 @@ public:
     if (ret == -ENOENT || old_bci.info.bucket.bucket_id != bci.info.bucket.bucket_id) {
       /* a new bucket, we need to select a new bucket placement for it */
       rgw_bucket bucket;
-      ret = store->select_bucket_placement(entry, bucket);
+      ret = store->set_bucket_location_by_rule(bci.info.placement_rule, entry, bucket);
       if (ret < 0) {
         ldout(store->ctx(), 0) << "ERROR: select_bucket_placement() returned " << ret << dendl;
         return ret;
diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h
index f4ff4ec15cc..a4c2fded5de 100644
--- a/src/rgw/rgw_bucket.h
+++ b/src/rgw/rgw_bucket.h
@@ -173,7 +173,7 @@ class RGWBucket
   RGWRados *store;
   RGWAccessHandle handle;
 
-  std::string user_id;
+  RGWUserInfo user_info;
   std::string bucket_name;
 
   bool failure;
@@ -184,8 +184,6 @@ public:
   RGWBucket() : store(NULL), failure(false) {}
   int init(RGWRados *storage, RGWBucketAdminOpState& op_state);
 
-  int create_bucket(string bucket_str, string& user_id, string& region_name, string& display_name);
-  
   int check_bad_index_multipart(RGWBucketAdminOpState& op_state,
           list<std::string>& objs_to_unlink, std::string *err_msg = NULL);
 
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 7f8bb87c986..4e898b80113 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -606,14 +606,16 @@ struct RGWBucketInfo
   uint32_t flags;
   string region;
   time_t creation_time;
+  string placement_rule;
 
   void encode(bufferlist& bl) const {
-     ENCODE_START(6, 4, bl);
+     ENCODE_START(7, 4, bl);
      ::encode(bucket, bl);
      ::encode(owner, bl);
      ::encode(flags, bl);
      ::encode(region, bl);
      ::encode(creation_time, bl);
+     ::encode(placement_rule, bl);
      ENCODE_FINISH(bl);
   }
   void decode(bufferlist::iterator& bl) {
@@ -627,6 +629,8 @@ struct RGWBucketInfo
        ::decode(region, bl);
      if (struct_v >= 6)
        ::decode(creation_time, bl);
+     if (struct_v >= 7)
+       ::decode(placement_rule, bl);
      DECODE_FINISH(bl);
   }
   void dump(Formatter *f) const;
diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc
index 442e2a42940..8555d825df5 100644
--- a/src/rgw/rgw_json_enc.cc
+++ b/src/rgw/rgw_json_enc.cc
@@ -443,6 +443,7 @@ void RGWBucketInfo::dump(Formatter *f) const
   encode_json("owner", owner, f);
   encode_json("flags", flags, f);
   encode_json("region", region, f);
+  encode_json("placement_rule", region, f);
 }
 
 void RGWBucketInfo::decode_json(JSONObj *obj) {
@@ -451,6 +452,7 @@ void RGWBucketInfo::decode_json(JSONObj *obj) {
   JSONDecoder::decode_json("owner", owner, obj);
   JSONDecoder::decode_json("flags", flags, obj);
   JSONDecoder::decode_json("region", region, obj);
+  JSONDecoder::decode_json("placement_rule", region, obj);
 }
 
 void RGWObjEnt::dump(Formatter *f) const
@@ -502,9 +504,9 @@ void RGWZoneParams::dump(Formatter *f) const
   encode_json("user_keys_pool", user_keys_pool.data_pool, f);
   encode_json("user_email_pool", user_email_pool.data_pool, f);
   encode_json("user_swift_pool", user_swift_pool.data_pool, f);
-  encode_json("user_uid_pool ", user_uid_pool.data_pool, f);
+  encode_json("user_uid_pool", user_uid_pool.data_pool, f);
   encode_json_plain("system_key", system_key, f);
-  encode_json("placement_pools ", placement_pools, f);
+  encode_json("placement_pools", placement_pools, f);
 }
 
 static void decode_json(const char *field, rgw_bucket& bucket, JSONObj *obj)
@@ -578,6 +580,7 @@ void RGWRegion::dump(Formatter *f) const
   encode_json("master_zone", master_zone, f);
   encode_json_map("zones", zones, f); /* more friendly representation */
   encode_json_map("placement_targets", placement_targets, f); /* more friendly representation */
+  encode_json("default_placement", default_placement, f);
 }
 
 static void decode_zones(map<string, RGWZone>& zones, JSONObj *o)
@@ -604,6 +607,7 @@ void RGWRegion::decode_json(JSONObj *obj)
   JSONDecoder::decode_json("master_zone", master_zone, obj);
   JSONDecoder::decode_json("zones", zones, decode_zones, obj);
   JSONDecoder::decode_json("placement_targets", placement_targets, decode_placement_targets, obj);
+  JSONDecoder::decode_json("default_placement", default_placement, obj);
 }
 
 
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 1f2cb9f5452..de45845f363 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -947,7 +947,7 @@ void RGWCreateBucket::execute()
   attrs[RGW_ATTR_ACL] = aclbl;
 
   s->bucket.name = s->bucket_name_str;
-  ret = store->create_bucket(s->user.user_id, s->bucket, region_name, attrs, objv_tracker, pobjv,
+  ret = store->create_bucket(s->user, s->bucket, region_name, placement_rule, attrs, objv_tracker, pobjv,
                              creation_time, pmaster_bucket, &info, true);
   /* continue if EEXIST and create_bucket will fail below.  this way we can recover
    * from a partial create by retrying it. */
diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 1632d35fcc8..462d58440c7 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -233,6 +233,7 @@ protected:
   int ret;
   RGWAccessControlPolicy policy;
   string location_constraint;
+  string placement_rule;
   RGWObjVersionTracker objv_tracker;
   RGWBucketInfo info;
 
diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 3e82053d6b1..c8ecb413976 100644
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -217,6 +217,11 @@ int RGWRegion::create_default()
 
   is_master = true;
 
+  RGWRegionPlacementTarget placement_target;
+  placement_target.name = "default-placement";
+  placement_targets[placement_target.name] = placement_target;
+  default_placement = "default-placement";
+
   RGWZone& default_zone = zones[zone_name];
   default_zone.name = zone_name;
 
@@ -267,6 +272,16 @@ void RGWZoneParams::init_default()
   user_email_pool = ".users.email";
   user_swift_pool = ".users.swift";
   user_uid_pool = ".users.uid";
+
+  RGWZonePlacementInfo default_placement;
+  default_placement.index_pool = ".rgw.buckets";
+  default_placement.data_pool = ".rgw.buckets";
+  placement_pools["default-placement"] = default_placement;
+
+  RGWZonePlacementInfo test_placement;
+  test_placement.index_pool = ".rgw.test.index";
+  test_placement.data_pool = ".rgw.test.data";
+  placement_pools["test"] = test_placement;
 }
 
 string RGWZoneParams::get_pool_name(CephContext *cct)
@@ -1723,8 +1738,9 @@ int RGWRados::init_bucket_index(rgw_bucket& bucket)
  * create a bucket with name bucket and the given list of attrs
  * returns 0 on success, -ERR# otherwise.
  */
-int RGWRados::create_bucket(string& owner, rgw_bucket& bucket,
+int RGWRados::create_bucket(RGWUserInfo& owner, rgw_bucket& bucket,
                             const string& region_name,
+                            const string& placement_rule,
 			    map<std::string, bufferlist>& attrs,
                             RGWObjVersionTracker& objv_tracker,
                             obj_version *pobjv,
@@ -1734,9 +1750,10 @@ int RGWRados::create_bucket(string& owner, rgw_bucket& bucket,
 			    bool exclusive)
 {
 #define MAX_CREATE_RETRIES 20 /* need to bound retries */
+  string selected_placement_rule;
   for (int i = 0; i < MAX_CREATE_RETRIES; i++) {
     int ret = 0;
-    ret = select_bucket_placement(bucket.name, bucket);
+    ret = select_bucket_placement(owner, region_name, placement_rule, bucket.name, bucket, &selected_placement_rule);
     if (ret < 0)
       return ret;
     bufferlist bl;
@@ -1777,8 +1794,9 @@ int RGWRados::create_bucket(string& owner, rgw_bucket& bucket,
 
     RGWBucketInfo info;
     info.bucket = bucket;
-    info.owner = owner;
+    info.owner = owner.user_id;
     info.region = region_name;
+    info.placement_rule = selected_placement_rule;
     if (!creation_time)
       time(&info.creation_time);
     else
@@ -1811,13 +1829,101 @@ int RGWRados::create_bucket(string& owner, rgw_bucket& bucket,
   return -ENOENT;
 }
 
-int RGWRados::select_bucket_placement(string& bucket_name, rgw_bucket& bucket)
+int RGWRados::select_new_bucket_location(RGWUserInfo& user_info, const string& region_name, const string& request_rule,
+                                         const string& bucket_name, rgw_bucket& bucket, string *pselected_rule)
+{
+  /* first check that rule exists within the specific region */
+  map<string, RGWRegion>::iterator riter = region_map.regions.find(region_name);
+  if (riter == region_map.regions.end()) {
+    ldout(cct, 0) << "could not find region " << region_name << " in region map" << dendl;
+    return -EINVAL;
+  }
+  /* now check that tag exists within region */
+  RGWRegion& region = riter->second;
+
+  /* find placement rule. Hierarchy: request rule > user default rule > region default rule */
+  string rule = request_rule;
+  if (rule.empty()) {
+    rule = user_info.default_placement;
+    if (rule.empty())
+      rule = region.default_placement;
+  }
+
+  if (rule.empty()) {
+    ldout(cct, 0) << "misconfiguration, should not have an empty placement rule name" << dendl;
+    return -EIO;
+  }
+
+  if (!rule.empty()) {
+    map<string, RGWRegionPlacementTarget>::iterator titer = region.placement_targets.find(rule);
+    if (titer == region.placement_targets.end()) {
+      ldout(cct, 0) << "could not find placement rule " << rule << " within region " << dendl;
+      return -EINVAL;
+    }
+
+    /* now check tag for the rule, whether user is permitted to use rule */
+    RGWRegionPlacementTarget& target_rule = titer->second;
+    if (!target_rule.user_permitted(user_info.placement_tags)) {
+      ldout(cct, 0) << "user not permitted to use placement rule" << dendl;
+      return -EPERM;
+    }
+  }
+
+  /* yay, user is permitted, now just make sure that zone has this rule configured. We're
+   * checking it for the local zone, because that's where this bucket object is going to
+   * reside.
+   */
+  map<string, RGWZonePlacementInfo>::iterator piter = zone.placement_pools.find(rule);
+  if (piter == zone.placement_pools.end()) {
+    /* couldn't find, means we cannot really place data for this bucket in this zone */
+    if ((region_name.empty() && region.is_master) ||
+        region_name == region.name) {
+      /* that's a configuration error, zone should have that rule, as we're within the requested
+       * region */
+      return -EINVAL;
+    } else {
+      /* oh, well, data is not going to be placed here, bucket object is just a placeholder */
+      return 0;
+    }
+  }
+
+  if (pselected_rule)
+    *pselected_rule = rule;
+  
+  return set_bucket_location_by_rule(rule, bucket_name, bucket);
+}
+
+int RGWRados::set_bucket_location_by_rule(const string& location_rule, const std::string& bucket_name, rgw_bucket& bucket)
+{
+  bucket.name = bucket_name;
+
+  map<string, RGWZonePlacementInfo>::iterator piter = zone.placement_pools.find(location_rule);
+  if (piter == zone.placement_pools.end()) {
+    /* silently ignore, bucket will not reside in this zone */
+    return 0;
+  }
+
+  RGWZonePlacementInfo& placement_info = piter->second;
+
+  bucket.data_pool = placement_info.data_pool;
+  bucket.index_pool = placement_info.index_pool;
+
+  return 0;
+
+}
+
+int RGWRados::select_bucket_placement(RGWUserInfo& user_info, const string& region_name, const string& placement_rule,
+                                      const string& bucket_name, rgw_bucket& bucket, string *pselected_rule)
 {
   bufferlist map_bl;
   map<string, bufferlist> m;
   string pool_name;
   bool write_map = false;
 
+  if (!zone.placement_pools.empty()) {
+    return select_new_bucket_location(user_info, region_name, placement_rule, bucket_name, bucket, pselected_rule);
+  }
+
   rgw_obj obj(zone.domain_root, avail_pools);
 
   int ret = rgw_get_system_obj(this, NULL, zone.domain_root, avail_pools, map_bl, NULL, NULL);
@@ -1882,7 +1988,6 @@ read_omap:
     pool_name = miter->first;
   }
   bucket.data_pool = pool_name;
-#warning FIXME
   bucket.index_pool = pool_name;
   bucket.name = bucket_name;
 
diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 3331f36b668..4a21ad00e02 100644
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -524,10 +524,16 @@ struct RGWRegionPlacementTarget {
   string name;
   list<string> tags;
 
-  bool tag_exists(const string& tag) {
-    for (list<string>::iterator iter = tags.begin(); iter != tags.end(); ++iter) {
-      if (tag == *iter) {
-        return true;
+  bool user_permitted(list<string>& user_tags) {
+    if (tags.empty()) {
+      return true;
+    }
+    for (list<string>::iterator uiter = user_tags.begin(); uiter != user_tags.end(); ++uiter) { /* we don't expect many of either, so we can handle this kind of lookup */
+      string& rule = *uiter;
+      for (list<string>::iterator iter = tags.begin(); iter != tags.end(); ++iter) {
+        if (rule == *iter) {
+          return true;
+        }
       }
     }
     return false;
@@ -562,6 +568,7 @@ struct RGWRegion {
   map<string, RGWZone> zones;
 
   map<string, RGWRegionPlacementTarget> placement_targets;
+  string default_placement;
 
   CephContext *cct;
   RGWRados *store;
@@ -576,6 +583,8 @@ struct RGWRegion {
     ::encode(endpoints, bl);
     ::encode(master_zone, bl);
     ::encode(zones, bl);
+    ::encode(placement_targets, bl);
+    ::encode(default_placement, bl);
     ENCODE_FINISH(bl);
   }
 
@@ -587,6 +596,8 @@ struct RGWRegion {
     ::decode(endpoints, bl);
     ::decode(master_zone, bl);
     ::decode(zones, bl);
+    ::decode(placement_targets, bl);
+    ::decode(default_placement, bl);
     DECODE_FINISH(bl);
   }
 
@@ -947,9 +958,14 @@ public:
    * returns 0 on success, -ERR# otherwise.
    */
   virtual int init_bucket_index(rgw_bucket& bucket);
-  int select_bucket_placement(std::string& bucket_name, rgw_bucket& bucket);
-  virtual int create_bucket(string& owner, rgw_bucket& bucket,
+  int select_bucket_placement(RGWUserInfo& user_info, const string& region_name, const std::string& rule,
+                              const std::string& bucket_name, rgw_bucket& bucket, string *pselected_rule);
+  int select_new_bucket_location(RGWUserInfo& user_info, const string& region_name, const string& rule,
+                                 const std::string& bucket_name, rgw_bucket& bucket, string *pselected_rule);
+  int set_bucket_location_by_rule(const string& location_rule, const std::string& bucket_name, rgw_bucket& bucket);
+  virtual int create_bucket(RGWUserInfo& owner, rgw_bucket& bucket,
                             const string& region_name,
+                            const string& placement_rule,
                             map<std::string,bufferlist>& attrs,
                             RGWObjVersionTracker& objv_tracker,
                             obj_version *pobjv,
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index bfb23f0be38..9442278f080 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -424,6 +424,12 @@ int RGWCreateBucket_ObjStore_S3::get_params()
     ldout(s->cct, 10) << "create bucket location constraint: " << location_constraint << dendl;
   }
 
+  int pos = location_constraint.find(':');
+  if (pos >= 0) {
+    placement_rule = location_constraint.substr(pos + 1);
+    location_constraint = location_constraint.substr(0, pos);
+  }
+
   return 0;
 }