Merge branch 'wip-crush-update'

Reviewed-by: Greg Farnum <greg@inktank.com>

12 files changed, 429 insertions, 21 deletions

diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc
index d8ddca2a0fd..a6141d4ebe4 100644
--- a/src/crush/CrushWrapper.cc
+++ b/src/crush/CrushWrapper.cc
@@ -44,6 +44,7 @@ int CrushWrapper::remove_item(CephContext *cct, int item)
 	  }	    
 	  was_bucket = t;
 	}
+	adjust_item_weight(cct, item, 0);
 	ldout(cct, 5) << "remove_device removing item " << item << " from bucket " << b->id << dendl;
 	crush_bucket_remove_item(b, item);
 	ret = 0;
@@ -59,13 +60,60 @@ int CrushWrapper::remove_item(CephContext *cct, int item)
     name_map.erase(item);
     have_rmaps = false;
     ret = 0;
-  }  
+  }
   
   return ret;
 }
 
+bool CrushWrapper::check_item_loc(CephContext *cct, int item, map<string,string>& loc,
+				  int *weight)
+{
+  ldout(cct, 5) << "check_item_loc item " << item << " loc " << loc << dendl;
+
+  int cur = item;
+  for (map<int,string>::const_iterator p = type_map.begin(); p != type_map.end(); p++) {
+    if (p->first == 0)
+      continue;
+
+    if (loc.count(p->second) == 0) {
+      ldout(cct, 2) << "warning: did not specify location for '" << p->second << "' level (levels are "
+		    << type_map << ")" << dendl;
+      continue;
+    }
+
+    int id = get_item_id(loc[p->second].c_str());
+    if (!id) {
+      ldout(cct, 5) << "check_item_loc bucket " << loc[p->second] << " of type " << p->second << " dne" << dendl;
+      return false;
+    }
+
+    if (id >= 0) {
+      ldout(cct, 5) << "check_item_loc requested " << loc[p->second] << " for type " << p->second
+		    << " is a device, not bucket" << dendl;
+      return false;
+    }
+
+    crush_bucket *b = get_bucket(id);
+    assert(b);
+
+    // see if item exists in this bucket
+    for (unsigned j=0; j<b->size; j++) {
+      if (b->items[j] == cur) {
+	ldout(cct, 2) << "check_item_loc " << cur << " exists in bucket " << b->id << dendl;
+	if (weight)
+	  *weight = crush_get_bucket_item_weight(b, j);
+	return true;
+      }
+    }
+    return false;
+  }
+  
+  ldout(cct, 1) << "check_item_loc item " << item << " loc " << loc << dendl;
+  return false;
+}
+
 int CrushWrapper::insert_item(CephContext *cct, int item, float weight, string name,
-				map<string,string>& loc)  // typename -> bucketname
+			      map<string,string>& loc)  // typename -> bucketname
 {
   ldout(cct, 5) << "insert_item item " << item << " weight " << weight
 		<< " name " << name << " loc " << loc << dendl;
@@ -130,6 +178,42 @@ int CrushWrapper::insert_item(CephContext *cct, int item, float weight, string n
   return -EINVAL;
 }
 
+int CrushWrapper::update_item(CephContext *cct, int item, float weight, string name,
+			      map<string,string>& loc)  // typename -> bucketname
+{
+  ldout(cct, 5) << "update_item item " << item << " weight " << weight
+		<< " name " << name << " loc " << loc << dendl;
+  int ret = 0;
+
+  // compare quantized (fixed-point integer) weights!  
+  int iweight = (int)(weight * (float)0x10000);
+  int old_iweight;
+  if (check_item_loc(cct, item, loc, &old_iweight)) {
+    ldout(cct, 5) << "update_item " << item << " already at " << loc << dendl;
+    if (old_iweight != iweight) {
+      ldout(cct, 5) << "update_item " << item << " adjusting weight "
+		    << ((float)old_iweight/(float)0x10000) << " -> " << weight << dendl;
+      adjust_item_weight(cct, item, iweight);
+      ret = 1;
+    }
+    if (get_item_name(item) != name) {
+      ldout(cct, 5) << "update_item setting " << item << " name to " << name << dendl;
+      set_item_name(item, name.c_str());
+      ret = 1;
+    }
+  } else {
+    if (item_exists(item)) {
+      remove_item(cct, item);
+    }
+    ldout(cct, 5) << "update_item adding " << item << " weight " << weight
+		  << " at " << loc << dendl;
+    int r = insert_item(cct, item, weight, name.c_str(), loc);
+    if (r == 0)
+      ret = 1;
+  }
+  return ret;
+}
+
 int CrushWrapper::adjust_item_weight(CephContext *cct, int id, int weight)
 {
   ldout(cct, 5) << "adjust_item_weight " << id << " weight " << weight << dendl;
diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h
index 5376a78ffc9..48d822281c8 100644
--- a/src/crush/CrushWrapper.h
+++ b/src/crush/CrushWrapper.h
@@ -164,7 +164,61 @@ public:
 
 
   void find_roots(set<int>& roots) const;
+
+  /**
+   * see if item is located where we think it is
+   *
+   * @param cct cct
+   * @param item item id
+   * @param loc location to check (map of type to bucket names)
+   * @param weight optional pointer to weight of item at that location
+   * @return true if item is at specified location
+   */
+  bool check_item_loc(CephContext *cct, int item, map<string,string>& loc, int *iweight);
+  bool check_item_loc(CephContext *cct, int item, map<string,string>& loc, float *weight) {
+    int iweight;
+    bool ret = check_item_loc(cct, item, loc, &iweight);
+    if (weight)
+      *weight = (float)iweight / (float)0x10000;
+    return ret;
+  }
+
+  /**
+   * insert an item into the map at a specific position
+   *
+   * If the item is already present in the map, we will return EEXIST or similar errors.
+   *
+   * @param cct cct
+   * @param id item id
+   * @param weight item weight
+   * @param name item name
+   * @param loc location (map of type to bucket names)
+   * @return 0 for success, negative on error
+   */
   int insert_item(CephContext *cct, int id, float weight, string name, map<string,string>& loc);
+
+  /**
+   * add or update an item's position in the map
+   *
+   * This is analogous to insert_item, except we will move an item if
+   * it is already present.
+   *
+   * @param cct cct
+   * @param id item id
+   * @param weight item weight
+   * @param name item name
+   * @param loc location (map of type to bucket names)
+   * @return 0 for no change, 1 for successful change, negative on error
+   */
+  int update_item(CephContext *cct, int id, float weight, string name, map<string,string>& loc);
+
+  /**
+   * remove an item from the map
+   *
+   * @param cct cct
+   * @param id item id to remove
+   * @return 0 on success, negative on error
+   */
   int remove_item(CephContext *cct, int id);
   int adjust_item_weight(CephContext *cct, int id, int weight);
   int adjust_item_weightf(CephContext *cct, int id, float weight) {
@@ -172,7 +226,6 @@ public:
   }
   void reweight(CephContext *cct);
 
-
   /*** devices ***/
   int get_max_devices() const {
     if (!crush) return 0;
diff --git a/src/crushtool.cc b/src/crushtool.cc
index 28a64acb95e..38f469a3a9f 100644
--- a/src/crushtool.cc
+++ b/src/crushtool.cc
@@ -63,6 +63,9 @@ void usage()
   cout << "   -i mapfn --add-item id weight name [--loc type name ...]\n";
   cout << "                         insert an item into the hierarchy at the\n";
   cout << "                         given location\n";
+  cout << "   -i mapfn --update-item id weight name [--loc type name ...]\n";
+  cout << "                         insert or move an item into the hierarchy at the\n";
+  cout << "                         given location\n";
   cout << "   -i mapfn --remove-item name\n"
        << "                         remove the given item\n";
   cout << "   -i mapfn --reweight-item name weight\n";
@@ -103,6 +106,7 @@ int main(int argc, const char **argv)
 
   bool reweight = false;
   int add_item = -1;
+  bool update_item = false;
   float add_weight = 0;
   map<string,string> add_loc;
   float reweight_weight = 0;
@@ -158,6 +162,20 @@ int main(int argc, const char **argv)
 	usage();
       add_name.assign(*i);
       i = args.erase(i);
+    } else if (ceph_argparse_withint(args, i, &add_item, &err, "--update_item", (char*)NULL)) {
+      update_item = true;
+      if (!err.str().empty()) {
+	cerr << err.str() << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      if (i == args.end())
+	usage();
+      add_weight = atof(*i);
+      i = args.erase(i);
+      if (i == args.end())
+	usage();
+      add_name.assign(*i);
+      i = args.erase(i);
     } else if (ceph_argparse_witharg(args, i, &val, "--loc", (char*)NULL)) {
       std::string type(val);
       if (i == args.end())
@@ -467,12 +485,15 @@ int main(int argc, const char **argv)
     }
   }
   if (add_item >= 0) {
-    cout << me << " adding item " << add_item << " weight " << add_weight
-	 << " at " << add_loc << std::endl;
-    int r = crush.insert_item(g_ceph_context, add_item, add_weight, add_name.c_str(), add_loc);
-    if (r == 0)
+    int r;
+    if (update_item) {
+      r = crush.update_item(g_ceph_context, add_item, add_weight, add_name.c_str(), add_loc);
+    } else {
+      r = crush.insert_item(g_ceph_context, add_item, add_weight, add_name.c_str(), add_loc);
+    }
+    if (r >= 0) {
       modified = true;
-    else {
+    } else {
       cerr << me << " " << cpp_strerror(r) << std::endl;
       return r;
     }
diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index d8648627ad4..32be93a221b 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -1755,9 +1755,9 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
       paxos->wait_for_commit(new Monitor::C_Command(mon, m, 0, rs, paxos->get_version()));
       return true;
     }
-    else if (m->cmd.size() >= 6 && m->cmd[1] == "crush" && m->cmd[2] == "add") {
+    else if (m->cmd.size() >= 6 && m->cmd[1] == "crush" && m->cmd[2] == "set") {
       do {
-	// osd crush add <id> <name> <weight> [<loc1> [<loc2> ...]]
+	// osd crush update <id> <name> <weight> [<loc1> [<loc2> ...]]
 	int id = atoi(m->cmd[3].c_str());
 	string name = m->cmd[4];
 	float weight = atof(m->cmd[5].c_str());
@@ -1772,7 +1772,7 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
 	  loc[key] = value;
 	}
 
-	dout(0) << "adding crush item id " << id << " name '" << name << "' weight " << weight
+	dout(0) << "adding/updating crush item id " << id << " name '" << name << "' weight " << weight
 		<< " at location " << loc << dendl;
 	bufferlist bl;
 	if (pending_inc.crush.length())
@@ -1784,17 +1784,16 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
 	bufferlist::iterator p = bl.begin();
 	newcrush.decode(p);
 
-	err = newcrush.insert_item(g_ceph_context, id, weight, name, loc);
+	err = newcrush.update_item(g_ceph_context, id, weight, name, loc);
 	if (err == 0) {
-	  if (newcrush.get_max_devices() > osdmap.get_max_osd()) {
-	    err = -ERANGE;
-	    ss << "crushmap max_devices " << newcrush.get_max_devices()
-	       << " > osdmap max_osd " << osdmap.get_max_osd();
-	    break;
-	  }
+	  ss << "updated item id " << id << " name '" << name << "' weight " << weight
+	     << " at location " << loc << " to crush map";
+	  break;
+	}
+	if (err > 0) {
 	  pending_inc.crush.clear();
 	  newcrush.encode(pending_inc.crush);
-	  ss << "added item id " << id << " name '" << name << "' weight " << weight
+	  ss << "updated item id " << id << " name '" << name << "' weight " << weight
 	     << " at location " << loc << " to crush map";
 	  getline(ss, rs);
 	  paxos->wait_for_commit(new Monitor::C_Command(mon, m, 0, rs, paxos->get_version()));
diff --git a/src/test/cli/crushtool/add-item.t b/src/test/cli/crushtool/add-item.t
index aa5c23ce24d..f058b639c61 100644
--- a/src/test/cli/crushtool/add-item.t
+++ b/src/test/cli/crushtool/add-item.t
@@ -1,4 +1,15 @@
   $ crushtool -i "$TESTDIR/simple.template" --add-item 0 1.0 device0 --loc host host0 --loc cluster cluster0 -o one > /dev/null
   $ crushtool -i one --add-item 1 1.0 device1 --loc host host0 --loc cluster cluster0 -o two > /dev/null
   $ crushtool -d two -o final
-  $ cmp final "$TESTDIR/simple.template.out"
+  $ cmp final "$TESTDIR/simple.template.two"
+  $ crushtool -i two --add-item 1 1.0 device1 --loc host host0 --loc cluster cluster0 -o three 2>/dev/null >/dev/null || echo FAIL
+  FAIL
+  $ crushtool -i two --remove-item device1 -o four > /dev/null
+  $ crushtool -d four -o final
+  $ cmp final "$TESTDIR/simple.template.four"
+  $ crushtool -i two --update-item 1 2.0 osd1 --loc host host1 --loc cluster cluster0 -o five > /dev/null
+  $ crushtool -d five -o final
+  $ cmp final "$TESTDIR/simple.template.five"
+  $ crushtool -i five --update-item 1 2.0 osd1 --loc host host1 --loc cluster cluster0 -o six > /dev/null
+  $ crushtool -d six -o final
+  $ cmp final "$TESTDIR/simple.template.five"
diff --git a/src/test/cli/crushtool/help.t b/src/test/cli/crushtool/help.t
index 7f91c1cbc3d..af411aee990 100644
--- a/src/test/cli/crushtool/help.t
+++ b/src/test/cli/crushtool/help.t
@@ -17,6 +17,9 @@
      -i mapfn --add-item id weight name [--loc type name ...]
                            insert an item into the hierarchy at the
                            given location
+     -i mapfn --update-item id weight name [--loc type name ...]
+                           insert or move an item into the hierarchy at the
+                           given location
      -i mapfn --remove-item name
                            remove the given item
      -i mapfn --reweight-item name weight
diff --git a/src/test/cli/crushtool/simple.template.five b/src/test/cli/crushtool/simple.template.five
new file mode 100644
index 00000000000..240e81de021
--- /dev/null
+++ b/src/test/cli/crushtool/simple.template.five
@@ -0,0 +1,65 @@
+# begin crush map
+
+# devices
+device 0 device0
+device 1 osd1
+
+# types
+type 0 device
+type 1 host
+type 2 cluster
+
+# buckets
+host host0 {
+	id -2		# do not change unnecessarily
+	# weight 1.000
+	alg straw
+	hash 0	# rjenkins1
+	item device0 weight 1.000
+}
+host host1 {
+	id -3		# do not change unnecessarily
+	# weight 2.000
+	alg straw
+	hash 0	# rjenkins1
+	item osd1 weight 2.000
+}
+cluster cluster0 {
+	id -1		# do not change unnecessarily
+	# weight 3.000
+	alg straw
+	hash 0	# rjenkins1
+	item host0 weight 1.000
+	item host1 weight 2.000
+}
+
+# rules
+rule data {
+	ruleset 0
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule metadata {
+	ruleset 1
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule rbd {
+	ruleset 2
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+
+# end crush map
diff --git a/src/test/cli/crushtool/simple.template.four b/src/test/cli/crushtool/simple.template.four
new file mode 100644
index 00000000000..aa16bbdedc3
--- /dev/null
+++ b/src/test/cli/crushtool/simple.template.four
@@ -0,0 +1,56 @@
+# begin crush map
+
+# devices
+device 0 device0
+
+# types
+type 0 device
+type 1 host
+type 2 cluster
+
+# buckets
+host host0 {
+	id -2		# do not change unnecessarily
+	# weight 1.000
+	alg straw
+	hash 0	# rjenkins1
+	item device0 weight 1.000
+}
+cluster cluster0 {
+	id -1		# do not change unnecessarily
+	# weight 1.000
+	alg straw
+	hash 0	# rjenkins1
+	item host0 weight 1.000
+}
+
+# rules
+rule data {
+	ruleset 0
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule metadata {
+	ruleset 1
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule rbd {
+	ruleset 2
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+
+# end crush map
diff --git a/src/test/cli/crushtool/simple.template.out b/src/test/cli/crushtool/simple.template.one
index 9a3aee7349d..9a3aee7349d 100644
--- a/src/test/cli/crushtool/simple.template.out
+++ b/src/test/cli/crushtool/simple.template.one
diff --git a/src/test/cli/crushtool/simple.template.three b/src/test/cli/crushtool/simple.template.three
new file mode 100644
index 00000000000..9a3aee7349d
--- /dev/null
+++ b/src/test/cli/crushtool/simple.template.three
@@ -0,0 +1,58 @@
+# begin crush map
+
+# devices
+device 0 device0
+device 1 device1
+
+# types
+type 0 device
+type 1 host
+type 2 cluster
+
+# buckets
+host host0 {
+	id -2		# do not change unnecessarily
+	# weight 2.000
+	alg straw
+	hash 0	# rjenkins1
+	item device0 weight 1.000
+	item device1 weight 1.000
+}
+cluster cluster0 {
+	id -1		# do not change unnecessarily
+	# weight 2.000
+	alg straw
+	hash 0	# rjenkins1
+	item host0 weight 2.000
+}
+
+# rules
+rule data {
+	ruleset 0
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule metadata {
+	ruleset 1
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule rbd {
+	ruleset 2
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+
+# end crush map
diff --git a/src/test/cli/crushtool/simple.template.two b/src/test/cli/crushtool/simple.template.two
new file mode 100644
index 00000000000..9a3aee7349d
--- /dev/null
+++ b/src/test/cli/crushtool/simple.template.two
@@ -0,0 +1,58 @@
+# begin crush map
+
+# devices
+device 0 device0
+device 1 device1
+
+# types
+type 0 device
+type 1 host
+type 2 cluster
+
+# buckets
+host host0 {
+	id -2		# do not change unnecessarily
+	# weight 2.000
+	alg straw
+	hash 0	# rjenkins1
+	item device0 weight 1.000
+	item device1 weight 1.000
+}
+cluster cluster0 {
+	id -1		# do not change unnecessarily
+	# weight 2.000
+	alg straw
+	hash 0	# rjenkins1
+	item host0 weight 2.000
+}
+
+# rules
+rule data {
+	ruleset 0
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule metadata {
+	ruleset 1
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule rbd {
+	ruleset 2
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+
+# end crush map
diff --git a/src/vstart.sh b/src/vstart.sh
index c4d43beab3c..5b0cb14b7aa 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -364,7 +364,7 @@ EOF
 	    uuid=`uuidgen`
 	    echo "add osd$osd $uuid"
 	    $SUDO $CEPH_ADM osd create $uuid
-	    $SUDO $CEPH_ADM osd crush add $osd osd.$osd 1.0 host=localhost rack=localrack pool=default
+	    $SUDO $CEPH_ADM osd crush set $osd osd.$osd 1.0 host=localhost rack=localrack pool=default
 	    $SUDO $CEPH_BIN/ceph-osd -i $osd $ARGS --mkfs --mkkey --osd-uuid $uuid
 
 	    if [ "$cephx" -eq 1 ]; then