Merge branch 'wip-osd-uuid'

Reviewed-by: Greg Farnum <greg@inktank.com>

14 files changed, 179 insertions, 68 deletions

diff --git a/ceph-object-corpus b/ceph-object-corpus
-Subproject 17d4afca1c84f5471c2824876099fef456a3209
+Subproject cddd50247fe2d578c1bd25d011594888865f70d
diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc
index e2d79606b84..57406e12ed6 100644
--- a/src/ceph_osd.cc
+++ b/src/ceph_osd.cc
@@ -107,9 +107,9 @@ int main(int argc, const char **argv)
       dump_journal = true;
     } else if (ceph_argparse_flag(args, i, "--get-cluster-fsid", (char*)NULL)) {
       get_cluster_fsid = true;
-    } else if (ceph_argparse_flag(args, i, "--get-osd-fsid", (char*)NULL)) {
+    } else if (ceph_argparse_flag(args, i, "--get-osd-fsid", "--get-osd-uuid", (char*)NULL)) {
       get_osd_fsid = true;
-    } else if (ceph_argparse_flag(args, i, "--get-journal-fsid", (char*)NULL)) {
+    } else if (ceph_argparse_flag(args, i, "--get-journal-fsid", "--get-journal-uuid", (char*)NULL)) {
       get_journal_fsid = true;
     } else {
       ++i;
diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 7b6d6f3c80c..6ae487bf002 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -250,6 +250,7 @@ OPTION(osd_auto_upgrade_tmap, OPT_BOOL, true)
 // If true, TMAPPUT sets uses_tmap DEBUGGING ONLY
 OPTION(osd_tmapput_sets_uses_tmap, OPT_BOOL, false)
 
+OPTION(osd_uuid, OPT_UUID, uuid_d())
 OPTION(osd_data, OPT_STR, "/var/lib/ceph/osd/$cluster-$id")
 OPTION(osd_journal, OPT_STR, "/var/lib/ceph/osd/$cluster-$id/journal")
 OPTION(osd_journal_size, OPT_INT, 0)         // in mb
diff --git a/src/include/rados.h b/src/include/rados.h
index 3a83c4f9f4e..679b90c353e 100644
--- a/src/include/rados.h
+++ b/src/include/rados.h
@@ -9,14 +9,6 @@
 #include "msgr.h"
 
 /*
- * osdmap encoding versions
- */
-#define CEPH_OSDMAP_INC_VERSION     5
-#define CEPH_OSDMAP_INC_VERSION_EXT 7
-#define CEPH_OSDMAP_VERSION         5
-#define CEPH_OSDMAP_VERSION_EXT     7
-
-/*
  * fs id
  */
 struct ceph_fsid {
diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 14878c4821e..d8648627ad4 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -812,6 +812,11 @@ bool OSDMonitor::prepare_boot(MOSDBoot *m)
     if (m->sb.weight)
       osd_weight[from] = m->sb.weight;
 
+    // set uuid?
+    dout(10) << " setting osd." << from << " uuid to " << m->sb.osd_fsid << dendl;
+    if (osdmap.get_uuid(from) != m->sb.osd_fsid)
+      pending_inc.new_uuid[from] = m->sb.osd_fsid;
+
     // fresh osd?
     if (m->sb.newest_map == 0 && osdmap.exists(from)) {
       const osd_info_t& i = osdmap.get_info(from);
@@ -2022,51 +2027,49 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
       }
     }
     else if (m->cmd[1] == "create") {
-      int i;
-      if (m->cmd.size() > 2) {
-	i = atoi(m->cmd[2].c_str());
-	if (i < 0) {
-	  ss << i << " is not a valid osd id";
-	  err = -ERANGE;
-	  goto out;
-	}
-	if (osdmap.exists(i)) {
-	  ss << i << " already exists";
-	  err = -EEXIST;
+      int i = -1;
+
+      // optional uuid provided?
+      uuid_d uuid;
+      if (m->cmd.size() > 2 && uuid.parse(m->cmd[2].c_str())) {
+	dout(10) << " osd create got uuid " << uuid << dendl;
+	i = osdmap.identify_osd(uuid);
+	if (i >= 0) {
+	  // osd already exists
+	  err = 0;
+	  ss << i;
+	  getline(ss, rs);
 	  goto out;
 	}
-	if (i >= osdmap.get_max_osd()) {
-	  if (i >= pending_inc.new_max_osd)
-	    pending_inc.new_max_osd = i + 1;
-	}
-	if (pending_inc.new_up_client.count(i) ||
-	    (pending_inc.new_state.count(i) &&
-	     (pending_inc.new_state[i] & CEPH_OSD_EXISTS))) {
-	  ss << i << " already exists";
-	  getline(ss, rs);
-	  paxos->wait_for_commit(new Monitor::C_Command(mon, m, -EEXIST, rs, paxos->get_version()));
+	i = pending_inc.identify_osd(uuid);
+	if (i >= 0) {
+	  // osd is about to exist
+	  paxos->wait_for_commit(new C_RetryMessage(this, m));
 	  return true;
 	}
-      } else {
-	// allocate a new id
-	for (i=0; i < osdmap.get_max_osd(); i++) {
-	  if (!osdmap.exists(i) &&
-	      pending_inc.new_up_client.count(i) == 0 &&
-	      (pending_inc.new_state.count(i) == 0 ||
-	       (pending_inc.new_state[i] & CEPH_OSD_EXISTS) == 0))
-	    goto done;
-	}
-	// hrm.  raise max_osd
-	if (pending_inc.new_max_osd < 0)
-	  pending_inc.new_max_osd = osdmap.get_max_osd() + 1;
-	else
-	  pending_inc.new_max_osd++;
-	i = pending_inc.new_max_osd - 1;
       }
 
+      // allocate a new id
+      for (i=0; i < osdmap.get_max_osd(); i++) {
+	if (!osdmap.exists(i) &&
+	    pending_inc.new_up_client.count(i) == 0 &&
+	    (pending_inc.new_state.count(i) == 0 ||
+	     (pending_inc.new_state[i] & CEPH_OSD_EXISTS) == 0))
+	  goto done;
+      }
+
+      // raise max_osd
+      if (pending_inc.new_max_osd < 0)
+	pending_inc.new_max_osd = osdmap.get_max_osd() + 1;
+      else
+	pending_inc.new_max_osd++;
+      i = pending_inc.new_max_osd - 1;
+
   done:
       dout(10) << " creating osd." << i << dendl;
       pending_inc.new_state[i] |= CEPH_OSD_EXISTS | CEPH_OSD_NEW;
+      if (!uuid.is_zero())
+	pending_inc.new_uuid[i] = uuid;
       ss << i;
       getline(ss, rs);
       paxos->wait_for_commit(new Monitor::C_Command(mon, m, 0, rs, paxos->get_version()));
diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc
index d4c4a061db5..3b8171b598c 100644
--- a/src/os/FileStore.cc
+++ b/src/os/FileStore.cc
@@ -984,7 +984,10 @@ int FileStore::mkfs()
   }
 
   // fsid
-  fsid.generate_random();
+  if (fsid.is_zero())
+    fsid.generate_random();
+  else
+    dout(1) << "mkfs using provided fsid " << fsid << dendl;
 
   char fsid_str[40];
   fsid.print(fsid_str);
diff --git a/src/os/FileStore.h b/src/os/FileStore.h
index b7da7d77b59..a2279e96404 100644
--- a/src/os/FileStore.h
+++ b/src/os/FileStore.h
@@ -383,6 +383,9 @@ public:
 
   int dump_journal(ostream& out);
 
+  void set_fsid(uuid_d u) {
+    fsid = u;
+  }
   uuid_d get_fsid() { return fsid; }
 
   int snapshot(const string& name);
diff --git a/src/os/ObjectStore.h b/src/os/ObjectStore.h
index 55b0dc6618b..56f5401a5af 100644
--- a/src/os/ObjectStore.h
+++ b/src/os/ObjectStore.h
@@ -739,6 +739,7 @@ public:
 
   virtual int snapshot(const string& name) { return -EOPNOTSUPP; }
     
+  virtual void set_fsid(uuid_d u) = 0;
   virtual uuid_d get_fsid() = 0;
 };
 
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 623680969ca..fdceb67064a 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -275,6 +275,10 @@ int OSD::mkfs(const std::string &dev, const std::string &jdev, uuid_d fsid, int
       ret = -ENOENT;
       goto out;
     }
+
+    // if we are fed a uuid for this osd, use it.
+    store->set_fsid(g_conf->osd_uuid);
+
     ret = store->mkfs();
     if (ret) {
       derr << "OSD::mkfs: FileStore::mkfs failed with error " << ret << dendl;
@@ -287,12 +291,6 @@ int OSD::mkfs(const std::string &dev, const std::string &jdev, uuid_d fsid, int
       derr << "OSD::mkfs: couldn't mount FileStore: error " << ret << dendl;
       goto free_store;
     }
-    store->sync_and_flush();
-    ret = write_meta(dev, sb.cluster_fsid, sb.osd_fsid, whoami);
-    if (ret) {
-      derr << "OSD::mkfs: failed to write fsid file: error " << ret << dendl;
-      goto umount_store;
-    }
 
     // age?
     if (g_conf->osd_age_time != 0) {
@@ -352,6 +350,21 @@ int OSD::mkfs(const std::string &dev, const std::string &jdev, uuid_d fsid, int
 	goto umount_store;
       }
     }
+
+    store->sync_and_flush();
+
+    ret = write_meta(dev, sb.cluster_fsid, sb.osd_fsid, whoami);
+    if (ret) {
+      derr << "OSD::mkfs: failed to write fsid file: error " << ret << dendl;
+      goto umount_store;
+    }
+
+    ret = write_meta(dev, "ready", "ready\n", 6);
+    if (ret) {
+      derr << "OSD::mkfs: failed to write ready file: error " << ret << dendl;
+      goto umount_store;
+    }
+
   }
   catch (const std::exception &se) {
     derr << "OSD::mkfs: caught exception " << se.what() << dendl;
@@ -407,29 +420,52 @@ int OSD::write_meta(const std::string &base, const std::string &file,
 {
   int ret;
   char fn[PATH_MAX];
+  char tmp[PATH_MAX];
   int fd;
 
   snprintf(fn, sizeof(fn), "%s/%s", base.c_str(), file.c_str());
-  fd = ::open(fn, O_WRONLY|O_CREAT|O_TRUNC, 0644);
+  snprintf(tmp, sizeof(tmp), "%s/%s.tmp", base.c_str(), file.c_str());
+  fd = ::open(tmp, O_WRONLY|O_CREAT|O_TRUNC, 0644);
   if (fd < 0) {
     ret = errno;
-    derr << "OSD::write_meta: error opening '" << fn << "': "
+    derr << "write_meta: error opening '" << tmp << "': "
 	 << cpp_strerror(ret) << dendl;
     return -ret;
   }
   ret = safe_write(fd, val, vallen);
   if (ret) {
-    derr << "OSD::write_meta: failed to write to '" << fn << "': "
+    derr << "write_meta: failed to write to '" << tmp << "': "
 	 << cpp_strerror(ret) << dendl;
     TEMP_FAILURE_RETRY(::close(fd));
     return ret;
   }
-  if (TEMP_FAILURE_RETRY(::close(fd)) < 0) {
-    ret = errno;
-    derr << "OSD::write_meta: close error writing to '" << fn << "': "
+
+  ret = ::fsync(fd);
+  TEMP_FAILURE_RETRY(::close(fd));
+  if (ret) {
+    ::unlink(tmp);
+    derr << "write_meta: failed to fsync to '" << tmp << "': "
+	 << cpp_strerror(ret) << dendl;
+    return ret;
+  }
+  ret = ::rename(tmp, fn);
+  if (ret) {
+    ::unlink(tmp);
+    derr << "write_meta: failed to rename '" << tmp << "' to '" << fn << "': "
 	 << cpp_strerror(ret) << dendl;
     return ret;
   }
+
+  fd = ::open(base.c_str(), O_RDONLY);
+  if (fd < 0) {
+    ret = errno;
+    derr << "write_meta: failed to open dir '" << base << "': "
+	 << cpp_strerror(ret) << dendl;
+    return -ret;
+  }
+  ::fsync(fd);
+  TEMP_FAILURE_RETRY(::close(fd));
+
   return 0;
 }
 
diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index fb73a83a98a..4ffda17b439 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -116,6 +116,16 @@ int OSDMap::Incremental::get_net_marked_down(const OSDMap *previous) const
   return n;
 }
 
+int OSDMap::Incremental::identify_osd(uuid_d u) const
+{
+  for (map<int32_t,uuid_d>::const_iterator p = new_uuid.begin();
+       p != new_uuid.end();
+       ++p)
+    if (p->second == u)
+      return p->first;
+  return -1;
+}
+
 void OSDMap::Incremental::encode_client_old(bufferlist& bl) const
 {
   __u16 v = 5;
@@ -198,7 +208,7 @@ void OSDMap::Incremental::encode(bufferlist& bl, uint64_t features) const
   ::encode(new_pg_temp, bl);
 
   // extended
-  __u16 ev = CEPH_OSDMAP_INC_VERSION_EXT;
+  __u16 ev = 8;
   ::encode(ev, bl);
   ::encode(new_hb_up, bl);
   ::encode(new_up_thru, bl);
@@ -208,6 +218,7 @@ void OSDMap::Incremental::encode(bufferlist& bl, uint64_t features) const
   ::encode(old_blacklist, bl);
   ::encode(new_up_internal, bl);
   ::encode(cluster_snapshot, bl);
+  ::encode(new_uuid, bl);
 }
 
 void OSDMap::Incremental::decode(bufferlist::iterator &p)
@@ -295,6 +306,8 @@ void OSDMap::Incremental::decode(bufferlist::iterator &p)
     ::decode(new_up_internal, p);
   if (ev >= 7)
     ::decode(cluster_snapshot, p);
+  if (ev >= 8)
+    ::decode(new_uuid, p);
 }
 
 void OSDMap::Incremental::dump(Formatter *f) const
@@ -433,6 +446,15 @@ void OSDMap::Incremental::dump(Formatter *f) const
 
   if (cluster_snapshot.size())
     f->dump_string("cluster_snapshot", cluster_snapshot);
+
+  f->open_array_section("new_uuid");
+  for (map<int32_t,uuid_d>::const_iterator p = new_uuid.begin(); p != new_uuid.end(); ++p) {
+    f->open_object_section("osd");
+    f->dump_int("osd", p->first);
+    f->dump_stream("uuid") << p->second;
+    f->close_section();
+  }
+  f->close_section();
 }
 
 void OSDMap::Incremental::generate_test_instances(list<Incremental*>& o)
@@ -482,6 +504,7 @@ void OSDMap::set_max_osd(int m)
   osd_addrs->client_addr.resize(m);
   osd_addrs->cluster_addr.resize(m);
   osd_addrs->hb_addr.resize(m);
+  osd_uuid->resize(m);
 
   calc_num_osds();
 }
@@ -554,6 +577,14 @@ int OSDMap::identify_osd(const entity_addr_t& addr) const
   return -1;
 }
 
+int OSDMap::identify_osd(const uuid_d& u) const
+{
+  for (int i=0; i<max_osd; i++)
+    if (exists(i) && get_uuid(i) == u)
+      return i;
+  return -1;
+}
+
 bool OSDMap::find_osd_on_ip(const entity_addr_t& ip) const
 {
   for (int i=0; i<max_osd; i++)
@@ -607,6 +638,11 @@ void OSDMap::dedup(const OSDMap *o, OSDMap *n)
     if (*o->pg_temp == *n->pg_temp)
       n->pg_temp = o->pg_temp;
   }
+
+  // do uuids match?
+  if (o->osd_uuid->size() == n->osd_uuid->size() &&
+      *o->osd_uuid == *n->osd_uuid)
+    n->osd_uuid = o->osd_uuid;
 }
 
 int OSDMap::apply_incremental(Incremental &inc)
@@ -675,6 +711,9 @@ int OSDMap::apply_incremental(Incremental &inc)
 	(s & CEPH_OSD_UP)) {
       osd_info[i->first].down_at = epoch;
     }
+    if ((osd_state[i->first] & CEPH_OSD_EXISTS) &&
+	(s & CEPH_OSD_EXISTS))
+      (*osd_uuid)[i->first] = uuid_d();
     osd_state[i->first] ^= s;
   }
   for (map<int32_t,entity_addr_t>::iterator i = inc.new_up_client.begin();
@@ -692,6 +731,7 @@ int OSDMap::apply_incremental(Incremental &inc)
        i != inc.new_up_internal.end();
        i++)
     osd_addrs->cluster_addr[i->first].reset(new entity_addr_t(i->second));
+
   // info
   for (map<int32_t,epoch_t>::iterator i = inc.new_up_thru.begin();
        i != inc.new_up_thru.end();
@@ -706,6 +746,10 @@ int OSDMap::apply_incremental(Incremental &inc)
   for (map<int32_t,epoch_t>::iterator p = inc.new_lost.begin(); p != inc.new_lost.end(); p++)
     osd_info[p->first].lost_at = p->second;
 
+  // uuid
+  for (map<int32_t,uuid_d>::iterator p = inc.new_uuid.begin(); p != inc.new_uuid.end(); ++p) 
+    (*osd_uuid)[p->first] = p->second;
+
   // pg rebuild
   for (map<pg_t, vector<int> >::iterator p = inc.new_pg_temp.begin(); p != inc.new_pg_temp.end(); p++) {
     if (p->second.empty())
@@ -968,7 +1012,7 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const
   ::encode(cbl, bl);
 
   // extended
-  __u16 ev = CEPH_OSDMAP_VERSION_EXT;
+  __u16 ev = 8;
   ::encode(ev, bl);
   ::encode(osd_addrs->hb_addr, bl);
   ::encode(osd_info, bl);
@@ -976,6 +1020,7 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const
   ::encode(osd_addrs->cluster_addr, bl);
   ::encode(cluster_snapshot_epoch, bl);
   ::encode(cluster_snapshot, bl);
+  ::encode(*osd_uuid, bl);
 }
 
 void OSDMap::decode(bufferlist& bl)
@@ -1075,6 +1120,12 @@ void OSDMap::decode(bufferlist::iterator& p)
     ::decode(cluster_snapshot, p);
   }
 
+  if (ev >= 8) {
+    ::decode(*osd_uuid, p);
+  } else {
+    osd_uuid->resize(max_osd);
+  }
+
   // index pool names
   name_pool.clear();
   for (map<int64_t,string>::iterator i = pool_name.begin(); i != pool_name.end(); i++)
@@ -1119,6 +1170,7 @@ void OSDMap::dump(Formatter *f) const
     if (exists(i)) {
       f->open_object_section("osd_info");
       f->dump_int("osd", i);
+      f->dump_stream("uuid") << get_uuid(i);
       f->dump_int("up", is_up(i));
       f->dump_int("in", is_in(i));
       get_info(i).dump(f);
@@ -1252,6 +1304,8 @@ void OSDMap::print(ostream& out) const
       set<string> st;
       get_state(i, st);
       out << " " << st;
+      if (!get_uuid(i).is_zero())
+	out << " " << get_uuid(i);
       out << "\n";
     }
   }
diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index f76f22d6db3..dc0162c3b04 100644
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -115,6 +115,7 @@ public:
     map<int32_t,epoch_t> new_up_thru;
     map<int32_t,pair<epoch_t,epoch_t> > new_last_clean_interval;
     map<int32_t,epoch_t> new_lost;
+    map<int32_t,uuid_d> new_uuid;
 
     map<entity_addr_t,utime_t> new_blacklist;
     vector<entity_addr_t> old_blacklist;
@@ -124,6 +125,7 @@ public:
 
     int get_net_marked_out(const OSDMap *previous) const;
     int get_net_marked_down(const OSDMap *previous) const;
+    int identify_osd(uuid_d u) const;
 
     void encode_client_old(bufferlist& bl) const;
     void encode(bufferlist& bl, uint64_t features=-1) const;
@@ -172,6 +174,8 @@ private:
   map<int64_t,string> pool_name;
   map<string,int64_t> name_pool;
 
+  std::tr1::shared_ptr< vector<uuid_d> > osd_uuid;
+
   hash_map<entity_addr_t,utime_t> blacklist;
 
   epoch_t cluster_snapshot_epoch;
@@ -191,6 +195,7 @@ private:
 	     num_osd(0), max_osd(0),
 	     osd_addrs(new addrs_s),
 	     pg_temp(new map<pg_t,vector<int> >),
+	     osd_uuid(new vector<uuid_d>),
 	     cluster_snapshot_epoch(0),
 	     crush(new CrushWrapper) {
     memset(&fsid, 0, sizeof(fsid));
@@ -292,6 +297,8 @@ private:
   }
   
   int identify_osd(const entity_addr_t& addr) const;
+  int identify_osd(const uuid_d& u) const;
+
   bool have_addr(const entity_addr_t& addr) const {
     return identify_osd(addr) >= 0;
   }
@@ -326,6 +333,11 @@ private:
     return entity_inst_t(entity_name_t::OSD(osd), get_hb_addr(osd));
   }
 
+  const uuid_d& get_uuid(int osd) const {
+    assert(exists(osd));
+    return (*osd_uuid)[osd];
+  }
+
   const epoch_t& get_up_from(int osd) const {
     assert(exists(osd));
     return osd_info[osd].up_from;
diff --git a/src/test/encoding/ceph_dencoder.cc b/src/test/encoding/ceph_dencoder.cc
index ae9535d0423..68d1ca8ae0a 100644
--- a/src/test/encoding/ceph_dencoder.cc
+++ b/src/test/encoding/ceph_dencoder.cc
@@ -9,9 +9,11 @@
 #include "msg/Message.h"
 
 #define TYPE(t)
+#define TYPEWITHSTRAYDATA(t)
 #define MESSAGE(t)
 #include "types.h"
 #undef TYPE
+#undef TYPEWITHSTRAYDATA
 #undef MESSAGE
 
 void usage(ostream &out)
@@ -51,9 +53,10 @@ template<class T>
 class DencoderImpl : public Dencoder {
   T* m_object;
   list<T*> m_list;
+  bool stray_okay;
 
 public:
-  DencoderImpl() : m_object(new T) {}
+  DencoderImpl(bool stray_okay=false) : m_object(new T), stray_okay(stray_okay) {}
 
   string decode(bufferlist bl) {
     bufferlist::iterator p = bl.begin();
@@ -63,7 +66,7 @@ public:
     catch (buffer::error& e) {
       return e.what();
     }
-    if (!p.end())
+    if (!stray_okay && !p.end())
       return "stray data at end of buffer";
     return string();
   }
@@ -178,9 +181,11 @@ int main(int argc, const char **argv)
 #define T_STR(x) #x
 #define T_STRINGIFY(x) T_STR(x)
 #define TYPE(t) dencoders[T_STRINGIFY(t)] = new DencoderImpl<t>;
+#define TYPEWITHSTRAYDATA(t) dencoders[T_STRINGIFY(t)] = new DencoderImpl<t>(true);
 #define MESSAGE(t) dencoders[T_STRINGIFY(t)] = new MessageDencoderImpl<t>;
 #include "types.h"
 #undef TYPE
+#undef TYPEWITHSTRAYDATA
 #undef T_STR
 #undef T_STRRINGIFY
 
diff --git a/src/test/encoding/types.h b/src/test/encoding/types.h
index 5b50147c520..45fe618faf1 100644
--- a/src/test/encoding/types.h
+++ b/src/test/encoding/types.h
@@ -19,8 +19,8 @@ TYPE(entity_addr_t)
 
 #include "osd/OSDMap.h"
 TYPE(osd_info_t)
-TYPE(OSDMap)
-TYPE(OSDMap::Incremental)
+TYPEWITHSTRAYDATA(OSDMap)
+TYPEWITHSTRAYDATA(OSDMap::Incremental)
 
 #include "crush/CrushWrapper.h"
 TYPE(CrushWrapper)
diff --git a/src/vstart.sh b/src/vstart.sh
index cddb7ebad04..c4d43beab3c 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -361,10 +361,11 @@ EOF
 EOF
 	    fi
 
-	    echo add osd$osd
-	    $SUDO $CEPH_ADM osd create $osd
+	    uuid=`uuidgen`
+	    echo "add osd$osd $uuid"
+	    $SUDO $CEPH_ADM osd create $uuid
 	    $SUDO $CEPH_ADM osd crush add $osd osd.$osd 1.0 host=localhost rack=localrack pool=default
-	    $SUDO $CEPH_BIN/ceph-osd -i $osd $ARGS --mkfs --mkkey
+	    $SUDO $CEPH_BIN/ceph-osd -i $osd $ARGS --mkfs --mkkey --osd-uuid $uuid
 
 	    if [ "$cephx" -eq 1 ]; then
 		key_fn=dev/osd$osd/keyring