osd/ReplicatedPG: set whiteout in cache pool on delete

If we delete an object in the cache pool, set the whiteout flag instead of
removing the on-disk object.

Signed-off-by: Sage Weil <sage@inktank.com>

2 files changed, 27 insertions, 8 deletions

diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index af8a523e5fe..0798f5a602b 100644
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -3789,22 +3789,30 @@ inline int ReplicatedPG::_delete_head(OpContext *ctx)
   if (!obs.exists || obs.oi.is_whiteout())
     return -ENOENT;
   
-  t.remove(coll, soid);
-
   if (oi.size > 0) {
     interval_set<uint64_t> ch;
     ch.insert(0, oi.size);
     ctx->modified_ranges.union_of(ch);
   }
 
-  ctx->delta_stats.num_objects--;
+  ctx->delta_stats.num_wr++;
   ctx->delta_stats.num_bytes -= oi.size;
-
   oi.size = 0;
+
+  // cache: writeback: set whiteout on delete?
+  if (pool.info.cache_mode == pg_pool_t::CACHEMODE_WRITEBACK) {
+    dout(20) << __func__ << " setting whiteout on " << soid << dendl;
+    oi.set_flag(object_info_t::FLAG_WHITEOUT);
+    t.truncate(coll, soid, 0);
+    t.omap_clear(coll, soid);
+    t.rmattrs(coll, soid);
+    return 0;
+  }
+
+  t.remove(coll, soid);
+  ctx->delta_stats.num_objects--;
   snapset.head_exists = false;
   obs.exists = false;
-
-  ctx->delta_stats.num_wr++;
   return 0;
 }
 
diff --git a/src/test/librados/tier.cc b/src/test/librados/tier.cc
index 3aaba6eafa8..466827e5e83 100644
--- a/src/test/librados/tier.cc
+++ b/src/test/librados/tier.cc
@@ -172,6 +172,15 @@ TEST(LibRadosTier, Whiteout) {
   IoCtx base_ioctx;
   ASSERT_EQ(0, cluster.ioctx_create(base_pool_name.c_str(), base_ioctx));
 
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, base_ioctx.operate("foo", &op));
+  }
+
   // configure cache
   bufferlist inbl;
   ASSERT_EQ(0, cluster.mon_command(
@@ -191,9 +200,9 @@ TEST(LibRadosTier, Whiteout) {
   cluster.wait_for_latest_map();
 
   // create some whiteouts, verify they behave
-  ASSERT_EQ(-ENOENT, base_ioctx.remove("foo"));
+  ASSERT_EQ(0, base_ioctx.remove("foo"));
+
   ASSERT_EQ(-ENOENT, base_ioctx.remove("bar"));
-  ASSERT_EQ(-ENOENT, base_ioctx.remove("foo"));
   ASSERT_EQ(-ENOENT, base_ioctx.remove("bar"));
 
   // verify the whiteouts are there in the cache tier
@@ -207,6 +216,8 @@ TEST(LibRadosTier, Whiteout) {
     ASSERT_TRUE(it == cache_ioctx.objects_end());
   }
 
+  ASSERT_EQ(-ENOENT, base_ioctx.remove("foo"));
+
   // tear down tiers
   ASSERT_EQ(0, cluster.mon_command(
     "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool_name +