rgw: update log APIs to use markers

a few APIs were missing markers, fixed that.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

12 files changed, 66 insertions, 29 deletions

diff --git a/src/cls/log/cls_log.cc b/src/cls/log/cls_log.cc
index b7a2a870f9a..74bd147320b 100644
--- a/src/cls/log/cls_log.cc
+++ b/src/cls/log/cls_log.cc
@@ -184,8 +184,16 @@ static int cls_log_trim(cls_method_context_t hctx, bufferlist *in, bufferlist *o
   string from_index;
   string to_index;
 
-  get_index_time_prefix(op.from_time, from_index);
-  get_index_time_prefix(op.to_time, to_index);
+  if (op.from_marker.empty()) {
+    get_index_time_prefix(op.from_time, from_index);
+  } else {
+    from_index = op.from_marker;
+  }
+  if (op.to_marker.empty()) {
+    get_index_time_prefix(op.to_time, to_index);
+  } else {
+    to_index = op.to_marker;
+  }
 
 #define MAX_TRIM_ENTRIES 1000
   size_t max_entries = MAX_TRIM_ENTRIES;
diff --git a/src/cls/log/cls_log_client.cc b/src/cls/log/cls_log_client.cc
index 00e37bb43aa..0e3aa85b827 100644
--- a/src/cls/log/cls_log_client.cc
+++ b/src/cls/log/cls_log_client.cc
@@ -45,24 +45,28 @@ void cls_log_add(librados::ObjectWriteOperation& op, const utime_t& timestamp,
   cls_log_add(op, entry);
 }
 
-void cls_log_trim(librados::ObjectWriteOperation& op, utime_t& from, utime_t& to)
+void cls_log_trim(librados::ObjectWriteOperation& op, const utime_t& from_time, const utime_t& to_time,
+                  const string& from_marker, const string& to_marker)
 {
   bufferlist in;
   cls_log_trim_op call;
-  call.from_time = from;
-  call.to_time = to;
+  call.from_time = from_time;
+  call.to_time = to_time;
+  call.from_marker = from_marker;
+  call.to_marker = to_marker;
   ::encode(call, in);
   op.exec("log", "trim", in);
 }
 
-int cls_log_trim(librados::IoCtx& io_ctx, const string& oid, utime_t& from, utime_t& to)
+int cls_log_trim(librados::IoCtx& io_ctx, const string& oid, const utime_t& from_time, const utime_t& to_time,
+                 const string& from_marker, const string& to_marker)
 {
   bool done = false;
 
   do {
     ObjectWriteOperation op;
 
-    cls_log_trim(op, from, to);
+    cls_log_trim(op, from_time, to_time, from_marker, to_marker);
 
     int r = io_ctx.operate(oid, &op);
     if (r == -ENODATA)
diff --git a/src/cls/log/cls_log_client.h b/src/cls/log/cls_log_client.h
index a035e58bde1..afda05597c2 100644
--- a/src/cls/log/cls_log_client.h
+++ b/src/cls/log/cls_log_client.h
@@ -21,7 +21,9 @@ void cls_log_list(librados::ObjectReadOperation& op, utime_t& from, utime_t& to,
                   string& in_marker, int max_entries, list<cls_log_entry>& entries,
                   string *out_marker, bool *truncated);
 
-void cls_log_trim(librados::ObjectWriteOperation& op, utime_t& from, utime_t& to);
-int cls_log_trim(librados::IoCtx& io_ctx, const string& oid, utime_t& from, utime_t& to);
+void cls_log_trim(librados::ObjectWriteOperation& op, const utime_t& from_time, const utime_t& to_time,
+                  const string& from_marker, const string& to_marker);
+int cls_log_trim(librados::IoCtx& io_ctx, const string& oid, const utime_t& from_time, const utime_t& to_time,
+                 const string& from_marker, const string& to_marker);
 
 #endif
diff --git a/src/cls/log/cls_log_ops.h b/src/cls/log/cls_log_ops.h
index 6a5366f5a66..dac232d76d2 100644
--- a/src/cls/log/cls_log_ops.h
+++ b/src/cls/log/cls_log_ops.h
@@ -88,20 +88,28 @@ WRITE_CLASS_ENCODER(cls_log_list_ret)
 struct cls_log_trim_op {
   utime_t from_time;
   utime_t to_time; /* inclusive */
+  string from_marker;
+  string to_marker;
 
   cls_log_trim_op() {}
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 1, bl);
     ::encode(from_time, bl);
     ::encode(to_time, bl);
+    ::encode(from_marker, bl);
+    ::encode(to_marker, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::iterator& bl) {
-    DECODE_START(1, bl);
+    DECODE_START(2, bl);
     ::decode(from_time, bl);
     ::decode(to_time, bl);
+    if (struct_v >= 2) {
+    ::decode(from_marker, bl);
+    ::decode(to_marker, bl);
+    }
     DECODE_FINISH(bl);
   }
 };
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 8d1536fcd66..810b18c6984 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -1899,7 +1899,7 @@ next:
 
     RGWMetadataLog *meta_log = store->meta_mgr->get_log();
 
-    ret = meta_log->trim(shard_id, start_time, end_time);
+    ret = meta_log->trim(shard_id, start_time, end_time, start_marker, end_marker);
     if (ret < 0) {
       cerr << "ERROR: meta_log->trim(): " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -2015,7 +2015,7 @@ next:
       return -ret;
 
     RGWDataChangesLog *log = store->data_log;
-    ret = log->trim_entries(start_time, end_time);
+    ret = log->trim_entries(start_time, end_time, start_marker, end_marker);
     if (ret < 0) {
       cerr << "ERROR: trim_entries(): " << cpp_strerror(-ret) << std::endl;
       return -ret;
diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index bd17b25562d..6be17e3fcdd 100644
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -1213,11 +1213,12 @@ int RGWDataChangesLog::list_entries(utime_t& start_time, utime_t& end_time, int
   return 0;
 }
 
-int RGWDataChangesLog::trim_entries(int shard_id, utime_t& start_time, utime_t& end_time)
+int RGWDataChangesLog::trim_entries(int shard_id, const utime_t& start_time, const utime_t& end_time,
+                                    const string& start_marker, const string& end_marker)
 {
   int ret;
 
-  ret = store->time_log_trim(oids[shard_id], start_time, end_time);
+  ret = store->time_log_trim(oids[shard_id], start_time, end_time, start_marker, end_marker);
 
   if (ret == -ENOENT)
     ret = 0;
@@ -1225,10 +1226,11 @@ int RGWDataChangesLog::trim_entries(int shard_id, utime_t& start_time, utime_t&
   return ret;
 }
 
-int RGWDataChangesLog::trim_entries(utime_t& start_time, utime_t& end_time)
+int RGWDataChangesLog::trim_entries(const utime_t& start_time, const utime_t& end_time,
+                                    const string& start_marker, const string& end_marker)
 {
   for (int shard = 0; shard < num_shards; shard++) {
-    int ret = store->time_log_trim(oids[shard], start_time, end_time);
+    int ret = store->time_log_trim(oids[shard], start_time, end_time, start_marker, end_marker);
     if (ret == -ENOENT) {
       continue;
     }
diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h
index c2280815928..50d51444eb0 100644
--- a/src/rgw/rgw_bucket.h
+++ b/src/rgw/rgw_bucket.h
@@ -345,8 +345,10 @@ public:
   int renew_entries();
   int list_entries(int shard, utime_t& start_time, utime_t& end_time, int max_entries,
                list<rgw_data_change>& entries, string& marker, bool *truncated);
-  int trim_entries(int shard_id, utime_t& start_time, utime_t& end_time);
-  int trim_entries(utime_t& start_time, utime_t& end_time);
+  int trim_entries(int shard_id, const utime_t& start_time, const utime_t& end_time,
+                   const string& start_marker, const string& end_marker);
+  int trim_entries(const utime_t& start_time, const utime_t& end_time,
+                   const string& start_marker, const string& end_marker);
   int lock_exclusive(int shard_id, utime_t& duration, string& zone_id, string& owner_id) {
     return store->lock_exclusive(store->zone.log_pool, oids[shard_id], duration, zone_id, owner_id);
   }
diff --git a/src/rgw/rgw_metadata.cc b/src/rgw/rgw_metadata.cc
index e0934193d5a..b1c133fe522 100644
--- a/src/rgw/rgw_metadata.cc
+++ b/src/rgw/rgw_metadata.cc
@@ -127,14 +127,15 @@ int RGWMetadataLog::list_entries(void *handle,
   return 0;
 }
 
-int RGWMetadataLog::trim(int shard_id, utime_t& from_time, utime_t& end_time)
+int RGWMetadataLog::trim(int shard_id, const utime_t& from_time, const utime_t& end_time,
+                         const string& start_marker, const string& end_marker)
 {
   string oid;
   get_shard_oid(shard_id, oid);
 
   int ret;
 
-  ret = store->time_log_trim(oid, from_time, end_time);
+  ret = store->time_log_trim(oid, from_time, end_time, start_marker, end_marker);
 
   if (ret == -ENOENT)
     ret = 0;
diff --git a/src/rgw/rgw_metadata.h b/src/rgw/rgw_metadata.h
index 43670007cd5..ff0e35f9438 100644
--- a/src/rgw/rgw_metadata.h
+++ b/src/rgw/rgw_metadata.h
@@ -103,7 +103,7 @@ public:
                    int max_entries,
                    list<cls_log_entry>& entries, bool *truncated);
 
-  int trim(int shard_id, utime_t& from_time, utime_t& end_time);
+  int trim(int shard_id, const utime_t& from_time, const utime_t& end_time, const string& start_marker, const string& end_marker);
   int lock_exclusive(int shard_id, utime_t& duration, string&zone_id, string& owner_id);
   int unlock(int shard_id, string& zone_id, string& owner_id);
 };
diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 4cd87a61073..ae263355575 100644
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -1522,7 +1522,8 @@ int RGWRados::time_log_list(const string& oid, utime_t& start_time, utime_t& end
   return 0;
 }
 
-int RGWRados::time_log_trim(const string& oid, utime_t& start_time, utime_t& end_time)
+int RGWRados::time_log_trim(const string& oid, const utime_t& start_time, const utime_t& end_time,
+			    const string& from_marker, const string& to_marker)
 {
   librados::IoCtx io_ctx;
 
@@ -1531,7 +1532,7 @@ int RGWRados::time_log_trim(const string& oid, utime_t& start_time, utime_t& end
   if (r < 0)
     return r;
 
-  return cls_log_trim(io_ctx, oid, start_time, end_time);
+  return cls_log_trim(io_ctx, oid, start_time, end_time, from_marker, to_marker);
 }
 
 
diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 54f4366ff3f..f8a0124b045 100644
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -1329,7 +1329,8 @@ public:
   int time_log_add(const string& oid, const utime_t& ut, const string& section, const string& key, bufferlist& bl);
   int time_log_list(const string& oid, utime_t& start_time, utime_t& end_time,
                     int max_entries, list<cls_log_entry>& entries, string& marker, bool *truncated);
-  int time_log_trim(const string& oid, utime_t& start_time, utime_t& end_time);
+  int time_log_trim(const string& oid, const utime_t& start_time, const utime_t& end_time,
+                    const string& from_marker, const string& to_marker);
   int lock_exclusive(rgw_bucket& pool, const string& oid, utime_t& duration, string& zone_id, string& owner_id);
   int unlock(rgw_bucket& pool, const string& oid, string& zone_id, string& owner_id);
 
diff --git a/src/rgw/rgw_rest_log.cc b/src/rgw/rgw_rest_log.cc
index 3e4f8873952..2696940f4ed 100644
--- a/src/rgw/rgw_rest_log.cc
+++ b/src/rgw/rgw_rest_log.cc
@@ -128,6 +128,8 @@ void RGWOp_MDLog_GetShardsInfo::send_response() {
 void RGWOp_MDLog_Delete::execute() {
   string   st = s->info.args.get("start-time"),
            et = s->info.args.get("end-time"),
+           start_marker = s->info.args.get("start-marker"),
+           end_marker = s->info.args.get("end-marker"),
            shard = s->info.args.get("id"),
            err;
   utime_t  ut_st, 
@@ -142,7 +144,9 @@ void RGWOp_MDLog_Delete::execute() {
     http_ret = -EINVAL;
     return;
   }
-  if (st.empty() || et.empty()) {
+  if ((st.empty() || et.empty()) && (start_marker.empty() || end_marker.empty())) {
+    /* need to have at least one pair, either start-time && end-time, or start-marker && end-marker
+     * */
     http_ret = -EINVAL;
     return;
   }
@@ -158,7 +162,7 @@ void RGWOp_MDLog_Delete::execute() {
   }
   RGWMetadataLog *meta_log = store->meta_mgr->get_log();
 
-  http_ret = meta_log->trim(shard_id, ut_st, ut_et);
+  http_ret = meta_log->trim(shard_id, ut_st, ut_et, start_marker, end_marker);
 }
 
 void RGWOp_MDLog_Lock::execute() {
@@ -511,6 +515,8 @@ void RGWOp_DATALog_Unlock::execute() {
 void RGWOp_DATALog_Delete::execute() {
   string   st = s->info.args.get("start-time"),
            et = s->info.args.get("end-time"),
+           start_marker = s->info.args.get("start-marker"),
+           end_marker = s->info.args.get("end-marker"),
            shard = s->info.args.get("id"),
            err;
   utime_t  ut_st, 
@@ -525,7 +531,9 @@ void RGWOp_DATALog_Delete::execute() {
     http_ret = -EINVAL;
     return;
   }
-  if (st.empty() || et.empty()) {
+  if ((st.empty() || et.empty()) && (start_marker.empty() || end_marker.empty())) {
+    /* need to have at least one pair, either start-time && end-time, or start-marker && end-marker
+     * */
     http_ret = -EINVAL;
     return;
   }
@@ -540,7 +548,7 @@ void RGWOp_DATALog_Delete::execute() {
     return;
   }
 
-  http_ret = store->data_log->trim_entries(shard_id, ut_st, ut_et);
+  http_ret = store->data_log->trim_entries(shard_id, ut_st, ut_et, start_marker, end_marker);
 }
 
 RGWOp *RGWHandler_Log::op_get() {