systest: fix race with pool deletion

The second test have pool deletion and object listing wait on the same
semaphore to connect and start. This led to errors sometimes when the
pool was deleted before it could be opened by the listing process. Add
another semaphore so the pool deletion happens only after the listing
has begun.

Fixes: #4147
Signed-off-by: Josh Durgin <josh.durgin@inktank.com>

4 files changed, 19 insertions, 13 deletions

diff --git a/src/test/system/rados_delete_pools_parallel.cc b/src/test/system/rados_delete_pools_parallel.cc
index 4bcf2124418..5347c53c296 100644
--- a/src/test/system/rados_delete_pools_parallel.cc
+++ b/src/test/system/rados_delete_pools_parallel.cc
@@ -91,9 +91,9 @@ int main(int argc, const char **argv)
   {
     StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
 			 pool, g_num_objects, ".obj");
-    StRadosDeletePool r2(argc, argv, pool_setup_sem, delete_pool_sem, pool);
+    StRadosDeletePool r2(argc, argv, delete_pool_sem, NULL, pool);
     StRadosListObjects r3(argc, argv, pool, true, g_num_objects / 2,
-			  pool_setup_sem, delete_pool_sem);
+			  pool_setup_sem, NULL, delete_pool_sem);
     vector < SysTestRunnable* > vec;
     vec.push_back(&r1);
     vec.push_back(&r2);
diff --git a/src/test/system/rados_list_parallel.cc b/src/test/system/rados_list_parallel.cc
index 3a68b943627..77df29e0e79 100644
--- a/src/test/system/rados_list_parallel.cc
+++ b/src/test/system/rados_list_parallel.cc
@@ -230,7 +230,7 @@ int main(int argc, const char **argv)
     StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
 			 pool, g_num_objects, ".obj");
     StRadosListObjects r2(argc, argv, pool, false, g_num_objects,
-			  pool_setup_sem, modify_sem);
+			  pool_setup_sem, modify_sem, NULL);
     vector < SysTestRunnable* > vec;
     vec.push_back(&r1);
     vec.push_back(&r2);
@@ -248,7 +248,7 @@ int main(int argc, const char **argv)
     StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
 			 pool, g_num_objects, ".obj");
     StRadosListObjects r2(argc, argv, pool, false, g_num_objects / 2,
-			  pool_setup_sem, modify_sem);
+			  pool_setup_sem, modify_sem, NULL);
     RadosDeleteObjectsR r3(argc, argv, pool);
     vector < SysTestRunnable* > vec;
     vec.push_back(&r1);
@@ -268,7 +268,7 @@ int main(int argc, const char **argv)
     StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
 			 pool, g_num_objects, ".obj");
     StRadosListObjects r2(argc, argv, pool, false, g_num_objects / 2,
-			  pool_setup_sem, modify_sem);
+			  pool_setup_sem, modify_sem, NULL);
     RadosAddObjectsR r3(argc, argv, pool, ".obj2");
     vector < SysTestRunnable* > vec;
     vec.push_back(&r1);
@@ -288,7 +288,7 @@ int main(int argc, const char **argv)
     StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
 			 pool, g_num_objects, ".obj");
     StRadosListObjects r2(argc, argv, pool, false, g_num_objects / 2,
-			  pool_setup_sem, modify_sem);
+			  pool_setup_sem, modify_sem, NULL);
     RadosAddObjectsR r3(argc, argv, pool, ".obj2");
     RadosAddObjectsR r4(argc, argv, pool, ".obj3");
     RadosDeleteObjectsR r5(argc, argv, pool);
@@ -312,7 +312,7 @@ int main(int argc, const char **argv)
     StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
 			 pool, g_num_objects, ".obj");
     StRadosListObjects r2(argc, argv, pool, false, g_num_objects / 2,
-			  pool_setup_sem, modify_sem);
+			  pool_setup_sem, modify_sem, NULL);
     // AddObjects with the same 'suffix' as used in StRadosCreatePool
     RadosAddObjectsR r3(argc, argv, pool, ".obj");
     vector < SysTestRunnable* > vec;
diff --git a/src/test/system/st_rados_list_objects.cc b/src/test/system/st_rados_list_objects.cc
index 8c0db947a04..9038db2ad4b 100644
--- a/src/test/system/st_rados_list_objects.cc
+++ b/src/test/system/st_rados_list_objects.cc
@@ -33,12 +33,14 @@ StRadosListObjects(int argc, const char **argv,
 		   bool accept_list_errors,
 		   int midway_cnt,
 		   CrossProcessSem *pool_setup_sem,
-		   CrossProcessSem *midway_sem)
+		   CrossProcessSem *midway_sem_wait,
+		   CrossProcessSem *midway_sem_post)
   : SysTestRunnable(argc, argv),
     m_accept_list_errors(accept_list_errors),
     m_midway_cnt(midway_cnt),
     m_pool_setup_sem(pool_setup_sem),
-    m_midway_sem(midway_sem)
+    m_midway_sem_wait(midway_sem_wait),
+    m_midway_sem_post(midway_sem_post)
 {
 }
 
@@ -85,8 +87,10 @@ run()
     }
     ++saw;
     if (saw == m_midway_cnt) {
-      if (m_midway_sem)
-	m_midway_sem->wait();
+      if (m_midway_sem_wait)
+	m_midway_sem_wait->wait();
+      if (m_midway_sem_post)
+	m_midway_sem_post->post();
     }
   }
   rados_objects_list_close(h);
diff --git a/src/test/system/st_rados_list_objects.h b/src/test/system/st_rados_list_objects.h
index e4698d7492d..ddb1342e8b6 100644
--- a/src/test/system/st_rados_list_objects.h
+++ b/src/test/system/st_rados_list_objects.h
@@ -37,7 +37,8 @@ public:
 		     bool accept_list_errors,
 		     int midway_cnt,
 		     CrossProcessSem *pool_setup_sem,
-		     CrossProcessSem *midway_sem);
+		     CrossProcessSem *midway_sem_wait,
+		     CrossProcessSem *midway_sem_post);
   ~StRadosListObjects();
   virtual int run();
 private:
@@ -45,7 +46,8 @@ private:
   bool m_accept_list_errors;
   int m_midway_cnt;
   CrossProcessSem *m_pool_setup_sem;
-  CrossProcessSem *m_midway_sem;
+  CrossProcessSem *m_midway_sem_wait;
+  CrossProcessSem *m_midway_sem_post;
 };
 
 #endif