spinlock: add generic spinlock implementation

Adds a ceph_spinlock_t implementation that will use pthread_spinlock_t
if available, and otherwise reverts to pthread_mutex_t. Note that this
spinlock is not intended to be used in process-shared memory.

Switches implementation in:

  ceph_context
  SimpleMessenger
  atomic_t

Only ceph_context initialized its spinlock with PTHREAD_PROCESS_SHARED.
However, there does not appear to be any instance in which CephContext
is allocated in shared memory, and thus can use the default private
memory space behavior.

Signed-off-by: Noah Watkins <noahwatkins@gmail.com>

8 files changed, 116 insertions, 33 deletions

diff --git a/configure.ac b/configure.ac
index 90e2c7d5eb4..f1ff6bc5a5f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -568,6 +568,22 @@ AC_CHECK_FUNCS([pipe2])
 
 AC_CHECK_HEADERS([sys/vfs.h sys/mount.h])
 
+#
+# Check for pthread spinlock.
+#
+# NOTE: virtually everythink in Makefile.am links against the pthread
+# libraries. It would be cleaner to just set LIBS and CFLAGS globally to
+# include the pthread bits.
+#
+saved_LIBS="$LIBS"
+saved_CFLAGS="$CFLAGS"
+LIBS="$LIBS $PTHREAD_LIBS"
+CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+AC_CHECK_FUNC([pthread_spin_init],
+  [AC_DEFINE(HAVE_PTHREAD_SPINLOCK, 1, [Define if you have pthread_spin_init])])
+LIBS="$saved_LIBS"
+CFLAGS="$saved_CFLAGS"
+
 # Checks for typedefs, structures, and compiler characteristics.
 #AC_HEADER_STDBOOL
 #AC_C_CONST
diff --git a/src/common/ceph_context.cc b/src/common/ceph_context.cc
index 53f9b4d6803..fad6f3e7c84 100644
--- a/src/common/ceph_context.cc
+++ b/src/common/ceph_context.cc
@@ -34,6 +34,8 @@
 #include <pthread.h>
 #include <semaphore.h>
 
+#include "include/spinlock.h"
+
 using ceph::HeartbeatMap;
 
 class CephContextServiceThread : public Thread
@@ -264,7 +266,7 @@ CephContext::CephContext(uint32_t module_type_)
     _crypto_none(NULL),
     _crypto_aes(NULL)
 {
-  pthread_spin_init(&_service_thread_lock, PTHREAD_PROCESS_SHARED);
+  ceph_spin_init(&_service_thread_lock);
 
   _log = new ceph::log::Log(&_conf->subsys);
   _log->start();
@@ -334,7 +336,7 @@ CephContext::~CephContext()
   _log = NULL;
 
   delete _conf;
-  pthread_spin_destroy(&_service_thread_lock);
+  ceph_spin_destroy(&_service_thread_lock);
 
   delete _crypto_none;
   delete _crypto_aes;
@@ -342,14 +344,14 @@ CephContext::~CephContext()
 
 void CephContext::start_service_thread()
 {
-  pthread_spin_lock(&_service_thread_lock);
+  ceph_spin_lock(&_service_thread_lock);
   if (_service_thread) {
-    pthread_spin_unlock(&_service_thread_lock);
+    ceph_spin_unlock(&_service_thread_lock);
     return;
   }
   _service_thread = new CephContextServiceThread(this);
   _service_thread->create();
-  pthread_spin_unlock(&_service_thread_lock);
+  ceph_spin_unlock(&_service_thread_lock);
 
   // make logs flush on_exit()
   if (_conf->log_flush_on_exit)
@@ -367,22 +369,22 @@ void CephContext::start_service_thread()
 
 void CephContext::reopen_logs()
 {
-  pthread_spin_lock(&_service_thread_lock);
+  ceph_spin_lock(&_service_thread_lock);
   if (_service_thread)
     _service_thread->reopen_logs();
-  pthread_spin_unlock(&_service_thread_lock);
+  ceph_spin_unlock(&_service_thread_lock);
 }
 
 void CephContext::join_service_thread()
 {
-  pthread_spin_lock(&_service_thread_lock);
+  ceph_spin_lock(&_service_thread_lock);
   CephContextServiceThread *thread = _service_thread;
   if (!thread) {
-    pthread_spin_unlock(&_service_thread_lock);
+    ceph_spin_unlock(&_service_thread_lock);
     return;
   }
   _service_thread = NULL;
-  pthread_spin_unlock(&_service_thread_lock);
+  ceph_spin_unlock(&_service_thread_lock);
 
   thread->exit_thread();
   thread->join();
diff --git a/src/common/ceph_context.h b/src/common/ceph_context.h
index 08efeceaa67..a381758523b 100644
--- a/src/common/ceph_context.h
+++ b/src/common/ceph_context.h
@@ -21,6 +21,7 @@
 #include "include/buffer.h"
 #include "include/atomic.h"
 #include "common/cmdparse.h"
+#include "include/spinlock.h"
 
 class AdminSocket;
 class CephContextServiceThread;
@@ -126,7 +127,7 @@ private:
   AdminSocket *_admin_socket;
 
   /* lock which protects service thread creation, destruction, etc. */
-  pthread_spinlock_t _service_thread_lock;
+  ceph_spinlock_t _service_thread_lock;
 
   /* The collection of profiling loggers associated with this context */
   PerfCountersCollection *_perf_counters_collection;
diff --git a/src/include/Makefile.am b/src/include/Makefile.am
index 1c7ba58f1ce..1798ea244e7 100644
--- a/src/include/Makefile.am
+++ b/src/include/Makefile.am
@@ -78,4 +78,5 @@ noinst_HEADERS += \
 	include/rbd/librbd.h \
 	include/rbd/librbd.hpp\
 	include/util.h\
-	include/stat.h
+	include/stat.h\
+	include/spinlock.h
diff --git a/src/include/atomic.h b/src/include/atomic.h
index 3ecbd287376..65230024f80 100644
--- a/src/include/atomic.h
+++ b/src/include/atomic.h
@@ -67,53 +67,52 @@ namespace ceph {
 /*
  * crappy slow implementation that uses a pthreads spinlock.
  */
-#include <pthread.h>
-#include "include/assert.h"
+#include "include/spinlock.h"
 
 namespace ceph {
   class atomic_t {
-    mutable pthread_spinlock_t lock;
+    mutable ceph_spinlock_t lock;
     signed long val;
   public:
     atomic_t(int i=0)
       : val(i) {
-      pthread_spin_init(&lock, PTHREAD_PROCESS_PRIVATE);
+      ceph_spin_init(&lock);
     }
     ~atomic_t() {
-      pthread_spin_destroy(&lock);
+      ceph_spin_destroy(&lock);
     }
     void set(size_t v) {
-      pthread_spin_lock(&lock);
+      ceph_spin_lock(&lock);
       val = v;
-      pthread_spin_unlock(&lock);
+      ceph_spin_unlock(&lock);
     }
     int inc() {
-      pthread_spin_lock(&lock);
+      ceph_spin_lock(&lock);
       int r = ++val;
-      pthread_spin_unlock(&lock);
+      ceph_spin_unlock(&lock);
       return r;
     }
     int dec() {
-      pthread_spin_lock(&lock);
+      ceph_spin_lock(&lock);
       int r = --val;
-      pthread_spin_unlock(&lock);
+      ceph_spin_unlock(&lock);
       return r;
     }
     void add(int d) {
-      pthread_spin_lock(&lock);
+      ceph_spin_lock(&lock);
       val += d;
-      pthread_spin_unlock(&lock);
+      ceph_spin_unlock(&lock);
     }
     void sub(int d) {
-      pthread_spin_lock(&lock);
+      ceph_spin_lock(&lock);
       val -= d;
-      pthread_spin_unlock(&lock);
+      ceph_spin_unlock(&lock);
     }
     int read() const {
       signed long ret;
-      pthread_spin_lock(&lock);
+      ceph_spin_lock(&lock);
       ret = val;
-      pthread_spin_unlock(&lock);
+      ceph_spin_unlock(&lock);
       return ret;
     }
   private:
diff --git a/src/include/spinlock.h b/src/include/spinlock.h
new file mode 100644
index 00000000000..505e89ec6b4
--- /dev/null
+++ b/src/include/spinlock.h
@@ -0,0 +1,62 @@
+#ifndef CEPH_SPINLOCK_H
+#define CEPH_SPINLOCK_H
+
+#include <pthread.h>
+
+#include "acconfig.h"
+
+typedef struct {
+#ifdef HAVE_PTHREAD_SPINLOCK
+  pthread_spinlock_t lock;
+#else
+  pthread_mutex_t lock;
+#endif
+} ceph_spinlock_t;
+
+#ifdef HAVE_PTHREAD_SPINLOCK
+
+static inline int ceph_spin_init(ceph_spinlock_t *l)
+{
+  return pthread_spin_init(&l->lock, PTHREAD_PROCESS_PRIVATE);
+}
+
+static inline int ceph_spin_destroy(ceph_spinlock_t *l)
+{
+  return pthread_spin_destroy(&l->lock);
+}
+
+static inline int ceph_spin_lock(ceph_spinlock_t *l)
+{
+  return pthread_spin_lock(&l->lock);
+}
+
+static inline int ceph_spin_unlock(ceph_spinlock_t *l)
+{
+  return pthread_spin_unlock(&l->lock);
+}
+
+#else /* !HAVE_PTHREAD_SPINLOCK */
+
+static inline int ceph_spin_init(ceph_spinlock_t *l)
+{
+  return pthread_mutex_init(&l->lock, NULL);
+}
+
+static inline int ceph_spin_destroy(ceph_spinlock_t *l)
+{
+  return pthread_mutex_destroy(&l->lock);
+}
+
+static inline int ceph_spin_lock(ceph_spinlock_t *l)
+{
+  return pthread_mutex_lock(&l->lock);
+}
+
+static inline int ceph_spin_unlock(ceph_spinlock_t *l)
+{
+  return pthread_mutex_unlock(&l->lock);
+}
+
+#endif
+
+#endif
diff --git a/src/msg/SimpleMessenger.cc b/src/msg/SimpleMessenger.cc
index 441ed432af0..6e967ddbf6c 100644
--- a/src/msg/SimpleMessenger.cc
+++ b/src/msg/SimpleMessenger.cc
@@ -23,6 +23,7 @@
 #include "common/Timer.h"
 #include "common/errno.h"
 #include "auth/Crypto.h"
+#include "include/spinlock.h"
 
 #define dout_subsys ceph_subsys_ms
 #undef dout_prefix
@@ -53,7 +54,7 @@ SimpleMessenger::SimpleMessenger(CephContext *cct, entity_name_t name,
     timeout(0),
     local_connection(new Connection(this))
 {
-  pthread_spin_init(&global_seq_lock, PTHREAD_PROCESS_PRIVATE);
+  ceph_spin_init(&global_seq_lock);
   init_local_connection();
 }
 
diff --git a/src/msg/SimpleMessenger.h b/src/msg/SimpleMessenger.h
index 6860c6c21a3..736af35b4fb 100644
--- a/src/msg/SimpleMessenger.h
+++ b/src/msg/SimpleMessenger.h
@@ -38,6 +38,7 @@ using namespace __gnu_cxx;
 
 #include "Pipe.h"
 #include "Accepter.h"
+#include "include/spinlock.h"
 
 /*
  * This class handles transmission and reception of messages. Generally
@@ -308,7 +309,7 @@ private:
   /// counter for the global seq our connection protocol uses
   __u32 global_seq;
   /// lock to protect the global_seq
-  pthread_spinlock_t global_seq_lock;
+  ceph_spinlock_t global_seq_lock;
 
   /**
    * hash map of addresses to Pipes
@@ -388,11 +389,11 @@ public:
    * @return a global sequence ID that nobody else has seen.
    */
   __u32 get_global_seq(__u32 old=0) {
-    pthread_spin_lock(&global_seq_lock);
+    ceph_spin_lock(&global_seq_lock);
     if (old > global_seq)
       global_seq = old;
     __u32 ret = ++global_seq;
-    pthread_spin_unlock(&global_seq_lock);
+    ceph_spin_unlock(&global_seq_lock);
     return ret;
   }
   /**