diff options
Diffstat (limited to 'cpp/src/qpid/sys')
115 files changed, 9498 insertions, 2209 deletions
diff --git a/cpp/src/qpid/sys/AggregateOutput.cpp b/cpp/src/qpid/sys/AggregateOutput.cpp index 2fad28c381..709d3bc640 100644 --- a/cpp/src/qpid/sys/AggregateOutput.cpp +++ b/cpp/src/qpid/sys/AggregateOutput.cpp @@ -7,9 +7,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -25,44 +25,67 @@ namespace qpid { namespace sys { - -void AggregateOutput::activateOutput() -{ - control.activateOutput(); -} + +AggregateOutput::AggregateOutput(OutputControl& c) : busy(false), control(c) {} + +void AggregateOutput::abort() { control.abort(); } + +void AggregateOutput::activateOutput() { control.activateOutput(); } + +void AggregateOutput::giveReadCredit(int32_t credit) { control.giveReadCredit(credit); } bool AggregateOutput::hasOutput() { - for (TaskList::const_iterator i = tasks.begin(); i != tasks.end(); ++i) - if ((*i)->hasOutput()) return true; - return false; + Mutex::ScopedLock l(lock); + return !tasks.empty(); } -bool AggregateOutput::doOutput() -{ - bool result = false; - if (!tasks.empty()) { - if (next >= tasks.size()) next = next % tasks.size(); - - size_t start = next; - //loop until a task generated some output - while (!result) { - result = tasks[next++]->doOutput(); - if (next >= tasks.size()) next = next % tasks.size(); - if (start == next) break; +// Clear the busy flag and notify waiting threads in destructor. +struct ScopedBusy { + bool& flag; + Monitor& monitor; + ScopedBusy(bool& f, Monitor& m) : flag(f), monitor(m) { f = true; } + ~ScopedBusy() { flag = false; monitor.notifyAll(); } +}; + +bool AggregateOutput::doOutput() { + Mutex::ScopedLock l(lock); + ScopedBusy sb(busy, lock); + + while (!tasks.empty()) { + OutputTask* t=tasks.front(); + tasks.pop_front(); + bool didOutput; + { + // Allow concurrent call to addOutputTask. + // removeOutputTask will wait till !busy before removing a task. + Mutex::ScopedUnlock u(lock); + didOutput = t->doOutput(); + } + if (didOutput) { + tasks.push_back(t); + return true; } } - return result; + return false; +} + +void AggregateOutput::addOutputTask(OutputTask* task) { + Mutex::ScopedLock l(lock); + tasks.push_back(task); } -void AggregateOutput::addOutputTask(OutputTask* t) -{ - tasks.push_back(t); +void AggregateOutput::removeOutputTask(OutputTask* task) { + Mutex::ScopedLock l(lock); + while (busy) lock.wait(); + tasks.erase(std::remove(tasks.begin(), tasks.end(), task), tasks.end()); } - -void AggregateOutput::removeOutputTask(OutputTask* t) + +void AggregateOutput::removeAll() { - TaskList::iterator i = std::find(tasks.begin(), tasks.end(), t); - if (i != tasks.end()) tasks.erase(i); + Mutex::ScopedLock l(lock); + while (busy) lock.wait(); + tasks.clear(); } + }} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/AggregateOutput.h b/cpp/src/qpid/sys/AggregateOutput.h index 02a53ed50b..71ad713eb7 100644 --- a/cpp/src/qpid/sys/AggregateOutput.h +++ b/cpp/src/qpid/sys/AggregateOutput.h @@ -7,9 +7,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -21,35 +21,58 @@ #ifndef _AggregateOutput_ #define _AggregateOutput_ -#include <vector> -#include "Mutex.h" -#include "OutputControl.h" -#include "OutputTask.h" +#include "qpid/sys/Monitor.h" +#include "qpid/sys/OutputControl.h" +#include "qpid/sys/OutputTask.h" +#include "qpid/CommonImportExport.h" + +#include <algorithm> +#include <deque> namespace qpid { namespace sys { - class AggregateOutput : public OutputTask, public OutputControl - { - typedef std::vector<OutputTask*> TaskList; - - TaskList tasks; - size_t next; - OutputControl& control; - - public: - AggregateOutput(OutputControl& c) : next(0), control(c) {}; - //this may be called on any thread - void activateOutput(); - //all the following will be called on the same thread - bool doOutput(); - bool hasOutput(); - void addOutputTask(OutputTask* t); - void removeOutputTask(OutputTask* t); - }; - -} -} +/** + * Holds a collection of output tasks, doOutput picks the next one to execute. + * + * Tasks are automatically removed if their doOutput() or hasOutput() returns false. + * + * Thread safe. addOutputTask may be called in one connection thread while + * doOutput is called in another. + */ + +class AggregateOutput : public OutputTask, public OutputControl +{ + typedef std::deque<OutputTask*> TaskList; + + Monitor lock; + TaskList tasks; + bool busy; + OutputControl& control; + + public: + QPID_COMMON_EXTERN AggregateOutput(OutputControl& c); + + // These may be called concurrently with any function. + QPID_COMMON_EXTERN void abort(); + QPID_COMMON_EXTERN void activateOutput(); + QPID_COMMON_EXTERN void giveReadCredit(int32_t); + QPID_COMMON_EXTERN void addOutputTask(OutputTask* t); + + // These functions must not be called concurrently with each other. + QPID_COMMON_EXTERN bool doOutput(); + QPID_COMMON_EXTERN bool hasOutput(); + QPID_COMMON_EXTERN void removeOutputTask(OutputTask* t); + QPID_COMMON_EXTERN void removeAll(); + + /** Apply f to each OutputTask* in the tasks list */ + template <class F> void eachOutput(F f) { + Mutex::ScopedLock l(lock); + std::for_each(tasks.begin(), tasks.end(), f); + } +}; + +}} // namespace qpid::sys #endif diff --git a/cpp/src/qpid/sys/AsynchIO.h b/cpp/src/qpid/sys/AsynchIO.h index ff7823e00d..2a41f0a7d1 100644 --- a/cpp/src/qpid/sys/AsynchIO.h +++ b/cpp/src/qpid/sys/AsynchIO.h @@ -21,15 +21,16 @@ * */ -#include "Dispatcher.h" - +#include "qpid/sys/IntegerTypes.h" +#include "qpid/CommonImportExport.h" #include <boost/function.hpp> -#include <deque> +#include <boost/shared_ptr.hpp> namespace qpid { namespace sys { class Socket; +class Poller; /* * Asynchronous acceptor: accepts connections then does a callback with the @@ -39,44 +40,36 @@ class AsynchAcceptor { public: typedef boost::function1<void, const Socket&> Callback; -private: - Callback acceptedCallback; - DispatchHandle handle; - const Socket& socket; - -public: - AsynchAcceptor(const Socket& s, Callback callback); - void start(Poller::shared_ptr poller); - -private: - void readable(DispatchHandle& handle); + QPID_COMMON_EXTERN static AsynchAcceptor* create(const Socket& s, Callback callback); + virtual ~AsynchAcceptor() {}; + virtual void start(boost::shared_ptr<Poller> poller) = 0; }; /* * Asynchronous connector: starts the process of initiating a connection and * invokes a callback when completed or failed. */ -class AsynchConnector : private DispatchHandle { +class AsynchConnector { public: typedef boost::function1<void, const Socket&> ConnectedCallback; - typedef boost::function2<void, int, std::string> FailedCallback; - -private: - ConnectedCallback connCallback; - FailedCallback failCallback; - const Socket& socket; - -public: - AsynchConnector(const Socket& socket, - Poller::shared_ptr poller, - std::string hostname, - uint16_t port, - ConnectedCallback connCb, - FailedCallback failCb = 0); - -private: - void connComplete(DispatchHandle& handle); - void failure(int, std::string); + typedef boost::function3<void, const Socket&, int, const std::string&> FailedCallback; + + // Call create() to allocate a new AsynchConnector object with the + // specified poller, addressing, and callbacks. + // This method is implemented in platform-specific code to + // create a correctly typed object. The platform code also manages + // deletes. To correctly manage heaps when needed, the allocate and + // delete should both be done from the same class/library. + QPID_COMMON_EXTERN static AsynchConnector* create(const Socket& s, + boost::shared_ptr<Poller> poller, + std::string hostname, + uint16_t port, + ConnectedCallback connCb, + FailedCallback failCb); + +protected: + AsynchConnector() {} + virtual ~AsynchConnector() {} }; struct AsynchIOBufferBase { @@ -99,16 +92,14 @@ struct AsynchIOBufferBase { /* * Asychronous reader/writer: * Reader accepts buffers to read into; reads into the provided buffers - * and then does a callback with the buffer and amount read. Optionally it can callback - * when there is something to read but no buffer to read it into. + * and then does a callback with the buffer and amount read. Optionally it + * can callback when there is something to read but no buffer to read it into. * * Writer accepts a buffer and queues it for writing; can also be given - * a callback for when writing is "idle" (ie fd is writable, but nothing to write) - * - * The class is implemented in terms of DispatchHandle to allow it to be deleted by deleting - * the contained DispatchHandle + * a callback for when writing is "idle" (ie fd is writable, but nothing + * to write). */ -class AsynchIO : private DispatchHandle { +class AsynchIO { public: typedef AsynchIOBufferBase BufferBase; @@ -118,47 +109,40 @@ public: typedef boost::function2<void, AsynchIO&, const Socket&> ClosedCallback; typedef boost::function1<void, AsynchIO&> BuffersEmptyCallback; typedef boost::function1<void, AsynchIO&> IdleCallback; - -private: - ReadCallback readCallback; - EofCallback eofCallback; - DisconnectCallback disCallback; - ClosedCallback closedCallback; - BuffersEmptyCallback emptyCallback; - IdleCallback idleCallback; - const Socket& socket; - std::deque<BufferBase*> bufferQueue; - std::deque<BufferBase*> writeQueue; - bool queuedClose; - /** - * This flag is used to detect and handle concurrency between - * calls to notifyPendingWrite() (which can be made from any thread) and - * the execution of the writeable() method (which is always on the - * thread processing this handle. - */ - volatile bool writePending; - + typedef boost::function1<void, AsynchIO&> RequestCallback; + + // Call create() to allocate a new AsynchIO object with the specified + // callbacks. This method is implemented in platform-specific code to + // create a correctly typed object. The platform code also manages + // deletes. To correctly manage heaps when needed, the allocate and + // delete should both be done from the same class/library. + QPID_COMMON_EXTERN static AsynchIO* create(const Socket& s, + ReadCallback rCb, + EofCallback eofCb, + DisconnectCallback disCb, + ClosedCallback cCb = 0, + BuffersEmptyCallback eCb = 0, + IdleCallback iCb = 0); public: - AsynchIO(const Socket& s, - ReadCallback rCb, EofCallback eofCb, DisconnectCallback disCb, - ClosedCallback cCb = 0, BuffersEmptyCallback eCb = 0, IdleCallback iCb = 0); - void queueForDeletion(); - - void start(Poller::shared_ptr poller); - void queueReadBuffer(BufferBase* buff); - void unread(BufferBase* buff); - void queueWrite(BufferBase* buff); - void notifyPendingWrite(); - void queueWriteClose(); - bool writeQueueEmpty() { return writeQueue.empty(); } - BufferBase* getQueuedBuffer(); - -private: - ~AsynchIO(); - void readable(DispatchHandle& handle); - void writeable(DispatchHandle& handle); - void disconnected(DispatchHandle& handle); - void close(DispatchHandle& handle); + virtual void queueForDeletion() = 0; + + virtual void start(boost::shared_ptr<Poller> poller) = 0; + virtual void queueReadBuffer(BufferBase* buff) = 0; + virtual void unread(BufferBase* buff) = 0; + virtual void queueWrite(BufferBase* buff) = 0; + virtual void notifyPendingWrite() = 0; + virtual void queueWriteClose() = 0; + virtual bool writeQueueEmpty() = 0; + virtual void startReading() = 0; + virtual void stopReading() = 0; + virtual void requestCallback(RequestCallback) = 0; + virtual BufferBase* getQueuedBuffer() = 0; + +protected: + // Derived class manages lifetime; must be constructed using the + // static create() method. Deletes not allowed from outside. + AsynchIO() {} + virtual ~AsynchIO() {} }; }} diff --git a/cpp/src/qpid/sys/AsynchIOHandler.cpp b/cpp/src/qpid/sys/AsynchIOHandler.cpp index 886dbc8f43..f658b7d50f 100644 --- a/cpp/src/qpid/sys/AsynchIOHandler.cpp +++ b/cpp/src/qpid/sys/AsynchIOHandler.cpp @@ -7,9 +7,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -19,13 +19,15 @@ * */ -#include "AsynchIOHandler.h" +#include "qpid/sys/AsynchIOHandler.h" #include "qpid/sys/AsynchIO.h" #include "qpid/sys/Socket.h" #include "qpid/framing/AMQP_HighestVersion.h" #include "qpid/framing/ProtocolInitiation.h" #include "qpid/log/Statement.h" +#include <boost/bind.hpp> + namespace qpid { namespace sys { @@ -44,7 +46,8 @@ AsynchIOHandler::AsynchIOHandler(std::string id, ConnectionCodec::Factory* f) : factory(f), codec(0), readError(false), - isClient(false) + isClient(false), + readCredit(InfiniteCredit) {} AsynchIOHandler::~AsynchIOHandler() { @@ -70,19 +73,61 @@ void AsynchIOHandler::write(const framing::ProtocolInitiation& data) buff = new Buff; framing::Buffer out(buff->bytes, buff->byteCount); data.encode(out); - buff->dataCount = data.size(); + buff->dataCount = data.encodedSize(); aio->queueWrite(buff); } +void AsynchIOHandler::abort() { + // Don't disconnect if we're already disconnecting + if (!readError) { + aio->requestCallback(boost::bind(&AsynchIOHandler::eof, this, _1)); + } +} + void AsynchIOHandler::activateOutput() { aio->notifyPendingWrite(); } // Input side +void AsynchIOHandler::giveReadCredit(int32_t credit) { + // Check whether we started in the don't about credit state + if (readCredit.boolCompareAndSwap(InfiniteCredit, credit)) + return; + // TODO In theory should be able to use an atomic operation before taking the lock + // but in practice there seems to be an unexplained race in that case + ScopedLock<Mutex> l(creditLock); + if (readCredit.fetchAndAdd(credit) != 0) + return; + assert(readCredit.get() >= 0); + if (readCredit.get() != 0) + aio->startReading(); +} + void AsynchIOHandler::readbuff(AsynchIO& , AsynchIO::BufferBase* buff) { if (readError) { return; } + + // Check here for read credit + if (readCredit.get() != InfiniteCredit) { + if (readCredit.get() == 0) { + // FIXME aconway 2009-10-01: Workaround to avoid "false wakeups". + // readbuff is sometimes called with no credit. + // This should be fixed somewhere else to avoid such calls. + aio->unread(buff); + return; + } + // TODO In theory should be able to use an atomic operation before taking the lock + // but in practice there seems to be an unexplained race in that case + ScopedLock<Mutex> l(creditLock); + if (--readCredit == 0) { + assert(readCredit.get() >= 0); + if (readCredit.get() == 0) { + aio->stopReading(); + } + } + } + size_t decoded = 0; if (codec) { // Already initiated try { @@ -99,13 +144,13 @@ void AsynchIOHandler::readbuff(AsynchIO& , AsynchIO::BufferBase* buff) { decoded = in.getPosition(); QPID_LOG(debug, "RECV [" << identifier << "] INIT(" << protocolInit << ")"); try { - codec = factory->create(protocolInit.getVersion(), *this, identifier); + codec = factory->create(protocolInit.getVersion(), *this, identifier, 0); if (!codec) { //TODO: may still want to revise this... //send valid version header & close connection. write(framing::ProtocolInitiation(framing::highestProtocolVersion)); readError = true; - aio->queueWriteClose(); + aio->queueWriteClose(); } } catch (const std::exception& e) { QPID_LOG(error, e.what()); @@ -130,11 +175,12 @@ void AsynchIOHandler::readbuff(AsynchIO& , AsynchIO::BufferBase* buff) { void AsynchIOHandler::eof(AsynchIO&) { QPID_LOG(debug, "DISCONNECTED [" << identifier << "]"); if (codec) codec->closed(); + readError = true; aio->queueWriteClose(); } void AsynchIOHandler::closedSocket(AsynchIO&, const Socket& s) { - // If we closed with data still to send log a warning + // If we closed with data still to send log a warning if (!aio->writeQueueEmpty()) { QPID_LOG(warning, "CLOSING [" << identifier << "] unsent data (probably due to client disconnect)"); } @@ -154,21 +200,29 @@ void AsynchIOHandler::nobuffs(AsynchIO&) { void AsynchIOHandler::idle(AsynchIO&){ if (isClient && codec == 0) { - codec = factory->create(*this, identifier); + codec = factory->create(*this, identifier, 0); write(framing::ProtocolInitiation(codec->getVersion())); return; } if (codec == 0) return; - if (codec->canEncode()) { - // Try and get a queued buffer if not then construct new one - AsynchIO::BufferBase* buff = aio->getQueuedBuffer(); - if (!buff) buff = new Buff; - size_t encoded=codec->encode(buff->bytes, buff->byteCount); - buff->dataCount = encoded; - aio->queueWrite(buff); - } - if (codec->isClosed()) + try { + if (codec->canEncode()) { + // Try and get a queued buffer if not then construct new one + AsynchIO::BufferBase* buff = aio->getQueuedBuffer(); + if (!buff) buff = new Buff; + size_t encoded=codec->encode(buff->bytes, buff->byteCount); + buff->dataCount = encoded; + aio->queueWrite(buff); + } + if (codec->isClosed()) { + readError = true; + aio->queueWriteClose(); + } + } catch (const std::exception& e) { + QPID_LOG(error, e.what()); + readError = true; aio->queueWriteClose(); + } } }} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/AsynchIOHandler.h b/cpp/src/qpid/sys/AsynchIOHandler.h index 26e2cf4c5c..e1885bac79 100644 --- a/cpp/src/qpid/sys/AsynchIOHandler.h +++ b/cpp/src/qpid/sys/AsynchIOHandler.h @@ -9,9 +9,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -21,8 +21,11 @@ * */ -#include "OutputControl.h" -#include "ConnectionCodec.h" +#include "qpid/sys/OutputControl.h" +#include "qpid/sys/ConnectionCodec.h" +#include "qpid/sys/AtomicValue.h" +#include "qpid/sys/Mutex.h" +#include "qpid/CommonImportExport.h" namespace qpid { @@ -33,7 +36,7 @@ namespace framing { namespace sys { class AsynchIO; -class AsynchIOBufferBase; +struct AsynchIOBufferBase; class Socket; class AsynchIOHandler : public OutputControl { @@ -43,29 +46,33 @@ class AsynchIOHandler : public OutputControl { ConnectionCodec* codec; bool readError; bool isClient; + AtomicValue<int32_t> readCredit; + static const int32_t InfiniteCredit = -1; + Mutex creditLock; void write(const framing::ProtocolInitiation&); public: - AsynchIOHandler(std::string id, ConnectionCodec::Factory* f); - ~AsynchIOHandler(); - void init(AsynchIO* a, int numBuffs); + QPID_COMMON_EXTERN AsynchIOHandler(std::string id, ConnectionCodec::Factory* f); + QPID_COMMON_EXTERN ~AsynchIOHandler(); + QPID_COMMON_EXTERN void init(AsynchIO* a, int numBuffs); - void setClient() { isClient = true; } + QPID_COMMON_EXTERN void setClient() { isClient = true; } // Output side - void close(); - void activateOutput(); + QPID_COMMON_EXTERN void abort(); + QPID_COMMON_EXTERN void activateOutput(); + QPID_COMMON_EXTERN void giveReadCredit(int32_t credit); // Input side - void readbuff(AsynchIO& aio, AsynchIOBufferBase* buff); - void eof(AsynchIO& aio); - void disconnect(AsynchIO& aio); - + QPID_COMMON_EXTERN void readbuff(AsynchIO& aio, AsynchIOBufferBase* buff); + QPID_COMMON_EXTERN void eof(AsynchIO& aio); + QPID_COMMON_EXTERN void disconnect(AsynchIO& aio); + // Notifications - void nobuffs(AsynchIO& aio); - void idle(AsynchIO& aio); - void closedSocket(AsynchIO& aio, const Socket& s); + QPID_COMMON_EXTERN void nobuffs(AsynchIO& aio); + QPID_COMMON_EXTERN void idle(AsynchIO& aio); + QPID_COMMON_EXTERN void closedSocket(AsynchIO& aio, const Socket& s); }; }} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/AtomicCount.h b/cpp/src/qpid/sys/AtomicCount.h index d598b49427..94580c61f3 100644 --- a/cpp/src/qpid/sys/AtomicCount.h +++ b/cpp/src/qpid/sys/AtomicCount.h @@ -20,7 +20,7 @@ */ #include <boost/detail/atomic_count.hpp> -#include "ScopedIncrement.h" +#include "qpid/sys/ScopedIncrement.h" namespace qpid { namespace sys { diff --git a/cpp/src/qpid/sys/AtomicValue_gcc.h b/cpp/src/qpid/sys/AtomicValue_gcc.h index da60edad65..d022b07c1d 100644 --- a/cpp/src/qpid/sys/AtomicValue_gcc.h +++ b/cpp/src/qpid/sys/AtomicValue_gcc.h @@ -57,7 +57,7 @@ class AtomicValue /** If current value == testval then set to newval. Returns true if the swap was performed. */ bool boolCompareAndSwap(T testval, T newval) { return __sync_bool_compare_and_swap(&value, testval, newval); } - T get() const { return const_cast<AtomicValue<T>*>(this)->fetchAndAdd(0); } + T get() const { return const_cast<AtomicValue<T>*>(this)->fetchAndAdd(static_cast<T>(0)); } private: T value; diff --git a/cpp/src/qpid/sys/AtomicValue_mutex.h b/cpp/src/qpid/sys/AtomicValue_mutex.h index 8871addbda..e4d433e7f5 100644 --- a/cpp/src/qpid/sys/AtomicValue_mutex.h +++ b/cpp/src/qpid/sys/AtomicValue_mutex.h @@ -53,6 +53,8 @@ class AtomicValue inline T operator++(int) { return fetchAndAdd(1); } inline T operator--(int) { return fetchAndSub(1); } + AtomicValue& operator=(T newval) { Lock l(lock); value = newval; return *this; } + /** If current value == testval then set to newval. Returns the old value. */ T valueCompareAndSwap(T testval, T newval) { Lock l(lock); diff --git a/cpp/src/qpid/sys/BlockingQueue.h b/cpp/src/qpid/sys/BlockingQueue.h index c6c6291b97..210cb4ad82 100644 --- a/cpp/src/qpid/sys/BlockingQueue.h +++ b/cpp/src/qpid/sys/BlockingQueue.h @@ -22,7 +22,7 @@ * */ -#include "Waitable.h" +#include "qpid/sys/Waitable.h" #include <queue> @@ -66,14 +66,17 @@ public: return true; } - T pop() { + T pop(Duration timeout=TIME_INFINITE) { T result; - bool ok = pop(result); - assert(ok); (void) ok; // Infinite wait. + bool ok = pop(result, timeout); + if (!ok) + throw Exception("Timed out waiting on a blocking queue"); return result; } - /** Push a value onto the queue */ + /** Push a value onto the queue. + * Note it is not an error to push onto a closed queue. + */ void push(const T& t) { Mutex::ScopedLock l(waitable); queue.push(t); diff --git a/cpp/src/qpid/sys/Runnable.h b/cpp/src/qpid/sys/Codec.h index fb3927c612..ace721fbcc 100644 --- a/cpp/src/qpid/sys/Runnable.h +++ b/cpp/src/qpid/sys/Codec.h @@ -1,5 +1,6 @@ -#ifndef _Runnable_ -#define _Runnable_ +#ifndef QPID_SYS_CODEC_H +#define QPID_SYS_CODEC_H + /* * * Licensed to the Apache Software Foundation (ASF) under one @@ -20,31 +21,32 @@ * under the License. * */ - -#include <boost/function.hpp> +#include <cstddef> namespace qpid { namespace sys { /** - * Interface for objects that can be run, e.g. in a thread. + * Generic codec interface */ -class Runnable +class Codec { public: - /** Type to represent a runnable as a Functor */ - typedef boost::function0<void> Functor; - - virtual ~Runnable(); + virtual ~Codec() {} - /** Derived classes override run(). */ - virtual void run() = 0; + /** Decode from buffer, return number of bytes decoded. + * @return may be less than size if there was incomplete + * data at the end of the buffer. + */ + virtual std::size_t decode(const char* buffer, std::size_t size) = 0; - /** Create a functor object that will call this->run(). */ - Functor functor(); -}; -}} + /** Encode into buffer, return number of bytes encoded */ + virtual std::size_t encode(const char* buffer, std::size_t size) = 0; + /** Return true if we have data to encode */ + virtual bool canEncode() = 0; +}; +}} // namespace qpid::sys -#endif +#endif /*!QPID_SYS_CODEC_H*/ diff --git a/cpp/src/qpid/sys/ConnectionCodec.h b/cpp/src/qpid/sys/ConnectionCodec.h index efc6839b60..7231b1daa6 100644 --- a/cpp/src/qpid/sys/ConnectionCodec.h +++ b/cpp/src/qpid/sys/ConnectionCodec.h @@ -21,54 +21,55 @@ * under the License. * */ +#include "qpid/sys/Codec.h" #include "qpid/framing/ProtocolVersion.h" -#include "OutputControl.h" -#include <memory> -#include <map> namespace qpid { namespace sys { +class InputHandlerFactory; +class OutputControl; + /** * Interface of coder/decoder for a connection of a specific protocol * version. */ -class ConnectionCodec { +class ConnectionCodec : public Codec { public: virtual ~ConnectionCodec() {} - /** Decode from buffer, return number of bytes decoded. - * @return may be less than size if there was incomplete - * data at the end of the buffer. - */ - virtual size_t decode(const char* buffer, size_t size) = 0; - - - /** Encode into buffer, return number of bytes encoded */ - virtual size_t encode(const char* buffer, size_t size) = 0; - - /** Return true if we have data to encode */ - virtual bool canEncode() = 0; - /** Network connection was closed from other end. */ virtual void closed() = 0; virtual bool isClosed() const = 0; virtual framing::ProtocolVersion getVersion() const = 0; - + struct Factory { virtual ~Factory() {} + /** Security Strength Factor - indicates the level of security provided + * by the underlying transport. If zero, the transport provides no + * security (e.g. TCP). If non-zero, the transport provides some level + * of security (e.g. SSL). The values for SSF can be interpreted as: + * + * 0 = No protection. + * 1 = Integrity checking only. + * >1 = Supports authentication, integrity and confidentiality. + * The number represents the encryption key length. + */ + /** Return 0 if version unknown */ virtual ConnectionCodec* create( - framing::ProtocolVersion, OutputControl&, const std::string& id + framing::ProtocolVersion, OutputControl&, const std::string& id, + unsigned int conn_ssf ) = 0; /** Return "preferred" codec for outbound connections. */ virtual ConnectionCodec* create( - OutputControl&, const std::string& id + OutputControl&, const std::string& id, + unsigned int conn_ssf ) = 0; }; }; diff --git a/cpp/src/qpid/sys/ConnectionInputHandler.h b/cpp/src/qpid/sys/ConnectionInputHandler.h index a2c18d6d9a..9e85c66924 100644 --- a/cpp/src/qpid/sys/ConnectionInputHandler.h +++ b/cpp/src/qpid/sys/ConnectionInputHandler.h @@ -22,8 +22,8 @@ #define _ConnectionInputHandler_ #include "qpid/framing/InputHandler.h" -#include "OutputTask.h" -#include "TimeoutHandler.h" +#include "qpid/sys/OutputTask.h" +#include "qpid/sys/TimeoutHandler.h" namespace qpid { namespace sys { @@ -33,6 +33,7 @@ namespace sys { public TimeoutHandler, public OutputTask { public: + virtual void closed() = 0; }; diff --git a/cpp/src/qpid/sys/ConnectionInputHandlerFactory.h b/cpp/src/qpid/sys/ConnectionInputHandlerFactory.h index 2b309b5758..9bb7e13686 100644 --- a/cpp/src/qpid/sys/ConnectionInputHandlerFactory.h +++ b/cpp/src/qpid/sys/ConnectionInputHandlerFactory.h @@ -42,7 +42,8 @@ class ConnectionInputHandlerFactory : private boost::noncopyable *@param id identify the connection for management purposes. */ virtual ConnectionInputHandler* create(ConnectionOutputHandler* out, - const std::string& id) = 0; + const std::string& id, + bool isClient) = 0; virtual ~ConnectionInputHandlerFactory(){} }; diff --git a/cpp/src/qpid/sys/ConnectionOutputHandler.h b/cpp/src/qpid/sys/ConnectionOutputHandler.h index 5a60ae4998..421dd7c269 100644 --- a/cpp/src/qpid/sys/ConnectionOutputHandler.h +++ b/cpp/src/qpid/sys/ConnectionOutputHandler.h @@ -22,7 +22,7 @@ #define _ConnectionOutputHandler_ #include "qpid/framing/OutputHandler.h" -#include "OutputControl.h" +#include "qpid/sys/OutputControl.h" namespace qpid { namespace sys { @@ -34,6 +34,7 @@ class ConnectionOutputHandler : public virtual qpid::framing::OutputHandler, pub { public: virtual void close() = 0; + virtual size_t getBuffered() const { return 0; } }; }} diff --git a/cpp/src/qpid/sys/ConnectionOutputHandlerPtr.h b/cpp/src/qpid/sys/ConnectionOutputHandlerPtr.h new file mode 100644 index 0000000000..95a08d15ae --- /dev/null +++ b/cpp/src/qpid/sys/ConnectionOutputHandlerPtr.h @@ -0,0 +1,56 @@ +#ifndef QPID_SYS_CONNECTIONOUTPUTHANDLERPTR_H +#define QPID_SYS_CONNECTIONOUTPUTHANDLERPTR_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/ConnectionOutputHandler.h" + +namespace qpid { +namespace sys { + +/** + * A ConnectionOutputHandler that delegates to another + * ConnectionOutputHandler. Allows the "real" ConnectionOutputHandler + * to be changed without updating all the pointers/references + * using the ConnectionOutputHandlerPtr + */ +class ConnectionOutputHandlerPtr : public ConnectionOutputHandler +{ + public: + ConnectionOutputHandlerPtr(ConnectionOutputHandler* p) : next(p) { assert(next); } + void set(ConnectionOutputHandler* p) { next = p; assert(next); } + ConnectionOutputHandler* get() { return next; } + const ConnectionOutputHandler* get() const { return next; } + + void close() { next->close(); } + size_t getBuffered() const { return next->getBuffered(); } + void abort() { next->abort(); } + void activateOutput() { next->activateOutput(); } + void giveReadCredit(int32_t credit) { next->giveReadCredit(credit); } + void send(framing::AMQFrame& f) { next->send(f); } + + private: + ConnectionOutputHandler* next; +}; +}} // namespace qpid::sys + +#endif /*!QPID_SYS_CONNECTIONOUTPUTHANDLERPTR_H*/ diff --git a/cpp/src/qpid/sys/CopyOnWriteArray.h b/cpp/src/qpid/sys/CopyOnWriteArray.h new file mode 100644 index 0000000000..e4ae3a6094 --- /dev/null +++ b/cpp/src/qpid/sys/CopyOnWriteArray.h @@ -0,0 +1,138 @@ +#ifndef QPID_SYS_COPYONWRITEARRAY_H +#define QPID_SYS_COPYONWRITEARRAY_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Mutex.h" +#include <algorithm> +#include <vector> +#include <boost/shared_ptr.hpp> + +namespace qpid { +namespace sys { + +/** + * An array that copies on adding/removing element allowing lock-free + * iteration. + */ +template <class T> +class CopyOnWriteArray +{ +public: + typedef boost::shared_ptr<const std::vector<T> > ConstPtr; + + CopyOnWriteArray() {} + CopyOnWriteArray(const CopyOnWriteArray& c) : array(c.array) {} + + void add(T& t) + { + Mutex::ScopedLock l(lock); + ArrayPtr copy(array ? new std::vector<T>(*array) : new std::vector<T>()); + copy->push_back(t); + array = copy; + } + + bool remove(T& t) + { + Mutex::ScopedLock l(lock); + if (array && std::find(array->begin(), array->end(), t) != array->end()) { + ArrayPtr copy(new std::vector<T>(*array)); + copy->erase(std::find(copy->begin(), copy->end(), t)); + array = copy; + return true; + } else { + return false; + } + } + + bool clear() + { + Mutex::ScopedLock l(lock); + if (array && !array->empty()) { + ArrayPtr copy; + array = copy; + return true; + } else { + return false; + } + } + + template <class F> + bool add_unless(T& t, F f) + { + Mutex::ScopedLock l(lock); + if (array && std::find_if(array->begin(), array->end(), f) != array->end()) { + return false; + } else { + ArrayPtr copy(array ? new std::vector<T>(*array) : new std::vector<T>()); + copy->push_back(t); + array = copy; + return true; + } + } + + template <class F> + bool remove_if(F f) + { + Mutex::ScopedLock l(lock); + if (array && std::find_if(array->begin(), array->end(), f) != array->end()) { + ArrayPtr copy(new std::vector<T>(*array)); + copy->erase(std::remove_if(copy->begin(), copy->end(), f), copy->end()); + array = copy; + return true; + } + return false; + } + + template <class F> + F for_each(F f) + { + ArrayPtr a; + { + Mutex::ScopedLock l(lock); + a = array; + } + if (!a) return f; + return std::for_each(a->begin(), a->end(), f); + } + + ConstPtr snapshot() + { + ConstPtr a; + { + Mutex::ScopedLock l(lock); + a = array; + } + return a; + } + +private: + typedef boost::shared_ptr< std::vector<T> > ArrayPtr; + Mutex lock; + ArrayPtr array; +}; + +}} + + + +#endif /*!QPID_SYS_COPYONWRITEARRAY_H*/ diff --git a/cpp/src/qpid/sys/DeletionManager.h b/cpp/src/qpid/sys/DeletionManager.h index 43154eb98e..c1fea19f30 100644 --- a/cpp/src/qpid/sys/DeletionManager.h +++ b/cpp/src/qpid/sys/DeletionManager.h @@ -54,6 +54,8 @@ struct deleter template <typename H> class DeletionManager { + struct ThreadStatus; + public: // Mark every thread as using the handle - it will be deleted // below after every thread marks the handle as unused @@ -65,6 +67,24 @@ public: // handles get deleted here when no one else // is using them either static void markAllUnusedInThisThread() { + ThreadStatus* threadStatus = getThreadStatus(); + ScopedLock<Mutex> l(threadStatus->lock); + + // The actual deletions will happen here when all the shared_ptr + // ref counts hit 0 (that is when every thread marks the handle unused) + threadStatus->handles.clear(); + } + + static void destroyThreadState() { + ThreadStatus* threadStatus = getThreadStatus(); + allThreadsStatuses.delThreadStatus(threadStatus); + delete threadStatus; + threadStatus = 0; + } + +private: + + static ThreadStatus*& getThreadStatus() { static __thread ThreadStatus* threadStatus = 0; // Thread local vars can't be dynamically constructed so we need @@ -75,14 +95,9 @@ public: allThreadsStatuses.addThreadStatus(threadStatus); } - ScopedLock<Mutex> l(threadStatus->lock); - - // The actual deletions will happen here when all the shared_ptr - // ref counts hit 0 (that is when every thread marks the handle unused) - threadStatus->handles.clear(); + return threadStatus; } -private: typedef boost::shared_ptr<H> shared_ptr; // In theory we know that we never need more handles than the number of @@ -125,6 +140,15 @@ private: statuses.push_back(t); } + void delThreadStatus(ThreadStatus* t) { + ScopedLock<Mutex> l(lock); + typename std::vector<ThreadStatus*>::iterator it = + std::find(statuses.begin(),statuses.end(), t); + if (it != statuses.end()) { + statuses.erase(it); + } + } + void addHandle(shared_ptr h) { ScopedLock<Mutex> l(lock); std::for_each(statuses.begin(), statuses.end(), handleAdder(h)); diff --git a/cpp/src/qpid/sys/DispatchHandle.cpp b/cpp/src/qpid/sys/DispatchHandle.cpp new file mode 100644 index 0000000000..605edabc64 --- /dev/null +++ b/cpp/src/qpid/sys/DispatchHandle.cpp @@ -0,0 +1,337 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/DispatchHandle.h" + +#include <algorithm> + +#include <boost/cast.hpp> + +#include <assert.h> + +namespace qpid { +namespace sys { + +DispatchHandle::DispatchHandle(const IOHandle& h, Callback rCb, Callback wCb, Callback dCb) : + PollerHandle(h), + readableCallback(rCb), + writableCallback(wCb), + disconnectedCallback(dCb), + state(IDLE) +{ +} + + +DispatchHandle::~DispatchHandle() { +} + +void DispatchHandle::startWatch(Poller::shared_ptr poller0) { + bool r = readableCallback; + bool w = writableCallback; + + ScopedLock<Mutex> lock(stateLock); + assert(state == IDLE); + + poller = poller0; + poller->registerHandle(*this); + state = WAITING; + Poller::Direction dir = r ? + ( w ? Poller::INOUT : Poller::INPUT ) : + ( w ? Poller::OUTPUT : Poller::NONE ); + poller->monitorHandle(*this, dir); +} + +void DispatchHandle::rewatch() { + bool r = readableCallback; + bool w = writableCallback; + if (!r && !w) { + return; + } + Poller::Direction dir = r ? + ( w ? Poller::INOUT : Poller::INPUT ) : + ( w ? Poller::OUTPUT : Poller::NONE ); + + ScopedLock<Mutex> lock(stateLock); + switch(state) { + case IDLE: + case STOPPING: + case DELETING: + return; + default: + break; + } + assert(poller); + poller->monitorHandle(*this, dir); +} + +void DispatchHandle::rewatchRead() { + if (!readableCallback) { + return; + } + + ScopedLock<Mutex> lock(stateLock); + switch(state) { + case IDLE: + case STOPPING: + case DELETING: + return; + default: + break; + } + assert(poller); + poller->monitorHandle(*this, Poller::INPUT); +} + +void DispatchHandle::rewatchWrite() { + if (!writableCallback) { + return; + } + + ScopedLock<Mutex> lock(stateLock); + switch(state) { + case IDLE: + case STOPPING: + case DELETING: + return; + default: + break; + } + assert(poller); + poller->monitorHandle(*this, Poller::OUTPUT); +} + +void DispatchHandle::unwatchRead() { + if (!readableCallback) { + return; + } + + ScopedLock<Mutex> lock(stateLock); + switch(state) { + case IDLE: + case STOPPING: + case DELETING: + return; + default: + break; + } + assert(poller); + poller->unmonitorHandle(*this, Poller::INPUT); +} + +void DispatchHandle::unwatchWrite() { + if (!writableCallback) { + return; + } + + ScopedLock<Mutex> lock(stateLock); + switch(state) { + case IDLE: + case STOPPING: + case DELETING: + return; + default: + break; + } + assert(poller); + poller->unmonitorHandle(*this, Poller::OUTPUT); +} + +void DispatchHandle::unwatch() { + ScopedLock<Mutex> lock(stateLock); + switch(state) { + case IDLE: + case STOPPING: + case DELETING: + return; + default: + break; + } + assert(poller); + poller->unmonitorHandle(*this, Poller::INOUT); +} + +void DispatchHandle::stopWatch() { + ScopedLock<Mutex> lock(stateLock); + switch (state) { + case IDLE: + assert(state != IDLE); + return; + case STOPPING: + assert(state != STOPPING); + return; + case CALLING: + state = STOPPING; + break; + case WAITING: + state = IDLE; + break; + case DELETING: + return; + } + assert(poller); + poller->unregisterHandle(*this); + poller.reset(); +} + +// If we are in the IDLE/STOPPING state we can't do the callback as we've +// not/no longer got the fd registered in any poller +void DispatchHandle::call(Callback iCb) { + assert(iCb); + ScopedLock<Mutex> lock(stateLock); + switch (state) { + case IDLE: + case STOPPING: + case DELETING: + return; + default: + interruptedCallbacks.push(iCb); + assert(poller); + (void) poller->interrupt(*this); + } +} + +// The slightly strange switch structure +// is to ensure that the lock is released before +// we do the delete +void DispatchHandle::doDelete() { + { + ScopedLock<Mutex> lock(stateLock); + // Ensure that we're no longer watching anything + switch (state) { + case IDLE: + state = DELETING; + break; + case STOPPING: + state = DELETING; + return; + case WAITING: + state = DELETING; + assert(poller); + (void) poller->interrupt(*this); + poller->unregisterHandle(*this); + return; + case CALLING: + state = DELETING; + assert(poller); + poller->unregisterHandle(*this); + return; + case DELETING: + return; + } + } + // If we're IDLE we can do this right away + delete this; +} + +void DispatchHandle::processEvent(Poller::EventType type) { + + // Phase I + { + ScopedLock<Mutex> lock(stateLock); + + switch(state) { + case IDLE: + // Can get here if a non connection thread stops watching + // whilst we were stuck in the above lock + return; + case WAITING: + state = CALLING; + break; + case CALLING: + assert(state!=CALLING); + return; + case STOPPING: + assert(state!=STOPPING); + return; + case DELETING: + // Need to make sure we clean up any pending callbacks in this case + std::swap(callbacks, interruptedCallbacks); + goto saybyebye; + } + + std::swap(callbacks, interruptedCallbacks); + } + + // Do callbacks - whilst we are doing the callbacks we are prevented from processing + // the same handle until we re-enable it. To avoid rentering the callbacks for a single + // handle re-enabling in the callbacks is actually deferred until they are complete. + switch (type) { + case Poller::READABLE: + readableCallback(*this); + break; + case Poller::WRITABLE: + writableCallback(*this); + break; + case Poller::READ_WRITABLE: + readableCallback(*this); + writableCallback(*this); + break; + case Poller::DISCONNECTED: + if (disconnectedCallback) { + disconnectedCallback(*this); + } + break; + case Poller::INTERRUPTED: + { + // We could only be interrupted if we also had a callback to do + assert(callbacks.size() > 0); + // We'll actually do the interrupt below + } + break; + default: + assert(false); + } + + // If we have any callbacks do them now - + // (because we use a copy from before the previous callbacks we won't + // do anything yet that was just added) + while (callbacks.size() > 0) { + Callback cb = callbacks.front(); + assert(cb); + cb(*this); + callbacks.pop(); + } + + { + ScopedLock<Mutex> lock(stateLock); + switch (state) { + case IDLE: + assert(state!=IDLE); + return; + case STOPPING: + state = IDLE; + return; + case WAITING: + assert(state!=WAITING); + return; + case CALLING: + state = WAITING; + return; + case DELETING: + break; + } + } + +saybyebye: + delete this; +} + +}} diff --git a/cpp/src/qpid/sys/DispatchHandle.h b/cpp/src/qpid/sys/DispatchHandle.h new file mode 100644 index 0000000000..115a3c44f7 --- /dev/null +++ b/cpp/src/qpid/sys/DispatchHandle.h @@ -0,0 +1,150 @@ +#ifndef _sys_DispatchHandle_h +#define _sys_DispatchHandle_h + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Poller.h" +#include "qpid/sys/Mutex.h" +#include "qpid/CommonImportExport.h" +#include <boost/function.hpp> + +#include <queue> + +namespace qpid { +namespace sys { + +class DispatchHandleRef; +/** + * In order to have your own handle (file descriptor on Unix) watched by the poller + * you need to: + * + * - Subclass IOHandle, in the constructor supply an appropriate + * IOHandlerPrivate object for the platform. + * + * - Construct a DispatchHandle passing it your IOHandle and + * callback functions for read, write and disconnect events. + * + * - Ensure the DispatchHandle is not deleted until the poller is no longer using it. + * TODO: astitcher document DispatchHandleRef to simplify this. + * + * When an event occurs on the handle, the poller calls the relevant callback and + * stops watching that handle. Your callback can call rewatch() or related functions + * to re-enable polling. + */ +class DispatchHandle : public PollerHandle { + friend class DispatchHandleRef; +public: + typedef boost::function1<void, DispatchHandle&> Callback; + typedef std::queue<Callback> CallbackQueue; + +private: + Callback readableCallback; + Callback writableCallback; + Callback disconnectedCallback; + CallbackQueue interruptedCallbacks; + CallbackQueue callbacks; // Double buffer + Poller::shared_ptr poller; + Mutex stateLock; + enum { + IDLE, + STOPPING, + WAITING, + CALLING, + DELETING + } state; + +public: + /** + * Provide a handle to poll and a set of callbacks. Note + * callbacks can be 0, meaning you are not interested in that + * event. + * + *@param h: the handle to watch. The IOHandle encapsulates a + * platfrom-specific handle to an IO object (e.g. a file descriptor + * on Unix.) + *@param rCb Callback called when the handle is readable. + *@param wCb Callback called when the handle is writable. + *@param dCb Callback called when the handle is disconnected. + */ + QPID_COMMON_EXTERN DispatchHandle(const IOHandle& h, Callback rCb, Callback wCb, Callback dCb); + QPID_COMMON_EXTERN ~DispatchHandle(); + + /** Add this DispatchHandle to the poller to be watched. */ + QPID_COMMON_EXTERN void startWatch(Poller::shared_ptr poller); + + /** Resume watching for all non-0 callbacks. */ + QPID_COMMON_EXTERN void rewatch(); + /** Resume watching for read only. */ + QPID_COMMON_EXTERN void rewatchRead(); + + /** Resume watching for write only. */ + QPID_COMMON_EXTERN void rewatchWrite(); + + /** Stop watching temporarily. The DispatchHandle remains + associated with the poller and can be re-activated using + rewatch. */ + QPID_COMMON_EXTERN void unwatch(); + /** Stop watching for read */ + QPID_COMMON_EXTERN void unwatchRead(); + /** Stop watching for write */ + QPID_COMMON_EXTERN void unwatchWrite(); + + /** Stop watching permanently. Disassociates from the poller. */ + QPID_COMMON_EXTERN void stopWatch(); + + /** Interrupt watching this handle and make a serialised callback that respects the + * same exclusivity guarantees as the other callbacks + */ + QPID_COMMON_EXTERN void call(Callback iCb); + +protected: + QPID_COMMON_EXTERN void doDelete(); + +private: + QPID_COMMON_EXTERN void processEvent(Poller::EventType dir); +}; + +class DispatchHandleRef { + DispatchHandle* ref; + +public: + typedef boost::function1<void, DispatchHandle&> Callback; + DispatchHandleRef(const IOHandle& h, Callback rCb, Callback wCb, Callback dCb) : + ref(new DispatchHandle(h, rCb, wCb, dCb)) + {} + + ~DispatchHandleRef() { ref->doDelete(); } + + void startWatch(Poller::shared_ptr poller) { ref->startWatch(poller); } + void rewatch() { ref->rewatch(); } + void rewatchRead() { ref->rewatchRead(); } + void rewatchWrite() { ref->rewatchWrite(); } + void unwatch() { ref->unwatch(); } + void unwatchRead() { ref->unwatchRead(); } + void unwatchWrite() { ref->unwatchWrite(); } + void stopWatch() { ref->stopWatch(); } + void call(Callback iCb) { ref->call(iCb); } +}; + +}} + +#endif // _sys_DispatchHandle_h diff --git a/cpp/src/qpid/sys/Dispatcher.cpp b/cpp/src/qpid/sys/Dispatcher.cpp index 02a62c8e66..5f52dcd990 100644 --- a/cpp/src/qpid/sys/Dispatcher.cpp +++ b/cpp/src/qpid/sys/Dispatcher.cpp @@ -19,9 +19,7 @@ * */ -#include "Dispatcher.h" - -#include <boost/cast.hpp> +#include "qpid/sys/Dispatcher.h" #include <assert.h> @@ -36,404 +34,7 @@ Dispatcher::~Dispatcher() { } void Dispatcher::run() { - do { - Poller::Event event = poller->wait(); - - // If can read/write then dispatch appropriate callbacks - if (event.handle) { - event.process(); - } else { - // Handle shutdown - switch (event.type) { - case Poller::SHUTDOWN: - goto dispatcher_shutdown; - default: - // This should be impossible - assert(false); - } - } - } while (true); - -dispatcher_shutdown: - ; -} - -DispatchHandle::~DispatchHandle() { - stopWatch(); -} - -void DispatchHandle::startWatch(Poller::shared_ptr poller0) { - bool r = readableCallback; - bool w = writableCallback; - - ScopedLock<Mutex> lock(stateLock); - assert(state == IDLE); - - // If no callbacks set then do nothing (that is what we were asked to do!) - // TODO: Maybe this should be an assert instead - if (!r && !w) { - state = INACTIVE; - return; - } - - Poller::Direction d = r ? - (w ? Poller::INOUT : Poller::IN) : - Poller::OUT; - - poller = poller0; - poller->addFd(*this, d); - - state = r ? - (w ? ACTIVE_RW : ACTIVE_R) : - ACTIVE_W; -} - -void DispatchHandle::rewatch() { - bool r = readableCallback; - bool w = writableCallback; - - ScopedLock<Mutex> lock(stateLock); - switch(state) { - case IDLE: - case DELAYED_IDLE: - break; - case DELAYED_R: - case DELAYED_W: - case DELAYED_INACTIVE: - state = r ? - (w ? DELAYED_RW : DELAYED_R) : - DELAYED_W; - break; - case DELAYED_DELETE: - break; - case INACTIVE: - case ACTIVE_R: - case ACTIVE_W: { - assert(poller); - Poller::Direction d = r ? - (w ? Poller::INOUT : Poller::IN) : - Poller::OUT; - poller->modFd(*this, d); - state = r ? - (w ? ACTIVE_RW : ACTIVE_R) : - ACTIVE_W; - break; - } - case DELAYED_RW: - case ACTIVE_RW: - // Don't need to do anything already waiting for readable/writable - break; - } -} - -void DispatchHandle::rewatchRead() { - if (!readableCallback) { - return; - } - - ScopedLock<Mutex> lock(stateLock); - switch(state) { - case IDLE: - case DELAYED_IDLE: - break; - case DELAYED_R: - case DELAYED_RW: - case DELAYED_DELETE: - break; - case DELAYED_W: - state = DELAYED_RW; - break; - case DELAYED_INACTIVE: - state = DELAYED_R; - break; - case ACTIVE_R: - case ACTIVE_RW: - // Nothing to do: already waiting for readable - break; - case INACTIVE: - assert(poller); - poller->modFd(*this, Poller::IN); - state = ACTIVE_R; - break; - case ACTIVE_W: - assert(poller); - poller->modFd(*this, Poller::INOUT); - state = ACTIVE_RW; - break; - } -} - -void DispatchHandle::rewatchWrite() { - if (!writableCallback) { - return; - } - - ScopedLock<Mutex> lock(stateLock); - switch(state) { - case IDLE: - case DELAYED_IDLE: - break; - case DELAYED_W: - case DELAYED_RW: - case DELAYED_DELETE: - break; - case DELAYED_R: - state = DELAYED_RW; - break; - case DELAYED_INACTIVE: - state = DELAYED_W; - break; - case INACTIVE: - assert(poller); - poller->modFd(*this, Poller::OUT); - state = ACTIVE_W; - break; - case ACTIVE_R: - assert(poller); - poller->modFd(*this, Poller::INOUT); - state = ACTIVE_RW; - break; - case ACTIVE_W: - case ACTIVE_RW: - // Nothing to do: already waiting for writable - break; - } -} - -void DispatchHandle::unwatchRead() { - if (!readableCallback) { - return; - } - - ScopedLock<Mutex> lock(stateLock); - switch(state) { - case IDLE: - case DELAYED_IDLE: - break; - case DELAYED_R: - state = DELAYED_INACTIVE; - break; - case DELAYED_RW: - state = DELAYED_W; - break; - case DELAYED_W: - case DELAYED_INACTIVE: - case DELAYED_DELETE: - break; - case ACTIVE_R: - assert(poller); - poller->modFd(*this, Poller::NONE); - state = INACTIVE; - break; - case ACTIVE_RW: - assert(poller); - poller->modFd(*this, Poller::OUT); - state = ACTIVE_W; - break; - case ACTIVE_W: - case INACTIVE: - break; - } -} - -void DispatchHandle::unwatchWrite() { - if (!writableCallback) { - return; - } - - ScopedLock<Mutex> lock(stateLock); - switch(state) { - case IDLE: - case DELAYED_IDLE: - break; - case DELAYED_W: - state = DELAYED_INACTIVE; - break; - case DELAYED_RW: - state = DELAYED_R; - break; - case DELAYED_R: - case DELAYED_INACTIVE: - case DELAYED_DELETE: - break; - case ACTIVE_W: - assert(poller); - poller->modFd(*this, Poller::NONE); - state = INACTIVE; - break; - case ACTIVE_RW: - assert(poller); - poller->modFd(*this, Poller::IN); - state = ACTIVE_R; - break; - case ACTIVE_R: - case INACTIVE: - break; - } -} - -void DispatchHandle::unwatch() { - ScopedLock<Mutex> lock(stateLock); - switch (state) { - case IDLE: - case DELAYED_IDLE: - break; - case DELAYED_R: - case DELAYED_W: - case DELAYED_RW: - case DELAYED_INACTIVE: - state = DELAYED_INACTIVE; - break; - case DELAYED_DELETE: - break; - default: - assert(poller); - poller->modFd(*this, Poller::NONE); - state = INACTIVE; - break; - } -} - -void DispatchHandle::stopWatch() { - ScopedLock<Mutex> lock(stateLock); - switch (state) { - case IDLE: - case DELAYED_IDLE: - case DELAYED_DELETE: - return; - case DELAYED_R: - case DELAYED_W: - case DELAYED_RW: - case DELAYED_INACTIVE: - state = DELAYED_IDLE; - break; - default: - state = IDLE; - break; - } - assert(poller); - poller->delFd(*this); - poller.reset(); -} - -// The slightly strange switch structure -// is to ensure that the lock is released before -// we do the delete -void DispatchHandle::doDelete() { - // Ensure that we're no longer watching anything - stopWatch(); - - // If we're in the middle of a callback defer the delete - { - ScopedLock<Mutex> lock(stateLock); - switch (state) { - case DELAYED_IDLE: - case DELAYED_DELETE: - state = DELAYED_DELETE; - return; - case IDLE: - break; - default: - // Can only get out of stopWatch() in DELAYED_IDLE/DELAYED_DELETE/IDLE states - assert(false); - } - } - // If we're not then do it right away - delete this; -} - -void DispatchHandle::processEvent(Poller::EventType type) { - // Note that we are now doing the callbacks - { - ScopedLock<Mutex> lock(stateLock); - - // Set up to wait for same events next time unless reset - switch(state) { - case ACTIVE_R: - state = DELAYED_R; - break; - case ACTIVE_W: - state = DELAYED_W; - break; - case ACTIVE_RW: - state = DELAYED_RW; - break; - // Can only get here in a DELAYED_* state in the rare case - // that we're already here for reading and we get activated for - // writing and we can write (it might be possible the other way - // round too). In this case we're already processing the handle - // in a different thread in this function so return right away - case DELAYED_R: - case DELAYED_W: - case DELAYED_RW: - case DELAYED_INACTIVE: - case DELAYED_IDLE: - case DELAYED_DELETE: - return; - default: - assert(false); - } - } - - // Do callbacks - whilst we are doing the callbacks we are prevented from processing - // the same handle until we re-enable it. To avoid rentering the callbacks for a single - // handle re-enabling in the callbacks is actually deferred until they are complete. - switch (type) { - case Poller::READABLE: - readableCallback(*this); - break; - case Poller::WRITABLE: - writableCallback(*this); - break; - case Poller::READ_WRITABLE: - readableCallback(*this); - writableCallback(*this); - break; - case Poller::DISCONNECTED: - { - ScopedLock<Mutex> lock(stateLock); - state = DELAYED_INACTIVE; - } - if (disconnectedCallback) { - disconnectedCallback(*this); - } - break; - default: - assert(false); - } - - // If any of the callbacks re-enabled reading/writing then actually - // do it now - { - ScopedLock<Mutex> lock(stateLock); - switch (state) { - case DELAYED_R: - poller->modFd(*this, Poller::IN); - state = ACTIVE_R; - return; - case DELAYED_W: - poller->modFd(*this, Poller::OUT); - state = ACTIVE_W; - return; - case DELAYED_RW: - poller->modFd(*this, Poller::INOUT); - state = ACTIVE_RW; - return; - case DELAYED_INACTIVE: - state = INACTIVE; - return; - case DELAYED_IDLE: - state = IDLE; - return; - default: - // This should be impossible - assert(false); - return; - case DELAYED_DELETE: - break; - } - } - delete this; + poller->run(); } }} diff --git a/cpp/src/qpid/sys/Dispatcher.h b/cpp/src/qpid/sys/Dispatcher.h index 8e34354f9e..e8213d0579 100644 --- a/cpp/src/qpid/sys/Dispatcher.h +++ b/cpp/src/qpid/sys/Dispatcher.h @@ -22,139 +22,21 @@ * */ -#include "Poller.h" -#include "Runnable.h" -#include "Mutex.h" - -#include <memory> -#include <queue> -#include <boost/function.hpp> - -#include <assert.h> - +#include "qpid/sys/Poller.h" +#include "qpid/sys/Runnable.h" +#include "qpid/CommonImportExport.h" namespace qpid { namespace sys { -class DispatchHandleRef; -/** - * In order to have your own handle (file descriptor on Unix) watched by the poller - * you need to: - * - * - Subclass IOHandle, in the constructor supply an appropriate - * IOHandlerPrivate object for the platform. - * - * - Construct a DispatchHandle passing it your IOHandle and - * callback functions for read, write and disconnect events. - * - * - Ensure the DispatchHandle is not deleted until the poller is no longer using it. - * TODO: astitcher document DispatchHandleRef to simplify this. - * - * When an event occurs on the handle, the poller calls the relevant callback and - * stops watching that handle. Your callback can call rewatch() or related functions - * to re-enable polling. - */ -class DispatchHandle : public PollerHandle { - friend class DispatchHandleRef; -public: - typedef boost::function1<void, DispatchHandle&> Callback; - -private: - Callback readableCallback; - Callback writableCallback; - Callback disconnectedCallback; - Poller::shared_ptr poller; - Mutex stateLock; - enum { - IDLE, INACTIVE, ACTIVE_R, ACTIVE_W, ACTIVE_RW, - DELAYED_IDLE, DELAYED_INACTIVE, DELAYED_R, DELAYED_W, DELAYED_RW, - DELAYED_DELETE - } state; - -public: - /** - * Provide a handle to poll and a set of callbacks. Note - * callbacks can be 0, meaning you are not interested in that - * event. - * - *@param h: the handle to watch. The IOHandle encapsulates a - * platfrom-specific handle to an IO object (e.g. a file descriptor - * on Unix.) - *@param rCb Callback called when the handle is readable. - *@param wCb Callback called when the handle is writable. - *@param dCb Callback called when the handle is disconnected. - */ - DispatchHandle(const IOHandle& h, Callback rCb, Callback wCb, Callback dCb) : - PollerHandle(h), - readableCallback(rCb), - writableCallback(wCb), - disconnectedCallback(dCb), - state(IDLE) - {} - - ~DispatchHandle(); - - /** Add this DispatchHandle to the poller to be watched. */ - void startWatch(Poller::shared_ptr poller); - - /** Resume watchingn for all non-0 callbacks. */ - void rewatch(); - /** Resume watchingn for read only. */ - void rewatchRead(); - - /** Resume watchingn for write only. */ - void rewatchWrite(); - - /** Stop watching temporarily. The DispatchHandle remains - associated with the poller and can be re-activated using - rewatch. */ - void unwatch(); - /** Stop watching for read */ - void unwatchRead(); - /** Stop watching for write */ - void unwatchWrite(); - - /** Stop watching permanently. Disassociates from the poller. */ - void stopWatch(); - -protected: - /** Override to get extra processing done when the DispatchHandle is deleted. */ - void doDelete(); - -private: - void processEvent(Poller::EventType dir); -}; - -class DispatchHandleRef { - DispatchHandle* ref; - -public: - typedef boost::function1<void, DispatchHandle&> Callback; - DispatchHandleRef(const IOHandle& h, Callback rCb, Callback wCb, Callback dCb) : - ref(new DispatchHandle(h, rCb, wCb, dCb)) - {} - - ~DispatchHandleRef() { ref->doDelete(); } - - void startWatch(Poller::shared_ptr poller) { ref->startWatch(poller); } - void rewatch() { ref->rewatch(); } - void rewatchRead() { ref->rewatchRead(); } - void rewatchWrite() { ref->rewatchWrite(); } - void unwatch() { ref->unwatch(); } - void unwatchRead() { ref->unwatchRead(); } - void unwatchWrite() { ref->unwatchWrite(); } - void stopWatch() { ref->stopWatch(); } -}; - - class Dispatcher : public Runnable { const Poller::shared_ptr poller; public: - Dispatcher(Poller::shared_ptr poller); - ~Dispatcher(); + QPID_COMMON_EXTERN Dispatcher(Poller::shared_ptr poller); + QPID_COMMON_EXTERN ~Dispatcher(); - void run(); + QPID_COMMON_EXTERN void run(); }; }} diff --git a/cpp/src/qpid/sys/ExceptionHolder.h b/cpp/src/qpid/sys/ExceptionHolder.h deleted file mode 100644 index cfb971411e..0000000000 --- a/cpp/src/qpid/sys/ExceptionHolder.h +++ /dev/null @@ -1,75 +0,0 @@ -#ifndef QPID_EXCEPTIONHOLDER_H -#define QPID_EXCEPTIONHOLDER_H - -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -#include "qpid/memory.h" -#include <boost/shared_ptr.hpp> - - -namespace qpid { -namespace sys { - -struct Raisable { - virtual ~Raisable() {}; - virtual void raise() const=0; - virtual std::string what() const=0; -}; - -/** - * Holder for exceptions. Allows the thread that notices an error condition to - * create an exception and store it to be thrown by another thread. - */ -class ExceptionHolder : public Raisable { - public: - ExceptionHolder() {} - // Use default copy & assign. - - /** Take ownership of ex */ - template <class Ex> ExceptionHolder(Ex* ex) { wrap(ex); } - template <class Ex> ExceptionHolder(const boost::shared_ptr<Ex>& ex) { wrap(ex.release()); } - - template <class Ex> ExceptionHolder& operator=(Ex* ex) { wrap(ex); return *this; } - template <class Ex> ExceptionHolder& operator=(boost::shared_ptr<Ex> ex) { wrap(ex.release()); return *this; } - - void raise() const { if (wrapper.get()) wrapper->raise() ; } - std::string what() const { return wrapper->what(); } - bool empty() const { return !wrapper.get(); } - operator bool() const { return !empty(); } - void reset() { wrapper.reset(); } - - private: - template <class Ex> struct Wrapper : public Raisable { - Wrapper(Ex* ptr) : exception(ptr) {} - void raise() const { throw *exception; } - std::string what() const { return exception->what(); } - boost::shared_ptr<Ex> exception; - }; - template <class Ex> void wrap(Ex* ex) { wrapper.reset(new Wrapper<Ex>(ex)); } - boost::shared_ptr<Raisable> wrapper; -}; - - -}} // namespace qpid::sys - - -#endif /*!QPID_EXCEPTIONHOLDER_H*/ diff --git a/cpp/src/qpid/sys/FileSysDir.h b/cpp/src/qpid/sys/FileSysDir.h new file mode 100755 index 0000000000..ffe7823f0a --- /dev/null +++ b/cpp/src/qpid/sys/FileSysDir.h @@ -0,0 +1,62 @@ +#ifndef QPID_SYS_FILESYSDIR_H +#define QPID_SYS_FILESYSDIR_H + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include <string> + +namespace qpid { +namespace sys { + +/** + * @class FileSysDir + * + * Represents a filesystem directory accessible from the local host. + * This class simply checks existence of, and creates, a directory. It could + * be added to later to list contents, etc. + */ +class FileSysDir +{ + const std::string dirPath; + + public: + + FileSysDir (std::string path) : dirPath(path) {} + ~FileSysDir () {} + + /** + * Check to see if the directory exists and is a directory. Throws an + * exception if there is an error checking existence or if the path + * exists but is not a directory. + * + * @retval true if the path exists and is a directory. + * @retval false if the path does not exist. + */ + bool exists (void) const; + + void mkdir(void); + + std::string getPath () { return dirPath; } +}; + +}} // namespace qpid::sys + +#endif /*!QPID_SYS_FILESYSDIR_H*/ diff --git a/cpp/src/qpid/sys/LockFile.h b/cpp/src/qpid/sys/LockFile.h index f06cd6a47d..14a76cbf3e 100644 --- a/cpp/src/qpid/sys/LockFile.h +++ b/cpp/src/qpid/sys/LockFile.h @@ -19,7 +19,44 @@ * */ -#include "posix/LockFile.h" +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <string> + +#include "qpid/CommonImportExport.h" +#include "qpid/sys/IntegerTypes.h" + +namespace qpid { +namespace sys { + +class LockFilePrivate; + +/** + * @class LockFile + * + * LockFile represents a locked file suitable for a coarse-grain system + * lock. For example, the broker uses this to ensure that only one broker + * runs. A common usage idiom is to store the current "owner" process ID + * in the lock file - if the lock file exists, but the stored process ID + * doesn't, the old owner has probably died without cleaning up the lock + * file. + */ +class LockFile : private boost::noncopyable +{ + std::string path; + bool created; + boost::shared_ptr<LockFilePrivate> impl; + +protected: + int read(void*, size_t) const; + int write(void*, size_t) const; + +public: + QPID_COMMON_EXTERN LockFile(const std::string& path_, bool create); + QPID_COMMON_EXTERN ~LockFile(); +}; + +}} /* namespace qpid::sys */ #endif /*!_sys_LockFile_h*/ diff --git a/cpp/src/qpid/sys/LockPtr.h b/cpp/src/qpid/sys/LockPtr.h new file mode 100644 index 0000000000..738a864317 --- /dev/null +++ b/cpp/src/qpid/sys/LockPtr.h @@ -0,0 +1,89 @@ +#ifndef QPID_SYS_LOCKPTR_H +#define QPID_SYS_LOCKPTR_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Mutex.h" +#include <boost/noncopyable.hpp> + +namespace qpid { +namespace sys { + +class Mutex; + +/** + * LockPtr is a smart pointer to T. It is constructed from a volatile + * T* and a Lock (by default a Mutex). It const_casts away the + * volatile qualifier and locks the Lock for the duration of its + * + * Used in conjuntion with the "volatile" keyword to get the compiler + * to help enforce correct concurrent use of mutli-threaded objects. + * See ochttp://www.ddj.com/cpp/184403766 for a detailed discussion. + * + * To summarize the convention: + * - Declare thread-safe member functions as volatile. + * - Declare instances of the class that may be called concurrently as volatile. + * - Use LockPtr to cast away the volatile qualifier while taking a lock. + * + * This means that code calling on a concurrently-used object + * (declared volatile) can only call thread-safe (volatile) member + * functions. Code that needs to use thread-unsafe members must use a + * LockPtr, thereby acquiring the lock and making it safe to do so. + * + * A good type-safe pattern is the internally-locked object: + * - It has it's own private lock member. + * - All public functions are thread safe and declared volatile. + * - Any thread-unsafe, non-volatile functions are private. + * - Only member function implementations use LockPtr to access private functions. + * + * This encapsulates all the locking logic inside the class. + * + * One nice feature of this convention is the common case where you + * need a public, locked version of some function foo() and also a + * private unlocked version to avoid recursive locks. They can be declared as + * volatile and non-volatile overloads of the same function: + * + * // public + * void Thing::foo() volatile { LockPtr<Thing>(this, myLock)->foo(); } + * // private + * void Thing::foo() { ... do stuff ...} + */ + +template <class T, class Lock> class LockPtr : public boost::noncopyable { + public: + LockPtr(volatile T* p, Lock& l) : ptr(const_cast<T*>(p)), lock(l) { lock.lock(); } + LockPtr(volatile T* p, volatile Lock& l) : ptr(const_cast<T*>(p)), lock(const_cast<Lock&>(l)) { lock.lock(); } + ~LockPtr() { lock.unlock(); } + + T& operator*() { return *ptr; } + T* operator->() { return ptr; } + + private: + T* ptr; + Lock& lock; +}; + + +}} // namespace qpid::sys + + +#endif /*!QPID_SYS_LOCKPTR_H*/ diff --git a/cpp/src/qpid/sys/Mutex.h b/cpp/src/qpid/sys/Mutex.h deleted file mode 100644 index b4bd3a9b4a..0000000000 --- a/cpp/src/qpid/sys/Mutex.h +++ /dev/null @@ -1,89 +0,0 @@ -#ifndef _sys_Mutex_h -#define _sys_Mutex_h - -/* - * - * Copyright (c) 2006 The Apache Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -namespace qpid { -namespace sys { - -/** - * Scoped lock template: calls lock() in ctor, unlock() in dtor. - * L can be any class with lock() and unlock() functions. - */ -template <class L> -class ScopedLock -{ - public: - ScopedLock(L& l) : mutex(l) { l.lock(); } - ~ScopedLock() { mutex.unlock(); } - private: - L& mutex; -}; - -template <class L> -class ScopedUnlock -{ - public: - ScopedUnlock(L& l) : mutex(l) { l.unlock(); } - ~ScopedUnlock() { mutex.lock(); } - private: - L& mutex; -}; - -template <class L> -class ScopedRlock -{ - public: - ScopedRlock(L& l) : mutex(l) { l.rlock(); } - ~ScopedRlock() { mutex.unlock(); } - private: - L& mutex; -}; - -template <class L> -class ScopedWlock -{ - public: - ScopedWlock(L& l) : mutex(l) { l.wlock(); } - ~ScopedWlock() { mutex.unlock(); } - private: - L& mutex; -}; - -template <class L> -class ConditionalScopedLock -{ - public: - ConditionalScopedLock(L& l) : mutex(l) { acquired = l.trylock(); } - ~ConditionalScopedLock() { if (acquired) mutex.unlock(); } - bool lockAcquired() { return acquired; } - private: - L& mutex; - bool acquired; -}; - -}} - -#ifdef USE_APR_PLATFORM -#include "apr/Mutex.h" -#else -#include "posix/Mutex.h" -#endif - -#endif /*!_sys_Mutex_h*/ diff --git a/cpp/src/qpid/sys/OutputControl.h b/cpp/src/qpid/sys/OutputControl.h index d922a0d85c..eae99beb0f 100644 --- a/cpp/src/qpid/sys/OutputControl.h +++ b/cpp/src/qpid/sys/OutputControl.h @@ -7,9 +7,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -18,17 +18,22 @@ * under the License. * */ + +#include "qpid/sys/IntegerTypes.h" + #ifndef _OutputControl_ #define _OutputControl_ namespace qpid { namespace sys { - class OutputControl + class OutputControl { public: virtual ~OutputControl() {} + virtual void abort() = 0; virtual void activateOutput() = 0; + virtual void giveReadCredit(int32_t credit) = 0; }; } diff --git a/cpp/src/qpid/sys/PipeHandle.h b/cpp/src/qpid/sys/PipeHandle.h new file mode 100755 index 0000000000..8aac76996b --- /dev/null +++ b/cpp/src/qpid/sys/PipeHandle.h @@ -0,0 +1,51 @@ +#ifndef _sys_PipeHandle_h +#define _sys_PipeHandle_h + +/* +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*/ + +#include "qpid/sys/IntegerTypes.h" +#include "qpid/CommonImportExport.h" +#include <string> + +// This class is a portability wrapper around pipe fds. +// It currently exists primarily and solely for the purpose of +// integration with single-threaded components that require QMF +// integration through a signalling fd. + +namespace qpid { +namespace sys { + + class PipeHandle { + private: + int writeFd; + int readFd; + public: + QPID_COMMON_EXTERN PipeHandle(bool nonBlocking=true); + QPID_COMMON_EXTERN ~PipeHandle(); + QPID_COMMON_EXTERN int read(void* buf, size_t bufSize); + QPID_COMMON_EXTERN int write(const void* buf, size_t bufSize); + QPID_COMMON_EXTERN int getReadHandle(); + }; + +}} + +#endif /*!_sys_PipeHandle_h*/ diff --git a/cpp/src/qpid/sys/PollableCondition.h b/cpp/src/qpid/sys/PollableCondition.h new file mode 100644 index 0000000000..2eb6f2d947 --- /dev/null +++ b/cpp/src/qpid/sys/PollableCondition.h @@ -0,0 +1,64 @@ +#ifndef QPID_SYS_POLLABLECONDITION_H +#define QPID_SYS_POLLABLECONDITION_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Poller.h" +#include "qpid/CommonImportExport.h" +#include <boost/function.hpp> +#include <boost/shared_ptr.hpp> + + +namespace qpid { +namespace sys { + +class PollableConditionPrivate; + +class PollableCondition { +public: + typedef boost::function1<void, PollableCondition&> Callback; + + QPID_COMMON_EXTERN PollableCondition(const Callback& cb, + const boost::shared_ptr<sys::Poller>& poller); + + QPID_COMMON_EXTERN ~PollableCondition(); + + /** + * Set the condition. Triggers callback to Callback from Poller. + */ + QPID_COMMON_EXTERN void set(); + + /** + * Clear the condition. Stops callbacks from Poller. + */ + QPID_COMMON_EXTERN void clear(); + + private: + PollableConditionPrivate *impl; + + Callback callback; + boost::shared_ptr<sys::Poller> poller; +}; + +}} // namespace qpid::sys + +#endif /*!QPID_SYS_POLLABLECONDITION_H*/ diff --git a/cpp/src/qpid/sys/PollableQueue.h b/cpp/src/qpid/sys/PollableQueue.h new file mode 100644 index 0000000000..0786b21610 --- /dev/null +++ b/cpp/src/qpid/sys/PollableQueue.h @@ -0,0 +1,176 @@ +#ifndef QPID_SYS_POLLABLEQUEUE_H +#define QPID_SYS_POLLABLEQUEUE_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/PollableCondition.h" +#include "qpid/sys/Monitor.h" +#include "qpid/sys/Thread.h" +#include <boost/function.hpp> +#include <boost/bind.hpp> +#include <algorithm> +#include <vector> + +namespace qpid { +namespace sys { + +class Poller; + +/** + * A queue whose item processing is dispatched by sys::Poller. + * Any thread can push to the queue; items pushed trigger an event the Poller + * recognizes. When a Poller I/O thread dispatches the event, a + * user-specified callback is invoked with all items on the queue. + */ +template <class T> +class PollableQueue { + public: + typedef std::vector<T> Batch; + typedef T value_type; + + /** + * Callback to process a batch of items from the queue. + * + * @param batch Queue of values to process. Any items remaining + * on return from Callback are put back on the queue. + * @return iterator pointing to the first un-processed item in batch. + * Items from this point up to batch.end() are put back on the queue. + */ + typedef boost::function<typename Batch::const_iterator (const Batch& batch)> Callback; + + /** + * Constructor; sets necessary parameters. + * + * @param cb Callback that will be called to process items on the + * queue. Will be called from a Poller I/O thread. + * @param poller Poller to use for dispatching queue events. + */ + PollableQueue(const Callback& cb, + const boost::shared_ptr<sys::Poller>& poller); + + ~PollableQueue(); + + /** Push a value onto the queue. Thread safe */ + void push(const T& t); + + /** Start polling. */ + void start(); + + /** Stop polling and wait for the current callback, if any, to complete. */ + void stop(); + + /** Are we currently stopped?*/ + bool isStopped() const { ScopedLock l(lock); return stopped; } + + size_t size() { ScopedLock l(lock); return queue.size(); } + bool empty() { ScopedLock l(lock); return queue.empty(); } + + /** + * Allow any queued events to be processed; intended for calling + * after all dispatch threads exit the Poller loop in order to + * ensure clean shutdown with no events left on the queue. + */ + void shutdown(); + + private: + typedef sys::Monitor::ScopedLock ScopedLock; + typedef sys::Monitor::ScopedUnlock ScopedUnlock; + + void dispatch(PollableCondition& cond); + void process(); + + mutable sys::Monitor lock; + Callback callback; + PollableCondition condition; + Batch queue, batch; + Thread dispatcher; + bool stopped; +}; + +template <class T> PollableQueue<T>::PollableQueue( + const Callback& cb, const boost::shared_ptr<sys::Poller>& p) + : callback(cb), + condition(boost::bind(&PollableQueue<T>::dispatch, this, _1), p), + stopped(true) +{ +} + +template <class T> void PollableQueue<T>::start() { + ScopedLock l(lock); + if (!stopped) return; + stopped = false; + if (!queue.empty()) condition.set(); +} + +template <class T> PollableQueue<T>::~PollableQueue() { +} + +template <class T> void PollableQueue<T>::push(const T& t) { + ScopedLock l(lock); + if (queue.empty()) condition.set(); + queue.push_back(t); +} + +template <class T> void PollableQueue<T>::dispatch(PollableCondition& cond) { + ScopedLock l(lock); + assert(dispatcher.id() == 0); + dispatcher = Thread::current(); + process(); + dispatcher = Thread(); + if (queue.empty()) cond.clear(); + if (stopped) lock.notifyAll(); +} + +template <class T> void PollableQueue<T>::process() { + // Called with lock held + while (!stopped && !queue.empty()) { + assert(batch.empty()); + batch.swap(queue); + typename Batch::const_iterator putBack; + { + ScopedUnlock u(lock); // Allow concurrent push to queue. + putBack = callback(batch); + } + // put back unprocessed items. + queue.insert(queue.begin(), putBack, typename Batch::const_iterator(batch.end())); + batch.clear(); + } +} + +template <class T> void PollableQueue<T>::shutdown() { + ScopedLock l(lock); + process(); +} + +template <class T> void PollableQueue<T>::stop() { + ScopedLock l(lock); + if (stopped) return; + condition.clear(); + stopped = true; + // Avoid deadlock if stop is called from the dispatch thread + if (dispatcher.id() != Thread::current().id()) + while (dispatcher.id()) lock.wait(); +} + +}} // namespace qpid::sys + +#endif /*!QPID_SYS_POLLABLEQUEUE_H*/ diff --git a/cpp/src/qpid/sys/Poller.h b/cpp/src/qpid/sys/Poller.h index e39528bdb5..413d4242b8 100644 --- a/cpp/src/qpid/sys/Poller.h +++ b/cpp/src/qpid/sys/Poller.h @@ -22,8 +22,9 @@ * */ -#include "Time.h" - +#include "qpid/sys/Time.h" +#include "qpid/sys/Runnable.h" +#include "qpid/CommonImportExport.h" #include <boost/shared_ptr.hpp> namespace qpid { @@ -37,7 +38,7 @@ namespace sys { */ class PollerHandle; class PollerPrivate; -class Poller { +class Poller : public Runnable { PollerPrivate* const impl; public: @@ -45,8 +46,8 @@ public: enum Direction { NONE = 0, - IN, - OUT, + INPUT, + OUTPUT, INOUT }; @@ -57,7 +58,8 @@ public: READ_WRITABLE, DISCONNECTED, SHUTDOWN, - TIMEOUT + TIMEOUT, + INTERRUPTED }; struct Event { @@ -72,16 +74,31 @@ public: void process(); }; - Poller(); - ~Poller(); + QPID_COMMON_EXTERN Poller(); + QPID_COMMON_EXTERN ~Poller(); /** Note: this function is async-signal safe */ - void shutdown(); + QPID_COMMON_EXTERN void shutdown(); + + // Interrupt waiting for a specific poller handle + // returns true if we could interrupt the handle + // - in this case on return the handle is no longer being monitored, + // but we will receive an event from some invocation of poller::wait + // with the handle and the INTERRUPTED event type + // if it returns false then the handle is not being monitored by the poller + // - This can either be because it has just received an event which has been + // reported and has not been reenabled since. + // - Because it was removed from the monitoring set + // - Or because it is already being interrupted + QPID_COMMON_EXTERN bool interrupt(PollerHandle& handle); + + // Poller run loop + QPID_COMMON_EXTERN void run(); - void addFd(PollerHandle& handle, Direction dir); - void delFd(PollerHandle& handle); - void modFd(PollerHandle& handle, Direction dir); - void rearmFd(PollerHandle& handle); - Event wait(Duration timeout = TIME_INFINITE); + QPID_COMMON_EXTERN void registerHandle(PollerHandle& handle); + QPID_COMMON_EXTERN void unregisterHandle(PollerHandle& handle); + QPID_COMMON_EXTERN void monitorHandle(PollerHandle& handle, Direction dir); + QPID_COMMON_EXTERN void unmonitorHandle(PollerHandle& handle, Direction dir); + QPID_COMMON_EXTERN Event wait(Duration timeout = TIME_INFINITE); }; /** @@ -91,14 +108,15 @@ class IOHandle; class PollerHandlePrivate; class PollerHandle { friend class Poller; + friend class PollerPrivate; friend struct Poller::Event; PollerHandlePrivate* const impl; - virtual void processEvent(Poller::EventType) {}; + QPID_COMMON_EXTERN virtual void processEvent(Poller::EventType) {}; public: - PollerHandle(const IOHandle& h); - virtual ~PollerHandle(); + QPID_COMMON_EXTERN PollerHandle(const IOHandle& h); + QPID_COMMON_EXTERN virtual ~PollerHandle(); }; inline void Poller::Event::process() { diff --git a/cpp/src/qpid/sys/ProtocolFactory.h b/cpp/src/qpid/sys/ProtocolFactory.h index 4aa14d2cf6..b233b2da1a 100644 --- a/cpp/src/qpid/sys/ProtocolFactory.h +++ b/cpp/src/qpid/sys/ProtocolFactory.h @@ -22,9 +22,9 @@ * */ -#include <stdint.h> +#include "qpid/sys/IntegerTypes.h" #include "qpid/SharedObject.h" -#include "ConnectionCodec.h" +#include "qpid/sys/ConnectionCodec.h" #include <boost/function.hpp> namespace qpid { @@ -46,6 +46,7 @@ class ProtocolFactory : public qpid::SharedObject<ProtocolFactory> const std::string& host, int16_t port, ConnectionCodec::Factory* codec, ConnectFailedCallback failed) = 0; + virtual bool supports(const std::string& /*capability*/) { return false; } }; inline ProtocolFactory::~ProtocolFactory() {} diff --git a/cpp/src/qpid/sys/RdmaIOPlugin.cpp b/cpp/src/qpid/sys/RdmaIOPlugin.cpp new file mode 100644 index 0000000000..bd19247124 --- /dev/null +++ b/cpp/src/qpid/sys/RdmaIOPlugin.cpp @@ -0,0 +1,353 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/ProtocolFactory.h" + +#include "qpid/Plugin.h" +#include "qpid/broker/Broker.h" +#include "qpid/framing/AMQP_HighestVersion.h" +#include "qpid/log/Statement.h" +#include "qpid/sys/rdma/RdmaIO.h" +#include "qpid/sys/OutputControl.h" + +#include <boost/bind.hpp> +#include <boost/lexical_cast.hpp> +#include <memory> + +#include <netdb.h> + +using std::auto_ptr; +using std::string; +using std::stringstream; + +namespace qpid { +namespace sys { + +class RdmaIOHandler : public OutputControl { + Rdma::Connection::intrusive_ptr connection; + std::string identifier; + Rdma::AsynchIO* aio; + ConnectionCodec::Factory* factory; + ConnectionCodec* codec; + bool readError; + + void write(const framing::ProtocolInitiation&); + + public: + RdmaIOHandler(Rdma::Connection::intrusive_ptr& c, ConnectionCodec::Factory* f); + ~RdmaIOHandler(); + void init(Rdma::AsynchIO* a); + void start(Poller::shared_ptr poller) {aio->start(poller);} + + // Output side + void close(); + void abort(); + void activateOutput(); + void giveReadCredit(int32_t credit); + void initProtocolOut(); + + // Input side + void readbuff(Rdma::AsynchIO& aio, Rdma::Buffer* buff); + void initProtocolIn(Rdma::Buffer* buff); + + // Notifications + void full(Rdma::AsynchIO& aio); + void idle(Rdma::AsynchIO& aio); + void error(Rdma::AsynchIO& aio); +}; + +RdmaIOHandler::RdmaIOHandler(Rdma::Connection::intrusive_ptr& c, qpid::sys::ConnectionCodec::Factory* f) : + connection(c), + identifier(c->getPeerName()), + factory(f), + codec(0), + readError(false) +{ +} + +void RdmaIOHandler::init(Rdma::AsynchIO* a) { + aio = a; +} + +RdmaIOHandler::~RdmaIOHandler() { + if (codec) + codec->closed(); + delete codec; + + aio->deferDelete(); +} + +void RdmaIOHandler::write(const framing::ProtocolInitiation& data) +{ + QPID_LOG(debug, "Rdma: SENT [" << identifier << "] INIT(" << data << ")"); + Rdma::Buffer* buff = aio->getBuffer(); + framing::Buffer out(buff->bytes, buff->byteCount); + data.encode(out); + buff->dataCount = data.encodedSize(); + aio->queueWrite(buff); +} + +void RdmaIOHandler::close() { + aio->queueWriteClose(); +} + +// TODO: Dummy implementation, need to fill this in for heartbeat timeout to work +void RdmaIOHandler::abort() { +} + +void RdmaIOHandler::activateOutput() { + aio->notifyPendingWrite(); +} + +void RdmaIOHandler::idle(Rdma::AsynchIO&) { + // TODO: Shouldn't need this test as idle() should only ever be called when + // the connection is writable anyway + if ( !(aio->writable() && aio->bufferAvailable()) ) { + return; + } + if (codec == 0) return; + if (codec->canEncode()) { + Rdma::Buffer* buff = aio->getBuffer(); + size_t encoded=codec->encode(buff->bytes, buff->byteCount); + buff->dataCount = encoded; + aio->queueWrite(buff); + } + if (codec->isClosed()) + aio->queueWriteClose(); +} + +void RdmaIOHandler::initProtocolOut() { + // We mustn't have already started the conversation + // but we must be able to send + assert( codec == 0 ); + assert( aio->writable() && aio->bufferAvailable() ); + codec = factory->create(*this, identifier, 0); + write(framing::ProtocolInitiation(codec->getVersion())); +} + +void RdmaIOHandler::error(Rdma::AsynchIO&) { + close(); +} + +void RdmaIOHandler::full(Rdma::AsynchIO&) { + QPID_LOG(debug, "Rdma: buffer full [" << identifier << "]"); +} + +// TODO: Dummy implementation of read throttling +void RdmaIOHandler::giveReadCredit(int32_t) { +} + +// The logic here is subtly different from TCP as RDMA is message oriented +// so we define that an RDMA message is a frame - in this case there is no putting back +// of any message remainder - there shouldn't be any. And what we read here can't be +// smaller than a frame +void RdmaIOHandler::readbuff(Rdma::AsynchIO&, Rdma::Buffer* buff) { + if (readError) { + return; + } + size_t decoded = 0; + try { + if (codec) { + decoded = codec->decode(buff->bytes+buff->dataStart, buff->dataCount); + }else{ + // Need to start protocol processing + initProtocolIn(buff); + } + }catch(const std::exception& e){ + QPID_LOG(error, e.what()); + readError = true; + aio->queueWriteClose(); + } +} + +void RdmaIOHandler::initProtocolIn(Rdma::Buffer* buff) { + framing::Buffer in(buff->bytes+buff->dataStart, buff->dataCount); + framing::ProtocolInitiation protocolInit; + size_t decoded = 0; + if (protocolInit.decode(in)) { + decoded = in.getPosition(); + QPID_LOG(debug, "Rdma: RECV [" << identifier << "] INIT(" << protocolInit << ")"); + + codec = factory->create(protocolInit.getVersion(), *this, identifier, 0); + + // If we failed to create the codec then we don't understand the offered protocol version + if (!codec) { + // send valid version header & close connection. + write(framing::ProtocolInitiation(framing::highestProtocolVersion)); + readError = true; + aio->queueWriteClose(); + } + } +} + +class RdmaIOProtocolFactory : public ProtocolFactory { + auto_ptr<Rdma::Listener> listener; + const uint16_t listeningPort; + + public: + RdmaIOProtocolFactory(int16_t port, int backlog); + void accept(Poller::shared_ptr, ConnectionCodec::Factory*); + void connect(Poller::shared_ptr, const string& host, int16_t port, ConnectionCodec::Factory*, ConnectFailedCallback); + + uint16_t getPort() const; + string getHost() const; + + private: + bool request(Rdma::Connection::intrusive_ptr&, const Rdma::ConnectionParams&, ConnectionCodec::Factory*); + void established(Poller::shared_ptr, Rdma::Connection::intrusive_ptr&); + void connected(Poller::shared_ptr, Rdma::Connection::intrusive_ptr&, const Rdma::ConnectionParams&, ConnectionCodec::Factory*); + void connectionError(Rdma::Connection::intrusive_ptr&, Rdma::ErrorType); + void disconnected(Rdma::Connection::intrusive_ptr&); + void rejected(Rdma::Connection::intrusive_ptr&, const Rdma::ConnectionParams&, ConnectFailedCallback); +}; + +// Static instance to initialise plugin +static class RdmaIOPlugin : public Plugin { + void earlyInitialize(Target&) { + } + + void initialize(Target& target) { + // Check whether we actually have any rdma devices + if ( Rdma::deviceCount() == 0 ) { + QPID_LOG(info, "Rdma: Disabled: no rdma devices found"); + return; + } + + broker::Broker* broker = dynamic_cast<broker::Broker*>(&target); + // Only provide to a Broker + if (broker) { + const broker::Broker::Options& opts = broker->getOptions(); + ProtocolFactory::shared_ptr protocol(new RdmaIOProtocolFactory(opts.port, opts.connectionBacklog)); + QPID_LOG(notice, "Rdma: Listening on RDMA port " << protocol->getPort()); + broker->registerProtocolFactory("rdma", protocol); + } + } +} rdmaPlugin; + +RdmaIOProtocolFactory::RdmaIOProtocolFactory(int16_t port, int /*backlog*/) : + listeningPort(port) +{} + +void RdmaIOProtocolFactory::established(Poller::shared_ptr poller, Rdma::Connection::intrusive_ptr& ci) { + RdmaIOHandler* async = ci->getContext<RdmaIOHandler>(); + async->start(poller); +} + +bool RdmaIOProtocolFactory::request(Rdma::Connection::intrusive_ptr& ci, const Rdma::ConnectionParams& cp, + ConnectionCodec::Factory* f) { + try { + RdmaIOHandler* async = new RdmaIOHandler(ci, f); + Rdma::AsynchIO* aio = + new Rdma::AsynchIO(ci->getQueuePair(), + cp.maxRecvBufferSize, cp.initialXmitCredit, Rdma::DEFAULT_WR_ENTRIES, + boost::bind(&RdmaIOHandler::readbuff, async, _1, _2), + boost::bind(&RdmaIOHandler::idle, async, _1), + 0, // boost::bind(&RdmaIOHandler::full, async, _1), + boost::bind(&RdmaIOHandler::error, async, _1)); + async->init(aio); + + // Record aio so we can get it back from a connection + ci->addContext(async); + return true; + } catch (const Rdma::Exception& e) { + QPID_LOG(error, "Rdma: Cannot accept new connection (Rdma exception): " << e.what()); + } catch (const std::exception& e) { + QPID_LOG(error, "Rdma: Cannot accept new connection (unknown exception): " << e.what()); + } + + // If we get here we caught an exception so reject connection + return false; +} + +void RdmaIOProtocolFactory::connectionError(Rdma::Connection::intrusive_ptr&, Rdma::ErrorType) { +} + +void RdmaIOProtocolFactory::disconnected(Rdma::Connection::intrusive_ptr& ci) { + // If we've got a connection already tear it down, otherwise ignore + RdmaIOHandler* async = ci->getContext<RdmaIOHandler>(); + if (async) { + async->close(); + } + delete async; +} + +uint16_t RdmaIOProtocolFactory::getPort() const { + return listeningPort; // Immutable no need for lock. +} + +string RdmaIOProtocolFactory::getHost() const { + //return listener.getSockname(); + return ""; +} + +void RdmaIOProtocolFactory::accept(Poller::shared_ptr poller, ConnectionCodec::Factory* fact) { + ::sockaddr_in sin; + + sin.sin_family = AF_INET; + sin.sin_port = htons(listeningPort); + sin.sin_addr.s_addr = INADDR_ANY; + + SocketAddress sa("",boost::lexical_cast<std::string>(listeningPort)); + listener.reset( + new Rdma::Listener(sa, + Rdma::ConnectionParams(65536, Rdma::DEFAULT_WR_ENTRIES), + boost::bind(&RdmaIOProtocolFactory::established, this, poller, _1), + boost::bind(&RdmaIOProtocolFactory::connectionError, this, _1, _2), + boost::bind(&RdmaIOProtocolFactory::disconnected, this, _1), + boost::bind(&RdmaIOProtocolFactory::request, this, _1, _2, fact))); + + listener->start(poller); +} + +// Only used for outgoing connections (in federation) +void RdmaIOProtocolFactory::rejected(Rdma::Connection::intrusive_ptr&, const Rdma::ConnectionParams&, ConnectFailedCallback failed) { + failed(-1, "Connection rejected"); +} + +// Do the same as connection request and established but mark a client too +void RdmaIOProtocolFactory::connected(Poller::shared_ptr poller, Rdma::Connection::intrusive_ptr& ci, const Rdma::ConnectionParams& cp, + ConnectionCodec::Factory* f) { + (void) request(ci, cp, f); + established(poller, ci); + RdmaIOHandler* async = ci->getContext<RdmaIOHandler>(); + async->initProtocolOut(); +} + +void RdmaIOProtocolFactory::connect( + Poller::shared_ptr poller, + const std::string& host, int16_t port, + ConnectionCodec::Factory* f, + ConnectFailedCallback failed) +{ + SocketAddress sa(host, boost::lexical_cast<std::string>(port)); + Rdma::Connector* c = + new Rdma::Connector( + sa, + Rdma::ConnectionParams(8000, Rdma::DEFAULT_WR_ENTRIES), + boost::bind(&RdmaIOProtocolFactory::connected, this, poller, _1, _2, f), + boost::bind(&RdmaIOProtocolFactory::connectionError, this, _1, _2), + boost::bind(&RdmaIOProtocolFactory::disconnected, this, _1), + boost::bind(&RdmaIOProtocolFactory::rejected, this, _1, _2, failed)); + + c->start(poller); +} + +}} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/Runnable.cpp b/cpp/src/qpid/sys/Runnable.cpp index 30122c682f..325d87c91b 100644 --- a/cpp/src/qpid/sys/Runnable.cpp +++ b/cpp/src/qpid/sys/Runnable.cpp @@ -16,7 +16,7 @@ * */ -#include "Runnable.h" +#include "qpid/sys/Runnable.h" #include <boost/bind.hpp> namespace qpid { diff --git a/cpp/src/qpid/sys/SystemInfo.h b/cpp/src/qpid/sys/SecurityLayer.h index 73c3ca3c17..52bc40e352 100644 --- a/cpp/src/qpid/sys/SystemInfo.h +++ b/cpp/src/qpid/sys/SecurityLayer.h @@ -1,7 +1,8 @@ -#ifndef QPID_SYS_SYSTEMINFO_H -#define QPID_SYS_SYSTEMINFO_H +#ifndef QPID_SYS_SECURITYLAYER_H +#define QPID_SYS_SECURITYLAYER_H /* + * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -9,9 +10,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -20,25 +21,22 @@ * under the License. * */ +#include "qpid/sys/Codec.h" namespace qpid { namespace sys { /** - * Retrieve information about the system we are running on. - * Results may be dependent on OS/hardware. + * Defines interface to a SASL negotiated Security Layer (for + * encryption/integrity) */ -class SystemInfo +class SecurityLayer : public Codec { public: - /** Estimate available concurrency, e.g. number of CPU cores. - * -1 means estimate not available on this platform. - */ - static long concurrency(); + virtual void init(Codec*) = 0; + virtual ~SecurityLayer() {} }; }} // namespace qpid::sys - - -#endif /*!QPID_SYS_SYSTEMINFO_H*/ +#endif /*!QPID_SYS_SECURITYLAYER_H*/ diff --git a/cpp/src/qpid/sys/Semaphore.h b/cpp/src/qpid/sys/Semaphore.h index 3efb7ce2df..9d70f89aeb 100644 --- a/cpp/src/qpid/sys/Semaphore.h +++ b/cpp/src/qpid/sys/Semaphore.h @@ -19,7 +19,7 @@ * */ -#include "Monitor.h" +#include "qpid/sys/Monitor.h" namespace qpid { namespace sys { @@ -51,10 +51,22 @@ public: count--; } + void release(uint n) + { + Monitor::ScopedLock l(monitor); + if (count==0) monitor.notifyAll(); + count+=n; + } + void release() { + release(1); + } + + void forceLock() + { Monitor::ScopedLock l(monitor); - if (!count++) monitor.notifyAll(); + count = 0; } private: diff --git a/cpp/src/qpid/sys/Shlib.cpp b/cpp/src/qpid/sys/Shlib.cpp index 8fd3f42cc6..342d726876 100644 --- a/cpp/src/qpid/sys/Shlib.cpp +++ b/cpp/src/qpid/sys/Shlib.cpp @@ -18,7 +18,7 @@ * */ -#include "Shlib.h" +#include "qpid/sys/Shlib.h" #include "qpid/log/Statement.h" diff --git a/cpp/src/qpid/sys/Shlib.h b/cpp/src/qpid/sys/Shlib.h index e2752dc7d6..7f66cfec14 100644 --- a/cpp/src/qpid/sys/Shlib.h +++ b/cpp/src/qpid/sys/Shlib.h @@ -21,10 +21,10 @@ * under the License. * */ - + +#include "qpid/CommonImportExport.h" #include <boost/noncopyable.hpp> #include <iostream> -#include <dlfcn.h> namespace qpid { namespace sys { @@ -41,10 +41,10 @@ class Shlib { Shlib(const std::string& libname) { load(libname.c_str()); } /** Unload shared library. */ - void unload(); + QPID_COMMON_EXTERN void unload(); /** Look up symbol. */ - void* getSymbol(const char* symbol); + QPID_COMMON_EXTERN void* getSymbol(const char* symbol); /** Look up symbol in shared library, cast it to the desired * pointer type, void* by default. @@ -58,7 +58,7 @@ class Shlib { private: void* handle; - void load(const char* libname); + QPID_COMMON_EXTERN void load(const char* libname); }; /** A shared library handle that unloads the shlib in it's dtor */ @@ -67,7 +67,7 @@ class AutoShlib : public Shlib { /** Load shared library */ AutoShlib(const std::string& libname) : Shlib(libname) {} /** Calls unload() */ - ~AutoShlib() throw(); + QPID_COMMON_EXTERN ~AutoShlib() throw(); }; diff --git a/cpp/src/qpid/sys/Socket.h b/cpp/src/qpid/sys/Socket.h index dd7ef9a96d..7b50c42a3c 100644 --- a/cpp/src/qpid/sys/Socket.h +++ b/cpp/src/qpid/sys/Socket.h @@ -22,47 +22,48 @@ * */ -#include "IOHandle.h" - +#include "qpid/sys/IOHandle.h" +#include "qpid/sys/IntegerTypes.h" +#include "qpid/CommonImportExport.h" #include <string> -struct sockaddr; - namespace qpid { namespace sys { class Duration; +class SocketAddress; class Socket : public IOHandle { public: /** Create a socket wrapper for descriptor. */ - Socket(); + QPID_COMMON_EXTERN Socket(); - /** Create an initialized TCP socket */ - void createTcp() const; - /** Set timeout for read and write */ void setTimeout(const Duration& interval) const; - + /** Set socket non blocking */ void setNonblocking() const; - void connect(const std::string& host, uint16_t port) const; + QPID_COMMON_EXTERN void setTcpNoDelay() const; + + QPID_COMMON_EXTERN void connect(const std::string& host, uint16_t port) const; + QPID_COMMON_EXTERN void connect(const SocketAddress&) const; - void close() const; + QPID_COMMON_EXTERN void close() const; /** Bind to a port and start listening. *@param port 0 means choose an available port. *@param backlog maximum number of pending connections. *@return The bound port. */ - int listen(uint16_t port = 0, int backlog = 10) const; - + QPID_COMMON_EXTERN int listen(uint16_t port = 0, int backlog = 10) const; + QPID_COMMON_EXTERN int listen(const SocketAddress&, int backlog = 10) const; + /** Returns the "socket name" ie the address bound to * the near end of the socket */ - std::string getSockname() const; + QPID_COMMON_EXTERN std::string getSockname() const; /** Returns the "peer name" ie the address bound to * the remote end of the socket @@ -73,14 +74,14 @@ public: * Returns an address (host and port) for the remote end of the * socket */ - std::string getPeerAddress() const; + QPID_COMMON_EXTERN std::string getPeerAddress() const; /** * Returns an address (host and port) for the local end of the * socket */ std::string getLocalAddress() const; - uint16_t getLocalPort() const; + QPID_COMMON_EXTERN uint16_t getLocalPort() const; uint16_t getRemotePort() const; /** @@ -92,17 +93,20 @@ public: /** Accept a connection from a socket that is already listening * and has an incoming connection */ - Socket* accept(struct sockaddr *addr, socklen_t *addrlen) const; + QPID_COMMON_EXTERN Socket* accept() const; // TODO The following are raw operations, maybe they need better wrapping? - int read(void *buf, size_t count) const; - int write(const void *buf, size_t count) const; - - void setTcpNoDelay(bool nodelay) const; + QPID_COMMON_EXTERN int read(void *buf, size_t count) const; + QPID_COMMON_EXTERN int write(const void *buf, size_t count) const; private: + /** Create socket */ + void createSocket(const SocketAddress&) const; + Socket(IOHandlePrivate*); mutable std::string connectname; + mutable bool nonblocking; + mutable bool nodelay; }; }} diff --git a/cpp/src/qpid/sys/posix/PrivatePosix.h b/cpp/src/qpid/sys/SocketAddress.h index 33c0cd81bc..27b9642f2c 100644 --- a/cpp/src/qpid/sys/posix/PrivatePosix.h +++ b/cpp/src/qpid/sys/SocketAddress.h @@ -1,5 +1,5 @@ -#ifndef _sys_posix_PrivatePosix_h -#define _sys_posix_PrivatePosix_h +#ifndef _sys_SocketAddress_h +#define _sys_SocketAddress_h /* * @@ -22,31 +22,32 @@ * */ -#include "qpid/sys/Time.h" +#include "qpid/sys/IntegerTypes.h" +#include "qpid/CommonImportExport.h" +#include <string> -struct timespec; -struct timeval; +struct addrinfo; namespace qpid { namespace sys { -// Private Time related implementation details -struct timespec& toTimespec(struct timespec& ts, const Duration& t); -struct timeval& toTimeval(struct timeval& tv, const Duration& t); -Duration toTime(const struct timespec& ts); +class SocketAddress { + friend const ::addrinfo& getAddrInfo(const SocketAddress&); -// Private fd related implementation details -class IOHandlePrivate { public: - IOHandlePrivate(int f = -1) : - fd(f) - {} - - int fd; + /** Create a SocketAddress from hostname and port*/ + QPID_COMMON_EXTERN SocketAddress(const std::string& host, const std::string& port); + QPID_COMMON_EXTERN SocketAddress(const SocketAddress&); + QPID_COMMON_EXTERN SocketAddress& operator=(const SocketAddress&); + QPID_COMMON_EXTERN ~SocketAddress(); + + std::string asString() const; + +private: + std::string host; + std::string port; + mutable ::addrinfo* addrInfo; }; -int toFd(const IOHandlePrivate* h); - }} - -#endif /*!_sys_posix_PrivatePosix_h*/ +#endif /*!_sys_SocketAddress_h*/ diff --git a/cpp/src/qpid/sys/SslPlugin.cpp b/cpp/src/qpid/sys/SslPlugin.cpp new file mode 100644 index 0000000000..c143f1f1d0 --- /dev/null +++ b/cpp/src/qpid/sys/SslPlugin.cpp @@ -0,0 +1,184 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/ProtocolFactory.h" + +#include "qpid/Plugin.h" +#include "qpid/sys/ssl/check.h" +#include "qpid/sys/ssl/util.h" +#include "qpid/sys/ssl/SslHandler.h" +#include "qpid/sys/ssl/SslIo.h" +#include "qpid/sys/ssl/SslSocket.h" +#include "qpid/broker/Broker.h" +#include "qpid/log/Statement.h" + +#include <boost/bind.hpp> +#include <memory> + + +namespace qpid { +namespace sys { + +struct SslServerOptions : ssl::SslOptions +{ + uint16_t port; + bool clientAuth; + + SslServerOptions() : port(5671), + clientAuth(false) + { + addOptions() + ("ssl-port", optValue(port, "PORT"), "Port on which to listen for SSL connections") + ("ssl-require-client-authentication", optValue(clientAuth), + "Forces clients to authenticate in order to establish an SSL connection"); + } +}; + +class SslProtocolFactory : public ProtocolFactory { + const bool tcpNoDelay; + qpid::sys::ssl::SslSocket listener; + const uint16_t listeningPort; + std::auto_ptr<qpid::sys::ssl::SslAcceptor> acceptor; + + public: + SslProtocolFactory(const SslServerOptions&, int backlog, bool nodelay); + void accept(Poller::shared_ptr, ConnectionCodec::Factory*); + void connect(Poller::shared_ptr, const std::string& host, int16_t port, + ConnectionCodec::Factory*, + boost::function2<void, int, std::string> failed); + + uint16_t getPort() const; + std::string getHost() const; + bool supports(const std::string& capability); + + private: + void established(Poller::shared_ptr, const qpid::sys::ssl::SslSocket&, ConnectionCodec::Factory*, + bool isClient); +}; + +// Static instance to initialise plugin +static struct SslPlugin : public Plugin { + SslServerOptions options; + + Options* getOptions() { return &options; } + + ~SslPlugin() { ssl::shutdownNSS(); } + + void earlyInitialize(Target&) { + } + + void initialize(Target& target) { + broker::Broker* broker = dynamic_cast<broker::Broker*>(&target); + // Only provide to a Broker + if (broker) { + if (options.certDbPath.empty()) { + QPID_LOG(info, "SSL plugin not enabled, you must set --ssl-cert-db to enable it."); + } else { + try { + ssl::initNSS(options, true); + + const broker::Broker::Options& opts = broker->getOptions(); + ProtocolFactory::shared_ptr protocol(new SslProtocolFactory(options, + opts.connectionBacklog, opts.tcpNoDelay)); + QPID_LOG(notice, "Listening for SSL connections on TCP port " << protocol->getPort()); + broker->registerProtocolFactory("ssl", protocol); + } catch (const std::exception& e) { + QPID_LOG(error, "Failed to initialise SSL plugin: " << e.what()); + } + } + } + } +} sslPlugin; + +SslProtocolFactory::SslProtocolFactory(const SslServerOptions& options, int backlog, bool nodelay) : + tcpNoDelay(nodelay), listeningPort(listener.listen(options.port, backlog, options.certName, options.clientAuth)) +{} + +void SslProtocolFactory::established(Poller::shared_ptr poller, const qpid::sys::ssl::SslSocket& s, + ConnectionCodec::Factory* f, bool isClient) { + qpid::sys::ssl::SslHandler* async = new qpid::sys::ssl::SslHandler(s.getPeerAddress(), f); + + if (tcpNoDelay) { + s.setTcpNoDelay(tcpNoDelay); + QPID_LOG(info, "Set TCP_NODELAY on connection to " << s.getPeerAddress()); + } + + if (isClient) + async->setClient(); + qpid::sys::ssl::SslIO* aio = new qpid::sys::ssl::SslIO(s, + boost::bind(&qpid::sys::ssl::SslHandler::readbuff, async, _1, _2), + boost::bind(&qpid::sys::ssl::SslHandler::eof, async, _1), + boost::bind(&qpid::sys::ssl::SslHandler::disconnect, async, _1), + boost::bind(&qpid::sys::ssl::SslHandler::closedSocket, async, _1, _2), + boost::bind(&qpid::sys::ssl::SslHandler::nobuffs, async, _1), + boost::bind(&qpid::sys::ssl::SslHandler::idle, async, _1)); + + async->init(aio, 4); + aio->start(poller); +} + +uint16_t SslProtocolFactory::getPort() const { + return listeningPort; // Immutable no need for lock. +} + +std::string SslProtocolFactory::getHost() const { + return listener.getSockname(); +} + +void SslProtocolFactory::accept(Poller::shared_ptr poller, + ConnectionCodec::Factory* fact) { + acceptor.reset( + new qpid::sys::ssl::SslAcceptor(listener, + boost::bind(&SslProtocolFactory::established, this, poller, _1, fact, false))); + acceptor->start(poller); +} + +void SslProtocolFactory::connect( + Poller::shared_ptr poller, + const std::string& host, int16_t port, + ConnectionCodec::Factory* fact, + ConnectFailedCallback failed) +{ + // Note that the following logic does not cause a memory leak. + // The allocated Socket is freed either by the SslConnector + // upon connection failure or by the SslIoHandle upon connection + // shutdown. The allocated SslConnector frees itself when it + // is no longer needed. + + qpid::sys::ssl::SslSocket* socket = new qpid::sys::ssl::SslSocket(); + new qpid::sys::ssl::SslConnector (*socket, poller, host, port, + boost::bind(&SslProtocolFactory::established, this, poller, _1, fact, true), + failed); +} + +namespace +{ +const std::string SSL = "ssl"; +} + +bool SslProtocolFactory::supports(const std::string& capability) +{ + std::string s = capability; + transform(s.begin(), s.end(), s.begin(), tolower); + return s == SSL; +} + +}} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/TCPIOPlugin.cpp b/cpp/src/qpid/sys/TCPIOPlugin.cpp index f38cf88e45..39ae12c189 100644 --- a/cpp/src/qpid/sys/TCPIOPlugin.cpp +++ b/cpp/src/qpid/sys/TCPIOPlugin.cpp @@ -19,12 +19,13 @@ * */ -#include "ProtocolFactory.h" -#include "AsynchIOHandler.h" -#include "AsynchIO.h" +#include "qpid/sys/ProtocolFactory.h" +#include "qpid/sys/AsynchIOHandler.h" +#include "qpid/sys/AsynchIO.h" #include "qpid/Plugin.h" #include "qpid/sys/Socket.h" +#include "qpid/sys/Poller.h" #include "qpid/broker/Broker.h" #include "qpid/log/Statement.h" @@ -45,7 +46,7 @@ class AsynchIOProtocolFactory : public ProtocolFactory { void accept(Poller::shared_ptr, ConnectionCodec::Factory*); void connect(Poller::shared_ptr, const std::string& host, int16_t port, ConnectionCodec::Factory*, - boost::function2<void, int, std::string> failed); + ConnectFailedCallback); uint16_t getPort() const; std::string getHost() const; @@ -53,6 +54,7 @@ class AsynchIOProtocolFactory : public ProtocolFactory { private: void established(Poller::shared_ptr, const Socket&, ConnectionCodec::Factory*, bool isClient); + void connectFailed(const Socket&, int, const std::string&, ConnectFailedCallback); }; // Static instance to initialise plugin @@ -65,9 +67,10 @@ static class TCPIOPlugin : public Plugin { // Only provide to a Broker if (broker) { const broker::Broker::Options& opts = broker->getOptions(); - ProtocolFactory::shared_ptr protocol(new AsynchIOProtocolFactory(opts.port, opts.connectionBacklog, opts.tcpNoDelay)); - QPID_LOG(info, "Listening on TCP port " << protocol->getPort()); - broker->registerProtocolFactory(protocol); + ProtocolFactory::shared_ptr protocol(new AsynchIOProtocolFactory(opts.port, opts.connectionBacklog, + opts.tcpNoDelay)); + QPID_LOG(notice, "Listening on TCP port " << protocol->getPort()); + broker->registerProtocolFactory("tcp", protocol); } } } tcpPlugin; @@ -81,19 +84,20 @@ void AsynchIOProtocolFactory::established(Poller::shared_ptr poller, const Socke AsynchIOHandler* async = new AsynchIOHandler(s.getPeerAddress(), f); if (tcpNoDelay) { - s.setTcpNoDelay(tcpNoDelay); + s.setTcpNoDelay(); QPID_LOG(info, "Set TCP_NODELAY on connection to " << s.getPeerAddress()); } if (isClient) async->setClient(); - AsynchIO* aio = new AsynchIO(s, - boost::bind(&AsynchIOHandler::readbuff, async, _1, _2), - boost::bind(&AsynchIOHandler::eof, async, _1), - boost::bind(&AsynchIOHandler::disconnect, async, _1), - boost::bind(&AsynchIOHandler::closedSocket, async, _1, _2), - boost::bind(&AsynchIOHandler::nobuffs, async, _1), - boost::bind(&AsynchIOHandler::idle, async, _1)); + AsynchIO* aio = AsynchIO::create + (s, + boost::bind(&AsynchIOHandler::readbuff, async, _1, _2), + boost::bind(&AsynchIOHandler::eof, async, _1), + boost::bind(&AsynchIOHandler::disconnect, async, _1), + boost::bind(&AsynchIOHandler::closedSocket, async, _1, _2), + boost::bind(&AsynchIOHandler::nobuffs, async, _1), + boost::bind(&AsynchIOHandler::idle, async, _1)); async->init(aio, 4); aio->start(poller); @@ -110,11 +114,20 @@ std::string AsynchIOProtocolFactory::getHost() const { void AsynchIOProtocolFactory::accept(Poller::shared_ptr poller, ConnectionCodec::Factory* fact) { acceptor.reset( - new AsynchAcceptor(listener, + AsynchAcceptor::create(listener, boost::bind(&AsynchIOProtocolFactory::established, this, poller, _1, fact, false))); acceptor->start(poller); } +void AsynchIOProtocolFactory::connectFailed( + const Socket& s, int ec, const std::string& emsg, + ConnectFailedCallback failedCb) +{ + failedCb(ec, emsg); + s.close(); + delete &s; +} + void AsynchIOProtocolFactory::connect( Poller::shared_ptr poller, const std::string& host, int16_t port, @@ -128,9 +141,14 @@ void AsynchIOProtocolFactory::connect( // is no longer needed. Socket* socket = new Socket(); - new AsynchConnector (*socket, poller, host, port, - boost::bind(&AsynchIOProtocolFactory::established, this, poller, _1, fact, true), - failed); + AsynchConnector::create(*socket, + poller, + host, + port, + boost::bind(&AsynchIOProtocolFactory::established, + this, poller, _1, fact, true), + boost::bind(&AsynchIOProtocolFactory::connectFailed, + this, _1, _2, _3, failed)); } }} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/Time.h b/cpp/src/qpid/sys/Time.h deleted file mode 100644 index 6501cd0806..0000000000 --- a/cpp/src/qpid/sys/Time.h +++ /dev/null @@ -1,154 +0,0 @@ -#ifndef _sys_Time_h -#define _sys_Time_h - -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -#include <stdint.h> -#include <limits> -#include <iosfwd> - -namespace qpid { -namespace sys { - -class Duration; - -/** Class to represent an instant in time: - * The time resolution is in nanosecs, and this is held with 64 bits - * giving a total time span from about 25 million years ago to 25 million - * years hence. As an aside the internal time can sensibly be negative - * meaning before the epoch (probably 1/1/1970 although this class doesn't - * care). - * - * The AbsTime class is a value class and so you don't need to add any accessors - * to its internal state. If you think you want to replace its value,i - * You need to construct a new AbsTime and assign it, viz: - * - * AbsTime when = AbsTime::now(); - * ... - * when = AbsTime(when, 2*TIME_SEC); // Advance timer 2 secs - * - * If for some reason you need access to the internal nanosec value you need - * to convert the AbsTime to a Duration and use its conversion to int64_t, viz: - * - * AbsTime now = AbsTime::now(); - * - * int64_t ns = Duration(now); - * - * However note that the nanosecond value that is returned here is not defined to be - * anything in particular and could vary from platform to platform. - * - * There are some sensible operations that are currently missing from AbsTime, but - * nearly all that's needed can be done with a mixture of AbsTimes and Durations. - * - * For example, convenience operators to add a Duration and AbsTime returning an AbsTime - * would fit here (although you can already perform the operation with one of the AbsTime - * constructors). However trying to add 2 AbsTimes doesn't make sense. - */ -class AbsTime { - static int64_t max() { return std::numeric_limits<int64_t>::max(); } - int64_t time_ns; - - friend class Duration; - -public: - inline AbsTime() {} - inline AbsTime(const AbsTime& time0, const Duration& duration); - // Default assignment operation fine - // Default copy constructor fine - - static AbsTime now(); - inline static AbsTime FarFuture(); - bool operator==(const AbsTime& t) const { return t.time_ns == time_ns; } - template <class S> void serialize(S& s) { s(time_ns); } - - friend bool operator<(const AbsTime& a, const AbsTime& b); - friend bool operator>(const AbsTime& a, const AbsTime& b); - friend std::ostream& operator << (std::ostream&, const AbsTime&); -}; - -std::ostream& operator << (std::ostream&, const AbsTime&); - -/** Class to represent the duration between instants of time: - * As AbsTime this class also uses nanosecs for its time - * resolution. For the most part a duration can be dealt with like a - * 64 bit integer, and indeed there is an implicit conversion which - * makes this quite conveient. - */ -class Duration { - static int64_t max() { return std::numeric_limits<int64_t>::max(); } - int64_t nanosecs; - - friend class AbsTime; - -public: - inline Duration(int64_t time0); - inline explicit Duration(const AbsTime& time0); - inline explicit Duration(const AbsTime& start, const AbsTime& finish); - inline operator int64_t() const; -}; - -std::ostream& operator << (std::ostream&, const Duration&); - -AbsTime::AbsTime(const AbsTime& t, const Duration& d) : - time_ns(d == Duration::max() ? max() : t.time_ns+d.nanosecs) -{} - -AbsTime AbsTime::FarFuture() { AbsTime ff; ff.time_ns = max(); return ff;} - -inline AbsTime now() { return AbsTime::now(); } - -inline bool operator<(const AbsTime& a, const AbsTime& b) { return a.time_ns < b.time_ns; } -inline bool operator>(const AbsTime& a, const AbsTime& b) { return a.time_ns > b.time_ns; } - -Duration::Duration(int64_t time0) : - nanosecs(time0) -{} - -Duration::Duration(const AbsTime& time0) : - nanosecs(time0.time_ns) -{} - -Duration::Duration(const AbsTime& start, const AbsTime& finish) : - nanosecs(finish.time_ns - start.time_ns) -{} - -Duration::operator int64_t() const -{ return nanosecs; } - -/** Nanoseconds per second. */ -const Duration TIME_SEC = 1000*1000*1000; -/** Nanoseconds per millisecond */ -const Duration TIME_MSEC = 1000*1000; -/** Nanoseconds per microseconds. */ -const Duration TIME_USEC = 1000; -/** Nanoseconds per nanosecond. */ -const Duration TIME_NSEC = 1; - -/** Value to represent an infinite timeout */ -const Duration TIME_INFINITE = std::numeric_limits<int64_t>::max(); - -/** Time greater than any other time */ -const AbsTime FAR_FUTURE = AbsTime::FarFuture(); - -}} - -#endif /*!_sys_Time_h*/ diff --git a/cpp/src/qpid/sys/Timer.cpp b/cpp/src/qpid/sys/Timer.cpp new file mode 100644 index 0000000000..c18fd93538 --- /dev/null +++ b/cpp/src/qpid/sys/Timer.cpp @@ -0,0 +1,179 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/Timer.h" +#include "qpid/sys/Mutex.h" +#include "qpid/log/Statement.h" + +#include <numeric> + +using boost::intrusive_ptr; +using std::max; + +namespace qpid { +namespace sys { + +TimerTask::TimerTask(Duration timeout) : + sortTime(AbsTime::FarFuture()), + period(timeout), + nextFireTime(AbsTime::now(), timeout), + cancelled(false) +{} + +TimerTask::TimerTask(AbsTime time) : + sortTime(AbsTime::FarFuture()), + period(0), + nextFireTime(time), + cancelled(false) +{} + +TimerTask::~TimerTask() {} + +bool TimerTask::readyToFire() const { + return !(nextFireTime > AbsTime::now()); +} + +void TimerTask::fireTask() { + cancelled = true; + fire(); +} + +// This can only be used to setup the next fire time. After the Timer has already fired +void TimerTask::setupNextFire() { + if (period && readyToFire()) { + nextFireTime = max(AbsTime::now(), AbsTime(nextFireTime, period)); + cancelled = false; + } else { + QPID_LOG(error, "Couldn't setup next timer firing: " << Duration(nextFireTime, AbsTime::now()) << "[" << period << "]"); + } +} + +// Only allow tasks to be delayed +void TimerTask::restart() { nextFireTime = max(nextFireTime, AbsTime(AbsTime::now(), period)); } + +void TimerTask::cancel() { + ScopedLock<Mutex> l(callbackLock); + cancelled = true; +} + +Timer::Timer() : + active(false) +{ + start(); +} + +Timer::~Timer() +{ + stop(); +} + +// TODO AStitcher 21/08/09 The threshholds for emitting warnings are a little arbitrary +void Timer::run() +{ + Monitor::ScopedLock l(monitor); + while (active) { + if (tasks.empty()) { + monitor.wait(); + } else { + intrusive_ptr<TimerTask> t = tasks.top(); + tasks.pop(); + assert(!(t->nextFireTime < t->sortTime)); + + // warn on extreme lateness + AbsTime start(AbsTime::now()); + Duration delay(t->sortTime, start); + { + ScopedLock<Mutex> l(t->callbackLock); + if (t->cancelled) { + if (delay > 500 * TIME_MSEC) { + QPID_LOG(debug, "cancelled Timer woken up " << delay / TIME_MSEC << "ms late"); + } + continue; + } else if(Duration(t->nextFireTime, start) >= 0) { + Monitor::ScopedUnlock u(monitor); + t->fireTask(); + // Warn on callback overrun + AbsTime end(AbsTime::now()); + Duration overrun(tasks.top()->nextFireTime, end); + bool late = delay > 50 * TIME_MSEC; + bool overran = overrun > 2 * TIME_MSEC; + if (late) + if (overran) { + QPID_LOG(warning, + "Timer woken up " << delay / TIME_MSEC << "ms late, " + "overrunning by " << overrun / TIME_MSEC << "ms [taking " + << Duration(start, end) << "]"); + } else { + QPID_LOG(warning, "Timer woken up " << delay / TIME_MSEC << "ms late"); + } else if (overran) { + QPID_LOG(warning, + "Timer callback overran by " << overrun / TIME_MSEC << "ms [taking " + << Duration(start, end) << "]"); + } + continue; + } else { + // If the timer was adjusted into the future it might no longer + // be the next event, so push and then get top to make sure + // You can only push events into the future + t->sortTime = t->nextFireTime; + tasks.push(t); + } + } + monitor.wait(tasks.top()->sortTime); + } + } +} + +void Timer::add(intrusive_ptr<TimerTask> task) +{ + Monitor::ScopedLock l(monitor); + task->sortTime = task->nextFireTime; + tasks.push(task); + monitor.notify(); +} + +void Timer::start() +{ + Monitor::ScopedLock l(monitor); + if (!active) { + active = true; + runner = Thread(this); + } +} + +void Timer::stop() +{ + { + Monitor::ScopedLock l(monitor); + if (!active) return; + active = false; + monitor.notifyAll(); + } + runner.join(); +} + +bool operator<(const intrusive_ptr<TimerTask>& a, + const intrusive_ptr<TimerTask>& b) +{ + // Lower priority if time is later + return a.get() && b.get() && a->sortTime > b->sortTime; +} + +}} diff --git a/cpp/src/qpid/sys/Timer.h b/cpp/src/qpid/sys/Timer.h new file mode 100644 index 0000000000..5748503841 --- /dev/null +++ b/cpp/src/qpid/sys/Timer.h @@ -0,0 +1,94 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#ifndef sys_Timer +#define sys_Timer + +#include "qpid/sys/Monitor.h" +#include "qpid/sys/Mutex.h" +#include "qpid/sys/Thread.h" +#include "qpid/sys/Runnable.h" +#include "qpid/RefCounted.h" +#include "qpid/CommonImportExport.h" +#include <memory> +#include <queue> + +#include <boost/intrusive_ptr.hpp> + +namespace qpid { +namespace sys { + +class Timer; + +class TimerTask : public RefCounted { + friend class Timer; + friend bool operator<(const boost::intrusive_ptr<TimerTask>&, + const boost::intrusive_ptr<TimerTask>&); + + AbsTime sortTime; + Duration period; + AbsTime nextFireTime; + Mutex callbackLock; + volatile bool cancelled; + + bool readyToFire() const; + void fireTask(); + +public: + QPID_COMMON_EXTERN TimerTask(Duration period); + QPID_COMMON_EXTERN TimerTask(AbsTime fireTime); + QPID_COMMON_EXTERN virtual ~TimerTask(); + + QPID_COMMON_EXTERN void setupNextFire(); + QPID_COMMON_EXTERN void restart(); + QPID_COMMON_EXTERN void cancel(); + +protected: + // Must be overridden with callback + virtual void fire() = 0; +}; + +// For the priority_queue order +bool operator<(const boost::intrusive_ptr<TimerTask>& a, + const boost::intrusive_ptr<TimerTask>& b); + +class Timer : private Runnable { + qpid::sys::Monitor monitor; + std::priority_queue<boost::intrusive_ptr<TimerTask> > tasks; + qpid::sys::Thread runner; + bool active; + + // Runnable interface + void run(); + +public: + QPID_COMMON_EXTERN Timer(); + QPID_COMMON_EXTERN ~Timer(); + + QPID_COMMON_EXTERN void add(boost::intrusive_ptr<TimerTask> task); + QPID_COMMON_EXTERN void start(); + QPID_COMMON_EXTERN void stop(); +}; + + +}} + + +#endif diff --git a/cpp/src/qpid/sys/Condition.h b/cpp/src/qpid/sys/alloca.h index 961c15e1ee..e989670e4f 100644 --- a/cpp/src/qpid/sys/Condition.h +++ b/cpp/src/qpid/sys/alloca.h @@ -1,8 +1,7 @@ -#ifndef _sys_Condition_h -#define _sys_Condition_h +#ifndef QPID_SYS_ALLOCA_H +#define QPID_SYS_ALLOCA_H /* - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -10,9 +9,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -22,10 +21,19 @@ * */ -#ifdef USE_APR_PLATFORM -#include "apr/Condition.h" -#else -#include "posix/Condition.h" +#if (defined(_WINDOWS) || defined (WIN32)) && defined(_MSC_VER) +#include <malloc.h> +#ifdef alloc +# undef alloc +#endif +#define alloc _alloc +#ifdef alloca +# undef alloca +#endif +#define alloca _alloca +#endif +#if !defined _WINDOWS && !defined WIN32 +#include <alloca.h> #endif -#endif /*!_sys_Condition_h*/ +#endif /*!QPID_SYS_ALLOCA_H*/ diff --git a/cpp/src/qpid/sys/apr/APRBase.cpp b/cpp/src/qpid/sys/apr/APRBase.cpp index 724c489303..8bdba66bdc 100644 --- a/cpp/src/qpid/sys/apr/APRBase.cpp +++ b/cpp/src/qpid/sys/apr/APRBase.cpp @@ -20,7 +20,7 @@ */ #include <iostream> #include "qpid/log/Statement.h" -#include "APRBase.h" +#include "qpid/sys/apr/APRBase.h" using namespace qpid::sys; diff --git a/cpp/src/qpid/sys/apr/APRPool.cpp b/cpp/src/qpid/sys/apr/APRPool.cpp index e8b71f6e8a..e221bfc2f1 100644 --- a/cpp/src/qpid/sys/apr/APRPool.cpp +++ b/cpp/src/qpid/sys/apr/APRPool.cpp @@ -19,8 +19,8 @@ * */ -#include "APRPool.h" -#include "APRBase.h" +#include "qpid/sys/apr/APRPool.h" +#include "qpid/sys/apr/APRBase.h" #include <boost/pool/detail/singleton.hpp> using namespace qpid::sys; diff --git a/cpp/src/qpid/sys/apr/Condition.h b/cpp/src/qpid/sys/apr/Condition.h index 5e544219ab..66d465ca75 100644 --- a/cpp/src/qpid/sys/apr/Condition.h +++ b/cpp/src/qpid/sys/apr/Condition.h @@ -22,7 +22,7 @@ * */ -#include "APRPool.h" +#include "qpid/sys/apr/APRPool.h" #include "qpid/sys/Mutex.h" #include "qpid/sys/Time.h" diff --git a/cpp/src/qpid/sys/apr/Mutex.h b/cpp/src/qpid/sys/apr/Mutex.h index 51089c98ff..cb75f5b339 100644 --- a/cpp/src/qpid/sys/apr/Mutex.h +++ b/cpp/src/qpid/sys/apr/Mutex.h @@ -19,8 +19,8 @@ * */ -#include "APRBase.h" -#include "APRPool.h" +#include "qpid/sys/apr/APRBase.h" +#include "qpid/sys/apr/APRPool.h" #include <boost/noncopyable.hpp> #include <apr_thread_mutex.h> diff --git a/cpp/src/qpid/sys/apr/Shlib.cpp b/cpp/src/qpid/sys/apr/Shlib.cpp index b0ba706713..b7ee13a03b 100644 --- a/cpp/src/qpid/sys/apr/Shlib.cpp +++ b/cpp/src/qpid/sys/apr/Shlib.cpp @@ -19,8 +19,8 @@ */ #include "qpid/sys/Shlib.h" -#include "APRBase.h" -#include "APRPool.h" +#include "qpid/sys/apr/APRBase.h" +#include "qpid/sys/apr/APRPool.h" #include <apr_dso.h> namespace qpid { diff --git a/cpp/src/qpid/sys/apr/Socket.cpp b/cpp/src/qpid/sys/apr/Socket.cpp index 577268844a..d9024d11c1 100644 --- a/cpp/src/qpid/sys/apr/Socket.cpp +++ b/cpp/src/qpid/sys/apr/Socket.cpp @@ -22,8 +22,8 @@ #include "qpid/sys/Socket.h" -#include "APRBase.h" -#include "APRPool.h" +#include "qpid/sys/apr/APRBase.h" +#include "qpid/sys/apr/APRPool.h" #include <apr_network_io.h> diff --git a/cpp/src/qpid/sys/apr/Thread.cpp b/cpp/src/qpid/sys/apr/Thread.cpp index 3369ef7eb1..b52d0e6ace 100644 --- a/cpp/src/qpid/sys/apr/Thread.cpp +++ b/cpp/src/qpid/sys/apr/Thread.cpp @@ -19,7 +19,7 @@ * */ -#include "Thread.h" +#include "qpid/sys/apr/Thread.h" #include "qpid/sys/Runnable.h" using namespace qpid::sys; diff --git a/cpp/src/qpid/sys/apr/Thread.h b/cpp/src/qpid/sys/apr/Thread.h index 8cbbc0456e..6cc63db5c9 100644 --- a/cpp/src/qpid/sys/apr/Thread.h +++ b/cpp/src/qpid/sys/apr/Thread.h @@ -22,8 +22,8 @@ * */ -#include "APRPool.h" -#include "APRBase.h" +#include "qpid/sys/apr/APRPool.h" +#include "qpid/sys/apr/APRBase.h" #include <apr_thread_proc.h> #include <apr_portable.h> diff --git a/cpp/src/qpid/sys/cyrus/CyrusSecurityLayer.cpp b/cpp/src/qpid/sys/cyrus/CyrusSecurityLayer.cpp new file mode 100644 index 0000000000..454ce62495 --- /dev/null +++ b/cpp/src/qpid/sys/cyrus/CyrusSecurityLayer.cpp @@ -0,0 +1,127 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/cyrus/CyrusSecurityLayer.h" +#include <algorithm> +#include "qpid/framing/reply_exceptions.h" +#include "qpid/log/Statement.h" +#include <string.h> + +namespace qpid { +namespace sys { +namespace cyrus { + +CyrusSecurityLayer::CyrusSecurityLayer(sasl_conn_t* c, uint16_t maxFrameSize) : + conn(c), decrypted(0), decryptedSize(0), encrypted(0), encryptedSize(0), codec(0), maxInputSize(0), + decodeBuffer(maxFrameSize), encodeBuffer(maxFrameSize), encoded(0) +{ + const void* value(0); + int result = sasl_getprop(conn, SASL_MAXOUTBUF, &value); + if (result != SASL_OK) { + throw framing::InternalErrorException(QPID_MSG("SASL encode error: " << sasl_errdetail(conn))); + } + maxInputSize = *(reinterpret_cast<const unsigned*>(value)); +} + +size_t CyrusSecurityLayer::decode(const char* input, size_t size) +{ + size_t inStart = 0; + do { + size_t inSize = std::min(size - inStart, maxInputSize); + int result = sasl_decode(conn, input + inStart, inSize, &decrypted, &decryptedSize); + if (result != SASL_OK) { + throw framing::InternalErrorException(QPID_MSG("SASL decode error: " << sasl_errdetail(conn))); + } + inStart += inSize; + size_t copied = 0; + do { + size_t count = std::min(decryptedSize - copied, decodeBuffer.size - decodeBuffer.position); + ::memcpy(decodeBuffer.data + decodeBuffer.position, decrypted + copied, count); + copied += count; + decodeBuffer.position += count; + size_t decodedSize = codec->decode(decodeBuffer.data, decodeBuffer.position); + if (decodedSize < decodeBuffer.position) { + ::memmove(decodeBuffer.data, decodeBuffer.data + decodedSize, decodeBuffer.position - decodedSize); + } + decodeBuffer.position -= decodedSize; + } while (copied < decryptedSize); + } while (inStart < size); + return size; +} + +size_t CyrusSecurityLayer::encode(const char* buffer, size_t size) +{ + size_t processed = 0;//records how many bytes have been written to buffer + do { + if (!encrypted) { + if (!encoded) { + encodeBuffer.position = 0; + encoded = codec->encode(encodeBuffer.data, encodeBuffer.size); + if (!encoded) break;//nothing more to do + } + + size_t encryptable = std::min(encoded, maxInputSize); + int result = sasl_encode(conn, encodeBuffer.data + encodeBuffer.position, encryptable, &encrypted, &encryptedSize); + if (result != SASL_OK) { + throw framing::InternalErrorException(QPID_MSG("SASL encode error: " << sasl_errdetail(conn))); + } + encodeBuffer.position += encryptable; + encoded -= encryptable; + } + size_t remaining = size - processed; + if (remaining < encryptedSize) { + //can't fit all encrypted data in the buffer we've + //been given, copy in what we can and hold on to the + //rest until the next call + ::memcpy(const_cast<char*>(buffer + processed), encrypted, remaining); + processed += remaining; + encrypted += remaining; + encryptedSize -= remaining; + } else { + ::memcpy(const_cast<char*>(buffer + processed), encrypted, encryptedSize); + processed += encryptedSize; + encrypted = 0; + encryptedSize = 0; + } + } while (processed < size); + return processed; +} + +bool CyrusSecurityLayer::canEncode() +{ + return encrypted || codec->canEncode(); +} + +void CyrusSecurityLayer::init(qpid::sys::Codec* c) +{ + codec = c; +} + +CyrusSecurityLayer::DataBuffer::DataBuffer(size_t s) : position(0), size(s) +{ + data = new char[size]; +} + +CyrusSecurityLayer::DataBuffer::~DataBuffer() +{ + delete[] data; +} + +}}} // namespace qpid::sys::cyrus diff --git a/cpp/src/qpid/sys/cyrus/CyrusSecurityLayer.h b/cpp/src/qpid/sys/cyrus/CyrusSecurityLayer.h new file mode 100644 index 0000000000..1645cf1a58 --- /dev/null +++ b/cpp/src/qpid/sys/cyrus/CyrusSecurityLayer.h @@ -0,0 +1,68 @@ +#ifndef QPID_SYS_CYRUS_CYRUSSECURITYLAYER_H +#define QPID_SYS_CYRUS_CYRUSSECURITYLAYER_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/IntegerTypes.h" +#include "qpid/sys/SecurityLayer.h" +#include <sasl/sasl.h> + +namespace qpid { +namespace sys { +namespace cyrus { + + +/** + * Implementation of SASL security layer using cyrus-sasl library + */ +class CyrusSecurityLayer : public qpid::sys::SecurityLayer +{ + public: + CyrusSecurityLayer(sasl_conn_t*, uint16_t maxFrameSize); + size_t decode(const char* buffer, size_t size); + size_t encode(const char* buffer, size_t size); + bool canEncode(); + void init(qpid::sys::Codec*); + private: + struct DataBuffer + { + char* data; + size_t position; + const size_t size; + DataBuffer(size_t); + ~DataBuffer(); + }; + + sasl_conn_t* conn; + const char* decrypted; + unsigned decryptedSize; + const char* encrypted; + unsigned encryptedSize; + qpid::sys::Codec* codec; + size_t maxInputSize; + DataBuffer decodeBuffer; + DataBuffer encodeBuffer; + size_t encoded; +}; +}}} // namespace qpid::sys::cyrus + +#endif /*!QPID_SYS_CYRUS_CYRUSSECURITYLAYER_H*/ diff --git a/cpp/src/qpid/sys/epoll/EpollPoller.cpp b/cpp/src/qpid/sys/epoll/EpollPoller.cpp index 5e20e312e0..d7f64f3b4c 100644 --- a/cpp/src/qpid/sys/epoll/EpollPoller.cpp +++ b/cpp/src/qpid/sys/epoll/EpollPoller.cpp @@ -7,9 +7,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -25,18 +25,20 @@ #include "qpid/sys/DeletionManager.h" #include "qpid/sys/posix/check.h" #include "qpid/sys/posix/PrivatePosix.h" +#include "qpid/log/Statement.h" #include <sys/epoll.h> #include <errno.h> +#include <signal.h> #include <assert.h> -#include <vector> +#include <queue> #include <exception> namespace qpid { namespace sys { -// Deletion manager to handle deferring deletion of PollerHandles to when they definitely aren't being used +// Deletion manager to handle deferring deletion of PollerHandles to when they definitely aren't being used DeletionManager<PollerHandlePrivate> PollerHandleDeletionManager; // Instantiate (and define) class static for DeletionManager @@ -45,6 +47,7 @@ DeletionManager<PollerHandlePrivate>::AllThreadsStatuses DeletionManager<PollerH class PollerHandlePrivate { friend class Poller; + friend class PollerPrivate; friend class PollerHandle; enum FDStat { @@ -53,28 +56,36 @@ class PollerHandlePrivate { INACTIVE, HUNGUP, MONITORED_HUNGUP, + INTERRUPTED, + INTERRUPTED_HUNGUP, DELETED }; - int fd; ::__uint32_t events; + const IOHandlePrivate* ioHandle; PollerHandle* pollerHandle; FDStat stat; Mutex lock; - PollerHandlePrivate(int f, PollerHandle* p) : - fd(f), + PollerHandlePrivate(const IOHandlePrivate* h, PollerHandle* p) : events(0), + ioHandle(h), pollerHandle(p), stat(ABSENT) { } + int fd() const { + return toFd(ioHandle); + } + bool isActive() const { return stat == MONITORED || stat == MONITORED_HUNGUP; } void setActive() { - stat = (stat == HUNGUP) ? MONITORED_HUNGUP : MONITORED; + stat = (stat == HUNGUP || stat == INTERRUPTED_HUNGUP) + ? MONITORED_HUNGUP + : MONITORED; } bool isInactive() const { @@ -94,14 +105,27 @@ class PollerHandlePrivate { } bool isHungup() const { - return stat == MONITORED_HUNGUP || stat == HUNGUP; + return + stat == MONITORED_HUNGUP || + stat == HUNGUP || + stat == INTERRUPTED_HUNGUP; } void setHungup() { assert(stat == MONITORED); stat = HUNGUP; } - + + bool isInterrupted() const { + return stat == INTERRUPTED || stat == INTERRUPTED_HUNGUP; + } + + void setInterrupted() { + stat = (stat == MONITORED_HUNGUP || stat == HUNGUP) + ? INTERRUPTED_HUNGUP + : INTERRUPTED; + } + bool isDeleted() const { return stat == DELETED; } @@ -112,13 +136,22 @@ class PollerHandlePrivate { }; PollerHandle::PollerHandle(const IOHandle& h) : - impl(new PollerHandlePrivate(toFd(h.impl), this)) + impl(new PollerHandlePrivate(h.impl, this)) {} PollerHandle::~PollerHandle() { + { ScopedLock<Mutex> l(impl->lock); - if (impl->isActive()) { + if (impl->isDeleted()) { + return; + } + impl->pollerHandle = 0; + if (impl->isInterrupted()) { impl->setDeleted(); + return; + } + assert(impl->isIdle()); + impl->setDeleted(); } PollerHandleDeletionManager.markForDeletion(impl); } @@ -134,7 +167,7 @@ class PollerPrivate { struct ReadablePipe { int fds[2]; - + /** * This encapsulates an always readable pipe which we can add * to the epoll set to force epoll_wait to return @@ -144,31 +177,69 @@ class PollerPrivate { // Just write the pipe's fds to the pipe QPID_POSIX_CHECK(::write(fds[1], fds, 2)); } - + ~ReadablePipe() { ::close(fds[0]); ::close(fds[1]); } - + int getFD() { return fds[0]; } }; - + static ReadablePipe alwaysReadable; - + static int alwaysReadableFd; + + class InterruptHandle: public PollerHandle { + std::queue<PollerHandle*> handles; + + void processEvent(Poller::EventType) { + PollerHandle* handle = handles.front(); + handles.pop(); + assert(handle); + + // Synthesise event + Poller::Event event(handle, Poller::INTERRUPTED); + + // Process synthesised event + event.process(); + } + + public: + InterruptHandle() : + PollerHandle(DummyIOHandle) + {} + + void addHandle(PollerHandle& h) { + handles.push(&h); + } + + PollerHandle* getHandle() { + PollerHandle* handle = handles.front(); + handles.pop(); + return handle; + } + + bool queuedHandles() { + return handles.size() > 0; + } + }; + const int epollFd; bool isShutdown; + InterruptHandle interruptHandle; + ::sigset_t sigMask; static ::__uint32_t directionToEpollEvent(Poller::Direction dir) { switch (dir) { - case Poller::IN: return ::EPOLLIN; - case Poller::OUT: return ::EPOLLOUT; - case Poller::INOUT: return ::EPOLLIN | ::EPOLLOUT; + case Poller::INPUT: return ::EPOLLIN; + case Poller::OUTPUT: return ::EPOLLOUT; + case Poller::INOUT: return ::EPOLLIN | ::EPOLLOUT; default: return 0; } } - + static Poller::EventType epollToDirection(::__uint32_t events) { // POLLOUT & POLLHUP are mutually exclusive really, but at least socketpairs // can give you both! @@ -188,81 +259,161 @@ class PollerPrivate { epollFd(::epoll_create(DefaultFds)), isShutdown(false) { QPID_POSIX_CHECK(epollFd); + ::sigemptyset(&sigMask); + // Add always readable fd into our set (but not listening to it yet) + ::epoll_event epe; + epe.events = 0; + epe.data.u64 = 1; + QPID_POSIX_CHECK(::epoll_ctl(epollFd, EPOLL_CTL_ADD, alwaysReadableFd, &epe)); } ~PollerPrivate() { // It's probably okay to ignore any errors here as there can't be data loss ::close(epollFd); + + // Need to put the interruptHandle in idle state to delete it + static_cast<PollerHandle&>(interruptHandle).impl->setIdle(); + } + + void resetMode(PollerHandlePrivate& handle); + + void interrupt() { + ::epoll_event epe; + // Use EPOLLONESHOT so we only wake a single thread + epe.events = ::EPOLLIN | ::EPOLLONESHOT; + epe.data.u64 = 0; // Keep valgrind happy + epe.data.ptr = &static_cast<PollerHandle&>(interruptHandle); + QPID_POSIX_CHECK(::epoll_ctl(epollFd, EPOLL_CTL_MOD, alwaysReadableFd, &epe)); + } + + void interruptAll() { + ::epoll_event epe; + // Not EPOLLONESHOT, so we eventually get all threads + epe.events = ::EPOLLIN; + epe.data.u64 = 2; // Keep valgrind happy + QPID_POSIX_CHECK(::epoll_ctl(epollFd, EPOLL_CTL_MOD, alwaysReadableFd, &epe)); } }; PollerPrivate::ReadablePipe PollerPrivate::alwaysReadable; +int PollerPrivate::alwaysReadableFd = alwaysReadable.getFD(); -void Poller::addFd(PollerHandle& handle, Direction dir) { +void Poller::registerHandle(PollerHandle& handle) { PollerHandlePrivate& eh = *handle.impl; ScopedLock<Mutex> l(eh.lock); + assert(eh.isIdle()); + ::epoll_event epe; - int op; - - if (eh.isIdle()) { - op = EPOLL_CTL_ADD; - epe.events = PollerPrivate::directionToEpollEvent(dir) | ::EPOLLONESHOT; - } else { - assert(eh.isActive()); - op = EPOLL_CTL_MOD; - epe.events = eh.events | PollerPrivate::directionToEpollEvent(dir); - } + epe.events = ::EPOLLONESHOT; + epe.data.u64 = 0; // Keep valgrind happy epe.data.ptr = &eh; - - QPID_POSIX_CHECK(::epoll_ctl(impl->epollFd, op, eh.fd, &epe)); - - // Record monitoring state of this fd - eh.events = epe.events; + + QPID_POSIX_CHECK(::epoll_ctl(impl->epollFd, EPOLL_CTL_ADD, eh.fd(), &epe)); + eh.setActive(); } -void Poller::delFd(PollerHandle& handle) { +void Poller::unregisterHandle(PollerHandle& handle) { PollerHandlePrivate& eh = *handle.impl; ScopedLock<Mutex> l(eh.lock); assert(!eh.isIdle()); - int rc = ::epoll_ctl(impl->epollFd, EPOLL_CTL_DEL, eh.fd, 0); + + int rc = ::epoll_ctl(impl->epollFd, EPOLL_CTL_DEL, eh.fd(), 0); // Ignore EBADF since deleting a nonexistent fd has the overall required result! // And allows the case where a sloppy program closes the fd and then does the delFd() if (rc == -1 && errno != EBADF) { - QPID_POSIX_CHECK(rc); + QPID_POSIX_CHECK(rc); } + eh.setIdle(); } -// modFd is equivalent to delFd followed by addFd -void Poller::modFd(PollerHandle& handle, Direction dir) { +void PollerPrivate::resetMode(PollerHandlePrivate& eh) { + PollerHandle* ph; + { + ScopedLock<Mutex> l(eh.lock); + assert(!eh.isActive()); + + if (eh.isIdle() || eh.isDeleted()) { + return; + } + + if (eh.events==0) { + eh.setActive(); + return; + } + + if (!eh.isInterrupted()) { + ::epoll_event epe; + epe.events = eh.events | ::EPOLLONESHOT; + epe.data.u64 = 0; // Keep valgrind happy + epe.data.ptr = &eh; + + QPID_POSIX_CHECK(::epoll_ctl(epollFd, EPOLL_CTL_MOD, eh.fd(), &epe)); + + eh.setActive(); + return; + } + ph = eh.pollerHandle; + } + + PollerHandlePrivate& ihp = *static_cast<PollerHandle&>(interruptHandle).impl; + ScopedLock<Mutex> l(ihp.lock); + interruptHandle.addHandle(*ph); + ihp.setActive(); + interrupt(); +} + +void Poller::monitorHandle(PollerHandle& handle, Direction dir) { PollerHandlePrivate& eh = *handle.impl; ScopedLock<Mutex> l(eh.lock); assert(!eh.isIdle()); - + + ::__uint32_t oldEvents = eh.events; + eh.events |= PollerPrivate::directionToEpollEvent(dir); + + // If no change nothing more to do - avoid unnecessary system call + if (oldEvents==eh.events) { + return; + } + + // If we're not actually listening wait till we are to perform change + if (!eh.isActive()) { + return; + } + ::epoll_event epe; - epe.events = PollerPrivate::directionToEpollEvent(dir) | ::EPOLLONESHOT; + epe.events = eh.events | ::EPOLLONESHOT; + epe.data.u64 = 0; // Keep valgrind happy epe.data.ptr = &eh; - - QPID_POSIX_CHECK(::epoll_ctl(impl->epollFd, EPOLL_CTL_MOD, eh.fd, &epe)); - - // Record monitoring state of this fd - eh.events = epe.events; - eh.setActive(); + + QPID_POSIX_CHECK(::epoll_ctl(impl->epollFd, EPOLL_CTL_MOD, eh.fd(), &epe)); } -void Poller::rearmFd(PollerHandle& handle) { +void Poller::unmonitorHandle(PollerHandle& handle, Direction dir) { PollerHandlePrivate& eh = *handle.impl; ScopedLock<Mutex> l(eh.lock); - assert(eh.isInactive()); + assert(!eh.isIdle()); + + ::__uint32_t oldEvents = eh.events; + eh.events &= ~PollerPrivate::directionToEpollEvent(dir); + + // If no change nothing more to do - avoid unnecessary system call + if (oldEvents==eh.events) { + return; + } + + // If we're not actually listening wait till we are to perform change + if (!eh.isActive()) { + return; + } ::epoll_event epe; - epe.events = eh.events; + epe.events = eh.events | ::EPOLLONESHOT; + epe.data.u64 = 0; // Keep valgrind happy epe.data.ptr = &eh; - QPID_POSIX_CHECK(::epoll_ctl(impl->epollFd, EPOLL_CTL_MOD, eh.fd, &epe)); - - eh.setActive(); + QPID_POSIX_CHECK(::epoll_ctl(impl->epollFd, EPOLL_CTL_MOD, eh.fd(), &epe)); } void Poller::shutdown() { @@ -273,63 +424,182 @@ void Poller::shutdown() { if (impl->isShutdown) return; - // Don't use any locking here - isshutdown will be visible to all + // Don't use any locking here - isShutdown will be visible to all // after the epoll_ctl() anyway (it's a memory barrier) impl->isShutdown = true; - - // Add always readable fd to epoll (not EPOLLONESHOT) - int fd = impl->alwaysReadable.getFD(); - ::epoll_event epe; - epe.events = ::EPOLLIN; - epe.data.ptr = 0; - QPID_POSIX_CHECK(::epoll_ctl(impl->epollFd, EPOLL_CTL_ADD, fd, &epe)); + + impl->interruptAll(); +} + +bool Poller::interrupt(PollerHandle& handle) { + { + PollerHandlePrivate& eh = *handle.impl; + ScopedLock<Mutex> l(eh.lock); + if (eh.isIdle() || eh.isDeleted()) { + return false; + } + + if (eh.isInterrupted()) { + return true; + } + + // Stop monitoring handle for read or write + ::epoll_event epe; + epe.events = 0; + epe.data.u64 = 0; // Keep valgrind happy + epe.data.ptr = &eh; + QPID_POSIX_CHECK(::epoll_ctl(impl->epollFd, EPOLL_CTL_MOD, eh.fd(), &epe)); + + if (eh.isInactive()) { + eh.setInterrupted(); + return true; + } + eh.setInterrupted(); + } + + PollerPrivate::InterruptHandle& ih = impl->interruptHandle; + PollerHandlePrivate& eh = *static_cast<PollerHandle&>(ih).impl; + ScopedLock<Mutex> l(eh.lock); + ih.addHandle(handle); + + impl->interrupt(); + eh.setActive(); + return true; +} + +void Poller::run() { + // Ensure that we exit thread responsibly under all circumstances + try { + // Make sure we can't be interrupted by signals at a bad time + ::sigset_t ss; + ::sigfillset(&ss); + ::pthread_sigmask(SIG_SETMASK, &ss, 0); + + do { + Event event = wait(); + + // If can read/write then dispatch appropriate callbacks + if (event.handle) { + event.process(); + } else { + // Handle shutdown + switch (event.type) { + case SHUTDOWN: + PollerHandleDeletionManager.destroyThreadState(); + return; + default: + // This should be impossible + assert(false); + } + } + } while (true); + } catch (const std::exception& e) { + QPID_LOG(error, "IO worker thread exiting with unhandled exception: " << e.what()); + } + PollerHandleDeletionManager.destroyThreadState(); } Poller::Event Poller::wait(Duration timeout) { + static __thread PollerHandlePrivate* lastReturnedHandle = 0; epoll_event epe; int timeoutMs = (timeout == TIME_INFINITE) ? -1 : timeout / TIME_MSEC; + AbsTime targetTimeout = + (timeout == TIME_INFINITE) ? + FAR_FUTURE : + AbsTime(now(), timeout); + + if (lastReturnedHandle) { + impl->resetMode(*lastReturnedHandle); + lastReturnedHandle = 0; + } - // Repeat until we weren't interupted + // Repeat until we weren't interrupted by signal do { PollerHandleDeletionManager.markAllUnusedInThisThread(); + // Need to run on kernels without epoll_pwait() + // - fortunately in this case we don't really need the atomicity of epoll_pwait() +#if 1 + sigset_t os; + pthread_sigmask(SIG_SETMASK, &impl->sigMask, &os); int rc = ::epoll_wait(impl->epollFd, &epe, 1, timeoutMs); - - if (impl->isShutdown) { - PollerHandleDeletionManager.markAllUnusedInThisThread(); - return Event(0, SHUTDOWN); - } - + pthread_sigmask(SIG_SETMASK, &os, 0); +#else + int rc = ::epoll_pwait(impl->epollFd, &epe, 1, timeoutMs, &impl->sigMask); +#endif + if (rc ==-1 && errno != EINTR) { QPID_POSIX_CHECK(rc); } else if (rc > 0) { assert(rc == 1); - PollerHandlePrivate& eh = *static_cast<PollerHandlePrivate*>(epe.data.ptr); - + void* dataPtr = epe.data.ptr; + + // Check if this is an interrupt + PollerPrivate::InterruptHandle& interruptHandle = impl->interruptHandle; + if (dataPtr == &interruptHandle) { + PollerHandle* wrappedHandle = 0; + { + ScopedLock<Mutex> l(interruptHandle.impl->lock); + if (interruptHandle.impl->isActive()) { + wrappedHandle = interruptHandle.getHandle(); + // If there is an interrupt queued behind this one we need to arm it + // We do it this way so that another thread can pick it up + if (interruptHandle.queuedHandles()) { + impl->interrupt(); + interruptHandle.impl->setActive(); + } else { + interruptHandle.impl->setInactive(); + } + } + } + if (wrappedHandle) { + PollerHandlePrivate& eh = *wrappedHandle->impl; + { + ScopedLock<Mutex> l(eh.lock); + if (!eh.isDeleted()) { + if (!eh.isIdle()) { + eh.setInactive(); + } + lastReturnedHandle = &eh; + assert(eh.pollerHandle == wrappedHandle); + return Event(wrappedHandle, INTERRUPTED); + } + } + PollerHandleDeletionManager.markForDeletion(&eh); + } + continue; + } + + // Check for shutdown + if (impl->isShutdown) { + PollerHandleDeletionManager.markAllUnusedInThisThread(); + return Event(0, SHUTDOWN); + } + + PollerHandlePrivate& eh = *static_cast<PollerHandlePrivate*>(dataPtr); ScopedLock<Mutex> l(eh.lock); - + // the handle could have gone inactive since we left the epoll_wait if (eh.isActive()) { PollerHandle* handle = eh.pollerHandle; + assert(handle); // If the connection has been hungup we could still be readable // (just not writable), allow us to readable until we get here again if (epe.events & ::EPOLLHUP) { if (eh.isHungup()) { + eh.setInactive(); + // Don't set up last Handle so that we don't reset this handle + // on re-entering Poller::wait. This means that we will never + // be set active again once we've returned disconnected, and so + // can never be returned again. return Event(handle, DISCONNECTED); } eh.setHungup(); } else { eh.setInactive(); } + lastReturnedHandle = &eh; return Event(handle, PollerPrivate::epollToDirection(epe.events)); - } else if (eh.isDeleted()) { - // The handle has been deleted whilst still active and so must be removed - // from the poller - int rc = ::epoll_ctl(impl->epollFd, EPOLL_CTL_DEL, eh.fd, 0); - // Ignore EBADF since it's quite likely that we could race with closing the fd - if (rc == -1 && errno != EBADF) { - QPID_POSIX_CHECK(rc); - } } } // We only get here if one of the following: @@ -340,10 +610,12 @@ Poller::Event Poller::wait(Duration timeout) { // The only things we can do here are return a timeout or wait more. // Obviously if we timed out we return timeout; if the wait was meant to // be indefinite then we should never return with a time out so we go again. - // If the wait wasn't indefinite, but we were interrupted then we have to return - // with a timeout as we don't know how long we've waited so far and so we can't - // continue the wait. - if (rc == 0 || timeoutMs != -1) { + // If the wait wasn't indefinite, we check whether we are after the target wait + // time or not + if (timeoutMs == -1) { + continue; + } + if (rc == 0 && now() > targetTimeout) { PollerHandleDeletionManager.markAllUnusedInThisThread(); return Event(0, TIMEOUT); } diff --git a/cpp/src/qpid/sys/posix/AsynchIO.cpp b/cpp/src/qpid/sys/posix/AsynchIO.cpp index 7598eefe83..67b5cf0534 100644 --- a/cpp/src/qpid/sys/posix/AsynchIO.cpp +++ b/cpp/src/qpid/sys/posix/AsynchIO.cpp @@ -21,13 +21,16 @@ #include "qpid/sys/AsynchIO.h" #include "qpid/sys/Socket.h" +#include "qpid/sys/SocketAddress.h" +#include "qpid/sys/Poller.h" +#include "qpid/sys/DispatchHandle.h" #include "qpid/sys/Time.h" #include "qpid/log/Statement.h" -#include "check.h" +#include "qpid/sys/posix/check.h" -// TODO The basic algorithm here is not really POSIX specific and with a bit more abstraction -// could (should) be promoted to be platform portable +// TODO The basic algorithm here is not really POSIX specific and with a +// bit more abstraction could (should) be promoted to be platform portable #include <unistd.h> #include <sys/socket.h> #include <signal.h> @@ -35,18 +38,21 @@ #include <string.h> #include <boost/bind.hpp> +#include <boost/lexical_cast.hpp> using namespace qpid::sys; namespace { -/* - * Make *process* not generate SIGPIPE when writing to closed - * pipe/socket (necessary as default action is to terminate process) - */ -void ignoreSigpipe() { - ::signal(SIGPIPE, SIG_IGN); -} +struct StaticInit { + StaticInit() { + /** + * Make *process* not generate SIGPIPE when writing to closed + * pipe/socket (necessary as default action is to terminate process) + */ + ::signal(SIGPIPE, SIG_IGN); + }; +} init; /* * We keep per thread state to avoid locking overhead. The assumption is that @@ -65,14 +71,37 @@ __thread int64_t threadMaxReadTimeNs = 2 * 1000000; // start at 2ms /* * Asynch Acceptor */ +namespace qpid { +namespace sys { +namespace posix { + +class AsynchAcceptor : public qpid::sys::AsynchAcceptor { +public: + AsynchAcceptor(const Socket& s, AsynchAcceptor::Callback callback); + ~AsynchAcceptor(); + void start(Poller::shared_ptr poller); + +private: + void readable(DispatchHandle& handle); + +private: + AsynchAcceptor::Callback acceptedCallback; + DispatchHandle handle; + const Socket& socket; -AsynchAcceptor::AsynchAcceptor(const Socket& s, Callback callback) : +}; + +AsynchAcceptor::AsynchAcceptor(const Socket& s, + AsynchAcceptor::Callback callback) : acceptedCallback(callback), handle(s, boost::bind(&AsynchAcceptor::readable, this, _1), 0, 0), socket(s) { s.setNonblocking(); - ignoreSigpipe(); +} + +AsynchAcceptor::~AsynchAcceptor() { + handle.stopWatch(); } void AsynchAcceptor::start(Poller::shared_ptr poller) { @@ -89,7 +118,7 @@ void AsynchAcceptor::readable(DispatchHandle& h) { // TODO: Currently we ignore the peers address, perhaps we should // log it or use it for connection acceptance. try { - s = socket.accept(0, 0); + s = socket.accept(); if (s) { acceptedCallback(*s); } else { @@ -97,6 +126,7 @@ void AsynchAcceptor::readable(DispatchHandle& h) { } } catch (const std::exception& e) { QPID_LOG(error, "Could not accept socket: " << e.what()); + break; } } while (true); @@ -104,8 +134,32 @@ void AsynchAcceptor::readable(DispatchHandle& h) { } /* - * Asynch Connector + * POSIX version of AsynchIO TCP socket connector. + * + * The class is implemented in terms of DispatchHandle to allow it to be + * deleted by deleting the contained DispatchHandle. */ +class AsynchConnector : public qpid::sys::AsynchConnector, + private DispatchHandle { + +private: + void connComplete(DispatchHandle& handle); + void failure(int, const std::string&); + +private: + ConnectedCallback connCallback; + FailedCallback failCallback; + std::string errMsg; + const Socket& socket; + +public: + AsynchConnector(const Socket& socket, + Poller::shared_ptr poller, + std::string hostname, + uint16_t port, + ConnectedCallback connCb, + FailedCallback failCb); +}; AsynchConnector::AsynchConnector(const Socket& s, Poller::shared_ptr poller, @@ -122,12 +176,17 @@ AsynchConnector::AsynchConnector(const Socket& s, socket(s) { socket.setNonblocking(); + SocketAddress sa(hostname, boost::lexical_cast<std::string>(port)); try { - socket.connect(hostname, port); - startWatch(poller); + socket.connect(sa); } catch(std::exception& e) { - failure(-1, std::string(e.what())); + // Defer reporting failure + startWatch(poller); + errMsg = e.what(); + DispatchHandle::call(boost::bind(&AsynchConnector::failure, this, -1, errMsg)); + return; } + startWatch(poller); } void AsynchConnector::connComplete(DispatchHandle& h) @@ -139,25 +198,87 @@ void AsynchConnector::connComplete(DispatchHandle& h) connCallback(socket); DispatchHandle::doDelete(); } else { - // TODO: This need to be fixed as strerror isn't thread safe - failure(errCode, std::string(::strerror(errCode))); + failure(errCode, strError(errCode)); } } -void AsynchConnector::failure(int errCode, std::string message) +void AsynchConnector::failure(int errCode, const std::string& message) { - if (failCallback) - failCallback(errCode, message); - - socket.close(); - delete &socket; + failCallback(socket, errCode, message); DispatchHandle::doDelete(); } /* - * Asynch reader/writer + * POSIX version of AsynchIO reader/writer + * + * The class is implemented in terms of DispatchHandle to allow it to be + * deleted by deleting the contained DispatchHandle. */ +class AsynchIO : public qpid::sys::AsynchIO, private DispatchHandle { + +public: + AsynchIO(const Socket& s, + ReadCallback rCb, + EofCallback eofCb, + DisconnectCallback disCb, + ClosedCallback cCb = 0, + BuffersEmptyCallback eCb = 0, + IdleCallback iCb = 0); + + // Methods inherited from qpid::sys::AsynchIO + + virtual void queueForDeletion(); + + virtual void start(Poller::shared_ptr poller); + virtual void queueReadBuffer(BufferBase* buff); + virtual void unread(BufferBase* buff); + virtual void queueWrite(BufferBase* buff); + virtual void notifyPendingWrite(); + virtual void queueWriteClose(); + virtual bool writeQueueEmpty(); + virtual void startReading(); + virtual void stopReading(); + virtual void requestCallback(RequestCallback); + virtual BufferBase* getQueuedBuffer(); + +private: + ~AsynchIO(); + + // Methods that are callback targets from Dispatcher. + void readable(DispatchHandle& handle); + void writeable(DispatchHandle& handle); + void disconnected(DispatchHandle& handle); + void requestedCall(RequestCallback); + void close(DispatchHandle& handle); + +private: + ReadCallback readCallback; + EofCallback eofCallback; + DisconnectCallback disCallback; + ClosedCallback closedCallback; + BuffersEmptyCallback emptyCallback; + IdleCallback idleCallback; + const Socket& socket; + std::deque<BufferBase*> bufferQueue; + std::deque<BufferBase*> writeQueue; + bool queuedClose; + /** + * This flag is used to detect and handle concurrency between + * calls to notifyPendingWrite() (which can be made from any thread) and + * the execution of the writeable() method (which is always on the + * thread processing this handle. + */ + volatile bool writePending; + /** + * This records whether we've been reading is flow controlled: + * it's safe as a simple boolean as the only way to be stopped + * is in calls only allowed in the callback context, the only calls + * checking it are also in calls only allowed in callback context. + */ + volatile bool readingStopped; +}; + AsynchIO::AsynchIO(const Socket& s, ReadCallback rCb, EofCallback eofCb, DisconnectCallback disCb, ClosedCallback cCb, BuffersEmptyCallback eCb, IdleCallback iCb) : @@ -174,7 +295,8 @@ AsynchIO::AsynchIO(const Socket& s, idleCallback(iCb), socket(s), queuedClose(false), - writePending(false) { + writePending(false), + readingStopped(false) { s.setNonblocking(); } @@ -202,8 +324,11 @@ void AsynchIO::queueReadBuffer(BufferBase* buff) { assert(buff); buff->dataStart = 0; buff->dataCount = 0; + + bool queueWasEmpty = bufferQueue.empty(); bufferQueue.push_back(buff); - DispatchHandle::rewatchRead(); + if (queueWasEmpty && !readingStopped) + DispatchHandle::rewatchRead(); } void AsynchIO::unread(BufferBase* buff) { @@ -212,15 +337,18 @@ void AsynchIO::unread(BufferBase* buff) { memmove(buff->bytes, buff->bytes+buff->dataStart, buff->dataCount); buff->dataStart = 0; } + + bool queueWasEmpty = bufferQueue.empty(); bufferQueue.push_front(buff); - DispatchHandle::rewatchRead(); + if (queueWasEmpty && !readingStopped) + DispatchHandle::rewatchRead(); } void AsynchIO::queueWrite(BufferBase* buff) { assert(buff); // If we've already closed the socket then throw the write away if (queuedClose) { - bufferQueue.push_front(buff); + queueReadBuffer(buff); return; } else { writeQueue.push_front(buff); @@ -229,6 +357,7 @@ void AsynchIO::queueWrite(BufferBase* buff) { DispatchHandle::rewatchWrite(); } +// This can happen outside the callback context void AsynchIO::notifyPendingWrite() { writePending = true; DispatchHandle::rewatchWrite(); @@ -239,6 +368,33 @@ void AsynchIO::queueWriteClose() { DispatchHandle::rewatchWrite(); } +bool AsynchIO::writeQueueEmpty() { + return writeQueue.empty(); +} + +// This can happen outside the callback context +void AsynchIO::startReading() { + readingStopped = false; + DispatchHandle::rewatchRead(); +} + +void AsynchIO::stopReading() { + readingStopped = true; + DispatchHandle::unwatchRead(); +} + +void AsynchIO::requestCallback(RequestCallback callback) { + // TODO creating a function object every time isn't all that + // efficient - if this becomes heavily used do something better (what?) + assert(callback); + DispatchHandle::call(boost::bind(&AsynchIO::requestedCall, this, callback)); +} + +void AsynchIO::requestedCall(RequestCallback callback) { + assert(callback); + callback(*this); +} + /** Return a queued buffer if there are enough * to spare */ @@ -277,6 +433,11 @@ void AsynchIO::readable(DispatchHandle& h) { readTotal += rc; readCallback(*this, buff); + if (readingStopped) { + // We have been flow controlled. + break; + } + if (rc != readCount) { // If we didn't fill the read buffer then time to stop reading break; @@ -303,7 +464,7 @@ void AsynchIO::readable(DispatchHandle& h) { break; } else { // Report error then just treat as a socket disconnect - QPID_LOG(error, "Error reading socket: " << qpid::sys::strError(rc) << "(" << rc << ")" ); + QPID_LOG(error, "Error reading socket: " << qpid::sys::strError(errno) << "(" << errno << ")" ); eofCallback(*this); h.unwatchRead(); break; @@ -427,3 +588,33 @@ void AsynchIO::close(DispatchHandle& h) { } } +} // namespace posix + +AsynchAcceptor* AsynchAcceptor::create(const Socket& s, + Callback callback) +{ + return new posix::AsynchAcceptor(s, callback); +} + +AsynchConnector* AsynchConnector::create(const Socket& s, + Poller::shared_ptr poller, + std::string hostname, + uint16_t port, + ConnectedCallback connCb, + FailedCallback failCb) +{ + return new posix::AsynchConnector(s, poller, hostname, port, connCb, failCb); +} + +AsynchIO* AsynchIO::create(const Socket& s, + AsynchIO::ReadCallback rCb, + AsynchIO::EofCallback eofCb, + AsynchIO::DisconnectCallback disCb, + AsynchIO::ClosedCallback cCb, + AsynchIO::BuffersEmptyCallback eCb, + AsynchIO::IdleCallback iCb) +{ + return new posix::AsynchIO(s, rCb, eofCb, disCb, cCb, eCb, iCb); +} + +}} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/posix/Condition.h b/cpp/src/qpid/sys/posix/Condition.h deleted file mode 100644 index 86d6500ee9..0000000000 --- a/cpp/src/qpid/sys/posix/Condition.h +++ /dev/null @@ -1,86 +0,0 @@ -#ifndef _sys_posix_Condition_h -#define _sys_posix_Condition_h - -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -#include "PrivatePosix.h" - -#include "qpid/sys/Mutex.h" -#include "qpid/sys/Time.h" - -#include <time.h> -#include <sys/errno.h> -#include <boost/noncopyable.hpp> - -namespace qpid { -namespace sys { - -/** - * A condition variable for thread synchronization. - */ -class Condition -{ - public: - inline Condition(); - inline ~Condition(); - inline void wait(Mutex&); - inline bool wait(Mutex&, const AbsTime& absoluteTime); - inline void notify(); - inline void notifyAll(); - - private: - pthread_cond_t condition; -}; - -Condition::Condition() { - QPID_POSIX_ASSERT_THROW_IF(pthread_cond_init(&condition, 0)); -} - -Condition::~Condition() { - QPID_POSIX_ASSERT_THROW_IF(pthread_cond_destroy(&condition)); -} - -void Condition::wait(Mutex& mutex) { - QPID_POSIX_ASSERT_THROW_IF(pthread_cond_wait(&condition, &mutex.mutex)); -} - -bool Condition::wait(Mutex& mutex, const AbsTime& absoluteTime){ - struct timespec ts; - toTimespec(ts, Duration(absoluteTime)); - int status = pthread_cond_timedwait(&condition, &mutex.mutex, &ts); - if (status != 0) { - if (status == ETIMEDOUT) return false; - throw QPID_POSIX_ERROR(status); - } - return true; -} - -void Condition::notify(){ - QPID_POSIX_ASSERT_THROW_IF(pthread_cond_signal(&condition)); -} - -void Condition::notifyAll(){ - QPID_POSIX_ASSERT_THROW_IF(pthread_cond_broadcast(&condition)); -} - -}} -#endif /*!_sys_posix_Condition_h*/ diff --git a/cpp/src/qpid/sys/posix/FileSysDir.cpp b/cpp/src/qpid/sys/posix/FileSysDir.cpp new file mode 100755 index 0000000000..22dc487e74 --- /dev/null +++ b/cpp/src/qpid/sys/posix/FileSysDir.cpp @@ -0,0 +1,54 @@ +/* + * + * Copyright (c) 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "qpid/sys/FileSysDir.h" +#include "qpid/sys/StrError.h" +#include "qpid/Exception.h" + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <cerrno> +#include <unistd.h> + +namespace qpid { +namespace sys { + +bool FileSysDir::exists (void) const +{ + const char *cpath = dirPath.c_str (); + struct stat s; + if (::stat(cpath, &s)) { + if (errno == ENOENT) { + return false; + } + throw qpid::Exception (strError(errno) + + ": Can't check directory: " + dirPath); + } + if (S_ISDIR(s.st_mode)) + return true; + throw qpid::Exception(dirPath + " is not a directory"); +} + +void FileSysDir::mkdir(void) +{ + if (::mkdir(dirPath.c_str(), 0755)) + throw Exception ("Can't create directory: " + dirPath); +} + +}} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/posix/Fork.cpp b/cpp/src/qpid/sys/posix/Fork.cpp index ec3af620ef..a0d404a16e 100644 --- a/cpp/src/qpid/sys/posix/Fork.cpp +++ b/cpp/src/qpid/sys/posix/Fork.cpp @@ -22,7 +22,9 @@ #include <errno.h> #include <fcntl.h> #include <signal.h> +#include <string.h> #include <sys/stat.h> +#include <sys/select.h> #include <sys/types.h> #include <unistd.h> diff --git a/cpp/src/qpid/sys/posix/IOHandle.cpp b/cpp/src/qpid/sys/posix/IOHandle.cpp index 80b487eadc..9c049ee1de 100644 --- a/cpp/src/qpid/sys/posix/IOHandle.cpp +++ b/cpp/src/qpid/sys/posix/IOHandle.cpp @@ -21,7 +21,7 @@ #include "qpid/sys/IOHandle.h" -#include "PrivatePosix.h" +#include "qpid/sys/posix/PrivatePosix.h" namespace qpid { namespace sys { @@ -31,6 +31,8 @@ int toFd(const IOHandlePrivate* h) return h->fd; } +NullIOHandle DummyIOHandle; + IOHandle::IOHandle(IOHandlePrivate* h) : impl(h) {} diff --git a/cpp/src/qpid/sys/posix/LockFile.cpp b/cpp/src/qpid/sys/posix/LockFile.cpp new file mode 100755 index 0000000000..1862ff6ac9 --- /dev/null +++ b/cpp/src/qpid/sys/posix/LockFile.cpp @@ -0,0 +1,108 @@ +/* + * + * Copyright (c) 2008 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "qpid/sys/LockFile.h" +#include "qpid/sys/posix/PidFile.h" + +#include <string> +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + +#include "qpid/sys/posix/check.h" + +namespace qpid { +namespace sys { + +class LockFilePrivate { + friend class LockFile; + friend class PidFile; + + int fd; + +public: + LockFilePrivate(int f) : fd(f) {} +}; + +LockFile::LockFile(const std::string& path_, bool create) + : path(path_), created(create) { + + errno = 0; + int flags=create ? O_WRONLY|O_CREAT|O_NOFOLLOW : O_RDWR; + int fd = ::open(path.c_str(), flags, 0644); + if (fd < 0) throw ErrnoException("Cannot open " + path, errno); + if (::lockf(fd, F_TLOCK, 0) < 0) { + ::close(fd); + throw ErrnoException("Cannot lock " + path, errno); + } + impl.reset(new LockFilePrivate(fd)); +} + +LockFile::~LockFile() { + if (impl) { + int f = impl->fd; + if (f >= 0) { + int unused_ret; + unused_ret = ::lockf(f, F_ULOCK, 0); // Suppress warnings about ignoring return value. + ::close(f); + impl->fd = -1; + } + } +} + +int LockFile::read(void* bytes, size_t len) const { + if (!impl) + throw Exception("Lock file not open: " + path); + + ssize_t rc = ::read(impl->fd, bytes, len); + if ((ssize_t)len > rc) { + throw Exception("Cannot read lock file: " + path); + } + return rc; +} + +int LockFile::write(void* bytes, size_t len) const { + if (!impl) + throw Exception("Lock file not open: " + path); + + ssize_t rc = ::write(impl->fd, bytes, len); + if ((ssize_t)len > rc) { + throw Exception("Cannot write lock file: " + path); + } + return rc; +} + +PidFile::PidFile(const std::string& path_, bool create): + LockFile(path_, create) +{} + +pid_t PidFile::readPid(void) const { + pid_t pid; + int desired_read = sizeof(pid_t); + read(&pid, desired_read); + return pid; +} + +void PidFile::writePid(void) { + pid_t pid = getpid(); + int desired_write = sizeof(pid_t); + write(&pid, desired_write); +} + +}} /* namespace qpid::sys */ diff --git a/cpp/src/qpid/sys/posix/LockFile.h b/cpp/src/qpid/sys/posix/LockFile.h deleted file mode 100644 index 027735e759..0000000000 --- a/cpp/src/qpid/sys/posix/LockFile.h +++ /dev/null @@ -1,58 +0,0 @@ -#ifndef _sys_posix_LockFile_h -#define _sys_posix_LockFile_h - -#include "check.h" - -#include <boost/noncopyable.hpp> -#include <string> -#include <unistd.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> - -/* - * - * Copyright (c) 2008 The Apache Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -namespace qpid { -namespace sys { - -class LockFile : private boost::noncopyable { -public: - LockFile(const std::string& path_, bool create): - path(path_), fd(-1), created(create) { - errno = 0; - int flags=create ? O_WRONLY|O_CREAT|O_NOFOLLOW : O_RDWR; - fd = ::open(path.c_str(), flags, 0644); - if (fd < 0) throw ErrnoException("Cannot open " + path, errno); - if (::lockf(fd, F_TLOCK, 0) < 0) throw ErrnoException("Cannot lock " + path, errno); - } - - ~LockFile() { - if (fd >= 0) { - (void) ::lockf(fd, F_ULOCK, 0); // Suppress warnings about ignoring return value. - ::close(fd); - } - } - - std::string path; - int fd; - bool created; -}; - -} -} -#endif /*!_sys_posix_LockFile_h*/ diff --git a/cpp/src/qpid/sys/posix/Mutex.h b/cpp/src/qpid/sys/posix/Mutex.h deleted file mode 100644 index cd5a8affd4..0000000000 --- a/cpp/src/qpid/sys/posix/Mutex.h +++ /dev/null @@ -1,158 +0,0 @@ -#ifndef _sys_posix_Mutex_h -#define _sys_posix_Mutex_h - -/* - * - * Copyright (c) 2006 The Apache Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include "check.h" - -#include <pthread.h> -#include <boost/noncopyable.hpp> - -namespace qpid { -namespace sys { - -class Condition; - -/** - * Mutex lock. - */ -class Mutex : private boost::noncopyable { - friend class Condition; - static const pthread_mutexattr_t* getAttribute(); - -public: - typedef ::qpid::sys::ScopedLock<Mutex> ScopedLock; - typedef ::qpid::sys::ScopedUnlock<Mutex> ScopedUnlock; - - inline Mutex(); - inline ~Mutex(); - inline void lock(); - inline void unlock(); - inline bool trylock(); - - -protected: - pthread_mutex_t mutex; -}; - -/** - * RW lock. - */ -class RWlock : private boost::noncopyable { - friend class Condition; - -public: - typedef ::qpid::sys::ScopedRlock<RWlock> ScopedRlock; - typedef ::qpid::sys::ScopedWlock<RWlock> ScopedWlock; - - inline RWlock(); - inline ~RWlock(); - inline void wlock(); // will write-lock - inline void rlock(); // will read-lock - inline void unlock(); - inline void trywlock(); // will write-try - inline void tryrlock(); // will read-try - -protected: - pthread_rwlock_t rwlock; -}; - - -/** - * PODMutex is a POD, can be static-initialized with - * PODMutex m = QPID_PODMUTEX_INITIALIZER - */ -struct PODMutex -{ - typedef ::qpid::sys::ScopedLock<PODMutex> ScopedLock; - - inline void lock(); - inline void unlock(); - inline bool trylock(); - - // Must be public to be a POD: - pthread_mutex_t mutex; -}; - -#define QPID_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER } - -void PODMutex::lock() { - QPID_POSIX_ASSERT_THROW_IF(pthread_mutex_lock(&mutex)); -} - -void PODMutex::unlock() { - QPID_POSIX_ASSERT_THROW_IF(pthread_mutex_unlock(&mutex)); -} - -bool PODMutex::trylock() { - return pthread_mutex_trylock(&mutex) == 0; -} - -Mutex::Mutex() { - QPID_POSIX_ASSERT_THROW_IF(pthread_mutex_init(&mutex, getAttribute())); -} - -Mutex::~Mutex(){ - QPID_POSIX_ASSERT_THROW_IF(pthread_mutex_destroy(&mutex)); -} - -void Mutex::lock() { - QPID_POSIX_ASSERT_THROW_IF(pthread_mutex_lock(&mutex)); -} - -void Mutex::unlock() { - QPID_POSIX_ASSERT_THROW_IF(pthread_mutex_unlock(&mutex)); -} - -bool Mutex::trylock() { - return pthread_mutex_trylock(&mutex) == 0; -} - - -RWlock::RWlock() { - QPID_POSIX_ASSERT_THROW_IF(pthread_rwlock_init(&rwlock, NULL)); -} - -RWlock::~RWlock(){ - QPID_POSIX_ASSERT_THROW_IF(pthread_rwlock_destroy(&rwlock)); -} - -void RWlock::wlock() { - QPID_POSIX_ASSERT_THROW_IF(pthread_rwlock_wrlock(&rwlock)); -} - -void RWlock::rlock() { - QPID_POSIX_ASSERT_THROW_IF(pthread_rwlock_rdlock(&rwlock)); -} - -void RWlock::unlock() { - QPID_POSIX_ASSERT_THROW_IF(pthread_rwlock_unlock(&rwlock)); -} - -void RWlock::trywlock() { - QPID_POSIX_ASSERT_THROW_IF(pthread_rwlock_trywrlock(&rwlock)); -} - -void RWlock::tryrlock() { - QPID_POSIX_ASSERT_THROW_IF(pthread_rwlock_tryrdlock(&rwlock)); -} - - -}} -#endif /*!_sys_posix_Mutex_h*/ diff --git a/cpp/src/qpid/sys/posix/PidFile.h b/cpp/src/qpid/sys/posix/PidFile.h new file mode 100644 index 0000000000..fb19d407f4 --- /dev/null +++ b/cpp/src/qpid/sys/posix/PidFile.h @@ -0,0 +1,62 @@ +#ifndef _sys_PidFile_h +#define _sys_PidFile_h + +/* + * + * Copyright (c) 2008 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "qpid/sys/LockFile.h" + +#include "qpid/CommonImportExport.h" +#include "qpid/sys/IntegerTypes.h" + +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <string> + +namespace qpid { +namespace sys { + +class PidFile : public LockFile +{ +public: + QPID_COMMON_EXTERN PidFile(const std::string& path_, bool create); + + /** + * Read the process ID from the lock file. This method assumes that + * if there is a process ID in the file, it was written there by + * writePid(); thus, it's at the start of the file. + * + * Throws an exception if there is an error reading the file. + * + * @returns The stored process ID. No validity check is done on it. + */ + QPID_COMMON_EXTERN pid_t readPid(void) const; + + /** + * Write the current process's ID to the lock file. It's written at + * the start of the file and will overwrite any other content that + * may be in the file. + * + * Throws an exception if the write fails. + */ + QPID_COMMON_EXTERN void writePid(void); +}; + +}} /* namespace qpid::sys */ + +#endif /*!_sys_PidFile_h*/ diff --git a/cpp/src/qpid/sys/posix/PipeHandle.cpp b/cpp/src/qpid/sys/posix/PipeHandle.cpp new file mode 100755 index 0000000000..4b19783338 --- /dev/null +++ b/cpp/src/qpid/sys/posix/PipeHandle.cpp @@ -0,0 +1,64 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +#include "qpid/sys/PipeHandle.h" +#include "qpid/sys/posix/check.h" +#include <unistd.h> +#include <fcntl.h> +#include <sys/socket.h> + +namespace qpid { +namespace sys { + +PipeHandle::PipeHandle(bool nonBlocking) { + + int pair[2]; + pair[0] = pair[1] = -1; + + if (socketpair(PF_UNIX, SOCK_STREAM, 0, pair) == -1) + throw qpid::Exception(QPID_MSG("Creation of pipe failed")); + + writeFd = pair[0]; + readFd = pair[1]; + + // Set the socket to non-blocking + if (nonBlocking) { + int flags = fcntl(readFd, F_GETFL); + fcntl(readFd, F_SETFL, flags | O_NONBLOCK); + } +} + +PipeHandle::~PipeHandle() { + close(readFd); + close(writeFd); +} + +int PipeHandle::read(void* buf, size_t bufSize) { + return ::read(readFd,buf,bufSize); +} + +int PipeHandle::write(const void* buf, size_t bufSize) { + return ::write(writeFd,buf,bufSize); +} + +int PipeHandle::getReadHandle() { + return readFd; +} + +}} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/posix/PollableCondition.cpp b/cpp/src/qpid/sys/posix/PollableCondition.cpp new file mode 100644 index 0000000000..b22a615a54 --- /dev/null +++ b/cpp/src/qpid/sys/posix/PollableCondition.cpp @@ -0,0 +1,124 @@ +#ifndef QPID_SYS_LINUX_POLLABLECONDITION_CPP +#define QPID_SYS_LINUX_POLLABLECONDITION_CPP + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/PollableCondition.h" +#include "qpid/sys/DispatchHandle.h" +#include "qpid/sys/IOHandle.h" +#include "qpid/sys/posix/PrivatePosix.h" +#include "qpid/Exception.h" + +#include <boost/bind.hpp> + +#include <unistd.h> +#include <fcntl.h> + +namespace qpid { +namespace sys { + +class PollableConditionPrivate : public sys::IOHandle { + friend class PollableCondition; + +private: + PollableConditionPrivate(const sys::PollableCondition::Callback& cb, + sys::PollableCondition& parent, + const boost::shared_ptr<sys::Poller>& poller); + ~PollableConditionPrivate(); + + void dispatch(sys::DispatchHandle& h); + void set(); + void clear(); + +private: + PollableCondition::Callback cb; + PollableCondition& parent; + boost::shared_ptr<sys::Poller> poller; + int writeFd; + std::auto_ptr<DispatchHandleRef> handle; +}; + +PollableConditionPrivate::PollableConditionPrivate( + const sys::PollableCondition::Callback& cb, + sys::PollableCondition& parent, + const boost::shared_ptr<sys::Poller>& poller +) : IOHandle(new sys::IOHandlePrivate), cb(cb), parent(parent) +{ + int fds[2]; + if (::pipe(fds) == -1) + throw ErrnoException(QPID_MSG("Can't create PollableCondition")); + impl->fd = fds[0]; + writeFd = fds[1]; + if (::fcntl(impl->fd, F_SETFL, O_NONBLOCK) == -1) + throw ErrnoException(QPID_MSG("Can't create PollableCondition")); + if (::fcntl(writeFd, F_SETFL, O_NONBLOCK) == -1) + throw ErrnoException(QPID_MSG("Can't create PollableCondition")); + handle.reset (new DispatchHandleRef( + *this, + boost::bind(&sys::PollableConditionPrivate::dispatch, this, _1), + 0, 0)); + handle->startWatch(poller); + handle->unwatch(); + + // Make the read FD readable + static const char dummy=0; + ssize_t n = ::write(writeFd, &dummy, 1); + if (n == -1 && errno != EAGAIN) + throw ErrnoException("Error setting PollableCondition"); +} + +PollableConditionPrivate::~PollableConditionPrivate() { + handle->stopWatch(); + close(writeFd); +} + +void PollableConditionPrivate::dispatch(sys::DispatchHandle&) { + cb(parent); +} + +void PollableConditionPrivate::set() { + handle->rewatch(); +} + +void PollableConditionPrivate::clear() { + handle->unwatch(); +} + + +PollableCondition::PollableCondition(const Callback& cb, + const boost::shared_ptr<sys::Poller>& poller +) : impl(new PollableConditionPrivate(cb, *this, poller)) +{ +} + +PollableCondition::~PollableCondition() +{ + delete impl; +} + +void PollableCondition::set() { impl->set(); } + +void PollableCondition::clear() { impl->clear(); } + +}} // namespace qpid::sys + +#endif /*!QPID_SYS_LINUX_POLLABLECONDITION_CPP*/ diff --git a/cpp/src/qpid/sys/posix/Shlib.cpp b/cpp/src/qpid/sys/posix/Shlib.cpp index 1552aa06b5..299331103c 100644 --- a/cpp/src/qpid/sys/posix/Shlib.cpp +++ b/cpp/src/qpid/sys/posix/Shlib.cpp @@ -27,11 +27,11 @@ namespace qpid { namespace sys { void Shlib::load(const char* name) { - dlerror(); + ::dlerror(); handle = ::dlopen(name, RTLD_NOW); const char* error = ::dlerror(); if (error) { - throw Exception(QPID_MSG(error)); + throw Exception(QPID_MSG(error << ": " << name)); } } @@ -52,7 +52,7 @@ void* Shlib::getSymbol(const char* name) { void* sym = ::dlsym(handle, name); const char* error = ::dlerror(); if (error) - throw Exception(QPID_MSG(error)); + throw Exception(QPID_MSG(error << ": " << name)); return sym; } diff --git a/cpp/src/qpid/sys/posix/Socket.cpp b/cpp/src/qpid/sys/posix/Socket.cpp index 415d5293ef..7b906f33e8 100644 --- a/cpp/src/qpid/sys/posix/Socket.cpp +++ b/cpp/src/qpid/sys/posix/Socket.cpp @@ -21,8 +21,9 @@ #include "qpid/sys/Socket.h" -#include "check.h" -#include "PrivatePosix.h" +#include "qpid/sys/SocketAddress.h" +#include "qpid/sys/posix/check.h" +#include "qpid/sys/posix/PrivatePosix.h" #include <fcntl.h> #include <sys/types.h> @@ -36,6 +37,7 @@ #include <iostream> #include <boost/format.hpp> +#include <boost/lexical_cast.hpp> namespace qpid { namespace sys { @@ -95,22 +97,33 @@ std::string getService(int fd, bool local) } Socket::Socket() : - IOHandle(new IOHandlePrivate) -{ - createTcp(); -} + IOHandle(new IOHandlePrivate), + nonblocking(false), + nodelay(false) +{} Socket::Socket(IOHandlePrivate* h) : - IOHandle(h) + IOHandle(h), + nonblocking(false), + nodelay(false) {} -void Socket::createTcp() const +void Socket::createSocket(const SocketAddress& sa) const { int& socket = impl->fd; if (socket != -1) Socket::close(); - int s = ::socket (PF_INET, SOCK_STREAM, 0); + int s = ::socket(getAddrInfo(sa).ai_family, getAddrInfo(sa).ai_socktype, 0); if (s < 0) throw QPID_POSIX_ERROR(errno); socket = s; + + try { + if (nonblocking) setNonblocking(); + if (nodelay) setTcpNoDelay(); + } catch (std::exception&) { + ::close(s); + socket = -1; + throw; + } } void Socket::setTimeout(const Duration& interval) const @@ -123,40 +136,42 @@ void Socket::setTimeout(const Duration& interval) const } void Socket::setNonblocking() const { - QPID_POSIX_CHECK(::fcntl(impl->fd, F_SETFL, O_NONBLOCK)); + int& socket = impl->fd; + nonblocking = true; + if (socket != -1) { + QPID_POSIX_CHECK(::fcntl(socket, F_SETFL, O_NONBLOCK)); + } } -namespace { -const char* h_errstr(int e) { - switch (e) { - case HOST_NOT_FOUND: return "Host not found"; - case NO_ADDRESS: return "Name does not have an IP address"; - case TRY_AGAIN: return "A temporary error occurred on an authoritative name server."; - case NO_RECOVERY: return "Non-recoverable name server error"; - default: return "Unknown error"; +void Socket::setTcpNoDelay() const +{ + int& socket = impl->fd; + nodelay = true; + if (socket != -1) { + int flag = 1; + int result = setsockopt(impl->fd, IPPROTO_TCP, TCP_NODELAY, (char *)&flag, sizeof(flag)); + QPID_POSIX_CHECK(result); } } -} void Socket::connect(const std::string& host, uint16_t port) const { - std::stringstream namestream; - namestream << host << ":" << port; - connectname = namestream.str(); + SocketAddress sa(host, boost::lexical_cast<std::string>(port)); + connect(sa); +} + +void Socket::connect(const SocketAddress& addr) const +{ + connectname = addr.asString(); + + createSocket(addr); const int& socket = impl->fd; - struct sockaddr_in name; - name.sin_family = AF_INET; - name.sin_port = htons(port); - // TODO: Be good to make this work for IPv6 as well as IPv4 - // Use more modern lookup functions - struct hostent* hp = gethostbyname ( host.c_str() ); - if (hp == 0) - throw Exception(QPID_MSG("Cannot resolve " << host << ": " << h_errstr(h_errno))); - ::memcpy(&name.sin_addr.s_addr, hp->h_addr_list[0], hp->h_length); - if ((::connect(socket, (struct sockaddr*)(&name), sizeof(name)) < 0) && - (errno != EINPROGRESS)) - throw qpid::Exception(QPID_MSG(strError(errno) << ": " << host << ":" << port)); + // TODO the correct thing to do here is loop on failure until you've used all the returned addresses + if ((::connect(socket, getAddrInfo(addr).ai_addr, getAddrInfo(addr).ai_addrlen) < 0) && + (errno != EINPROGRESS)) { + throw Exception(QPID_MSG(strError(errno) << ": " << connectname)); + } } void @@ -170,18 +185,24 @@ Socket::close() const int Socket::listen(uint16_t port, int backlog) const { + SocketAddress sa("", boost::lexical_cast<std::string>(port)); + return listen(sa, backlog); +} + +int Socket::listen(const SocketAddress& sa, int backlog) const +{ + createSocket(sa); + const int& socket = impl->fd; int yes=1; QPID_POSIX_CHECK(setsockopt(socket,SOL_SOCKET,SO_REUSEADDR,&yes,sizeof(yes))); - struct sockaddr_in name; - name.sin_family = AF_INET; - name.sin_port = htons(port); - name.sin_addr.s_addr = 0; - if (::bind(socket, (struct sockaddr*)&name, sizeof(name)) < 0) - throw Exception(QPID_MSG("Can't bind to port " << port << ": " << strError(errno))); + + if (::bind(socket, getAddrInfo(sa).ai_addr, getAddrInfo(sa).ai_addrlen) < 0) + throw Exception(QPID_MSG("Can't bind to port " << sa.asString() << ": " << strError(errno))); if (::listen(socket, backlog) < 0) - throw Exception(QPID_MSG("Can't listen on port " << port << ": " << strError(errno))); - + throw Exception(QPID_MSG("Can't listen on port " << sa.asString() << ": " << strError(errno))); + + struct sockaddr_in name; socklen_t namelen = sizeof(name); if (::getsockname(socket, (struct sockaddr*)&name, &namelen) < 0) throw QPID_POSIX_ERROR(errno); @@ -189,9 +210,9 @@ int Socket::listen(uint16_t port, int backlog) const return ntohs(name.sin_port); } -Socket* Socket::accept(struct sockaddr *addr, socklen_t *addrlen) const +Socket* Socket::accept() const { - int afd = ::accept(impl->fd, addr, addrlen); + int afd = ::accept(impl->fd, 0, 0); if ( afd >= 0) return new Socket(new IOHandlePrivate(afd)); else if (errno == EAGAIN) @@ -221,9 +242,10 @@ std::string Socket::getPeername() const std::string Socket::getPeerAddress() const { - if (!connectname.empty()) - return std::string (connectname); - return getName(impl->fd, false, true); + if (connectname.empty()) { + connectname = getName(impl->fd, false, true); + } + return connectname; } std::string Socket::getLocalAddress() const @@ -238,7 +260,7 @@ uint16_t Socket::getLocalPort() const uint16_t Socket::getRemotePort() const { - return atoi(getService(impl->fd, true).c_str()); + return std::atoi(getService(impl->fd, true).c_str()); } int Socket::getError() const @@ -252,13 +274,4 @@ int Socket::getError() const return result; } -void Socket::setTcpNoDelay(bool nodelay) const -{ - if (nodelay) { - int flag = 1; - int result = setsockopt(impl->fd, IPPROTO_TCP, TCP_NODELAY, (char *)&flag, sizeof(flag)); - QPID_POSIX_CHECK(result); - } -} - }} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/posix/SocketAddress.cpp b/cpp/src/qpid/sys/posix/SocketAddress.cpp new file mode 100644 index 0000000000..cb44f8e41f --- /dev/null +++ b/cpp/src/qpid/sys/posix/SocketAddress.cpp @@ -0,0 +1,97 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/SocketAddress.h" + +#include "qpid/sys/posix/check.h" + +#include <sys/socket.h> +#include <string.h> +#include <netdb.h> + +namespace qpid { +namespace sys { + +SocketAddress::SocketAddress(const std::string& host0, const std::string& port0) : + host(host0), + port(port0), + addrInfo(0) +{ +} + +SocketAddress::SocketAddress(const SocketAddress& sa) : + host(sa.host), + port(sa.port), + addrInfo(0) +{ +} + +SocketAddress& SocketAddress::operator=(const SocketAddress& sa) +{ + if (&sa != this) { + host = sa.host; + port = sa.port; + + if (addrInfo) { + ::freeaddrinfo(addrInfo); + addrInfo = 0; + } + } + return *this; +} + +SocketAddress::~SocketAddress() +{ + if (addrInfo) { + ::freeaddrinfo(addrInfo); + } +} + +std::string SocketAddress::asString() const +{ + return host + ":" + port; +} + +const ::addrinfo& getAddrInfo(const SocketAddress& sa) +{ + if (!sa.addrInfo) { + ::addrinfo hints; + ::memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_INET; // In order to allow AF_INET6 we'd have to change createTcp() as well + hints.ai_socktype = SOCK_STREAM; + + const char* node = 0; + if (sa.host.empty()) { + hints.ai_flags |= AI_PASSIVE; + } else { + node = sa.host.c_str(); + } + const char* service = sa.port.empty() ? "0" : sa.port.c_str(); + + int n = ::getaddrinfo(node, service, &hints, &sa.addrInfo); + if (n != 0) + throw Exception(QPID_MSG("Cannot resolve " << sa.host << ": " << ::gai_strerror(n))); + } + + return *sa.addrInfo; +} + +}} diff --git a/cpp/src/qpid/sys/posix/SystemInfo.cpp b/cpp/src/qpid/sys/posix/SystemInfo.cpp new file mode 100755 index 0000000000..55737fcfcc --- /dev/null +++ b/cpp/src/qpid/sys/posix/SystemInfo.cpp @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/SystemInfo.h" + +#include "qpid/sys/posix/check.h" + +#include <sys/ioctl.h> +#include <sys/utsname.h> +#include <net/if.h> +#include <unistd.h> +#include <arpa/inet.h> +#include <stdio.h> +#include <unistd.h> +#include <iostream> +#include <fstream> +#include <sstream> +#include <netdb.h> + +#ifndef HOST_NAME_MAX +# define HOST_NAME_MAX 256 +#endif + +using namespace std; + +namespace qpid { +namespace sys { + +long SystemInfo::concurrency() { +#ifdef _SC_NPROCESSORS_ONLN // Linux specific. + return sysconf(_SC_NPROCESSORS_ONLN); +#else + return -1; +#endif +} + +bool SystemInfo::getLocalHostname (TcpAddress &address) { + char name[HOST_NAME_MAX]; + if (::gethostname(name, sizeof(name)) != 0) + return false; + address.host = name; + return true; +} + +static const string LOCALHOST("127.0.0.1"); + +void SystemInfo::getLocalIpAddresses (uint16_t port, + std::vector<Address> &addrList) { + int s = ::socket(PF_INET, SOCK_STREAM, 0); + for (int i=1;;i++) { + ::ifreq ifr; + ifr.ifr_ifindex = i; + if (::ioctl (s, SIOCGIFNAME, &ifr) < 0) + break; + /* now ifr.ifr_name is set */ + if (::ioctl (s, SIOCGIFADDR, &ifr) < 0) + continue; + ::sockaddr *saddr = (::sockaddr *) &ifr.ifr_addr; + char dispName[NI_MAXHOST]; + if (int rc=::getnameinfo(saddr, sizeof(ifr.ifr_addr), dispName, sizeof(dispName), 0, 0, NI_NUMERICHOST) != 0) + throw QPID_POSIX_ERROR(rc); + string addr(dispName); + if (addr != LOCALHOST) + addrList.push_back(TcpAddress(addr, port)); + } + if (addrList.empty()) { + addrList.push_back(TcpAddress(LOCALHOST, port)); + } + close (s); +} + +void SystemInfo::getSystemId (std::string &osName, + std::string &nodeName, + std::string &release, + std::string &version, + std::string &machine) +{ + struct utsname _uname; + if (uname (&_uname) == 0) + { + osName = _uname.sysname; + nodeName = _uname.nodename; + release = _uname.release; + version = _uname.version; + machine = _uname.machine; + } +} + +uint32_t SystemInfo::getProcessId() +{ + return (uint32_t) ::getpid(); +} + +uint32_t SystemInfo::getParentProcessId() +{ + return (uint32_t) ::getppid(); +} + +// Linux specific (Solaris has quite different stuff in /proc) +string SystemInfo::getProcessName() +{ + string value; + + ifstream input("/proc/self/status"); + if (input.good()) { + while (!input.eof()) { + string key; + input >> key; + if (key == "Name:") { + input >> value; + break; + } + } + input.close(); + } + + return value; +} + +}} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/posix/Thread.cpp b/cpp/src/qpid/sys/posix/Thread.cpp index bb5641bc53..a784e63195 100644 --- a/cpp/src/qpid/sys/posix/Thread.cpp +++ b/cpp/src/qpid/sys/posix/Thread.cpp @@ -22,7 +22,7 @@ #include "qpid/sys/Thread.h" #include "qpid/sys/Runnable.h" -#include "check.h" +#include "qpid/sys/posix/check.h" #include <pthread.h> diff --git a/cpp/src/qpid/sys/posix/Time.cpp b/cpp/src/qpid/sys/posix/Time.cpp index 8aa9fd9946..0734abd1df 100644 --- a/cpp/src/qpid/sys/posix/Time.cpp +++ b/cpp/src/qpid/sys/posix/Time.cpp @@ -19,25 +19,46 @@ * */ -#include "PrivatePosix.h" +#include "qpid/sys/posix/PrivatePosix.h" #include "qpid/sys/Time.h" #include <ostream> #include <time.h> #include <stdio.h> #include <sys/time.h> +#include <unistd.h> + +namespace { +int64_t max_abstime() { return std::numeric_limits<int64_t>::max(); } +} namespace qpid { namespace sys { +AbsTime::AbsTime(const AbsTime& t, const Duration& d) : + timepoint(d == Duration::max() ? max_abstime() : t.timepoint+d.nanosecs) +{} + +AbsTime AbsTime::FarFuture() { + AbsTime ff; ff.timepoint = max_abstime(); return ff; +} + AbsTime AbsTime::now() { struct timespec ts; ::clock_gettime(CLOCK_REALTIME, &ts); AbsTime time_now; - time_now.time_ns = toTime(ts).nanosecs; + time_now.timepoint = toTime(ts).nanosecs; return time_now; } +Duration::Duration(const AbsTime& time0) : + nanosecs(time0.timepoint) +{} + +Duration::Duration(const AbsTime& start, const AbsTime& finish) : + nanosecs(finish.timepoint - start.timepoint) +{} + struct timespec& toTimespec(struct timespec& ts, const Duration& t) { ts.tv_sec = t / TIME_SEC; ts.tv_nsec = t % TIME_SEC; @@ -58,25 +79,35 @@ std::ostream& operator<<(std::ostream& o, const Duration& d) { return o << int64_t(d) << "ns"; } -std::ostream& operator<<(std::ostream& o, const AbsTime& t) { - static const char * month_abbrevs[] = { - "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec" - }; - struct tm * timeinfo; - time_t rawtime(t.time_ns/TIME_SEC); - timeinfo = localtime (&rawtime); +namespace { +inline std::ostream& outputFormattedTime(std::ostream& o, const ::time_t* time) { + ::tm timeinfo; char time_string[100]; - sprintf ( time_string, - "%d-%s-%02d %02d:%02d:%02d", - 1900 + timeinfo->tm_year, - month_abbrevs[timeinfo->tm_mon], - timeinfo->tm_mday, - timeinfo->tm_hour, - timeinfo->tm_min, - timeinfo->tm_sec - ); + ::strftime(time_string, 100, + "%Y-%m-%d %H:%M:%S", + localtime_r(time, &timeinfo)); return o << time_string; } +} -}} +std::ostream& operator<<(std::ostream& o, const AbsTime& t) { + ::time_t rawtime(t.timepoint/TIME_SEC); + return outputFormattedTime(o, &rawtime); +} +void outputFormattedNow(std::ostream& o) { + ::time_t rawtime; + ::time(&rawtime); + outputFormattedTime(o, &rawtime); + o << " "; +} + +void sleep(int secs) { + ::sleep(secs); +} + +void usleep(uint64_t usecs) { + ::usleep(usecs); +} + +}} diff --git a/cpp/src/qpid/sys/posix/check.h b/cpp/src/qpid/sys/posix/check.h deleted file mode 100644 index f3031b7593..0000000000 --- a/cpp/src/qpid/sys/posix/check.h +++ /dev/null @@ -1,49 +0,0 @@ -#ifndef _posix_check_h -#define _posix_check_h - -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -#include "qpid/Exception.h" - -#include <cerrno> -#include <assert.h> -#include <stdio.h> - -#define QPID_POSIX_ERROR(ERRNO) qpid::Exception(QPID_MSG(qpid::sys::strError(ERRNO))) - -/** THROW QPID_POSIX_ERROR(errno) if RESULT is less than zero */ -#define QPID_POSIX_CHECK(RESULT) \ - if ((RESULT) < 0) throw QPID_POSIX_ERROR((errno)) - -/** Throw a posix error if ERRNO is non-zero */ -#define QPID_POSIX_THROW_IF(ERRNO) \ - do { int e=(ERRNO); if (e) throw QPID_POSIX_ERROR(e); } while(0) - -/** Same as _THROW_IF in a release build, but abort a debug build */ -#ifdef NDEBUG -#define QPID_POSIX_ASSERT_THROW_IF(ERRNO) QPID_POSIX_THROW_IF(ERRNO) -#else -#define QPID_POSIX_ASSERT_THROW_IF(ERRNO) \ - do { int e=(ERRNO); if (e) { errno=e; ::perror(0); assert(0); } } while(0) -#endif - -#endif /*!_posix_check_h*/ diff --git a/cpp/src/qpid/sys/rdma/RdmaClient.cpp b/cpp/src/qpid/sys/rdma/RdmaClient.cpp index afff96b72f..d39f7885a5 100644 --- a/cpp/src/qpid/sys/rdma/RdmaClient.cpp +++ b/cpp/src/qpid/sys/rdma/RdmaClient.cpp @@ -1,4 +1,24 @@ -#include "RdmaIO.h" +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/rdma/RdmaIO.h" #include "qpid/sys/Time.h" #include <netdb.h> @@ -20,18 +40,20 @@ using std::rand; using qpid::sys::Poller; using qpid::sys::Dispatcher; +using qpid::sys::SocketAddress; using qpid::sys::AbsTime; using qpid::sys::Duration; using qpid::sys::TIME_SEC; using qpid::sys::TIME_INFINITE; +namespace qpid { +namespace tests { + // count of messages int64_t smsgs = 0; int64_t sbytes = 0; int64_t rmsgs = 0; int64_t rbytes = 0; -int64_t cmsgs = 0; -int writable = true; int target = 1000000; int msgsize = 200; @@ -42,17 +64,15 @@ Duration fullTestDuration(TIME_INFINITE); vector<char> testString; void write(Rdma::AsynchIO& aio) { - if ((cmsgs - rmsgs) < Rdma::DEFAULT_WR_ENTRIES/2) { - while (writable) { - if (smsgs >= target) - return; - Rdma::Buffer* b = aio.getBuffer(); - std::copy(testString.begin(), testString.end(), b->bytes); - b->dataCount = msgsize; - aio.queueWrite(b); - ++smsgs; - sbytes += b->byteCount; - } + while (aio.writable()) { + if (smsgs >= target) + return; + Rdma::Buffer* b = aio.getBuffer(); + std::copy(testString.begin(), testString.end(), b->bytes); + b->dataCount = msgsize; + aio.queueWrite(b); + ++smsgs; + sbytes += msgsize; } } @@ -62,39 +82,46 @@ void dataError(Rdma::AsynchIO&) { void data(Poller::shared_ptr p, Rdma::AsynchIO& aio, Rdma::Buffer* b) { ++rmsgs; - rbytes += b->byteCount; + rbytes += b->dataCount; // When all messages have been recvd stop if (rmsgs < target) { write(aio); } else { fullTestDuration = std::min(fullTestDuration, Duration(startTime, AbsTime::now())); - if (cmsgs >= target) + if (aio.incompletedWrites() == 0) p->shutdown(); } } -void full(Rdma::AsynchIO&) { - writable = false; +void full(Rdma::AsynchIO& a, Rdma::Buffer* b) { + // Warn as we shouldn't get here anymore + cerr << "!"; + + // Don't need to keep buffer just adjust the counts + --smsgs; + sbytes -= b->dataCount; + + // Give buffer back + a.returnBuffer(b); } void idle(Poller::shared_ptr p, Rdma::AsynchIO& aio) { - writable = true; - ++cmsgs; if (smsgs < target) { write(aio); } else { sendingDuration = std::min(sendingDuration, Duration(startTime, AbsTime::now())); - if (rmsgs >= target && cmsgs >= target) + if (rmsgs >= target && aio.incompletedWrites() == 0) p->shutdown(); } } -void connected(Poller::shared_ptr poller, Rdma::Connection::intrusive_ptr& ci) { +void connected(Poller::shared_ptr poller, Rdma::Connection::intrusive_ptr& ci, const Rdma::ConnectionParams& cp) { cout << "Connected\n"; Rdma::QueuePair::intrusive_ptr q = ci->getQueuePair(); - Rdma::AsynchIO* aio = new Rdma::AsynchIO(ci->getQueuePair(), msgsize, + Rdma::AsynchIO* aio = new Rdma::AsynchIO(ci->getQueuePair(), + cp.maxRecvBufferSize, cp.initialXmitCredit , Rdma::DEFAULT_WR_ENTRIES, boost::bind(&data, poller, _1, _2), boost::bind(&idle, poller, _1), &full, @@ -111,31 +138,25 @@ void disconnected(boost::shared_ptr<Poller> p, Rdma::Connection::intrusive_ptr&) p->shutdown(); } -void connectionError(boost::shared_ptr<Poller> p, Rdma::Connection::intrusive_ptr&) { +void connectionError(boost::shared_ptr<Poller> p, Rdma::Connection::intrusive_ptr&, const Rdma::ErrorType) { cout << "Connection error\n"; p->shutdown(); } -void rejected(boost::shared_ptr<Poller> p, Rdma::Connection::intrusive_ptr&) { +void rejected(boost::shared_ptr<Poller> p, Rdma::Connection::intrusive_ptr&, const Rdma::ConnectionParams&) { cout << "Connection rejected\n"; p->shutdown(); } +}} // namespace qpid::tests + +using namespace qpid::tests; + int main(int argc, char* argv[]) { vector<string> args(&argv[0], &argv[argc]); - ::addrinfo *res; - ::addrinfo hints = {}; - hints.ai_family = AF_INET; - hints.ai_socktype = SOCK_STREAM; + string host = args[1]; string port = (args.size() < 3) ? "20079" : args[2]; - int n = ::getaddrinfo(args[1].c_str(), port.c_str(), &hints, &res); - if (n<0) { - cerr << "Can't find information for: " << args[1] << "\n"; - return 1; - } else { - cout << "Connecting to: " << args[1] << ":" << port <<"\n"; - } if (args.size() > 3) msgsize = atoi(args[3].c_str()); @@ -144,19 +165,22 @@ int main(int argc, char* argv[]) { // Make a random message of that size testString.resize(msgsize); for (int i = 0; i < msgsize; ++i) { - testString[i] = 32 + rand() & 0x3f; + testString[i] = 32 + (rand() & 0x3f); } try { boost::shared_ptr<Poller> p(new Poller()); Dispatcher d(p); + SocketAddress sa(host, port); + cout << "Connecting to: " << sa.asString() <<"\n"; Rdma::Connector c( - *res->ai_addr, - boost::bind(&connected, p, _1), - boost::bind(&connectionError, p, _1), + sa, + Rdma::ConnectionParams(msgsize, Rdma::DEFAULT_WR_ENTRIES), + boost::bind(&connected, p, _1, _2), + boost::bind(&connectionError, p, _1, _2), boost::bind(&disconnected, p, _1), - boost::bind(&rejected, p, _1)); + boost::bind(&rejected, p, _1, _2)); c.start(p); d.run(); diff --git a/cpp/src/qpid/sys/rdma/RdmaIO.cpp b/cpp/src/qpid/sys/rdma/RdmaIO.cpp index 755d6f17c4..8d06fccba1 100644 --- a/cpp/src/qpid/sys/rdma/RdmaIO.cpp +++ b/cpp/src/qpid/sys/rdma/RdmaIO.cpp @@ -1,12 +1,41 @@ -#include "RdmaIO.h" +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/rdma/RdmaIO.h" + +#include "qpid/log/Statement.h" + #include <iostream> #include <boost/bind.hpp> +using qpid::sys::SocketAddress; +using qpid::sys::DispatchHandle; +using qpid::sys::Poller; + namespace Rdma { AsynchIO::AsynchIO( QueuePair::intrusive_ptr q, - int s, + int size, + int xCredit, + int rCount, ReadCallback rc, IdleCallback ic, FullCallback fc, @@ -14,10 +43,15 @@ namespace Rdma { ) : qp(q), dataHandle(*qp, boost::bind(&AsynchIO::dataEvent, this, _1), 0, 0), - bufferSize(s), - recvBufferCount(DEFAULT_WR_ENTRIES), - xmitBufferCount(DEFAULT_WR_ENTRIES), + bufferSize(size), + recvCredit(0), + xmitCredit(xCredit), + recvBufferCount(rCount), + xmitBufferCount(xCredit), outstandingWrites(0), + closed(false), + deleting(false), + state(IDLE), readCallback(rc), idleCallback(ic), fullCallback(fc), @@ -29,80 +63,281 @@ namespace Rdma { // Prepost some recv buffers before we go any further for (int i = 0; i<recvBufferCount; ++i) { + // Allocate recv buffer Buffer* b = qp->createBuffer(bufferSize); buffers.push_front(b); b->dataCount = b->byteCount; qp->postRecv(b); } + + for (int i = 0; i<xmitBufferCount; ++i) { + // Allocate xmit buffer + Buffer* b = qp->createBuffer(bufferSize); + buffers.push_front(b); + bufferQueue.push_front(b); + b->dataCount = 0; + b->dataStart = 0; + } } AsynchIO::~AsynchIO() { + // Warn if we are deleting whilst there are still unreclaimed write buffers + if ( outstandingWrites>0 ) + QPID_LOG(error, "RDMA: qp=" << qp << ": Deleting queue before all write buffers finished"); + + // Turn off callbacks (before doing the deletes) + dataHandle.stopWatch(); + // The buffers ptr_deque automatically deletes all the buffers we've allocated + // TODO: It might turn out to be more efficient in high connection loads to reuse the + // buffers rather than having to reregister them all the time (this would be straightforward if all + // connections haver the same buffer size and harder otherwise) } void AsynchIO::start(Poller::shared_ptr poller) { dataHandle.startWatch(poller); } - // TODO: Currently we don't prevent write buffer overrun we just advise - // when to stop writing. - void AsynchIO::queueWrite(Buffer* buff) { - qp->postSend(buff); - ++outstandingWrites; - if (outstandingWrites >= xmitBufferCount) { - fullCallback(*this); + // Mark for deletion/Delete this object when we have no outstanding writes + void AsynchIO::deferDelete() { + State oldState; + State newState; + bool doReturn; + //qpid::sys::ScopedLock<qpid::sys::Mutex> l(stateLock); + // It is safe to assign to deleting here as we either delete ourselves + // before leaving this function or deleting is set on exit + do { + newState = oldState = state.get(); + doReturn = false; + if (outstandingWrites > 0 || oldState != IDLE) { + deleting = true; + doReturn = true; + } else{ + newState = DELETED; // Stop any read callback before the dataHandle.stopWatch() in the destructor + } + } while (!state.boolCompareAndSwap(oldState, newState)); + if (doReturn) { + return; } + delete this; } - void AsynchIO::notifyPendingWrite() { - // Just perform the idle callback (if possible) - if (outstandingWrites < xmitBufferCount) { - idleCallback(*this); + void AsynchIO::queueWrite(Buffer* buff) { + // Make sure we don't overrun our available buffers + // either at our end or the known available at the peers end + if (writable()) { + // TODO: We might want to batch up sending credit + if (recvCredit > 0) { + int creditSent = recvCredit & ~FlagsMask; + qp->postSend(creditSent, buff); + recvCredit -= creditSent; + } else { + qp->postSend(buff); + } + ++outstandingWrites; + --xmitCredit; + } else { + if (fullCallback) { + fullCallback(*this, buff); + } else { + QPID_LOG(error, "RDMA: qp=" << qp << ": Write queue full, but no callback, throwing buffer away"); + returnBuffer(buff); + } } } + // Mark now closed (so we don't accept any more writes or make any idle callbacks) void AsynchIO::queueWriteClose() { + // Don't think we actually need to lock here as transition is 1 way only to closed + closed = true; } - Buffer* AsynchIO::getBuffer() { - qpid::sys::ScopedLock<qpid::sys::Mutex> l(bufferQueueLock); - if (bufferQueue.empty()) { - Buffer* b = qp->createBuffer(bufferSize); - buffers.push_front(b); - b->dataCount = 0; - return b; - } else { - Buffer* b = bufferQueue.front(); - bufferQueue.pop_front(); - b->dataCount = 0; - b->dataStart = 0; - return b; + void AsynchIO::notifyPendingWrite() { + // As notifyPendingWrite can be called on an arbitrary thread it must check whether we are processing or not. + // If we are then we just return as we know that we will eventually do the idle callback anyway. + // + // qpid::sys::ScopedLock<qpid::sys::Mutex> l(stateLock); + // We can get here in any state (as the caller could be in any thread) + State oldState; + State newState; + bool doReturn; + do { + newState = oldState = state.get(); + doReturn = false; + switch (oldState) { + case NOTIFY_WRITE: + case PENDING_NOTIFY: + // We only need to note a pending notify if we're already doing a notify as data processing + // is always followed by write notification processing + newState = PENDING_NOTIFY; + doReturn = true; + break; + case PENDING_DATA: + doReturn = true; + break; + case DATA: + // Only need to return here as data processing will do the idleCallback itself anyway + doReturn = true; + break; + case IDLE: + newState = NOTIFY_WRITE; + break; + case DELETED: + assert(oldState!=DELETED); + doReturn = true; + }; + } while (!state.boolCompareAndSwap(oldState, newState)); + if (doReturn) { + return; + } + + doWriteCallback(); + + // Keep track of what we need to do so that we can release the lock + enum {COMPLETION, NOTIFY, RETURN, EXIT} action; + // If there was pending data whilst we were doing this, process it now + // + // Using NOTIFY_WRITE for both NOTIFY & COMPLETION is a bit strange, but we're making sure we get the + // correct result if we reenter notifyPendingWrite(), in which case we want to + // end up in PENDING_NOTIFY (entering dataEvent doesn't matter as it only checks + // not IDLE) + do { + //qpid::sys::ScopedLock<qpid::sys::Mutex> l(stateLock); + do { + newState = oldState = state.get(); + action = RETURN; // Anything but COMPLETION + switch (oldState) { + case NOTIFY_WRITE: + newState = IDLE; + action = (action == COMPLETION) ? EXIT : RETURN; + break; + case PENDING_DATA: + newState = NOTIFY_WRITE; + action = COMPLETION; + break; + case PENDING_NOTIFY: + newState = NOTIFY_WRITE; + action = NOTIFY; + break; + default: + assert(oldState!=IDLE && oldState!=DATA && oldState!=DELETED); + action = RETURN; + } + } while (!state.boolCompareAndSwap(oldState, newState)); + + // Note we only get here if we were in the PENDING_DATA or PENDING_NOTIFY state + // so that we do need to process completions or notifications now + switch (action) { + case COMPLETION: + processCompletions(); + // Fall through + case NOTIFY: + doWriteCallback(); + break; + case RETURN: + return; + case EXIT: + // If we just processed completions we might need to delete ourselves + if (deleting && outstandingWrites == 0) { + delete this; + } + return; + } + } while (true); + } + + void AsynchIO::dataEvent(qpid::sys::DispatchHandle&) { + // Keep track of writable notifications + // qpid::sys::ScopedLock<qpid::sys::Mutex> l(stateLock); + State oldState; + State newState; + bool doReturn; + do { + newState = oldState = state.get(); + doReturn = false; + // We're already processing a notification + switch (oldState) { + case IDLE: + newState = DATA; + break; + default: + // Can't get here in DATA state as that would violate the serialisation rules + assert( oldState!=DATA ); + newState = PENDING_DATA; + doReturn = true; + } + } while (!state.boolCompareAndSwap(oldState, newState)); + if (doReturn) { + return; } + processCompletions(); + + //qpid::sys::ScopedLock<qpid::sys::Mutex> l(stateLock); + do { + newState = oldState = state.get(); + assert( oldState==DATA ); + newState = NOTIFY_WRITE; + } while (!state.boolCompareAndSwap(oldState, newState)); + + do { + doWriteCallback(); + + // qpid::sys::ScopedLock<qpid::sys::Mutex> l(stateLock); + bool doBreak; + do { + newState = oldState = state.get(); + doBreak = false; + if ( oldState==NOTIFY_WRITE ) { + newState = IDLE; + doBreak = true; + } else { + // Can't get DATA/PENDING_DATA here as dataEvent cannot be reentered + assert( oldState==PENDING_NOTIFY ); + newState = NOTIFY_WRITE; + } + } while (!state.boolCompareAndSwap(oldState, newState)); + if (doBreak) { + break; + } + } while (true); + + // We might need to delete ourselves + if (deleting && outstandingWrites == 0) { + delete this; + } } - void AsynchIO::dataEvent(DispatchHandle&) { + void AsynchIO::processCompletions() { QueuePair::intrusive_ptr q = qp->getNextChannelEvent(); + // Re-enable notification for queue: + // This needs to happen before we could do anything that could generate more work completion + // events (ie the callbacks etc. in the following). + // This can't make us reenter this code as the handle attached to the completion queue will still be + // disabled by the poller until we leave this code + qp->notifyRecv(); + qp->notifySend(); + + int recvEvents = 0; + int sendEvents = 0; + // If no event do nothing if (!q) return; assert(q == qp); - // Re-enable notification for queue - qp->notifySend(); - qp->notifyRecv(); - // Repeat until no more events do { QueuePairEvent e(qp->getNextEvent()); if (!e) - return; + break; ::ibv_wc_status status = e.getEventStatus(); if (status != IBV_WC_SUCCESS) { errorCallback(*this); + // TODO: Probably need to flush queues at this point return; } @@ -111,46 +346,143 @@ namespace Rdma { Buffer* b = e.getBuffer(); QueueDirection dir = e.getDirection(); if (dir == RECV) { - readCallback(*this, b); + ++recvEvents; + + // Get our xmitCredit if it was sent + bool dataPresent = true; + if (e.immPresent() ) { + xmitCredit += (e.getImm() & ~FlagsMask); + dataPresent = ((e.getImm() & IgnoreData) == 0); + } + + // if there was no data sent then the message was only to update our credit + if ( dataPresent ) { + readCallback(*this, b); + } + // At this point the buffer has been consumed so put it back on the recv queue + b->dataStart = 0; + b->dataCount = 0; qp->postRecv(b); + + // Received another message + ++recvCredit; + + // Send recvCredit if it is large enough (it will have got this large because we've not sent anything recently) + if (recvCredit > recvBufferCount/2) { + // TODO: This should use RDMA write with imm as there might not ever be a buffer to receive this message + // but this is a little unlikely, as to get in this state we have to have received messages without sending any + // for a while so its likely we've received an credit update from the far side. + if (writable()) { + Buffer* ob = getBuffer(); + // Have to send something as adapters hate it when you try to transfer 0 bytes + *reinterpret_cast< uint32_t* >(ob->bytes) = htonl(recvCredit); + ob->dataCount = sizeof(uint32_t); + + int creditSent = recvCredit & ~FlagsMask; + qp->postSend(creditSent | IgnoreData, ob); + recvCredit -= creditSent; + ++outstandingWrites; + --xmitCredit; + } else { + QPID_LOG(warning, "RDMA: qp=" << qp << ": Unable to send unsolicited credit"); + } + } } else { + ++sendEvents; { qpid::sys::ScopedLock<qpid::sys::Mutex> l(bufferQueueLock); bufferQueue.push_front(b); } --outstandingWrites; - // TODO: maybe don't call idle unless we're low on write buffers - idleCallback(*this); } } while (true); + + // Not sure if this is expected or not + if (recvEvents == 0 && sendEvents == 0) { + QPID_LOG(debug, "RDMA: qp=" << qp << ": Got channel event with no recv/send completions"); + } + } + + void AsynchIO::doWriteCallback() { + // TODO: maybe don't call idle unless we're low on write buffers + // Keep on calling the idle routine as long as we are writable and we got something to write last call + while (writable()) { + int xc = xmitCredit; + idleCallback(*this); + // Check whether we actually wrote anything + if (xmitCredit == xc) { + QPID_LOG(debug, "RDMA: qp=" << qp << ": Called for data, but got none: xmitCredit=" << xmitCredit); + return; + } + } + } + + Buffer* AsynchIO::getBuffer() { + qpid::sys::ScopedLock<qpid::sys::Mutex> l(bufferQueueLock); + assert(!bufferQueue.empty()); + Buffer* b = bufferQueue.front(); + bufferQueue.pop_front(); + b->dataCount = 0; + b->dataStart = 0; + return b; + } + + void AsynchIO::returnBuffer(Buffer* b) { + qpid::sys::ScopedLock<qpid::sys::Mutex> l(bufferQueueLock); + bufferQueue.push_front(b); + b->dataCount = 0; + b->dataStart = 0; + } + + ConnectionManager::ConnectionManager( + ErrorCallback errc, + DisconnectedCallback dc + ) : + ci(Connection::make()), + handle(*ci, boost::bind(&ConnectionManager::event, this, _1), 0, 0), + errorCallback(errc), + disconnectedCallback(dc) + { + ci->nonblocking(); + } + + ConnectionManager::~ConnectionManager() + { + handle.stopWatch(); + } + + void ConnectionManager::start(Poller::shared_ptr poller) { + startConnection(ci); + handle.startWatch(poller); + } + + void ConnectionManager::event(DispatchHandle&) { + connectionEvent(ci); } Listener::Listener( - const sockaddr& src, - ConnectedCallback cc, + const SocketAddress& src, + const ConnectionParams& cp, + EstablishedCallback ec, ErrorCallback errc, DisconnectedCallback dc, ConnectionRequestCallback crc ) : + ConnectionManager(errc, dc), src_addr(src), - ci(Connection::make()), - handle(*ci, boost::bind(&Listener::connectionEvent, this, _1), 0, 0), - connectedCallback(cc), - errorCallback(errc), - disconnectedCallback(dc), - connectionRequestCallback(crc) + checkConnectionParams(cp), + connectionRequestCallback(crc), + establishedCallback(ec) { - ci->nonblocking(); } - void Listener::start(Poller::shared_ptr poller) { + void Listener::startConnection(Connection::intrusive_ptr ci) { ci->bind(src_addr); ci->listen(); - handle.startWatch(poller); } - void Listener::connectionEvent(DispatchHandle&) { + void Listener::connectionEvent(Connection::intrusive_ptr ci) { ConnectionEvent e(ci->getNextEvent()); // If (for whatever reason) there was no event do nothing @@ -161,65 +493,75 @@ namespace Rdma { // you get from CONNECT_REQUEST has the same context info // as its parent listening rdma_cm_id ::rdma_cm_event_type eventType = e.getEventType(); + ::rdma_conn_param conn_param = e.getConnectionParam(); Rdma::Connection::intrusive_ptr id = e.getConnection(); switch (eventType) { case RDMA_CM_EVENT_CONNECT_REQUEST: { - bool accept = true; - // Extract connection parameters and private data from event - ::rdma_conn_param conn_param = e.getConnectionParam(); + // Make sure peer has sent params we can use + if (!conn_param.private_data || conn_param.private_data_len < sizeof(ConnectionParams)) { + id->reject(); + break; + } + ConnectionParams cp = *static_cast<const ConnectionParams*>(conn_param.private_data); + + // Reject if requested msg size is bigger than we allow + if (cp.maxRecvBufferSize > checkConnectionParams.maxRecvBufferSize) { + id->reject(&checkConnectionParams); + break; + } + bool accept = true; if (connectionRequestCallback) - //TODO: pass private data to callback (and accept new private data for accept somehow) - accept = connectionRequestCallback(id); + accept = connectionRequestCallback(id, cp); + if (accept) { // Accept connection - id->accept(conn_param); + cp.initialXmitCredit = checkConnectionParams.initialXmitCredit; + id->accept(conn_param, &cp); } else { - //Reject connection + // Reject connection id->reject(); } - break; } case RDMA_CM_EVENT_ESTABLISHED: - connectedCallback(id); + establishedCallback(id); break; case RDMA_CM_EVENT_DISCONNECTED: disconnectedCallback(id); break; case RDMA_CM_EVENT_CONNECT_ERROR: - errorCallback(id); + errorCallback(id, CONNECT_ERROR); break; default: - std::cerr << "Warning: unexpected response to listen - " << eventType << "\n"; + // Unexpected response + errorCallback(id, UNKNOWN); + //std::cerr << "Warning: unexpected response to listen - " << eventType << "\n"; } } Connector::Connector( - const sockaddr& dst, + const SocketAddress& dst, + const ConnectionParams& cp, ConnectedCallback cc, ErrorCallback errc, DisconnectedCallback dc, RejectedCallback rc ) : + ConnectionManager(errc, dc), dst_addr(dst), - ci(Connection::make()), - handle(*ci, boost::bind(&Connector::connectionEvent, this, _1), 0, 0), - connectedCallback(cc), - errorCallback(errc), - disconnectedCallback(dc), - rejectedCallback(rc) + connectionParams(cp), + rejectedCallback(rc), + connectedCallback(cc) { - ci->nonblocking(); } - void Connector::start(Poller::shared_ptr poller) { + void Connector::startConnection(Connection::intrusive_ptr ci) { ci->resolve_addr(dst_addr); - handle.startWatch(poller); } - void Connector::connectionEvent(DispatchHandle&) { + void Connector::connectionEvent(Connection::intrusive_ptr ci) { ConnectionEvent e(ci->getNextEvent()); // If (for whatever reason) there was no event do nothing @@ -227,6 +569,8 @@ namespace Rdma { return; ::rdma_cm_event_type eventType = e.getEventType(); + ::rdma_conn_param conn_param = e.getConnectionParam(); + Rdma::Connection::intrusive_ptr id = e.getConnection(); switch (eventType) { case RDMA_CM_EVENT_ADDR_RESOLVED: // RESOLVE_ADDR @@ -234,38 +578,46 @@ namespace Rdma { break; case RDMA_CM_EVENT_ADDR_ERROR: // RESOLVE_ADDR - errorCallback(ci); + errorCallback(ci, ADDR_ERROR); break; case RDMA_CM_EVENT_ROUTE_RESOLVED: // RESOLVE_ROUTE: - ci->connect(); + ci->connect(&connectionParams); break; case RDMA_CM_EVENT_ROUTE_ERROR: // RESOLVE_ROUTE: - errorCallback(ci); + errorCallback(ci, ROUTE_ERROR); break; case RDMA_CM_EVENT_CONNECT_ERROR: // CONNECTING - errorCallback(ci); + errorCallback(ci, CONNECT_ERROR); break; case RDMA_CM_EVENT_UNREACHABLE: // CONNECTING - errorCallback(ci); + errorCallback(ci, UNREACHABLE); break; - case RDMA_CM_EVENT_REJECTED: + case RDMA_CM_EVENT_REJECTED: { // CONNECTING - rejectedCallback(ci); + // Extract private data from event + assert(conn_param.private_data && conn_param.private_data_len >= sizeof(ConnectionParams)); + ConnectionParams cp = *static_cast<const ConnectionParams*>(conn_param.private_data); + rejectedCallback(ci, cp); break; - case RDMA_CM_EVENT_ESTABLISHED: + } + case RDMA_CM_EVENT_ESTABLISHED: { // CONNECTING - connectedCallback(ci); + // Extract private data from event + assert(conn_param.private_data && conn_param.private_data_len >= sizeof(ConnectionParams)); + ConnectionParams cp = *static_cast<const ConnectionParams*>(conn_param.private_data); + connectedCallback(ci, cp); break; + } case RDMA_CM_EVENT_DISCONNECTED: // ESTABLISHED disconnectedCallback(ci); break; default: - std::cerr << "Warning: unexpected event in connect: " << eventType << "\n"; + QPID_LOG(warning, "RDMA: Unexpected event in connect: " << eventType); } } } diff --git a/cpp/src/qpid/sys/rdma/RdmaIO.h b/cpp/src/qpid/sys/rdma/RdmaIO.h index 5c8d49607c..12a1b98d24 100644 --- a/cpp/src/qpid/sys/rdma/RdmaIO.h +++ b/cpp/src/qpid/sys/rdma/RdmaIO.h @@ -1,10 +1,33 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ #ifndef Rdma_Acceptor_h #define Rdma_Acceptor_h -#include "rdma_wrap.h" +#include "qpid/sys/rdma/rdma_wrap.h" +#include "qpid/sys/AtomicValue.h" #include "qpid/sys/Dispatcher.h" +#include "qpid/sys/DispatchHandle.h" #include "qpid/sys/Mutex.h" +#include "qpid/sys/SocketAddress.h" #include <netinet/in.h> @@ -12,32 +35,30 @@ #include <boost/ptr_container/ptr_deque.hpp> #include <deque> -using qpid::sys::DispatchHandle; -using qpid::sys::Poller; - namespace Rdma { class Connection; - typedef boost::function1<void, Rdma::Connection::intrusive_ptr&> ConnectedCallback; - typedef boost::function1<void, Rdma::Connection::intrusive_ptr&> ErrorCallback; - typedef boost::function1<void, Rdma::Connection::intrusive_ptr&> DisconnectedCallback; - typedef boost::function1<bool, Rdma::Connection::intrusive_ptr&> ConnectionRequestCallback; - typedef boost::function1<void, Rdma::Connection::intrusive_ptr&> RejectedCallback; - class AsynchIO { typedef boost::function1<void, AsynchIO&> ErrorCallback; typedef boost::function2<void, AsynchIO&, Buffer*> ReadCallback; typedef boost::function1<void, AsynchIO&> IdleCallback; - typedef boost::function1<void, AsynchIO&> FullCallback; + typedef boost::function2<void, AsynchIO&, Buffer*> FullCallback; QueuePair::intrusive_ptr qp; - DispatchHandle dataHandle; + qpid::sys::DispatchHandleRef dataHandle; int bufferSize; + int recvCredit; + int xmitCredit; int recvBufferCount; int xmitBufferCount; int outstandingWrites; + bool closed; // TODO: Perhaps (probably) this state can be merged with the following... + bool deleting; // TODO: Perhaps (probably) this state can be merged with the following... + enum State { IDLE, DATA, PENDING_DATA, NOTIFY_WRITE, PENDING_NOTIFY, DELETED }; + qpid::sys::AtomicValue<State> state; + //qpid::sys::Mutex stateLock; std::deque<Buffer*> bufferQueue; qpid::sys::Mutex bufferQueueLock; boost::ptr_deque<Buffer> buffers; @@ -48,72 +69,154 @@ namespace Rdma { ErrorCallback errorCallback; public: + // TODO: Instead of specifying a buffer size specify the amount of memory the AsynchIO class can use + // for buffers both read and write (allocate half to each up front) and fail if we cannot allocate that much + // locked memory AsynchIO( QueuePair::intrusive_ptr q, - int s, + int size, + int xCredit, + int rCount, ReadCallback rc, IdleCallback ic, FullCallback fc, ErrorCallback ec ); - ~AsynchIO(); - void start(Poller::shared_ptr poller); + void start(qpid::sys::Poller::shared_ptr poller); + bool writable() const; + bool bufferAvailable() const; void queueWrite(Buffer* buff); void notifyPendingWrite(); void queueWriteClose(); + void deferDelete(); + int incompletedWrites() const; Buffer* getBuffer(); + void returnBuffer(Buffer*); private: - void dataEvent(DispatchHandle& handle); + // Don't let anyone else delete us to make sure there can't be outstanding callbacks + ~AsynchIO(); + + // Constants for the peer-peer command messages + // These are sent in the high bits if the imm data of an rdma message + // The low bits are used to send the credit + const static int FlagsMask = 0x10000000; // Mask for all flag bits - be sure to update this if you add more command bits + const static int IgnoreData = 0x10000000; // Message contains no application data + + void dataEvent(qpid::sys::DispatchHandle& handle); + void processCompletions(); + void doWriteCallback(); }; - class Listener - { - sockaddr src_addr; + inline bool AsynchIO::writable() const { + return (!closed && outstandingWrites < xmitBufferCount && xmitCredit > 0); + } + + inline int AsynchIO::incompletedWrites() const { + return outstandingWrites; + } + + inline bool AsynchIO::bufferAvailable() const { + return !bufferQueue.empty(); + } + // These are the parameters necessary to start the conversation + // * Each peer HAS to allocate buffers of the size of the maximum receive from its peer + // * Each peer HAS to know the initial "credit" it has for transmitting to its peer + struct ConnectionParams { + int maxRecvBufferSize; + int initialXmitCredit ; + + ConnectionParams(int s, int c) : + maxRecvBufferSize(s), + initialXmitCredit(c) + {} + }; + + enum ErrorType { + ADDR_ERROR, + ROUTE_ERROR, + CONNECT_ERROR, + UNREACHABLE, + UNKNOWN + }; + + typedef boost::function2<void, Rdma::Connection::intrusive_ptr&, ErrorType> ErrorCallback; + typedef boost::function1<void, Rdma::Connection::intrusive_ptr&> DisconnectedCallback; + + class ConnectionManager { Connection::intrusive_ptr ci; - DispatchHandle handle; - ConnectedCallback connectedCallback; + qpid::sys::DispatchHandle handle; + + protected: ErrorCallback errorCallback; DisconnectedCallback disconnectedCallback; + + public: + ConnectionManager( + ErrorCallback errc, + DisconnectedCallback dc + ); + + virtual ~ConnectionManager(); + + void start(qpid::sys::Poller::shared_ptr poller); + + private: + void event(qpid::sys::DispatchHandle& handle); + + virtual void startConnection(Connection::intrusive_ptr ci) = 0; + virtual void connectionEvent(Connection::intrusive_ptr ci) = 0; + }; + + typedef boost::function2<bool, Rdma::Connection::intrusive_ptr&, const ConnectionParams&> ConnectionRequestCallback; + typedef boost::function1<void, Rdma::Connection::intrusive_ptr&> EstablishedCallback; + + class Listener : public ConnectionManager + { + qpid::sys::SocketAddress src_addr; + ConnectionParams checkConnectionParams; ConnectionRequestCallback connectionRequestCallback; + EstablishedCallback establishedCallback; public: Listener( - const sockaddr& src, - ConnectedCallback cc, + const qpid::sys::SocketAddress& src, + const ConnectionParams& cp, + EstablishedCallback ec, ErrorCallback errc, DisconnectedCallback dc, ConnectionRequestCallback crc = 0 ); - void start(Poller::shared_ptr poller); private: - void connectionEvent(DispatchHandle& handle); + void startConnection(Connection::intrusive_ptr ci); + void connectionEvent(Connection::intrusive_ptr ci); }; - class Connector + typedef boost::function2<void, Rdma::Connection::intrusive_ptr&, const ConnectionParams&> RejectedCallback; + typedef boost::function2<void, Rdma::Connection::intrusive_ptr&, const ConnectionParams&> ConnectedCallback; + + class Connector : public ConnectionManager { - sockaddr dst_addr; - Connection::intrusive_ptr ci; - DispatchHandle handle; - ConnectedCallback connectedCallback; - ErrorCallback errorCallback; - DisconnectedCallback disconnectedCallback; + qpid::sys::SocketAddress dst_addr; + ConnectionParams connectionParams; RejectedCallback rejectedCallback; + ConnectedCallback connectedCallback; public: Connector( - const sockaddr& dst, + const qpid::sys::SocketAddress& dst, + const ConnectionParams& cp, ConnectedCallback cc, ErrorCallback errc, DisconnectedCallback dc, RejectedCallback rc = 0 ); - void start(Poller::shared_ptr poller); private: - void connectionEvent(DispatchHandle& handle); + void startConnection(Connection::intrusive_ptr ci); + void connectionEvent(Connection::intrusive_ptr ci); }; } diff --git a/cpp/src/qpid/sys/rdma/RdmaServer.cpp b/cpp/src/qpid/sys/rdma/RdmaServer.cpp index f7f739d6c2..4c11ba23eb 100644 --- a/cpp/src/qpid/sys/rdma/RdmaServer.cpp +++ b/cpp/src/qpid/sys/rdma/RdmaServer.cpp @@ -1,4 +1,24 @@ -#include "RdmaIO.h" +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/rdma/RdmaIO.h" #include <arpa/inet.h> @@ -15,19 +35,21 @@ using std::string; using std::cout; using std::cerr; +using qpid::sys::SocketAddress; using qpid::sys::Poller; using qpid::sys::Dispatcher; // All the accepted connections +namespace qpid { +namespace tests { + struct ConRec { Rdma::Connection::intrusive_ptr connection; Rdma::AsynchIO* data; - bool writable; queue<Rdma::Buffer*> queuedWrites; ConRec(Rdma::Connection::intrusive_ptr c) : - connection(c), - writable(true) + connection(c) {} }; @@ -35,50 +57,53 @@ void dataError(Rdma::AsynchIO&) { cout << "Data error:\n"; } +void idle(ConRec* cr, Rdma::AsynchIO& a) { + // Need to make sure full is not called as it would reorder messages + while (!cr->queuedWrites.empty() && a.writable()) { + Rdma::Buffer* buf = cr->queuedWrites.front(); + cr->queuedWrites.pop(); + a.queueWrite(buf); + } +} + void data(ConRec* cr, Rdma::AsynchIO& a, Rdma::Buffer* b) { // Echo data back Rdma::Buffer* buf = a.getBuffer(); std::copy(b->bytes+b->dataStart, b->bytes+b->dataStart+b->dataCount, buf->bytes); buf->dataCount = b->dataCount; - if (cr->queuedWrites.empty() && cr->writable) { + if (cr->queuedWrites.empty()) { + // If can't write then full will be called and push buffer on back of queue a.queueWrite(buf); } else { cr->queuedWrites.push(buf); + // Try to empty queue + idle(cr, a); } } -void full(ConRec* cr, Rdma::AsynchIO&) { - cr->writable = false; -} - -void idle(ConRec* cr, Rdma::AsynchIO& a) { - cr->writable = true; - while (!cr->queuedWrites.empty() && cr->writable) { - Rdma::Buffer* buf = cr->queuedWrites.front(); - cr->queuedWrites.pop(); - a.queueWrite(buf); - } +void full(ConRec* cr, Rdma::AsynchIO&, Rdma::Buffer* buf) { + cr->queuedWrites.push(buf); } void disconnected(Rdma::Connection::intrusive_ptr& ci) { ConRec* cr = ci->getContext<ConRec>(); cr->connection->disconnect(); - delete cr->data; + cr->data->queueWriteClose(); delete cr; cout << "Disconnected: " << cr << "\n"; } -void connectionError(Rdma::Connection::intrusive_ptr& ci) { +void connectionError(Rdma::Connection::intrusive_ptr& ci, Rdma::ErrorType) { ConRec* cr = ci->getContext<ConRec>(); cr->connection->disconnect(); if (cr) { - delete cr->data; + cr->data->queueWriteClose(); delete cr; } cout << "Connection error: " << cr << "\n"; } -bool connectionRequest(Rdma::Connection::intrusive_ptr& ci) { +bool connectionRequest(Rdma::Connection::intrusive_ptr& ci, const Rdma::ConnectionParams& cp) { cout << "Incoming connection: "; // For fun reject alternate connection attempts @@ -89,10 +114,11 @@ bool connectionRequest(Rdma::Connection::intrusive_ptr& ci) { if (x) { ConRec* cr = new ConRec(ci); Rdma::AsynchIO* aio = - new Rdma::AsynchIO(ci->getQueuePair(), 8000, + new Rdma::AsynchIO(ci->getQueuePair(), + cp.maxRecvBufferSize, cp.initialXmitCredit, Rdma::DEFAULT_WR_ENTRIES, boost::bind(data, cr, _1, _2), boost::bind(idle, cr, _1), - boost::bind(full, cr, _1), + boost::bind(full, cr, _1, _2), dataError); ci->addContext(cr); cr->data = aio; @@ -112,23 +138,23 @@ void connected(Poller::shared_ptr poller, Rdma::Connection::intrusive_ptr& ci) { cr->data->start(poller); } +}} // namespace qpid::tests + +using namespace qpid::tests; + int main(int argc, char* argv[]) { vector<string> args(&argv[0], &argv[argc]); - ::sockaddr_in sin; - - int port = (args.size() < 2) ? 20079 : atoi(args[1].c_str()); + std::string port = (args.size() < 2) ? "20079" : args[1]; cout << "Listening on port: " << port << "\n"; - sin.sin_family = AF_INET; - sin.sin_port = htons(port); - sin.sin_addr.s_addr = INADDR_ANY; - try { boost::shared_ptr<Poller> p(new Poller()); Dispatcher d(p); - Rdma::Listener a((const sockaddr&)(sin), + SocketAddress sa("", port); + Rdma::Listener a(sa, + Rdma::ConnectionParams(16384, Rdma::DEFAULT_WR_ENTRIES), boost::bind(connected, p, _1), connectionError, disconnected, diff --git a/cpp/src/qpid/sys/rdma/rdma_exception.h b/cpp/src/qpid/sys/rdma/rdma_exception.h index 2773f25917..7867aef2e4 100644 --- a/cpp/src/qpid/sys/rdma/rdma_exception.h +++ b/cpp/src/qpid/sys/rdma/rdma_exception.h @@ -1,3 +1,23 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ #ifndef RDMA_EXCEPTION_H #define RDMA_EXCEPTION_H diff --git a/cpp/src/qpid/sys/rdma/rdma_factories.cpp b/cpp/src/qpid/sys/rdma/rdma_factories.cpp index 53dc4f8935..a5e9ebd23c 100644 --- a/cpp/src/qpid/sys/rdma/rdma_factories.cpp +++ b/cpp/src/qpid/sys/rdma/rdma_factories.cpp @@ -1,4 +1,24 @@ -#include "rdma_factories.h" +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/rdma/rdma_factories.h" namespace Rdma { void acker(::rdma_cm_event* e) throw () { @@ -36,4 +56,9 @@ namespace Rdma { (void) ::ibv_destroy_cq(cq); } + void destroyQp(::ibv_qp* qp) throw () { + if (qp) + (void) ::ibv_destroy_qp(qp); + } + } diff --git a/cpp/src/qpid/sys/rdma/rdma_factories.h b/cpp/src/qpid/sys/rdma/rdma_factories.h index 26a93bb494..783181cf1b 100644 --- a/cpp/src/qpid/sys/rdma/rdma_factories.h +++ b/cpp/src/qpid/sys/rdma/rdma_factories.h @@ -1,7 +1,27 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ #ifndef RDMA_FACTORIES_H #define RDMA_FACTORIES_H -#include "rdma_exception.h" +#include "qpid/sys/rdma/rdma_exception.h" #include <rdma/rdma_cma.h> @@ -15,10 +35,11 @@ namespace Rdma { void deallocPd(::ibv_pd* p) throw (); void destroyCChannel(::ibv_comp_channel* c) throw (); void destroyCq(::ibv_cq* cq) throw (); + void destroyQp(::ibv_qp* qp) throw (); inline boost::shared_ptr< ::rdma_event_channel > mkEChannel() { - return - boost::shared_ptr< ::rdma_event_channel >(::rdma_create_event_channel(), destroyEChannel); + ::rdma_event_channel* c = CHECK_NULL(::rdma_create_event_channel()); + return boost::shared_ptr< ::rdma_event_channel >(c, destroyEChannel); } inline boost::shared_ptr< ::rdma_cm_id > diff --git a/cpp/src/qpid/sys/rdma/rdma_wrap.cpp b/cpp/src/qpid/sys/rdma/rdma_wrap.cpp new file mode 100644 index 0000000000..53e31ca766 --- /dev/null +++ b/cpp/src/qpid/sys/rdma/rdma_wrap.cpp @@ -0,0 +1,185 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/rdma/rdma_wrap.h" + +namespace Rdma { + const ::rdma_conn_param DEFAULT_CONNECT_PARAM = { + 0, // .private_data + 0, // .private_data_len + 4, // .responder_resources + 4, // .initiator_depth + 0, // .flow_control + 5, // .retry_count + 7 // .rnr_retry_count + }; + + // This is moderately inefficient so don't use in a critical path + int deviceCount() { + int count; + ::ibv_free_device_list(::ibv_get_device_list(&count)); + return count; + } + + ::rdma_conn_param ConnectionEvent::getConnectionParam() const { + // It's badly documented, but it seems from the librdma source code that all the following + // event types have a valid param.conn + switch (event->event) { + case RDMA_CM_EVENT_CONNECT_REQUEST: + case RDMA_CM_EVENT_ESTABLISHED: + case RDMA_CM_EVENT_REJECTED: + case RDMA_CM_EVENT_DISCONNECTED: + case RDMA_CM_EVENT_CONNECT_ERROR: + return event->param.conn; + default: + ::rdma_conn_param p = {}; + return p; + } + } + + QueuePair::QueuePair(boost::shared_ptr< ::rdma_cm_id > i) : + qpid::sys::IOHandle(new qpid::sys::IOHandlePrivate), + pd(allocPd(i->verbs)), + cchannel(mkCChannel(i->verbs)), + scq(mkCq(i->verbs, DEFAULT_CQ_ENTRIES, 0, cchannel.get())), + rcq(mkCq(i->verbs, DEFAULT_CQ_ENTRIES, 0, cchannel.get())), + outstandingSendEvents(0), + outstandingRecvEvents(0) + { + impl->fd = cchannel->fd; + + // Set cq context to this QueuePair object so we can find + // ourselves again + scq->cq_context = this; + rcq->cq_context = this; + + ::ibv_qp_init_attr qp_attr = {}; + + // TODO: make a default struct for this + qp_attr.cap.max_send_wr = DEFAULT_WR_ENTRIES; + qp_attr.cap.max_send_sge = 4; + qp_attr.cap.max_recv_wr = DEFAULT_WR_ENTRIES; + qp_attr.cap.max_recv_sge = 4; + + qp_attr.send_cq = scq.get(); + qp_attr.recv_cq = rcq.get(); + qp_attr.qp_type = IBV_QPT_RC; + + CHECK(::rdma_create_qp(i.get(), pd.get(), &qp_attr)); + qp = boost::shared_ptr< ::ibv_qp >(i->qp, destroyQp); + + // Set the qp context to this so we can find ourselves again + qp->qp_context = this; + } + + QueuePair::~QueuePair() { + if (outstandingSendEvents > 0) + ::ibv_ack_cq_events(scq.get(), outstandingSendEvents); + if (outstandingRecvEvents > 0) + ::ibv_ack_cq_events(rcq.get(), outstandingRecvEvents); + + // Reset back pointer in case someone else has the qp + qp->qp_context = 0; + } + + void QueuePair::postRecv(Buffer* buf) { + ::ibv_recv_wr rwr = {}; + ::ibv_sge sge; + + sge.addr = (uintptr_t) buf->bytes+buf->dataStart; + sge.length = buf->dataCount; + sge.lkey = buf->mr->lkey; + + rwr.wr_id = reinterpret_cast<uint64_t>(buf); + rwr.sg_list = &sge; + rwr.num_sge = 1; + + ::ibv_recv_wr* badrwr = 0; + CHECK_IBV(::ibv_post_recv(qp.get(), &rwr, &badrwr)); + if (badrwr) + throw std::logic_error("ibv_post_recv(): Bad rwr"); + } + + void QueuePair::postSend(Buffer* buf) { + ::ibv_send_wr swr = {}; + ::ibv_sge sge; + + sge.addr = (uintptr_t) buf->bytes+buf->dataStart; + sge.length = buf->dataCount; + sge.lkey = buf->mr->lkey; + + swr.wr_id = reinterpret_cast<uint64_t>(buf); + swr.opcode = IBV_WR_SEND; + swr.send_flags = IBV_SEND_SIGNALED; + swr.sg_list = &sge; + swr.num_sge = 1; + + ::ibv_send_wr* badswr = 0; + CHECK_IBV(::ibv_post_send(qp.get(), &swr, &badswr)); + if (badswr) + throw std::logic_error("ibv_post_send(): Bad swr"); + } + + void QueuePair::postSend(uint32_t imm, Buffer* buf) { + ::ibv_send_wr swr = {}; + ::ibv_sge sge; + + sge.addr = (uintptr_t) buf->bytes+buf->dataStart; + sge.length = buf->dataCount; + sge.lkey = buf->mr->lkey; + swr.send_flags = IBV_SEND_SIGNALED; + + swr.wr_id = reinterpret_cast<uint64_t>(buf); + swr.imm_data = htonl(imm); + swr.opcode = IBV_WR_SEND_WITH_IMM; + swr.sg_list = &sge; + swr.num_sge = 1; + + ::ibv_send_wr* badswr = 0; + CHECK_IBV(::ibv_post_send(qp.get(), &swr, &badswr)); + if (badswr) + throw std::logic_error("ibv_post_send(): Bad swr"); + } +} + +std::ostream& operator<<(std::ostream& o, ::rdma_cm_event_type t) { +# define CHECK_TYPE(t) case t: o << #t; break; + switch(t) { + CHECK_TYPE(RDMA_CM_EVENT_ADDR_RESOLVED) + CHECK_TYPE(RDMA_CM_EVENT_ADDR_ERROR) + CHECK_TYPE(RDMA_CM_EVENT_ROUTE_RESOLVED) + CHECK_TYPE(RDMA_CM_EVENT_ROUTE_ERROR) + CHECK_TYPE(RDMA_CM_EVENT_CONNECT_REQUEST) + CHECK_TYPE(RDMA_CM_EVENT_CONNECT_RESPONSE) + CHECK_TYPE(RDMA_CM_EVENT_CONNECT_ERROR) + CHECK_TYPE(RDMA_CM_EVENT_UNREACHABLE) + CHECK_TYPE(RDMA_CM_EVENT_REJECTED) + CHECK_TYPE(RDMA_CM_EVENT_ESTABLISHED) + CHECK_TYPE(RDMA_CM_EVENT_DISCONNECTED) + CHECK_TYPE(RDMA_CM_EVENT_DEVICE_REMOVAL) + CHECK_TYPE(RDMA_CM_EVENT_MULTICAST_JOIN) + CHECK_TYPE(RDMA_CM_EVENT_MULTICAST_ERROR) + default: + o << "UNKNOWN_EVENT"; + } +# undef CHECK_TYPE + return o; +} diff --git a/cpp/src/qpid/sys/rdma/rdma_wrap.h b/cpp/src/qpid/sys/rdma/rdma_wrap.h index 421c8fc41b..e11497dc02 100644 --- a/cpp/src/qpid/sys/rdma/rdma_wrap.h +++ b/cpp/src/qpid/sys/rdma/rdma_wrap.h @@ -1,7 +1,27 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ #ifndef RDMA_WRAP_H #define RDMA_WRAP_H -#include "rdma_factories.h" +#include "qpid/sys/rdma/rdma_factories.h" #include <rdma/rdma_cma.h> @@ -11,6 +31,8 @@ #include <fcntl.h> +#include <netdb.h> + #include <vector> #include <algorithm> #include <iostream> @@ -23,15 +45,9 @@ namespace Rdma { const int DEFAULT_BACKLOG = 100; const int DEFAULT_CQ_ENTRIES = 256; const int DEFAULT_WR_ENTRIES = 64; - const ::rdma_conn_param DEFAULT_CONNECT_PARAM = { - 0, // .private_data - 0, // .private_data_len - 4, // .responder_resources - 4, // .initiator_depth - 0, // .flow_control - 5, // .retry_count - 7 // .rnr_retry_count - }; + extern const ::rdma_conn_param DEFAULT_CONNECT_PARAM; + + int deviceCount(); struct Buffer { friend class QueuePair; @@ -95,6 +111,14 @@ namespace Rdma { return dir != NONE; } + bool immPresent() const { + return wc.wc_flags & IBV_WC_WITH_IMM; + } + + uint32_t getImm() const { + return ntohl(wc.imm_data); + } + QueueDirection getDirection() const { return dir; } @@ -117,15 +141,12 @@ namespace Rdma { // Wrapper for a queue pair - this has the functionality for // putting buffers on the receive queue and for sending buffers // to the other end of the connection. - // - // Currently QueuePairs are contained inside Connections and have no - // separate lifetime class QueuePair : public qpid::sys::IOHandle, public qpid::RefCounted { boost::shared_ptr< ::ibv_pd > pd; boost::shared_ptr< ::ibv_comp_channel > cchannel; boost::shared_ptr< ::ibv_cq > scq; boost::shared_ptr< ::ibv_cq > rcq; - boost::shared_ptr< ::rdma_cm_id > id; + boost::shared_ptr< ::ibv_qp > qp; int outstandingSendEvents; int outstandingRecvEvents; @@ -187,6 +208,7 @@ namespace Rdma { void postRecv(Buffer* buf); void postSend(Buffer* buf); + void postSend(uint32_t imm, Buffer* buf); void notifyRecv(); void notifySend(); }; @@ -213,14 +235,7 @@ namespace Rdma { return event->event; } - ::rdma_conn_param getConnectionParam() const { - if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) { - return event->param.conn; - } else { - ::rdma_conn_param p = {}; - return p; - } - } + ::rdma_conn_param getConnectionParam() const; boost::intrusive_ptr<Connection> getConnection () const { return id; @@ -271,6 +286,11 @@ namespace Rdma { impl->fd = channel->fd; } + ~Connection() { + // Reset the id context in case someone else has it + id->context = 0; + } + // Default destructor fine void ensureQueuePair() { @@ -330,9 +350,9 @@ namespace Rdma { return ConnectionEvent(e); } - void bind(sockaddr& src_addr) const { + void bind(qpid::sys::SocketAddress& src_addr) const { assert(id.get()); - CHECK(::rdma_bind_addr(id.get(), &src_addr)); + CHECK(::rdma_bind_addr(id.get(), getAddrInfo(src_addr).ai_addr)); } void listen(int backlog = DEFAULT_BACKLOG) const { @@ -341,12 +361,11 @@ namespace Rdma { } void resolve_addr( - sockaddr& dst_addr, - sockaddr* src_addr = 0, + qpid::sys::SocketAddress& dst_addr, int timeout_ms = DEFAULT_TIMEOUT) const { assert(id.get()); - CHECK(::rdma_resolve_addr(id.get(), src_addr, &dst_addr, timeout_ms)); + CHECK(::rdma_resolve_addr(id.get(), 0, getAddrInfo(dst_addr).ai_addr, timeout_ms)); } void resolve_route(int timeout_ms = DEFAULT_TIMEOUT) const { @@ -425,52 +444,38 @@ namespace Rdma { return qp; } + + std::string getLocalName() const { + ::sockaddr* addr = ::rdma_get_local_addr(id.get()); + char hostName[NI_MAXHOST]; + char portName[NI_MAXSERV]; + CHECK_IBV(::getnameinfo( + addr, sizeof(::sockaddr_storage), + hostName, sizeof(hostName), + portName, sizeof(portName), + NI_NUMERICHOST | NI_NUMERICSERV)); + std::string r(hostName); + r += ":"; + r += portName; + return r; + } + + std::string getPeerName() const { + ::sockaddr* addr = ::rdma_get_peer_addr(id.get()); + char hostName[NI_MAXHOST]; + char portName[NI_MAXSERV]; + CHECK_IBV(::getnameinfo( + addr, sizeof(::sockaddr_storage), + hostName, sizeof(hostName), + portName, sizeof(portName), + NI_NUMERICHOST | NI_NUMERICSERV)); + std::string r(hostName); + r += ":"; + r += portName; + return r; + } }; - inline QueuePair::QueuePair(boost::shared_ptr< ::rdma_cm_id > i) : - qpid::sys::IOHandle(new qpid::sys::IOHandlePrivate), - pd(allocPd(i->verbs)), - cchannel(mkCChannel(i->verbs)), - scq(mkCq(i->verbs, DEFAULT_CQ_ENTRIES, 0, cchannel.get())), - rcq(mkCq(i->verbs, DEFAULT_CQ_ENTRIES, 0, cchannel.get())), - id(i), - outstandingSendEvents(0), - outstandingRecvEvents(0) - { - impl->fd = cchannel->fd; - - // Set cq context to this QueuePair object so we can find - // ourselves again - scq->cq_context = this; - rcq->cq_context = this; - - ::ibv_qp_init_attr qp_attr = {}; - - // TODO: make a default struct for this - qp_attr.cap.max_send_wr = DEFAULT_WR_ENTRIES; - qp_attr.cap.max_send_sge = 4; - qp_attr.cap.max_recv_wr = DEFAULT_WR_ENTRIES; - qp_attr.cap.max_recv_sge = 4; - - qp_attr.send_cq = scq.get(); - qp_attr.recv_cq = rcq.get(); - qp_attr.qp_type = IBV_QPT_RC; - - CHECK(::rdma_create_qp(id.get(), pd.get(), &qp_attr)); - - // Set the qp context to this so we can find ourselves again - id->qp->qp_context = this; - } - - inline QueuePair::~QueuePair() { - if (outstandingSendEvents > 0) - ::ibv_ack_cq_events(scq.get(), outstandingSendEvents); - if (outstandingRecvEvents > 0) - ::ibv_ack_cq_events(rcq.get(), outstandingRecvEvents); - - ::rdma_destroy_qp(id.get()); - } - inline void QueuePair::notifyRecv() { CHECK_IBV(ibv_req_notify_cq(rcq.get(), 0)); } @@ -479,44 +484,6 @@ namespace Rdma { CHECK_IBV(ibv_req_notify_cq(scq.get(), 0)); } - inline void QueuePair::postRecv(Buffer* buf) { - ::ibv_recv_wr rwr = {}; - ::ibv_sge sge; - - sge.addr = (uintptr_t) buf->bytes+buf->dataStart; - sge.length = buf->dataCount; - sge.lkey = buf->mr->lkey; - - rwr.wr_id = reinterpret_cast<uint64_t>(buf); - rwr.sg_list = &sge; - rwr.num_sge = 1; - - ::ibv_recv_wr* badrwr = 0; - CHECK_IBV(::ibv_post_recv(id->qp, &rwr, &badrwr)); - if (badrwr) - throw std::logic_error("ibv_post_recv(): Bad rwr"); - } - - inline void QueuePair::postSend(Buffer* buf) { - ::ibv_send_wr swr = {}; - ::ibv_sge sge; - - sge.addr = (uintptr_t) buf->bytes+buf->dataStart; - sge.length = buf->dataCount; - sge.lkey = buf->mr->lkey; - - swr.wr_id = reinterpret_cast<uint64_t>(buf); - swr.opcode = IBV_WR_SEND; - swr.send_flags = IBV_SEND_SIGNALED; - swr.sg_list = &sge; - swr.num_sge = 1; - - ::ibv_send_wr* badswr = 0; - CHECK_IBV(::ibv_post_send(id->qp, &swr, &badswr)); - if (badswr) - throw std::logic_error("ibv_post_send(): Bad swr"); - } - inline ConnectionEvent::ConnectionEvent(::rdma_cm_event* e) : id((e->event != RDMA_CM_EVENT_CONNECT_REQUEST) ? Connection::find(e->id) : new Connection(e->id)), @@ -525,26 +492,6 @@ namespace Rdma { {} } -inline std::ostream& operator<<(std::ostream& o, ::rdma_cm_event_type t) { -# define CHECK_TYPE(t) case t: o << #t; break; - switch(t) { - CHECK_TYPE(RDMA_CM_EVENT_ADDR_RESOLVED) - CHECK_TYPE(RDMA_CM_EVENT_ADDR_ERROR) - CHECK_TYPE(RDMA_CM_EVENT_ROUTE_RESOLVED) - CHECK_TYPE(RDMA_CM_EVENT_ROUTE_ERROR) - CHECK_TYPE(RDMA_CM_EVENT_CONNECT_REQUEST) - CHECK_TYPE(RDMA_CM_EVENT_CONNECT_RESPONSE) - CHECK_TYPE(RDMA_CM_EVENT_CONNECT_ERROR) - CHECK_TYPE(RDMA_CM_EVENT_UNREACHABLE) - CHECK_TYPE(RDMA_CM_EVENT_REJECTED) - CHECK_TYPE(RDMA_CM_EVENT_ESTABLISHED) - CHECK_TYPE(RDMA_CM_EVENT_DISCONNECTED) - CHECK_TYPE(RDMA_CM_EVENT_DEVICE_REMOVAL) - CHECK_TYPE(RDMA_CM_EVENT_MULTICAST_JOIN) - CHECK_TYPE(RDMA_CM_EVENT_MULTICAST_ERROR) - } -# undef CHECK_TYPE - return o; -} +std::ostream& operator<<(std::ostream& o, ::rdma_cm_event_type t); #endif // RDMA_WRAP_H diff --git a/cpp/src/qpid/sys/solaris/ECFPoller.cpp b/cpp/src/qpid/sys/solaris/ECFPoller.cpp index fefa21dae1..f12012cbb0 100644 --- a/cpp/src/qpid/sys/solaris/ECFPoller.cpp +++ b/cpp/src/qpid/sys/solaris/ECFPoller.cpp @@ -30,9 +30,11 @@ #include <port.h> #include <poll.h> #include <errno.h> +#include <pthread.h> +#include <signal.h> #include <assert.h> -#include <vector> +#include <queue> #include <exception> @@ -43,11 +45,11 @@ namespace qpid { namespace sys { // Deletion manager to handle deferring deletion of PollerHandles to when they definitely aren't being used -DeletionManager<PollerHandle> PollerHandleDeletionManager; +DeletionManager<PollerHandlePrivate> PollerHandleDeletionManager; // Instantiate (and define) class static for DeletionManager template <> -DeletionManager<PollerHandle>::AllThreadsStatuses DeletionManager<PollerHandle>::allThreadsStatuses(0); +DeletionManager<PollerHandlePrivate>::AllThreadsStatuses DeletionManager<PollerHandlePrivate>::allThreadsStatuses(0); class PollerHandlePrivate { friend class Poller; @@ -58,7 +60,8 @@ class PollerHandlePrivate { MONITORED, INACTIVE, HUNGUP, - MONITORED_HUNGUP + MONITORED_HUNGUP, + DELETED }; int fd; @@ -104,6 +107,14 @@ class PollerHandlePrivate { assert(stat == MONITORED); stat = HUNGUP; } + + bool isDeleted() const { + return stat == DELETED; + } + + void setDeleted() { + stat = DELETED; + } }; PollerHandle::PollerHandle(const IOHandle& h) : @@ -111,11 +122,16 @@ PollerHandle::PollerHandle(const IOHandle& h) : {} PollerHandle::~PollerHandle() { - delete impl; -} - -void PollerHandle::deferDelete() { - PollerHandleDeletionManager.markForDeletion(this); + { + ScopedLock<Mutex> l(impl->lock); + if (impl->isDeleted()) { + return; + } + if (impl->isActive()) { + impl->setDeleted(); + } + } + PollerHandleDeletionManager.markForDeletion(impl); } /** @@ -125,35 +141,82 @@ void PollerHandle::deferDelete() { class PollerPrivate { friend class Poller; - const int portId; + class InterruptHandle: public PollerHandle { + std::queue<PollerHandle*> handles; + + void processEvent(Poller::EventType) { + PollerHandle* handle = handles.front(); + handles.pop(); + assert(handle); + + //Synthesise event + Poller::Event event(handle, Poller::INTERRUPTED); + + //Process synthesised event + event.process(); + } + public: + InterruptHandle() : PollerHandle(DummyIOHandle) {} + + void addHandle(PollerHandle& h) { + handles.push(&h); + } + + PollerHandle *getHandle() { + PollerHandle* handle = handles.front(); + handles.pop(); + return handle; + } + + bool queuedHandles() { + return handles.size() > 0; + } + }; + + const int portId; + bool isShutdown; + InterruptHandle interruptHandle; + static uint32_t directionToPollEvent(Poller::Direction dir) { switch (dir) { - case Poller::IN: return POLLIN; - case Poller::OUT: return POLLOUT; - case Poller::INOUT: return POLLIN | POLLOUT; - default: return 0; + case Poller::INPUT: return POLLIN; + case Poller::OUTPUT: return POLLOUT; + case Poller::INOUT: return POLLIN | POLLOUT; + default: return 0; } } static Poller::EventType pollToDirection(uint32_t events) { uint32_t e = events & (POLLIN | POLLOUT); switch (e) { - case POLLIN: return Poller::READABLE; - case POLLOUT: return Poller::WRITABLE; - case POLLIN | POLLOUT: return Poller::READ_WRITABLE; - default: - return (events & (POLLHUP | POLLERR)) ? - Poller::DISCONNECTED : Poller::INVALID; + case POLLIN: return Poller::READABLE; + case POLLOUT: return Poller::WRITABLE; + case POLLIN | POLLOUT: return Poller::READ_WRITABLE; + default: + return (events & (POLLHUP | POLLERR)) ? + Poller::DISCONNECTED : Poller::INVALID; } } - + PollerPrivate() : - portId(::port_create()) { + portId(::port_create()), + isShutdown(false) { + QPID_POSIX_CHECK(portId); + QPID_LOG(trace, "port_create returned port Id: " << portId); } ~PollerPrivate() { } + + void interrupt() { + //Send an Alarm to the port + //We need to send a nonzero event mask, using POLLHUP, + //nevertheless the wait method will only look for a PORT_ALERT_SET + QPID_LOG(trace, "Sending a port_alert to " << portId); + QPID_POSIX_CHECK(::port_alert(portId, PORT_ALERT_SET, POLLHUP, + &static_cast<PollerHandle&>(interruptHandle))); + } }; void Poller::addFd(PollerHandle& handle, Direction dir) { @@ -177,7 +240,6 @@ void Poller::addFd(PollerHandle& handle, Direction dir) { QPID_LOG(trace, "Poller::addFd(handle=" << &handle << "[" << typeid(&handle).name() << "], fd=" << eh.fd << ")"); - //assert(dynamic_cast<DispatchHandle*>(&handle)); } void Poller::delFd(PollerHandle& handle) { @@ -223,17 +285,56 @@ void Poller::rearmFd(PollerHandle& handle) { } void Poller::shutdown() { - //Send an Alarm to the port - //We need to send a nonzero event mask, using POLLHUP, but - //The wait method will only look for a PORT_ALERT_SET - QPID_POSIX_CHECK(::port_alert(impl->portId, PORT_ALERT_SET, POLLHUP, NULL)); - QPID_LOG(trace, "Poller::shutdown"); + //Allow sloppy code to shut us down more than once + if (impl->isShutdown) + return; + + impl->isShutdown = true; + impl->interrupt(); +} + +bool Poller::interrupt(PollerHandle& handle) { + PollerPrivate::InterruptHandle& ih = impl->interruptHandle; + PollerHandlePrivate& eh = *static_cast<PollerHandle&>(ih).impl; + ScopedLock<Mutex> l(eh.lock); + ih.addHandle(handle); + impl->interrupt(); + eh.setActive(); + return true; +} + +void Poller::run() { + // Make sure we can't be interrupted by signals at a bad time + ::sigset_t ss; + ::sigfillset(&ss); + ::pthread_sigmask(SIG_SETMASK, &ss, 0); + + do { + Event event = wait(); + + // If can read/write then dispatch appropriate callbacks + if (event.handle) { + event.process(); + } else { + // Handle shutdown + switch (event.type) { + case SHUTDOWN: + return; + default: + // This should be impossible + assert(false); + } + } + } while (true); } Poller::Event Poller::wait(Duration timeout) { timespec_t tout; timespec_t* ptout = NULL; port_event_t pe; + + AbsTime targetTimeout = (timeout == TIME_INFINITE) ? FAR_FUTURE : + AbsTime(now(), timeout); if (timeout != TIME_INFINITE) { tout.tv_sec = 0; @@ -243,12 +344,21 @@ Poller::Event Poller::wait(Duration timeout) { do { PollerHandleDeletionManager.markAllUnusedInThisThread(); - QPID_LOG(trace, "About to enter port_get. Thread " - << pthread_self() + QPID_LOG(trace, "About to enter port_get on " << impl->portId + << ". Thread " << pthread_self() << ", timeout=" << timeout); - + + int rc = ::port_get(impl->portId, &pe, ptout); + QPID_LOG(trace, "port_get on " << impl->portId + << " returned " << rc); + + if (impl->isShutdown) { + PollerHandleDeletionManager.markAllUnusedInThisThread(); + return Event(0, SHUTDOWN); + } + if (rc < 0) { switch (errno) { case EINTR: @@ -259,33 +369,61 @@ Poller::Event Poller::wait(Duration timeout) { QPID_POSIX_CHECK(rc); } } else { - //We use alert mode to notify the shutdown of the Poller - if (pe.portev_source == PORT_SOURCE_ALERT) { - return Event(0, SHUTDOWN); - } - if (pe.portev_source == PORT_SOURCE_FD) { - PollerHandle *handle = static_cast<PollerHandle*>(pe.portev_user); - PollerHandlePrivate& eh = *handle->impl; - ScopedLock<Mutex> l(eh.lock); - QPID_LOG(trace, "About to send handle: " << handle); + PollerHandle* handle = static_cast<PollerHandle*>(pe.portev_user); + PollerHandlePrivate& eh = *handle->impl; + ScopedLock<Mutex> l(eh.lock); + + if (eh.isActive()) { + QPID_LOG(trace, "Handle is active"); + //We use alert mode to notify interrupts + if (pe.portev_source == PORT_SOURCE_ALERT && + handle == &impl->interruptHandle) { + QPID_LOG(trace, "Interrupt notified"); + + PollerHandle* wrappedHandle = impl->interruptHandle.getHandle(); + + if (impl->interruptHandle.queuedHandles()) { + impl->interrupt(); + eh.setActive(); + } else { + eh.setInactive(); + } + return Event(wrappedHandle, INTERRUPTED); + } - if (eh.isActive()) { - if (pe.portev_events & POLLHUP) { - if (eh.isHungup()) { - return Event(handle, DISCONNECTED); + if (pe.portev_source == PORT_SOURCE_FD) { + QPID_LOG(trace, "About to send handle: " << handle); + if (pe.portev_events & POLLHUP) { + if (eh.isHungup()) { + return Event(handle, DISCONNECTED); + } + eh.setHungup(); + } else { + eh.setInactive(); } - eh.setHungup(); - } else { - eh.setInactive(); - } - QPID_LOG(trace, "Sending event (thread: " - << pthread_self() << ") for handle " << handle - << ", direction= " - << PollerPrivate::pollToDirection(pe.portev_events)); - return Event(handle, PollerPrivate::pollToDirection(pe.portev_events)); + QPID_LOG(trace, "Sending event (thread: " + << pthread_self() << ") for handle " << handle + << ", direction= " + << PollerPrivate::pollToDirection(pe.portev_events)); + return Event(handle, PollerPrivate::pollToDirection(pe.portev_events)); + } + } else if (eh.isDeleted()) { + //Remove the handle from the poller + int rc = ::port_dissociate(impl->portId, PORT_SOURCE_FD, + (uintptr_t) eh.fd); + if (rc == -1 && errno != EBADFD) { + QPID_POSIX_CHECK(rc); } } } + + if (timeout == TIME_INFINITE) { + continue; + } + if (rc == 0 && now() > targetTimeout) { + PollerHandleDeletionManager.markAllUnusedInThisThread(); + return Event(0, TIMEOUT); + } } while (true); } diff --git a/cpp/src/qpid/sys/solaris/SystemInfo.cpp b/cpp/src/qpid/sys/solaris/SystemInfo.cpp new file mode 100755 index 0000000000..0075a89021 --- /dev/null +++ b/cpp/src/qpid/sys/solaris/SystemInfo.cpp @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/SystemInfo.h" + +#define BSD_COMP +#include <sys/ioctl.h> +#include <netdb.h> +#undef BDS_COMP + + +#include <unistd.h> +#include <net/if.h> +#include <sys/types.h> +#include <sys/utsname.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <stdio.h> +#include <errno.h> +#include <limits.h> +#include <procfs.h> +#include <fcntl.h> +#include <sys/types.h> + +using namespace std; + +namespace qpid { +namespace sys { + +long SystemInfo::concurrency() { + return sysconf(_SC_NPROCESSORS_ONLN); +} + +bool SystemInfo::getLocalHostname(TcpAddress &address) { + char name[MAXHOSTNAMELEN]; + if (::gethostname(name, sizeof(name)) != 0) + return false; + address.host = name; + return true; +} + +static const string LOCALHOST("127.0.0.1"); + +void SystemInfo::getLocalIpAddresses(uint16_t port, + std::vector<Address> &addrList) { + int s = socket(PF_INET, SOCK_STREAM, 0); + for (int i=1;;i++) { + struct lifreq ifr; + ifr.lifr_index = i; + if (::ioctl(s, SIOCGIFADDR, &ifr) < 0) { + break; + } + struct sockaddr_in *sin = (struct sockaddr_in *) &ifr.lifr_addr; + std::string addr(inet_ntoa(sin->sin_addr)); + if (addr != LOCALHOST) + addrList.push_back(TcpAddress(addr, port)); + } + if (addrList.empty()) { + addrList.push_back(TcpAddress(LOCALHOST, port)); + } + close (s); +} + +void SystemInfo::getSystemId(std::string &osName, + std::string &nodeName, + std::string &release, + std::string &version, + std::string &machine) { + struct utsname _uname; + if (uname (&_uname) == 0) { + osName = _uname.sysname; + nodeName = _uname.nodename; + release = _uname.release; + version = _uname.version; + machine = _uname.machine; + } +} + +uint32_t SystemInfo::getProcessId() +{ + return (uint32_t) ::getpid(); +} + +uint32_t SystemInfo::getParentProcessId() +{ + return (uint32_t) ::getppid(); +} + +string SystemInfo::getProcessName() +{ + psinfo processInfo; + char procfile[PATH_MAX]; + int fd; + string value; + + snprintf(procfile, PATH_MAX, "/proc/%d/psinfo", getProcessId()); + if ((fd = open(procfile, O_RDONLY)) >= 0) { + if (read(fd, (void *) &processInfo, sizeof(processInfo)) == sizeof(processInfo)) { + value = processInfo.pr_fname; + } + } + return value; +} + +}} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/ssl/SslHandler.cpp b/cpp/src/qpid/sys/ssl/SslHandler.cpp new file mode 100644 index 0000000000..3469f88c0f --- /dev/null +++ b/cpp/src/qpid/sys/ssl/SslHandler.cpp @@ -0,0 +1,187 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/ssl/SslHandler.h" + +#include "qpid/sys/ssl/SslIo.h" +#include "qpid/sys/ssl/SslSocket.h" +#include "qpid/framing/AMQP_HighestVersion.h" +#include "qpid/framing/ProtocolInitiation.h" +#include "qpid/log/Statement.h" + +#include <boost/bind.hpp> + +namespace qpid { +namespace sys { +namespace ssl { + + +// Buffer definition +struct Buff : public SslIO::BufferBase { + Buff() : + SslIO::BufferBase(new char[65536], 65536) + {} + ~Buff() + { delete [] bytes;} +}; + +SslHandler::SslHandler(std::string id, ConnectionCodec::Factory* f) : + identifier(id), + aio(0), + factory(f), + codec(0), + readError(false), + isClient(false) +{} + +SslHandler::~SslHandler() { + if (codec) + codec->closed(); + delete codec; +} + +void SslHandler::init(SslIO* a, int numBuffs) { + aio = a; + + // Give connection some buffers to use + for (int i = 0; i < numBuffs; i++) { + aio->queueReadBuffer(new Buff); + } +} + +void SslHandler::write(const framing::ProtocolInitiation& data) +{ + QPID_LOG(debug, "SENT [" << identifier << "] INIT(" << data << ")"); + SslIO::BufferBase* buff = aio->getQueuedBuffer(); + if (!buff) + buff = new Buff; + framing::Buffer out(buff->bytes, buff->byteCount); + data.encode(out); + buff->dataCount = data.encodedSize(); + aio->queueWrite(buff); +} + +void SslHandler::abort() { + // TODO: can't implement currently as underlying functionality not implemented + // aio->requestCallback(boost::bind(&SslHandler::eof, this, _1)); +} +void SslHandler::activateOutput() { + aio->notifyPendingWrite(); +} + +void SslHandler::giveReadCredit(int32_t) { + // FIXME aconway 2008-12-05: not yet implemented. +} + +// Input side +void SslHandler::readbuff(SslIO& , SslIO::BufferBase* buff) { + if (readError) { + return; + } + size_t decoded = 0; + if (codec) { // Already initiated + try { + decoded = codec->decode(buff->bytes+buff->dataStart, buff->dataCount); + }catch(const std::exception& e){ + QPID_LOG(error, e.what()); + readError = true; + aio->queueWriteClose(); + } + }else{ + framing::Buffer in(buff->bytes+buff->dataStart, buff->dataCount); + framing::ProtocolInitiation protocolInit; + if (protocolInit.decode(in)) { + decoded = in.getPosition(); + QPID_LOG(debug, "RECV [" << identifier << "] INIT(" << protocolInit << ")"); + try { + codec = factory->create(protocolInit.getVersion(), *this, identifier, aio->getKeyLen()); + if (!codec) { + //TODO: may still want to revise this... + //send valid version header & close connection. + write(framing::ProtocolInitiation(framing::highestProtocolVersion)); + readError = true; + aio->queueWriteClose(); + } + } catch (const std::exception& e) { + QPID_LOG(error, e.what()); + readError = true; + aio->queueWriteClose(); + } + } + } + // TODO: unreading needs to go away, and when we can cope + // with multiple sub-buffers in the general buffer scheme, it will + if (decoded != size_t(buff->dataCount)) { + // Adjust buffer for used bytes and then "unread them" + buff->dataStart += decoded; + buff->dataCount -= decoded; + aio->unread(buff); + } else { + // Give whole buffer back to aio subsystem + aio->queueReadBuffer(buff); + } +} + +void SslHandler::eof(SslIO&) { + QPID_LOG(debug, "DISCONNECTED [" << identifier << "]"); + if (codec) codec->closed(); + aio->queueWriteClose(); +} + +void SslHandler::closedSocket(SslIO&, const SslSocket& s) { + // If we closed with data still to send log a warning + if (!aio->writeQueueEmpty()) { + QPID_LOG(warning, "CLOSING [" << identifier << "] unsent data (probably due to client disconnect)"); + } + delete &s; + aio->queueForDeletion(); + delete this; +} + +void SslHandler::disconnect(SslIO& a) { + // treat the same as eof + eof(a); +} + +// Notifications +void SslHandler::nobuffs(SslIO&) { +} + +void SslHandler::idle(SslIO&){ + if (isClient && codec == 0) { + codec = factory->create(*this, identifier, aio->getKeyLen()); + write(framing::ProtocolInitiation(codec->getVersion())); + return; + } + if (codec == 0) return; + if (codec->canEncode()) { + // Try and get a queued buffer if not then construct new one + SslIO::BufferBase* buff = aio->getQueuedBuffer(); + if (!buff) buff = new Buff; + size_t encoded=codec->encode(buff->bytes, buff->byteCount); + buff->dataCount = encoded; + aio->queueWrite(buff); + } + if (codec->isClosed()) + aio->queueWriteClose(); +} + + +}}} // namespace qpid::sys::ssl diff --git a/cpp/src/qpid/sys/ssl/SslHandler.h b/cpp/src/qpid/sys/ssl/SslHandler.h new file mode 100644 index 0000000000..8f6b8e732a --- /dev/null +++ b/cpp/src/qpid/sys/ssl/SslHandler.h @@ -0,0 +1,76 @@ +#ifndef QPID_SYS_SSL_SSLHANDLER_H +#define QPID_SYS_SSL_SSLHANDLER_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/ConnectionCodec.h" +#include "qpid/sys/OutputControl.h" + +namespace qpid { + +namespace framing { + class ProtocolInitiation; +} + +namespace sys { +namespace ssl { + +class SslIO; +class SslIOBufferBase; +class SslSocket; + +class SslHandler : public OutputControl { + std::string identifier; + SslIO* aio; + ConnectionCodec::Factory* factory; + ConnectionCodec* codec; + bool readError; + bool isClient; + + void write(const framing::ProtocolInitiation&); + + public: + SslHandler(std::string id, ConnectionCodec::Factory* f); + ~SslHandler(); + void init(SslIO* a, int numBuffs); + + void setClient() { isClient = true; } + + // Output side + void abort(); + void activateOutput(); + void giveReadCredit(int32_t); + + // Input side + void readbuff(SslIO& aio, SslIOBufferBase* buff); + void eof(SslIO& aio); + void disconnect(SslIO& aio); + + // Notifications + void nobuffs(SslIO& aio); + void idle(SslIO& aio); + void closedSocket(SslIO& aio, const SslSocket& s); +}; + +}}} // namespace qpid::sys::ssl + +#endif /*!QPID_SYS_SSL_SSLHANDLER_H*/ diff --git a/cpp/src/qpid/sys/ssl/SslIo.cpp b/cpp/src/qpid/sys/ssl/SslIo.cpp new file mode 100644 index 0000000000..c149d6ea74 --- /dev/null +++ b/cpp/src/qpid/sys/ssl/SslIo.cpp @@ -0,0 +1,439 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/ssl/SslIo.h" +#include "qpid/sys/ssl/SslSocket.h" + +#include "qpid/sys/Time.h" +#include "qpid/sys/posix/check.h" +#include "qpid/log/Statement.h" + +// TODO The basic algorithm here is not really POSIX specific and with a bit more abstraction +// could (should) be promoted to be platform portable +#include <unistd.h> +#include <sys/socket.h> +#include <signal.h> +#include <errno.h> +#include <string.h> + +#include <boost/bind.hpp> + +using namespace qpid::sys; +using namespace qpid::sys::ssl; + +namespace { + +/* + * Make *process* not generate SIGPIPE when writing to closed + * pipe/socket (necessary as default action is to terminate process) + */ +void ignoreSigpipe() { + ::signal(SIGPIPE, SIG_IGN); +} + +/* + * We keep per thread state to avoid locking overhead. The assumption is that + * on average all the connections are serviced by all the threads so the state + * recorded in each thread is about the same. If this turns out not to be the + * case we could rebalance the info occasionally. + */ +__thread int threadReadTotal = 0; +__thread int threadMaxRead = 0; +__thread int threadReadCount = 0; +__thread int threadWriteTotal = 0; +__thread int threadWriteCount = 0; +__thread int64_t threadMaxReadTimeNs = 2 * 1000000; // start at 2ms +} + +/* + * Asynch Acceptor + */ + +SslAcceptor::SslAcceptor(const SslSocket& s, Callback callback) : + acceptedCallback(callback), + handle(s, boost::bind(&SslAcceptor::readable, this, _1), 0, 0), + socket(s) { + + s.setNonblocking(); + ignoreSigpipe(); +} + +SslAcceptor::~SslAcceptor() +{ + handle.stopWatch(); +} + +void SslAcceptor::start(Poller::shared_ptr poller) { + handle.startWatch(poller); +} + +/* + * We keep on accepting as long as there is something to accept + */ +void SslAcceptor::readable(DispatchHandle& h) { + SslSocket* s; + do { + errno = 0; + // TODO: Currently we ignore the peers address, perhaps we should + // log it or use it for connection acceptance. + try { + s = socket.accept(); + if (s) { + acceptedCallback(*s); + } else { + break; + } + } catch (const std::exception& e) { + QPID_LOG(error, "Could not accept socket: " << e.what()); + } + } while (true); + + h.rewatch(); +} + +/* + * Asynch Connector + */ + +SslConnector::SslConnector(const SslSocket& s, + Poller::shared_ptr poller, + std::string hostname, + uint16_t port, + ConnectedCallback connCb, + FailedCallback failCb) : + DispatchHandle(s, + 0, + boost::bind(&SslConnector::connComplete, this, _1), + boost::bind(&SslConnector::connComplete, this, _1)), + connCallback(connCb), + failCallback(failCb), + socket(s) +{ + //TODO: would be better for connect to be performed on a + //non-blocking socket, but that doesn't work at present so connect + //blocks until complete + try { + socket.connect(hostname, port); + socket.setNonblocking(); + startWatch(poller); + } catch(std::exception& e) { + failure(-1, std::string(e.what())); + } +} + +void SslConnector::connComplete(DispatchHandle& h) +{ + int errCode = socket.getError(); + + h.stopWatch(); + if (errCode == 0) { + connCallback(socket); + DispatchHandle::doDelete(); + } else { + // TODO: This need to be fixed as strerror isn't thread safe + failure(errCode, std::string(::strerror(errCode))); + } +} + +void SslConnector::failure(int errCode, std::string message) +{ + if (failCallback) + failCallback(errCode, message); + + socket.close(); + delete &socket; + + DispatchHandle::doDelete(); +} + +/* + * Asynch reader/writer + */ +SslIO::SslIO(const SslSocket& s, + ReadCallback rCb, EofCallback eofCb, DisconnectCallback disCb, + ClosedCallback cCb, BuffersEmptyCallback eCb, IdleCallback iCb) : + + DispatchHandle(s, + boost::bind(&SslIO::readable, this, _1), + boost::bind(&SslIO::writeable, this, _1), + boost::bind(&SslIO::disconnected, this, _1)), + readCallback(rCb), + eofCallback(eofCb), + disCallback(disCb), + closedCallback(cCb), + emptyCallback(eCb), + idleCallback(iCb), + socket(s), + queuedClose(false), + writePending(false) { + + s.setNonblocking(); +} + +struct deleter +{ + template <typename T> + void operator()(T *ptr){ delete ptr;} +}; + +SslIO::~SslIO() { + std::for_each( bufferQueue.begin(), bufferQueue.end(), deleter()); + std::for_each( writeQueue.begin(), writeQueue.end(), deleter()); +} + +void SslIO::queueForDeletion() { + DispatchHandle::doDelete(); +} + +void SslIO::start(Poller::shared_ptr poller) { + DispatchHandle::startWatch(poller); +} + +void SslIO::queueReadBuffer(BufferBase* buff) { + assert(buff); + buff->dataStart = 0; + buff->dataCount = 0; + bufferQueue.push_back(buff); + DispatchHandle::rewatchRead(); +} + +void SslIO::unread(BufferBase* buff) { + assert(buff); + if (buff->dataStart != 0) { + memmove(buff->bytes, buff->bytes+buff->dataStart, buff->dataCount); + buff->dataStart = 0; + } + bufferQueue.push_front(buff); + DispatchHandle::rewatchRead(); +} + +void SslIO::queueWrite(BufferBase* buff) { + assert(buff); + // If we've already closed the socket then throw the write away + if (queuedClose) { + bufferQueue.push_front(buff); + return; + } else { + writeQueue.push_front(buff); + } + writePending = false; + DispatchHandle::rewatchWrite(); +} + +void SslIO::notifyPendingWrite() { + writePending = true; + DispatchHandle::rewatchWrite(); +} + +void SslIO::queueWriteClose() { + queuedClose = true; + DispatchHandle::rewatchWrite(); +} + +/** Return a queued buffer if there are enough + * to spare + */ +SslIO::BufferBase* SslIO::getQueuedBuffer() { + // Always keep at least one buffer (it might have data that was "unread" in it) + if (bufferQueue.size()<=1) + return 0; + BufferBase* buff = bufferQueue.back(); + assert(buff); + buff->dataStart = 0; + buff->dataCount = 0; + bufferQueue.pop_back(); + return buff; +} + +/* + * We keep on reading as long as we have something to read and a buffer to put + * it in + */ +void SslIO::readable(DispatchHandle& h) { + int readTotal = 0; + AbsTime readStartTime = AbsTime::now(); + do { + // (Try to) get a buffer + if (!bufferQueue.empty()) { + // Read into buffer + BufferBase* buff = bufferQueue.front(); + assert(buff); + bufferQueue.pop_front(); + errno = 0; + int readCount = buff->byteCount-buff->dataCount; + int rc = socket.read(buff->bytes + buff->dataCount, readCount); + if (rc > 0) { + buff->dataCount += rc; + threadReadTotal += rc; + readTotal += rc; + + readCallback(*this, buff); + if (rc != readCount) { + // If we didn't fill the read buffer then time to stop reading + break; + } + + // Stop reading if we've overrun our timeslot + if (Duration(readStartTime, AbsTime::now()) > threadMaxReadTimeNs) { + break; + } + + } else { + // Put buffer back (at front so it doesn't interfere with unread buffers) + bufferQueue.push_front(buff); + assert(buff); + + // Eof or other side has gone away + if (rc == 0 || errno == ECONNRESET) { + eofCallback(*this); + h.unwatchRead(); + break; + } else if (errno == EAGAIN) { + // We have just put a buffer back so we know + // we can carry on watching for reads + break; + } else { + // Report error then just treat as a socket disconnect + QPID_LOG(error, "Error reading socket: " << qpid::sys::strError(rc) << "(" << rc << ")" ); + eofCallback(*this); + h.unwatchRead(); + break; + } + } + } else { + // Something to read but no buffer + if (emptyCallback) { + emptyCallback(*this); + } + // If we still have no buffers we can't do anything more + if (bufferQueue.empty()) { + h.unwatchRead(); + break; + } + + } + } while (true); + + ++threadReadCount; + threadMaxRead = std::max(threadMaxRead, readTotal); + return; +} + +/* + * We carry on writing whilst we have data to write and we can write + */ +void SslIO::writeable(DispatchHandle& h) { + int writeTotal = 0; + do { + // See if we've got something to write + if (!writeQueue.empty()) { + // Write buffer + BufferBase* buff = writeQueue.back(); + writeQueue.pop_back(); + errno = 0; + assert(buff->dataStart+buff->dataCount <= buff->byteCount); + int rc = socket.write(buff->bytes+buff->dataStart, buff->dataCount); + if (rc >= 0) { + threadWriteTotal += rc; + writeTotal += rc; + + // If we didn't write full buffer put rest back + if (rc != buff->dataCount) { + buff->dataStart += rc; + buff->dataCount -= rc; + writeQueue.push_back(buff); + break; + } + + // Recycle the buffer + queueReadBuffer(buff); + + // If we've already written more than the max for reading then stop + // (this is to stop writes dominating reads) + if (writeTotal > threadMaxRead) + break; + } else { + // Put buffer back + writeQueue.push_back(buff); + if (errno == ECONNRESET || errno == EPIPE) { + // Just stop watching for write here - we'll get a + // disconnect callback soon enough + h.unwatchWrite(); + break; + } else if (errno == EAGAIN) { + // We have just put a buffer back so we know + // we can carry on watching for writes + break; + } else { + QPID_POSIX_CHECK(rc); + } + } + } else { + // If we're waiting to close the socket then can do it now as there is nothing to write + if (queuedClose) { + close(h); + break; + } + // Fd is writable, but nothing to write + if (idleCallback) { + writePending = false; + idleCallback(*this); + } + // If we still have no buffers to write we can't do anything more + if (writeQueue.empty() && !writePending && !queuedClose) { + h.unwatchWrite(); + // The following handles the case where writePending is + // set to true after the test above; in this case its + // possible that the unwatchWrite overwrites the + // desired rewatchWrite so we correct that here + if (writePending) + h.rewatchWrite(); + break; + } + } + } while (true); + + ++threadWriteCount; + return; +} + +void SslIO::disconnected(DispatchHandle& h) { + // If we've already queued close do it instead of disconnected callback + if (queuedClose) { + close(h); + } else if (disCallback) { + disCallback(*this); + h.unwatch(); + } +} + +/* + * Close the socket and callback to say we've done it + */ +void SslIO::close(DispatchHandle& h) { + h.stopWatch(); + socket.close(); + if (closedCallback) { + closedCallback(*this, socket); + } +} + +int SslIO::getKeyLen() {return socket.getKeyLen();} diff --git a/cpp/src/qpid/sys/ssl/SslIo.h b/cpp/src/qpid/sys/ssl/SslIo.h new file mode 100644 index 0000000000..3162abac40 --- /dev/null +++ b/cpp/src/qpid/sys/ssl/SslIo.h @@ -0,0 +1,171 @@ +#ifndef _sys_ssl_SslIO +#define _sys_ssl_SslIO +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/DispatchHandle.h" + +#include <boost/function.hpp> +#include <deque> + +namespace qpid { +namespace sys { +namespace ssl { + +class SslSocket; + +/* + * Asynchronous ssl acceptor: accepts connections then does a callback + * with the accepted fd + */ +class SslAcceptor { +public: + typedef boost::function1<void, const SslSocket&> Callback; + +private: + Callback acceptedCallback; + qpid::sys::DispatchHandle handle; + const SslSocket& socket; + +public: + SslAcceptor(const SslSocket& s, Callback callback); + ~SslAcceptor(); + void start(qpid::sys::Poller::shared_ptr poller); + +private: + void readable(qpid::sys::DispatchHandle& handle); +}; + +/* + * Asynchronous ssl connector: starts the process of initiating a + * connection and invokes a callback when completed or failed. + */ +class SslConnector : private qpid::sys::DispatchHandle { +public: + typedef boost::function1<void, const SslSocket&> ConnectedCallback; + typedef boost::function2<void, int, std::string> FailedCallback; + +private: + ConnectedCallback connCallback; + FailedCallback failCallback; + const SslSocket& socket; + +public: + SslConnector(const SslSocket& socket, + Poller::shared_ptr poller, + std::string hostname, + uint16_t port, + ConnectedCallback connCb, + FailedCallback failCb = 0); + +private: + void connComplete(DispatchHandle& handle); + void failure(int, std::string); +}; + +struct SslIOBufferBase { + char* const bytes; + const int32_t byteCount; + int32_t dataStart; + int32_t dataCount; + + SslIOBufferBase(char* const b, const int32_t s) : + bytes(b), + byteCount(s), + dataStart(0), + dataCount(0) + {} + + virtual ~SslIOBufferBase() + {} +}; + +/* + * Asychronous reader/writer: + * Reader accepts buffers to read into; reads into the provided buffers + * and then does a callback with the buffer and amount read. Optionally it can callback + * when there is something to read but no buffer to read it into. + * + * Writer accepts a buffer and queues it for writing; can also be given + * a callback for when writing is "idle" (ie fd is writable, but nothing to write) + * + * The class is implemented in terms of DispatchHandle to allow it to be deleted by deleting + * the contained DispatchHandle + */ +class SslIO : private qpid::sys::DispatchHandle { +public: + typedef SslIOBufferBase BufferBase; + + typedef boost::function2<void, SslIO&, BufferBase*> ReadCallback; + typedef boost::function1<void, SslIO&> EofCallback; + typedef boost::function1<void, SslIO&> DisconnectCallback; + typedef boost::function2<void, SslIO&, const SslSocket&> ClosedCallback; + typedef boost::function1<void, SslIO&> BuffersEmptyCallback; + typedef boost::function1<void, SslIO&> IdleCallback; + + +private: + ReadCallback readCallback; + EofCallback eofCallback; + DisconnectCallback disCallback; + ClosedCallback closedCallback; + BuffersEmptyCallback emptyCallback; + IdleCallback idleCallback; + const SslSocket& socket; + std::deque<BufferBase*> bufferQueue; + std::deque<BufferBase*> writeQueue; + bool queuedClose; + /** + * This flag is used to detect and handle concurrency between + * calls to notifyPendingWrite() (which can be made from any thread) and + * the execution of the writeable() method (which is always on the + * thread processing this handle. + */ + volatile bool writePending; + +public: + SslIO(const SslSocket& s, + ReadCallback rCb, EofCallback eofCb, DisconnectCallback disCb, + ClosedCallback cCb = 0, BuffersEmptyCallback eCb = 0, IdleCallback iCb = 0); + void queueForDeletion(); + + void start(qpid::sys::Poller::shared_ptr poller); + void queueReadBuffer(BufferBase* buff); + void unread(BufferBase* buff); + void queueWrite(BufferBase* buff); + void notifyPendingWrite(); + void queueWriteClose(); + bool writeQueueEmpty() { return writeQueue.empty(); } + BufferBase* getQueuedBuffer(); + + int getKeyLen(); + +private: + ~SslIO(); + void readable(qpid::sys::DispatchHandle& handle); + void writeable(qpid::sys::DispatchHandle& handle); + void disconnected(qpid::sys::DispatchHandle& handle); + void close(qpid::sys::DispatchHandle& handle); +}; + +}}} + +#endif // _sys_ssl_SslIO diff --git a/cpp/src/qpid/sys/ssl/SslSocket.cpp b/cpp/src/qpid/sys/ssl/SslSocket.cpp new file mode 100644 index 0000000000..aa8cf127d7 --- /dev/null +++ b/cpp/src/qpid/sys/ssl/SslSocket.cpp @@ -0,0 +1,297 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/ssl/SslSocket.h" +#include "qpid/sys/ssl/check.h" +#include "qpid/sys/ssl/util.h" +#include "qpid/Exception.h" +#include "qpid/sys/posix/check.h" +#include "qpid/sys/posix/PrivatePosix.h" + +#include <fcntl.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <sys/errno.h> +#include <netinet/in.h> +#include <netinet/tcp.h> +#include <netdb.h> +#include <cstdlib> +#include <string.h> +#include <iostream> + +#include <private/pprio.h> +#include <nss.h> +#include <pk11pub.h> +#include <ssl.h> +#include <key.h> + +#include <boost/format.hpp> + +namespace qpid { +namespace sys { +namespace ssl { + +namespace { +std::string getName(int fd, bool local, bool includeService = false) +{ + ::sockaddr_storage name; // big enough for any socket address + ::socklen_t namelen = sizeof(name); + + int result = -1; + if (local) { + result = ::getsockname(fd, (::sockaddr*)&name, &namelen); + } else { + result = ::getpeername(fd, (::sockaddr*)&name, &namelen); + } + + QPID_POSIX_CHECK(result); + + char servName[NI_MAXSERV]; + char dispName[NI_MAXHOST]; + if (includeService) { + if (int rc=::getnameinfo((::sockaddr*)&name, namelen, dispName, sizeof(dispName), + servName, sizeof(servName), + NI_NUMERICHOST | NI_NUMERICSERV) != 0) + throw QPID_POSIX_ERROR(rc); + return std::string(dispName) + ":" + std::string(servName); + + } else { + if (int rc=::getnameinfo((::sockaddr*)&name, namelen, dispName, sizeof(dispName), 0, 0, NI_NUMERICHOST) != 0) + throw QPID_POSIX_ERROR(rc); + return dispName; + } +} + +std::string getService(int fd, bool local) +{ + ::sockaddr_storage name; // big enough for any socket address + ::socklen_t namelen = sizeof(name); + + int result = -1; + if (local) { + result = ::getsockname(fd, (::sockaddr*)&name, &namelen); + } else { + result = ::getpeername(fd, (::sockaddr*)&name, &namelen); + } + + QPID_POSIX_CHECK(result); + + char servName[NI_MAXSERV]; + if (int rc=::getnameinfo((::sockaddr*)&name, namelen, 0, 0, + servName, sizeof(servName), + NI_NUMERICHOST | NI_NUMERICSERV) != 0) + throw QPID_POSIX_ERROR(rc); + return servName; +} + +} + +SslSocket::SslSocket() : IOHandle(new IOHandlePrivate()), socket(0), prototype(0) +{ + impl->fd = ::socket (PF_INET, SOCK_STREAM, 0); + if (impl->fd < 0) throw QPID_POSIX_ERROR(errno); + socket = SSL_ImportFD(0, PR_ImportTCPSocket(impl->fd)); +} + +/** + * This form of the constructor is used with the server-side sockets + * returned from accept. Because we use posix accept rather than + * PR_Accept, we have to reset the handshake. + */ +SslSocket::SslSocket(IOHandlePrivate* ioph, PRFileDesc* model) : IOHandle(ioph), socket(0), prototype(0) +{ + socket = SSL_ImportFD(model, PR_ImportTCPSocket(impl->fd)); + NSS_CHECK(SSL_ResetHandshake(socket, true)); +} + +void SslSocket::setNonblocking() const +{ + PRSocketOptionData option; + option.option = PR_SockOpt_Nonblocking; + option.value.non_blocking = true; + PR_SetSocketOption(socket, &option); +} + +void SslSocket::connect(const std::string& host, uint16_t port) const +{ + std::stringstream namestream; + namestream << host << ":" << port; + connectname = namestream.str(); + + void* arg = SslOptions::global.certName.empty() ? 0 : const_cast<char*>(SslOptions::global.certName.c_str()); + NSS_CHECK(SSL_GetClientAuthDataHook(socket, NSS_GetClientAuthData, arg)); + NSS_CHECK(SSL_SetURL(socket, host.data())); + + char hostBuffer[PR_NETDB_BUF_SIZE]; + PRHostEnt hostEntry; + PR_CHECK(PR_GetHostByName(host.data(), hostBuffer, PR_NETDB_BUF_SIZE, &hostEntry)); + PRNetAddr address; + int value = PR_EnumerateHostEnt(0, &hostEntry, port, &address); + if (value < 0) { + throw Exception(QPID_MSG("Error getting address for host: " << ErrorString())); + } else if (value == 0) { + throw Exception(QPID_MSG("Could not resolve address for host.")); + } + PR_CHECK(PR_Connect(socket, &address, PR_INTERVAL_NO_TIMEOUT)); +} + +void SslSocket::close() const +{ + if (impl->fd > 0) { + PR_Close(socket); + impl->fd = -1; + } +} + +int SslSocket::listen(uint16_t port, int backlog, const std::string& certName, bool clientAuth) const +{ + //configure prototype socket: + prototype = SSL_ImportFD(0, PR_NewTCPSocket()); + if (clientAuth) { + NSS_CHECK(SSL_OptionSet(prototype, SSL_REQUEST_CERTIFICATE, PR_TRUE)); + NSS_CHECK(SSL_OptionSet(prototype, SSL_REQUIRE_CERTIFICATE, PR_TRUE)); + } + + //get certificate and key (is this the correct way?) + CERTCertificate *cert = PK11_FindCertFromNickname(const_cast<char*>(certName.c_str()), 0); + if (!cert) throw Exception(QPID_MSG("Failed to load certificate '" << certName << "'")); + SECKEYPrivateKey *key = PK11_FindKeyByAnyCert(cert, 0); + if (!key) throw Exception(QPID_MSG("Failed to retrieve private key from certificate")); + NSS_CHECK(SSL_ConfigSecureServer(prototype, cert, key, NSS_FindCertKEAType(cert))); + SECKEY_DestroyPrivateKey(key); + CERT_DestroyCertificate(cert); + + //bind and listen + const int& socket = impl->fd; + int yes=1; + QPID_POSIX_CHECK(setsockopt(socket,SOL_SOCKET,SO_REUSEADDR,&yes,sizeof(yes))); + struct sockaddr_in name; + name.sin_family = AF_INET; + name.sin_port = htons(port); + name.sin_addr.s_addr = 0; + if (::bind(socket, (struct sockaddr*)&name, sizeof(name)) < 0) + throw Exception(QPID_MSG("Can't bind to port " << port << ": " << strError(errno))); + if (::listen(socket, backlog) < 0) + throw Exception(QPID_MSG("Can't listen on port " << port << ": " << strError(errno))); + + socklen_t namelen = sizeof(name); + if (::getsockname(socket, (struct sockaddr*)&name, &namelen) < 0) + throw QPID_POSIX_ERROR(errno); + + return ntohs(name.sin_port); +} + +SslSocket* SslSocket::accept() const +{ + int afd = ::accept(impl->fd, 0, 0); + if ( afd >= 0) { + return new SslSocket(new IOHandlePrivate(afd), prototype); + } else if (errno == EAGAIN) { + return 0; + } else { + throw QPID_POSIX_ERROR(errno); + } +} + +int SslSocket::read(void *buf, size_t count) const +{ + return PR_Read(socket, buf, count); +} + +int SslSocket::write(const void *buf, size_t count) const +{ + return PR_Write(socket, buf, count); +} + +std::string SslSocket::getSockname() const +{ + return getName(impl->fd, true); +} + +std::string SslSocket::getPeername() const +{ + return getName(impl->fd, false); +} + +std::string SslSocket::getPeerAddress() const +{ + if (!connectname.empty()) + return connectname; + return getName(impl->fd, false, true); +} + +std::string SslSocket::getLocalAddress() const +{ + return getName(impl->fd, true, true); +} + +uint16_t SslSocket::getLocalPort() const +{ + return std::atoi(getService(impl->fd, true).c_str()); +} + +uint16_t SslSocket::getRemotePort() const +{ + return atoi(getService(impl->fd, true).c_str()); +} + +int SslSocket::getError() const +{ + int result; + socklen_t rSize = sizeof (result); + + if (::getsockopt(impl->fd, SOL_SOCKET, SO_ERROR, &result, &rSize) < 0) + throw QPID_POSIX_ERROR(errno); + + return result; +} + +void SslSocket::setTcpNoDelay(bool nodelay) const +{ + if (nodelay) { + PRSocketOptionData option; + option.option = PR_SockOpt_NoDelay; + option.value.no_delay = true; + PR_SetSocketOption(socket, &option); + } +} + + +/** get the bit length of the current cipher's key */ +int SslSocket::getKeyLen() const +{ + int enabled = 0; + int keySize = 0; + SECStatus rc; + + rc = SSL_SecurityStatus( socket, + &enabled, + NULL, + NULL, + &keySize, + NULL, NULL ); + if (rc == SECSuccess && enabled) { + return keySize; + } + return 0; +} + +}}} // namespace qpid::sys::ssl diff --git a/cpp/src/qpid/sys/ssl/SslSocket.h b/cpp/src/qpid/sys/ssl/SslSocket.h new file mode 100644 index 0000000000..f1f05e7a98 --- /dev/null +++ b/cpp/src/qpid/sys/ssl/SslSocket.h @@ -0,0 +1,119 @@ +#ifndef _sys_ssl_Socket_h +#define _sys_ssl_Socket_h + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/IOHandle.h" +#include <nspr.h> + +#include <string> + +struct sockaddr; + +namespace qpid { +namespace sys { + +class Duration; + +namespace ssl { + +class SslSocket : public qpid::sys::IOHandle +{ +public: + /** Create a socket wrapper for descriptor. */ + SslSocket(); + + /** Set socket non blocking */ + void setNonblocking() const; + + /** Set tcp-nodelay */ + void setTcpNoDelay(bool nodelay) const; + + void connect(const std::string& host, uint16_t port) const; + + void close() const; + + /** Bind to a port and start listening. + *@param port 0 means choose an available port. + *@param backlog maximum number of pending connections. + *@param certName name of certificate to use to identify the server + *@return The bound port. + */ + int listen(uint16_t port = 0, int backlog = 10, const std::string& certName = "localhost.localdomain", bool clientAuth = false) const; + + /** + * Accept a connection from a socket that is already listening + * and has an incoming connection + */ + SslSocket* accept() const; + + // TODO The following are raw operations, maybe they need better wrapping? + int read(void *buf, size_t count) const; + int write(const void *buf, size_t count) const; + + /** Returns the "socket name" ie the address bound to + * the near end of the socket + */ + std::string getSockname() const; + + /** Returns the "peer name" ie the address bound to + * the remote end of the socket + */ + std::string getPeername() const; + + /** + * Returns an address (host and port) for the remote end of the + * socket + */ + std::string getPeerAddress() const; + /** + * Returns an address (host and port) for the local end of the + * socket + */ + std::string getLocalAddress() const; + + uint16_t getLocalPort() const; + uint16_t getRemotePort() const; + + /** + * Returns the error code stored in the socket. This may be used + * to determine the result of a non-blocking connect. + */ + int getError() const; + + int getKeyLen() const; + +private: + mutable std::string connectname; + mutable PRFileDesc* socket; + /** + * 'model' socket, with configuration to use when importing + * accepted sockets for use as ssl sockets. Set on listen(), used + * in accept to pass through to newly created socket instances. + */ + mutable PRFileDesc* prototype; + + SslSocket(IOHandlePrivate* ioph, PRFileDesc* model); +}; + +}}} +#endif /*!_sys_ssl_Socket_h*/ diff --git a/cpp/src/qpid/sys/ssl/check.cpp b/cpp/src/qpid/sys/ssl/check.cpp new file mode 100644 index 0000000000..c5e6005e03 --- /dev/null +++ b/cpp/src/qpid/sys/ssl/check.cpp @@ -0,0 +1,72 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/ssl/check.h" +#include <secerr.h> +#include <sslerr.h> +#include <boost/format.hpp> + +using boost::format; +using boost::str; + +namespace qpid { +namespace sys { +namespace ssl { + +const std::string SSL_ERROR_BAD_CERT_DOMAIN_STR = + "Unable to communicate securely with peer: requested domain name does not match the server's certificate."; +const std::string SSL_ERROR_BAD_CERT_ALERT_STR = "SSL peer cannot verify your certificate."; +const std::string SEC_ERROR_BAD_DATABASE_STR = "Security library: bad database."; +const std::string SSL_ERROR_NO_CERTIFICATE_STR = "Unable to find the certificate or key necessary for authentication."; +const std::string SSL_ERROR_UNKNOWN = "Unknown NSS error code."; + +ErrorString::ErrorString() : code(PR_GetError()), buffer(new char[PR_GetErrorTextLength()]), used(PR_GetErrorText(buffer)) {} + +ErrorString::~ErrorString() +{ + delete[] buffer; +} + +std::string ErrorString::getString() const +{ + std::string msg = std::string(buffer, used); + if (!used) { + //seems most of the NSPR/NSS errors don't have text set for + //them, add a few specific ones in here. (TODO: more complete + //list?): + switch (code) { + case SSL_ERROR_BAD_CERT_DOMAIN: msg = SSL_ERROR_BAD_CERT_DOMAIN_STR; break; + case SSL_ERROR_BAD_CERT_ALERT: msg = SSL_ERROR_BAD_CERT_ALERT_STR; break; + case SEC_ERROR_BAD_DATABASE: msg = SEC_ERROR_BAD_DATABASE_STR; break; + case SSL_ERROR_NO_CERTIFICATE: msg = SSL_ERROR_NO_CERTIFICATE_STR; break; + default: msg = SSL_ERROR_UNKNOWN; break; + } + } + return str(format("%1% [%2%]") % msg % code); +} + +std::ostream& operator<<(std::ostream& out, const ErrorString& err) +{ + out << err.getString(); + return out; +} + + +}}} // namespace qpid::sys::ssl diff --git a/cpp/src/qpid/sys/Monitor.h b/cpp/src/qpid/sys/ssl/check.h index 1d9835675c..984c338a18 100644 --- a/cpp/src/qpid/sys/Monitor.h +++ b/cpp/src/qpid/sys/ssl/check.h @@ -1,5 +1,5 @@ -#ifndef _sys_Monitor_h -#define _sys_Monitor_h +#ifndef QPID_SYS_SSL_CHECK_H +#define QPID_SYS_SSL_CHECK_H /* * @@ -21,31 +21,33 @@ * under the License. * */ - -#include <sys/errno.h> -#include "Condition.h" +#include <iostream> +#include <string> +#include <nspr.h> +#include <nss.h> namespace qpid { namespace sys { +namespace ssl { -/** - * A monitor is a condition variable and a mutex - */ -class Monitor : public Mutex, public Condition { +class ErrorString +{ public: - using Condition::wait; - inline void wait(); - inline bool wait(const AbsTime& absoluteTime); + ErrorString(); + ~ErrorString(); + std::string getString() const; + private: + const int code; + char* const buffer; + const size_t used; }; +std::ostream& operator<<(std::ostream& out, const ErrorString& err); + +}}} // namespace qpid::sys::ssl -void Monitor::wait() { - Condition::wait(*this); -} -bool Monitor::wait(const AbsTime& absoluteTime) { - return Condition::wait(*this, absoluteTime); -} +#define NSS_CHECK(value) if (value != SECSuccess) { throw Exception(QPID_MSG("Failed: " << qpid::sys::ssl::ErrorString())); } +#define PR_CHECK(value) if (value != PR_SUCCESS) { throw Exception(QPID_MSG("Failed: " << qpid::sys::ssl::ErrorString())); } -}} -#endif /*!_sys_Monitor_h*/ +#endif /*!QPID_SYS_SSL_CHECK_H*/ diff --git a/cpp/src/qpid/sys/ssl/util.cpp b/cpp/src/qpid/sys/ssl/util.cpp new file mode 100644 index 0000000000..53326e2f55 --- /dev/null +++ b/cpp/src/qpid/sys/ssl/util.cpp @@ -0,0 +1,118 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/ssl/util.h" +#include "qpid/sys/ssl/check.h" +#include "qpid/Exception.h" +#include "qpid/sys/SystemInfo.h" + +#include <unistd.h> +#include <nspr.h> +#include <nss.h> +#include <pk11pub.h> +#include <ssl.h> + +#include <iostream> +#include <fstream> +#include <boost/filesystem/operations.hpp> +#include <boost/filesystem/path.hpp> + +namespace qpid { +namespace sys { +namespace ssl { + +std::string defaultCertName() +{ + TcpAddress address; + if (SystemInfo::getLocalHostname(address)) { + return address.host; + } else { + return "localhost"; + } +} + +SslOptions::SslOptions() : qpid::Options("SSL Settings"), + certName(defaultCertName()), + exportPolicy(false) +{ + addOptions() + ("ssl-use-export-policy", optValue(exportPolicy), "Use NSS export policy") + ("ssl-cert-password-file", optValue(certPasswordFile, "PATH"), "File containing password to use for accessing certificate database") + ("ssl-cert-db", optValue(certDbPath, "PATH"), "Path to directory containing certificate database") + ("ssl-cert-name", optValue(certName, "NAME"), "Name of the certificate to use"); +} + +SslOptions& SslOptions::operator=(const SslOptions& o) +{ + certDbPath = o.certDbPath; + certName = o.certName; + certPasswordFile = o.certPasswordFile; + exportPolicy = o.exportPolicy; + return *this; +} + +char* promptForPassword(PK11SlotInfo*, PRBool retry, void*) +{ + if (retry) return 0; + //TODO: something else? + return PL_strdup(getpass("Please enter the password for accessing the certificate database:")); +} + +SslOptions SslOptions::global; + +char* readPasswordFromFile(PK11SlotInfo*, PRBool retry, void*) +{ + const std::string& passwordFile = SslOptions::global.certPasswordFile; + if (retry || passwordFile.empty() || !boost::filesystem::exists(passwordFile)) { + return 0; + } else { + std::ifstream file(passwordFile.c_str()); + std::string password; + file >> password; + return PL_strdup(password.c_str()); + } +} + +void initNSS(const SslOptions& options, bool server) +{ + SslOptions::global = options; + if (options.certPasswordFile.empty()) { + PK11_SetPasswordFunc(promptForPassword); + } else { + PK11_SetPasswordFunc(readPasswordFromFile); + } + NSS_CHECK(NSS_Init(options.certDbPath.c_str())); + if (options.exportPolicy) { + NSS_CHECK(NSS_SetExportPolicy()); + } else { + NSS_CHECK(NSS_SetDomesticPolicy()); + } + if (server) { + //use defaults for all args, TODO: may want to make this configurable + SSL_ConfigServerSessionIDCache(0, 0, 0, 0); + } +} + +void shutdownNSS() +{ + NSS_Shutdown(); +} + +}}} // namespace qpid::sys::ssl diff --git a/cpp/src/qpid/sys/StrError.h b/cpp/src/qpid/sys/ssl/util.h index 3843f2abe1..f34adab7be 100644 --- a/cpp/src/qpid/sys/StrError.h +++ b/cpp/src/qpid/sys/ssl/util.h @@ -1,5 +1,5 @@ -#ifndef _sys_StrError_h -#define _sys_StrError_h +#ifndef QPID_SYS_SSL_UTIL_H +#define QPID_SYS_SSL_UTIL_H /* * @@ -22,14 +22,29 @@ * */ +#include "qpid/Options.h" #include <string> namespace qpid { namespace sys { +namespace ssl { -/** Get the error message for a system number err, e.g. errno. */ -std::string strError(int err); +struct SslOptions : qpid::Options +{ + static SslOptions global; -}} // namespace qpid - -#endif // _sys_StrError_h + std::string certDbPath; + std::string certName; + std::string certPasswordFile; + bool exportPolicy; + + SslOptions(); + SslOptions& operator=(const SslOptions&); +}; + +void initNSS(const SslOptions& options, bool server = false); +void shutdownNSS(); + +}}} // namespace qpid::sys::ssl + +#endif /*!QPID_SYS_SSL_UTIL_H*/ diff --git a/cpp/src/qpid/sys/uuid.h b/cpp/src/qpid/sys/uuid.h new file mode 100644 index 0000000000..804ab34463 --- /dev/null +++ b/cpp/src/qpid/sys/uuid.h @@ -0,0 +1,28 @@ +#ifndef _sys_uuid_h +#define _sys_uuid_h + +/* + * + * Copyright (c) 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifdef _WIN32 +# include "qpid/sys/windows/uuid.h" +#else +# include <uuid/uuid.h> +#endif /* _WIN32 */ + +#endif /* _sys_uuid_h */ diff --git a/cpp/src/qpid/sys/windows/AsynchIO.cpp b/cpp/src/qpid/sys/windows/AsynchIO.cpp new file mode 100644 index 0000000000..e4174ee445 --- /dev/null +++ b/cpp/src/qpid/sys/windows/AsynchIO.cpp @@ -0,0 +1,751 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/windows/AsynchIoResult.h" +#include "qpid/sys/windows/IoHandlePrivate.h" +#include "qpid/sys/AsynchIO.h" +#include "qpid/sys/Mutex.h" +#include "qpid/sys/Socket.h" +#include "qpid/sys/Poller.h" +#include "qpid/sys/Thread.h" +#include "qpid/sys/Time.h" +#include "qpid/log/Statement.h" + +#include "qpid/sys/windows/check.h" + +#include <boost/thread/once.hpp> + +#include <queue> +#include <winsock2.h> +#include <mswsock.h> +#include <windows.h> + +#include <boost/bind.hpp> + +namespace { + + typedef qpid::sys::ScopedLock<qpid::sys::Mutex> QLock; + +/* + * The function pointers for AcceptEx and ConnectEx need to be looked up + * at run time. Make sure this is done only once. + */ +boost::once_flag lookUpAcceptExOnce = BOOST_ONCE_INIT; +LPFN_ACCEPTEX fnAcceptEx = 0; +typedef void (*lookUpFunc)(const qpid::sys::Socket &); + +void lookUpAcceptEx() { + SOCKET h = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + GUID guidAcceptEx = WSAID_ACCEPTEX; + DWORD dwBytes = 0; + WSAIoctl(h, + SIO_GET_EXTENSION_FUNCTION_POINTER, + &guidAcceptEx, + sizeof(guidAcceptEx), + &fnAcceptEx, + sizeof(fnAcceptEx), + &dwBytes, + NULL, + NULL); + closesocket(h); + if (fnAcceptEx == 0) + throw qpid::Exception(QPID_MSG("Failed to look up AcceptEx")); +} + +} + +namespace qpid { +namespace sys { +namespace windows { + +/* + * Asynch Acceptor + * + */ +class AsynchAcceptor : public qpid::sys::AsynchAcceptor { + + friend class AsynchAcceptResult; + +public: + AsynchAcceptor(const Socket& s, AsynchAcceptor::Callback callback); + ~AsynchAcceptor(); + void start(Poller::shared_ptr poller); + +private: + void restart(void); + + AsynchAcceptor::Callback acceptedCallback; + const Socket& socket; +}; + +AsynchAcceptor::AsynchAcceptor(const Socket& s, Callback callback) + : acceptedCallback(callback), + socket(s) { + + s.setNonblocking(); +#if (BOOST_VERSION >= 103500) /* boost 1.35 or later reversed the args */ + boost::call_once(lookUpAcceptExOnce, lookUpAcceptEx); +#else + boost::call_once(lookUpAcceptEx, lookUpAcceptExOnce); +#endif +} + +AsynchAcceptor::~AsynchAcceptor() +{ + socket.close(); +} + +void AsynchAcceptor::start(Poller::shared_ptr poller) { + poller->monitorHandle(PollerHandle(socket), Poller::INPUT); + restart (); +} + +void AsynchAcceptor::restart(void) { + DWORD bytesReceived = 0; // Not used, needed for AcceptEx API + AsynchAcceptResult *result = new AsynchAcceptResult(acceptedCallback, + this, + toSocketHandle(socket)); + BOOL status; + status = ::fnAcceptEx(toSocketHandle(socket), + toSocketHandle(*result->newSocket), + result->addressBuffer, + 0, + AsynchAcceptResult::SOCKADDRMAXLEN, + AsynchAcceptResult::SOCKADDRMAXLEN, + &bytesReceived, + result->overlapped()); + QPID_WINDOWS_CHECK_ASYNC_START(status); +} + + +AsynchAcceptResult::AsynchAcceptResult(AsynchAcceptor::Callback cb, + AsynchAcceptor *acceptor, + SOCKET listener) + : callback(cb), acceptor(acceptor), listener(listener) { + newSocket.reset (new Socket()); +} + +void AsynchAcceptResult::success(size_t /*bytesTransferred*/) { + ::setsockopt (toSocketHandle(*newSocket), + SOL_SOCKET, + SO_UPDATE_ACCEPT_CONTEXT, + (char*)&listener, + sizeof (listener)); + callback(*(newSocket.release())); + acceptor->restart (); + delete this; +} + +void AsynchAcceptResult::failure(int status) { + //if (status != WSA_OPERATION_ABORTED) + // Can there be anything else? ; + delete this; +} + +/* + * AsynchConnector does synchronous connects for now... to do asynch the + * IocpPoller will need some extension to register an event handle as a + * CONNECT-type "direction", the connect completion/result will need an + * event handle to associate with the connecting handle. But there's no + * time for that right now... + */ +class AsynchConnector : public qpid::sys::AsynchConnector { +private: + ConnectedCallback connCallback; + FailedCallback failCallback; + const Socket& socket; + +public: + AsynchConnector(const Socket& socket, + Poller::shared_ptr poller, + std::string hostname, + uint16_t port, + ConnectedCallback connCb, + FailedCallback failCb = 0); +}; + +AsynchConnector::AsynchConnector(const Socket& sock, + Poller::shared_ptr poller, + std::string hostname, + uint16_t port, + ConnectedCallback connCb, + FailedCallback failCb) + : connCallback(connCb), failCallback(failCb), socket(sock) { + try { + socket.connect(hostname, port); + socket.setNonblocking(); + connCallback(socket); + } catch(std::exception& e) { + if (failCallback) + failCallback(socket, -1, std::string(e.what())); + socket.close(); + delete &socket; + } +} + +} // namespace windows + +AsynchAcceptor* AsynchAcceptor::create(const Socket& s, + Callback callback) +{ + return new windows::AsynchAcceptor(s, callback); +} + +AsynchConnector* qpid::sys::AsynchConnector::create(const Socket& s, + Poller::shared_ptr poller, + std::string hostname, + uint16_t port, + ConnectedCallback connCb, + FailedCallback failCb) +{ + return new windows::AsynchConnector(s, + poller, + hostname, + port, + connCb, + failCb); +} + + +/* + * Asynch reader/writer + */ + +namespace windows { + +class AsynchIO : public qpid::sys::AsynchIO { +public: + AsynchIO(const Socket& s, + ReadCallback rCb, + EofCallback eofCb, + DisconnectCallback disCb, + ClosedCallback cCb = 0, + BuffersEmptyCallback eCb = 0, + IdleCallback iCb = 0); + ~AsynchIO(); + + // Methods inherited from qpid::sys::AsynchIO + + /** + * Notify the object is should delete itself as soon as possible. + */ + virtual void queueForDeletion(); + + /// Take any actions needed to prepare for working with the poller. + virtual void start(Poller::shared_ptr poller); + virtual void queueReadBuffer(BufferBase* buff); + virtual void unread(BufferBase* buff); + virtual void queueWrite(BufferBase* buff); + virtual void notifyPendingWrite(); + virtual void queueWriteClose(); + virtual bool writeQueueEmpty(); + virtual void startReading(); + virtual void stopReading(); + virtual void requestCallback(RequestCallback); + + /** + * getQueuedBuffer returns a buffer from the buffer queue, if one is + * available. + * + * @retval Pointer to BufferBase buffer; 0 if none is available. + */ + virtual BufferBase* getQueuedBuffer(); + +private: + ReadCallback readCallback; + EofCallback eofCallback; + DisconnectCallback disCallback; + ClosedCallback closedCallback; + BuffersEmptyCallback emptyCallback; + IdleCallback idleCallback; + const Socket& socket; + Poller::shared_ptr poller; + + std::deque<BufferBase*> bufferQueue; + std::deque<BufferBase*> writeQueue; + /* The MSVC-supplied deque is not thread-safe; keep locks to serialize + * access to the buffer queue and write queue. + */ + Mutex bufferQueueLock; + + // Number of outstanding I/O operations. + volatile LONG opsInProgress; + // Is there a write in progress? + volatile bool writeInProgress; + // Deletion requested, but there are callbacks in progress. + volatile bool queuedDelete; + // Socket close requested, but there are operations in progress. + volatile bool queuedClose; + +private: + // Dispatch events that have completed. + void notifyEof(void); + void notifyDisconnect(void); + void notifyClosed(void); + void notifyBuffersEmpty(void); + void notifyIdle(void); + + /** + * Initiate a write of the specified buffer. There's no callback for + * write completion to the AsynchIO object. + */ + void startWrite(AsynchIO::BufferBase* buff); + + void close(void); + + /** + * readComplete is called when a read request is complete. + * + * @param result Results of the operation. + */ + void readComplete(AsynchReadResult *result); + + /** + * writeComplete is called when a write request is complete. + * + * @param result Results of the operation. + */ + void writeComplete(AsynchWriteResult *result); + + /** + * Queue of completions to run. This queue enforces the requirement + * from upper layers that only one thread at a time is allowed to act + * on any given connection. Once a thread is busy processing a completion + * on this object, other threads that dispatch completions queue the + * completions here for the in-progress thread to handle when done. + * Thus, any threads can dispatch a completion from the IocpPoller, but + * this class ensures that actual processing at the connection level is + * only on one thread at a time. + */ + std::queue<AsynchIoResult *> completionQueue; + volatile bool working; + Mutex completionLock; + + /** + * Called when there's a completion to process. + */ + void completion(AsynchIoResult *result); +}; + +// This is used to encapsulate pure callbacks into a handle +class CallbackHandle : public IOHandle { +public: + CallbackHandle(AsynchIoResult::Completer completeCb, + AsynchIO::RequestCallback reqCb = 0) : + IOHandle(new IOHandlePrivate (INVALID_SOCKET, completeCb, reqCb)) + {} +}; + +AsynchIO::AsynchIO(const Socket& s, + ReadCallback rCb, + EofCallback eofCb, + DisconnectCallback disCb, + ClosedCallback cCb, + BuffersEmptyCallback eCb, + IdleCallback iCb) : + + readCallback(rCb), + eofCallback(eofCb), + disCallback(disCb), + closedCallback(cCb), + emptyCallback(eCb), + idleCallback(iCb), + socket(s), + opsInProgress(0), + writeInProgress(false), + queuedDelete(false), + queuedClose(false), + working(false) { +} + +struct deleter +{ + template <typename T> + void operator()(T *ptr){ delete ptr;} +}; + +AsynchIO::~AsynchIO() { + std::for_each( bufferQueue.begin(), bufferQueue.end(), deleter()); + std::for_each( writeQueue.begin(), writeQueue.end(), deleter()); +} + +void AsynchIO::queueForDeletion() { + queuedDelete = true; + if (opsInProgress > 0) { + QPID_LOG(info, "Delete AsynchIO queued; ops in progress"); + // AsynchIOHandler calls this then deletes itself; don't do any more + // callbacks. + readCallback = 0; + eofCallback = 0; + disCallback = 0; + closedCallback = 0; + emptyCallback = 0; + idleCallback = 0; + } + else { + delete this; + } +} + +void AsynchIO::start(Poller::shared_ptr poller0) { + poller = poller0; + poller->monitorHandle(PollerHandle(socket), Poller::INPUT); + if (writeQueue.size() > 0) // Already have data queued for write + notifyPendingWrite(); + startReading(); +} + +void AsynchIO::queueReadBuffer(AsynchIO::BufferBase* buff) { + assert(buff); + buff->dataStart = 0; + buff->dataCount = 0; + QLock l(bufferQueueLock); + bufferQueue.push_back(buff); +} + +void AsynchIO::unread(AsynchIO::BufferBase* buff) { + assert(buff); + if (buff->dataStart != 0) { + memmove(buff->bytes, buff->bytes+buff->dataStart, buff->dataCount); + buff->dataStart = 0; + } + QLock l(bufferQueueLock); + bufferQueue.push_front(buff); +} + +void AsynchIO::queueWrite(AsynchIO::BufferBase* buff) { + assert(buff); + QLock l(bufferQueueLock); + writeQueue.push_back(buff); + if (!writeInProgress) + notifyPendingWrite(); +} + +void AsynchIO::notifyPendingWrite() { + // This method is generally called from a processing thread; transfer + // work on this to an I/O thread. Much of the upper layer code assumes + // that all I/O-related things happen in an I/O thread. + if (poller == 0) // Not really going yet... + return; + + InterlockedIncrement(&opsInProgress); + PollerHandle ph(CallbackHandle(boost::bind(&AsynchIO::completion, this, _1))); + poller->monitorHandle(ph, Poller::OUTPUT); +} + +void AsynchIO::queueWriteClose() { + queuedClose = true; + if (!writeInProgress) + notifyPendingWrite(); +} + +bool AsynchIO::writeQueueEmpty() { + QLock l(bufferQueueLock); + return writeQueue.size() == 0; +} + +/* + * Initiate a read operation. AsynchIO::readComplete() will be + * called when the read is complete and data is available. + */ +void AsynchIO::startReading() { + if (queuedDelete) + return; + + // (Try to) get a buffer; look on the front since there may be an + // "unread" one there with data remaining from last time. + AsynchIO::BufferBase *buff = 0; + { + QLock l(bufferQueueLock); + + if (!bufferQueue.empty()) { + buff = bufferQueue.front(); + assert(buff); + bufferQueue.pop_front(); + } + } + if (buff != 0) { + int readCount = buff->byteCount - buff->dataCount; + AsynchReadResult *result = + new AsynchReadResult(boost::bind(&AsynchIO::completion, this, _1), + buff, + readCount); + DWORD bytesReceived = 0, flags = 0; + InterlockedIncrement(&opsInProgress); + int status = WSARecv(toSocketHandle(socket), + const_cast<LPWSABUF>(result->getWSABUF()), 1, + &bytesReceived, + &flags, + result->overlapped(), + 0); + if (status != 0) { + int error = WSAGetLastError(); + if (error != WSA_IO_PENDING) { + result->failure(error); + result = 0; // result is invalid here + return; + } + } + // On status 0 or WSA_IO_PENDING, completion will handle the rest. + } + else { + notifyBuffersEmpty(); + } + return; +} + +// stopReading was added to prevent a race condition with read-credit on Linux. +// It may or may not be required on windows. +// +// AsynchIOHandler::readbuff() calls stopReading() inside the same +// critical section that protects startReading() in +// AsynchIOHandler::giveReadCredit(). +// +void AsynchIO::stopReading() {} + +// Queue the specified callback for invocation from an I/O thread. +void AsynchIO::requestCallback(RequestCallback callback) { + // This method is generally called from a processing thread; transfer + // work on this to an I/O thread. Much of the upper layer code assumes + // that all I/O-related things happen in an I/O thread. + if (poller == 0) // Not really going yet... + return; + + InterlockedIncrement(&opsInProgress); + PollerHandle ph(CallbackHandle( + boost::bind(&AsynchIO::completion, this, _1), + callback)); + poller->monitorHandle(ph, Poller::INPUT); +} + +/** + * Return a queued buffer if there are enough to spare. + */ +AsynchIO::BufferBase* AsynchIO::getQueuedBuffer() { + QLock l(bufferQueueLock); + // Always keep at least one buffer (it might have data that was + // "unread" in it). + if (bufferQueue.size() <= 1) + return 0; + BufferBase* buff = bufferQueue.back(); + assert(buff); + bufferQueue.pop_back(); + return buff; +} + +void AsynchIO::notifyEof(void) { + if (eofCallback) + eofCallback(*this); +} + +void AsynchIO::notifyDisconnect(void) { + if (disCallback) + disCallback(*this); +} + +void AsynchIO::notifyClosed(void) { + if (closedCallback) + closedCallback(*this, socket); +} + +void AsynchIO::notifyBuffersEmpty(void) { + if (emptyCallback) + emptyCallback(*this); +} + +void AsynchIO::notifyIdle(void) { + if (idleCallback) + idleCallback(*this); +} + +/* + * Asynch reader/writer using overlapped I/O + */ + +void AsynchIO::startWrite(AsynchIO::BufferBase* buff) { + writeInProgress = true; + InterlockedIncrement(&opsInProgress); + int writeCount = buff->byteCount-buff->dataCount; + AsynchWriteResult *result = + new AsynchWriteResult(boost::bind(&AsynchIO::completion, this, _1), + buff, + buff->dataCount); + DWORD bytesSent = 0; + int status = WSASend(toSocketHandle(socket), + const_cast<LPWSABUF>(result->getWSABUF()), 1, + &bytesSent, + 0, + result->overlapped(), + 0); + if (status != 0) { + int error = WSAGetLastError(); + if (error != WSA_IO_PENDING) { + result->failure(error); // Also decrements in-progress count + result = 0; // result is invalid here + return; + } + } + // On status 0 or WSA_IO_PENDING, completion will handle the rest. + return; +} + +/* + * Close the socket and callback to say we've done it + */ +void AsynchIO::close(void) { + socket.close(); + notifyClosed(); +} + +void AsynchIO::readComplete(AsynchReadResult *result) { + int status = result->getStatus(); + size_t bytes = result->getTransferred(); + if (status == 0 && bytes > 0) { + bool restartRead = true; // May not if receiver doesn't want more + if (readCallback) + readCallback(*this, result->getBuff()); + if (restartRead) + startReading(); + } + else { + // No data read, so put the buffer back. It may be partially filled, + // so "unread" it back to the front of the queue. + unread(result->getBuff()); + if (status == 0) + notifyEof(); + else + notifyDisconnect(); + } +} + +/* + * NOTE - this completion is called for completed writes and also when + * a write is desired. The difference is in the buff - if a write is desired + * the buff is 0. + */ +void AsynchIO::writeComplete(AsynchWriteResult *result) { + int status = result->getStatus(); + size_t bytes = result->getTransferred(); + AsynchIO::BufferBase *buff = result->getBuff(); + if (buff != 0) { + writeInProgress = false; + if (status == 0 && bytes > 0) { + if (bytes < result->getRequested()) // Still more to go; resubmit + startWrite(buff); + else + queueReadBuffer(buff); // All done; back to the pool + } + else { + // An error... if it's a connection close, ignore it - it will be + // noticed and handled on a read completion any moment now. + // What to do with real error??? Save the Buffer? + } + } + + // If there are no writes outstanding, check for more writes to initiate + // (either queued or via idle). The opsInProgress count is handled in + // completion() + if (!writeInProgress) { + bool writing = false; + { + QLock l(bufferQueueLock); + if (writeQueue.size() > 0) { + buff = writeQueue.front(); + assert(buff); + writeQueue.pop_front(); + startWrite(buff); + writing = true; + } + } + if (!writing && !queuedClose) { + notifyIdle(); + } + } + return; +} + +void AsynchIO::completion(AsynchIoResult *result) { + { + ScopedLock<Mutex> l(completionLock); + if (working) { + completionQueue.push(result); + return; + } + + // First thread in with something to do; note we're working then keep + // handling completions. + working = true; + while (result != 0) { + // New scope to unlock temporarily. + { + ScopedUnlock<Mutex> ul(completionLock); + AsynchReadResult *r = dynamic_cast<AsynchReadResult*>(result); + if (r != 0) + readComplete(r); + else { + AsynchWriteResult *w = + dynamic_cast<AsynchWriteResult*>(result); + if (w != 0) + writeComplete(w); + else { + AsynchCallbackRequest *req = + dynamic_cast<AsynchCallbackRequest*>(result); + req->reqCallback(*this); + } + } + delete result; + result = 0; + InterlockedDecrement(&opsInProgress); + } + // Lock is held again. + if (completionQueue.empty()) + continue; + result = completionQueue.front(); + completionQueue.pop(); + } + working = false; + } + // Lock released; ok to close if ops are done and close requested. + // Layer above will call back to queueForDeletion() if it hasn't + // already been done. If it already has, go ahead and delete. + if (opsInProgress == 0) { + if (queuedClose) + // close() may cause a delete; don't trust 'this' on return + close(); + else if (queuedDelete) + delete this; + } +} + +} // namespace windows + +AsynchIO* qpid::sys::AsynchIO::create(const Socket& s, + AsynchIO::ReadCallback rCb, + AsynchIO::EofCallback eofCb, + AsynchIO::DisconnectCallback disCb, + AsynchIO::ClosedCallback cCb, + AsynchIO::BuffersEmptyCallback eCb, + AsynchIO::IdleCallback iCb) +{ + return new qpid::sys::windows::AsynchIO(s, rCb, eofCb, disCb, cCb, eCb, iCb); +} + +}} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/windows/AsynchIoResult.h b/cpp/src/qpid/sys/windows/AsynchIoResult.h new file mode 100755 index 0000000000..66c89efc11 --- /dev/null +++ b/cpp/src/qpid/sys/windows/AsynchIoResult.h @@ -0,0 +1,204 @@ +#ifndef _windows_asynchIoResult_h +#define _windows_asynchIoResult_h + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/AsynchIO.h" +#include "qpid/sys/Socket.h" +#include <memory.h> +#include <winsock2.h> +#include <ws2tcpip.h> + +namespace qpid { +namespace sys { +namespace windows { + +/* + * AsynchIoResult defines the class that receives the result of an + * asynchronous I/O operation, either send/recv or accept/connect. + * + * Operation factories should set one of these up before beginning the + * operation. Poller knows how to dispatch completion to this class. + * This class must be subclassed for needed operations; this class provides + * an interface only and cannot be instantiated. + * + * This class is tied to Windows; it inherits from OVERLAPPED so that the + * IocpPoller can cast OVERLAPPED pointers back to AsynchIoResult and call + * the completion handler. + */ +class AsynchResult : private OVERLAPPED { +public: + LPOVERLAPPED overlapped(void) { return this; } + static AsynchResult* from_overlapped(LPOVERLAPPED ol) { + return static_cast<AsynchResult*>(ol); + } + virtual void success (size_t bytesTransferred) { + bytes = bytesTransferred; + status = 0; + complete(); + } + virtual void failure (int error) { + bytes = 0; + status = error; + complete(); + } + size_t getTransferred(void) const { return bytes; } + int getStatus(void) const { return status; } + +protected: + AsynchResult() : bytes(0), status(0) + { memset(overlapped(), 0, sizeof(OVERLAPPED)); } + ~AsynchResult() {} + virtual void complete(void) = 0; + + size_t bytes; + int status; +}; + +class AsynchAcceptor; + +class AsynchAcceptResult : public AsynchResult { + + friend class AsynchAcceptor; + +public: + AsynchAcceptResult(qpid::sys::AsynchAcceptor::Callback cb, + AsynchAcceptor *acceptor, + SOCKET listener); + virtual void success (size_t bytesTransferred); + virtual void failure (int error); + +private: + virtual void complete(void) {} // No-op for this class. + + std::auto_ptr<qpid::sys::Socket> newSocket; + qpid::sys::AsynchAcceptor::Callback callback; + AsynchAcceptor *acceptor; + SOCKET listener; + + // AcceptEx needs a place to write the local and remote addresses + // when accepting the connection. Place those here; get enough for + // IPv6 addresses, even if the socket is IPv4. + enum { SOCKADDRMAXLEN = sizeof sockaddr_in6 + 16, + SOCKADDRBUFLEN = 2 * SOCKADDRMAXLEN }; + char addressBuffer[SOCKADDRBUFLEN]; +}; + +class AsynchIoResult : public AsynchResult { +public: + typedef boost::function1<void, AsynchIoResult *> Completer; + + virtual ~AsynchIoResult() {} + qpid::sys::AsynchIO::BufferBase *getBuff(void) const { return iobuff; } + size_t getRequested(void) const { return requested; } + const WSABUF *getWSABUF(void) const { return &wsabuf; } + +protected: + void setBuff (qpid::sys::AsynchIO::BufferBase *buffer) { iobuff = buffer; } + +protected: + AsynchIoResult(Completer cb, + qpid::sys::AsynchIO::BufferBase *buff, size_t length) + : completionCallback(cb), iobuff(buff), requested(length) {} + + virtual void complete(void) = 0; + WSABUF wsabuf; + Completer completionCallback; + +private: + qpid::sys::AsynchIO::BufferBase *iobuff; + size_t requested; // Number of bytes in original I/O request +}; + +class AsynchReadResult : public AsynchIoResult { + + // complete() updates buffer then does completion callback. + virtual void complete(void) { + getBuff()->dataCount += bytes; + completionCallback(this); + } + +public: + AsynchReadResult(AsynchIoResult::Completer cb, + qpid::sys::AsynchIO::BufferBase *buff, + size_t length) + : AsynchIoResult(cb, buff, length) { + wsabuf.buf = buff->bytes + buff->dataCount; + wsabuf.len = length; + } +}; + +class AsynchWriteResult : public AsynchIoResult { + + // complete() updates buffer then does completion callback. + virtual void complete(void) { + qpid::sys::AsynchIO::BufferBase *b = getBuff(); + b->dataStart += bytes; + b->dataCount -= bytes; + completionCallback(this); + } + +public: + AsynchWriteResult(AsynchIoResult::Completer cb, + qpid::sys::AsynchIO::BufferBase *buff, + size_t length) + : AsynchIoResult(cb, buff, length) { + wsabuf.buf = buff ? buff->bytes : 0; + wsabuf.len = length; + } +}; + +class AsynchWriteWanted : public AsynchWriteResult { + + // complete() just does completion callback; no buffers used. + virtual void complete(void) { + completionCallback(this); + } + +public: + AsynchWriteWanted(AsynchIoResult::Completer cb) + : AsynchWriteResult(cb, 0, 0) { + wsabuf.buf = 0; + wsabuf.len = 0; + } +}; + +class AsynchCallbackRequest : public AsynchIoResult { + // complete() needs to simply call the completionCallback; no buffers. + virtual void complete(void) { + completionCallback(this); + } + +public: + AsynchCallbackRequest(AsynchIoResult::Completer cb, + qpid::sys::AsynchIO::RequestCallback reqCb) + : AsynchIoResult(cb, 0, 0), reqCallback(reqCb) { + wsabuf.buf = 0; + wsabuf.len = 0; + } + + qpid::sys::AsynchIO::RequestCallback reqCallback; +}; + +}}} // qpid::sys::windows + +#endif /*!_windows_asynchIoResult_h*/ diff --git a/cpp/src/qpid/sys/windows/FileSysDir.cpp b/cpp/src/qpid/sys/windows/FileSysDir.cpp new file mode 100644 index 0000000000..88f1637d48 --- /dev/null +++ b/cpp/src/qpid/sys/windows/FileSysDir.cpp @@ -0,0 +1,53 @@ +/* + * + * Copyright (c) 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "qpid/sys/FileSysDir.h" +#include "qpid/sys/StrError.h" +#include "qpid/Exception.h" + +#include <sys/types.h> +#include <sys/stat.h> +#include <direct.h> +#include <errno.h> + +namespace qpid { +namespace sys { + +bool FileSysDir::exists (void) const +{ + const char *cpath = dirPath.c_str (); + struct _stat s; + if (::_stat(cpath, &s)) { + if (errno == ENOENT) { + return false; + } + throw qpid::Exception (strError(errno) + + ": Can't check directory: " + dirPath); + } + if (s.st_mode & _S_IFDIR) + return true; + throw qpid::Exception(dirPath + " is not a directory"); +} + +void FileSysDir::mkdir(void) +{ + if (::_mkdir(dirPath.c_str()) == -1) + throw Exception ("Can't create directory: " + dirPath); +} + +}} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/SystemInfo.cpp b/cpp/src/qpid/sys/windows/IOHandle.cpp index dcc7ad9985..250737cb99 100644..100755 --- a/cpp/src/qpid/sys/SystemInfo.cpp +++ b/cpp/src/qpid/sys/windows/IOHandle.cpp @@ -1,4 +1,5 @@ /* + * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -6,9 +7,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -18,18 +19,24 @@ * */ -#include "SystemInfo.h" -#include <unistd.h> +#include "qpid/sys/IOHandle.h" +#include "qpid/sys/windows/IoHandlePrivate.h" +#include <windows.h> namespace qpid { namespace sys { -long SystemInfo::concurrency() { -#ifdef _SC_NPROCESSORS_ONLN // Linux specific. - return sysconf(_SC_NPROCESSORS_ONLN); -#else - return -1; -#endif +SOCKET toFd(const IOHandlePrivate* h) +{ + return h->fd; +} + +IOHandle::IOHandle(IOHandlePrivate* h) : + impl(h) +{} + +IOHandle::~IOHandle() { + delete impl; } }} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/windows/IoHandlePrivate.h b/cpp/src/qpid/sys/windows/IoHandlePrivate.h new file mode 100755 index 0000000000..5943db5cc7 --- /dev/null +++ b/cpp/src/qpid/sys/windows/IoHandlePrivate.h @@ -0,0 +1,61 @@ +#ifndef _sys_windows_IoHandlePrivate_h +#define _sys_windows_IoHandlePrivate_h + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/AsynchIO.h" +#include "qpid/sys/windows/AsynchIoResult.h" +#include "qpid/CommonImportExport.h" + +#include <winsock2.h> + +namespace qpid { +namespace sys { + +// Private fd related implementation details +// There should be either a valid socket handle or a completer callback. +// Handle is used to associate with poller's iocp; completer is used to +// inject a completion that will very quickly trigger a callback to the +// completer from an I/O thread. If the callback mechanism is used, there +// can be a RequestCallback set - this carries the callback object through +// from AsynchIO::requestCallback() through to the I/O completion processing. +class IOHandlePrivate { + friend QPID_COMMON_EXTERN SOCKET toSocketHandle(const Socket& s); + static IOHandlePrivate* getImpl(const IOHandle& h); + +public: + IOHandlePrivate(SOCKET f = INVALID_SOCKET, + windows::AsynchIoResult::Completer cb = 0, + AsynchIO::RequestCallback reqCallback = 0) : + fd(f), event(cb), cbRequest(reqCallback) + {} + + SOCKET fd; + windows::AsynchIoResult::Completer event; + AsynchIO::RequestCallback cbRequest; +}; + +QPID_COMMON_EXTERN SOCKET toSocketHandle(const Socket& s); + +}} + +#endif /* _sys_windows_IoHandlePrivate_h */ diff --git a/cpp/src/qpid/sys/windows/IocpPoller.cpp b/cpp/src/qpid/sys/windows/IocpPoller.cpp new file mode 100755 index 0000000000..4fcc9155f1 --- /dev/null +++ b/cpp/src/qpid/sys/windows/IocpPoller.cpp @@ -0,0 +1,214 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Poller.h" +#include "qpid/sys/Mutex.h" +#include "qpid/sys/Dispatcher.h" + +#include "qpid/sys/windows/AsynchIoResult.h" +#include "qpid/sys/windows/IoHandlePrivate.h" +#include "qpid/sys/windows/check.h" + +#include <winsock2.h> +#include <windows.h> + +#include <assert.h> +#include <vector> +#include <exception> + +namespace qpid { +namespace sys { + +class PollerHandlePrivate { + friend class Poller; + friend class PollerHandle; + + SOCKET fd; + windows::AsynchIoResult::Completer cb; + AsynchIO::RequestCallback cbRequest; + + PollerHandlePrivate(SOCKET f, + windows::AsynchIoResult::Completer cb0 = 0, + AsynchIO::RequestCallback rcb = 0) + : fd(f), cb(cb0), cbRequest(rcb) + { + } + +}; + +PollerHandle::PollerHandle(const IOHandle& h) : + impl(new PollerHandlePrivate(toSocketHandle(static_cast<const Socket&>(h)), h.impl->event, h.impl->cbRequest)) +{} + +PollerHandle::~PollerHandle() { + delete impl; +} + +/** + * Concrete implementation of Poller to use the Windows I/O Completion + * port (IOCP) facility. + */ +class PollerPrivate { + friend class Poller; + + const HANDLE iocp; + + // The number of threads running the event loop. + volatile LONG threadsRunning; + + // Shutdown request is handled by setting isShutdown and injecting a + // well-formed completion event into the iocp. + bool isShutdown; + + PollerPrivate() : + iocp(::CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0)), + threadsRunning(0), + isShutdown(false) { + QPID_WINDOWS_CHECK_NULL(iocp); + } + + ~PollerPrivate() { + // It's probably okay to ignore any errors here as there can't be + // data loss + ::CloseHandle(iocp); + } +}; + +void Poller::shutdown() { + // Allow sloppy code to shut us down more than once. + if (impl->isShutdown) + return; + ULONG_PTR key = 1; // Tell wait() it's a shutdown, not I/O + PostQueuedCompletionStatus(impl->iocp, 0, key, 0); +} + +bool Poller::interrupt(PollerHandle&) { + return false; // There's no concept of a registered handle. +} + +void Poller::run() { + do { + Poller::Event event = this->wait(); + + // Handle shutdown + switch (event.type) { + case Poller::SHUTDOWN: + return; + break; + case Poller::INVALID: // On any type of success or fail completion + break; + default: + // This should be impossible + assert(false); + } + } while (true); +} + +void Poller::monitorHandle(PollerHandle& handle, Direction dir) { + HANDLE h = (HANDLE)(handle.impl->fd); + if (h != INVALID_HANDLE_VALUE) { + HANDLE iocpHandle = ::CreateIoCompletionPort (h, impl->iocp, 0, 0); + QPID_WINDOWS_CHECK_NULL(iocpHandle); + } + else { + // INPUT is used to request a callback; OUTPUT to request a write + assert(dir == Poller::INPUT || dir == Poller::OUTPUT); + + if (dir == Poller::OUTPUT) { + windows::AsynchWriteWanted *result = + new windows::AsynchWriteWanted(handle.impl->cb); + PostQueuedCompletionStatus(impl->iocp, 0, 0, result->overlapped()); + } + else { + windows::AsynchCallbackRequest *result = + new windows::AsynchCallbackRequest(handle.impl->cb, + handle.impl->cbRequest); + PostQueuedCompletionStatus(impl->iocp, 0, 0, result->overlapped()); + } + } +} + +// All no-ops... +void Poller::unmonitorHandle(PollerHandle& handle, Direction dir) {} +void Poller::registerHandle(PollerHandle& handle) {} +void Poller::unregisterHandle(PollerHandle& handle) {} + +Poller::Event Poller::wait(Duration timeout) { + DWORD timeoutMs = 0; + DWORD numTransferred = 0; + ULONG_PTR completionKey = 0; + OVERLAPPED *overlapped = 0; + windows::AsynchResult *result = 0; + + // Wait for either an I/O operation to finish (thus signaling the + // IOCP handle) or a shutdown request to be made (thus signaling the + // shutdown event). + if (timeout == TIME_INFINITE) + timeoutMs = INFINITE; + else + timeoutMs = static_cast<DWORD>(timeout / TIME_MSEC); + + InterlockedIncrement(&impl->threadsRunning); + bool goodOp = ::GetQueuedCompletionStatus (impl->iocp, + &numTransferred, + &completionKey, + &overlapped, + timeoutMs); + LONG remainingThreads = InterlockedDecrement(&impl->threadsRunning); + if (goodOp) { + // Dequeued a successful completion. If it's a posted packet from + // shutdown() the overlapped ptr is 0 and key is 1. Else downcast + // the OVERLAPPED pointer to an AsynchIoResult and call the + // completion handler. + if (overlapped == 0 && completionKey == 1) { + // If there are other threads still running this wait, re-post + // the completion. + if (remainingThreads > 0) + PostQueuedCompletionStatus(impl->iocp, 0, completionKey, 0); + return Event(0, SHUTDOWN); + } + + result = windows::AsynchResult::from_overlapped(overlapped); + result->success (static_cast<size_t>(numTransferred)); + } + else { + if (overlapped != 0) { + // Dequeued a completion for a failed operation. Downcast back + // to the result object and inform it that the operation failed. + DWORD status = ::GetLastError(); + result = windows::AsynchResult::from_overlapped(overlapped); + result->failure (static_cast<int>(status)); + } + } + return Event(0, INVALID); // TODO - this may need to be changed. + +} + +// Concrete constructors +Poller::Poller() : + impl(new PollerPrivate()) +{} + +Poller::~Poller() { + delete impl; +} + +}} diff --git a/cpp/src/qpid/sys/windows/LockFile.cpp b/cpp/src/qpid/sys/windows/LockFile.cpp new file mode 100755 index 0000000000..048c2d5b18 --- /dev/null +++ b/cpp/src/qpid/sys/windows/LockFile.cpp @@ -0,0 +1,64 @@ +/* + * + * Copyright (c) 2008 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "qpid/sys/LockFile.h" +#include "qpid/sys/windows/check.h" + +#include <windows.h> + +namespace qpid { +namespace sys { + +class LockFilePrivate { + friend class LockFile; + + HANDLE fd; + +public: + LockFilePrivate(HANDLE f) : fd(f) {} +}; + +LockFile::LockFile(const std::string& path_, bool create) + : path(path_), created(create) { + + HANDLE h = ::CreateFile(path.c_str(), + create ? (GENERIC_READ|GENERIC_WRITE) : GENERIC_READ, + FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, + 0, /* Default security */ + create ? OPEN_ALWAYS : OPEN_EXISTING, + FILE_FLAG_DELETE_ON_CLOSE, /* Delete file when closed */ + NULL); + if (h == INVALID_HANDLE_VALUE) + throw qpid::Exception(path + ": " + qpid::sys::strError(GetLastError())); + + // Lock up to 4Gb + if (!::LockFile(h, 0, 0, 0xffffffff, 0)) + throw qpid::Exception(path + ": " + qpid::sys::strError(GetLastError())); + impl.reset(new LockFilePrivate(h)); +} + +LockFile::~LockFile() { + if (impl) { + if (impl->fd != INVALID_HANDLE_VALUE) { + ::UnlockFile(impl->fd, 0, 0, 0xffffffff, 0); + ::CloseHandle(impl->fd); + } + } +} + +}} /* namespace qpid::sys */ diff --git a/cpp/src/qpid/sys/windows/PipeHandle.cpp b/cpp/src/qpid/sys/windows/PipeHandle.cpp new file mode 100755 index 0000000000..062458ae5f --- /dev/null +++ b/cpp/src/qpid/sys/windows/PipeHandle.cpp @@ -0,0 +1,101 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +#include "qpid/sys/PipeHandle.h" +#include "qpid/sys/windows/check.h" +#include <winsock2.h> + +namespace qpid { +namespace sys { + +PipeHandle::PipeHandle(bool nonBlocking) { + + SOCKET listener, pair[2]; + struct sockaddr_in addr; + int err; + int addrlen = sizeof(addr); + pair[0] = pair[1] = INVALID_SOCKET; + if ((listener = socket(AF_INET, SOCK_STREAM, 0)) == INVALID_SOCKET) + throw QPID_WINDOWS_ERROR(WSAGetLastError()); + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = 0; + + err = bind(listener, (const struct sockaddr*) &addr, sizeof(addr)); + if (err == SOCKET_ERROR) { + err = WSAGetLastError(); + closesocket(listener); + throw QPID_WINDOWS_ERROR(err); + } + + err = getsockname(listener, (struct sockaddr*) &addr, &addrlen); + if (err == SOCKET_ERROR) { + err = WSAGetLastError(); + closesocket(listener); + throw QPID_WINDOWS_ERROR(err); + } + + try { + if (listen(listener, 1) == SOCKET_ERROR) + throw QPID_WINDOWS_ERROR(WSAGetLastError()); + if ((pair[0] = socket(AF_INET, SOCK_STREAM, 0)) == INVALID_SOCKET) + throw QPID_WINDOWS_ERROR(WSAGetLastError()); + if (connect(pair[0], (const struct sockaddr*)&addr, sizeof(addr)) == SOCKET_ERROR) + throw QPID_WINDOWS_ERROR(WSAGetLastError()); + if ((pair[1] = accept(listener, NULL, NULL)) == INVALID_SOCKET) + throw QPID_WINDOWS_ERROR(WSAGetLastError()); + + closesocket(listener); + writeFd = pair[0]; + readFd = pair[1]; + } + catch (...) { + closesocket(listener); + if (pair[0] != INVALID_SOCKET) + closesocket(pair[0]); + throw; + } + + // Set the socket to non-blocking + if (nonBlocking) { + unsigned long nonblock = 1; + ioctlsocket(readFd, FIONBIO, &nonblock); + } +} + +PipeHandle::~PipeHandle() { + closesocket(readFd); + closesocket(writeFd); +} + +int PipeHandle::read(void* buf, size_t bufSize) { + return ::recv(readFd, (char *)buf, bufSize, 0); +} + +int PipeHandle::write(const void* buf, size_t bufSize) { + return ::send(writeFd, (const char *)buf, bufSize, 0); +} + +int PipeHandle::getReadHandle() { + return readFd; +} + +}} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/windows/PollableCondition.cpp b/cpp/src/qpid/sys/windows/PollableCondition.cpp new file mode 100644 index 0000000000..5ccc136bd1 --- /dev/null +++ b/cpp/src/qpid/sys/windows/PollableCondition.cpp @@ -0,0 +1,116 @@ +#ifndef QPID_SYS_WINDOWS_POLLABLECONDITION_CPP +#define QPID_SYS_WINDOWS_POLLABLECONDITION_CPP + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/PollableCondition.h" +#include "qpid/sys/IOHandle.h" +#include "qpid/sys/windows/AsynchIoResult.h" +#include "qpid/sys/windows/IoHandlePrivate.h" + +#include <boost/bind.hpp> +#include <windows.h> + +namespace qpid { +namespace sys { + +// PollableConditionPrivate will reuse the IocpPoller's ability to queue +// a completion to the IOCP and have it dispatched to the completer callback +// noted in the IOHandlePrivate when the request is queued. The +// AsynchCallbackRequest object is not really used - we already have the +// desired callback for the user of PollableCondition. +class PollableConditionPrivate : private IOHandle { + friend class PollableCondition; + +private: + PollableConditionPrivate(const sys::PollableCondition::Callback& cb, + sys::PollableCondition& parent, + const boost::shared_ptr<sys::Poller>& poller); + ~PollableConditionPrivate(); + + void poke(); + void dispatch(windows::AsynchIoResult *result); + +private: + PollableCondition::Callback cb; + PollableCondition& parent; + boost::shared_ptr<sys::Poller> poller; + LONG isSet; + LONG armed; +}; + +PollableConditionPrivate::PollableConditionPrivate(const sys::PollableCondition::Callback& cb, + sys::PollableCondition& parent, + const boost::shared_ptr<sys::Poller>& poller) + : IOHandle(new sys::IOHandlePrivate(INVALID_SOCKET, + boost::bind(&PollableConditionPrivate::dispatch, this, _1))), + cb(cb), parent(parent), poller(poller), isSet(0), armed(0) +{ +} + +PollableConditionPrivate::~PollableConditionPrivate() +{ +} + +void PollableConditionPrivate::poke() +{ + if (!armed) + return; + + // monitorHandle will queue a completion for the IOCP; when it's handled, a + // poller thread will call back to dispatch() below. + PollerHandle ph(*this); + poller->monitorHandle(ph, Poller::INPUT); +} + +void PollableConditionPrivate::dispatch(windows::AsynchIoResult *result) +{ + delete result; // Poller::monitorHandle() allocates this + cb(parent); +} + + /* PollableCondition */ + +PollableCondition::PollableCondition(const Callback& cb, + const boost::shared_ptr<sys::Poller>& poller) + : impl(new PollableConditionPrivate(cb, *this, poller)) +{ +} + +PollableCondition::~PollableCondition() +{ + delete impl; +} + +void PollableCondition::set() { + // Add one to the set count and poke it to provoke a callback + ::InterlockedIncrement(&impl->isSet); + impl->poke(); +} + +void PollableCondition::clear() { + ::InterlockedExchange(&impl->isSet, 0); +} + +}} // namespace qpid::sys + +#endif /*!QPID_SYS_WINDOWS_POLLABLECONDITION_CPP*/ diff --git a/cpp/src/qpid/sys/windows/Shlib.cpp b/cpp/src/qpid/sys/windows/Shlib.cpp new file mode 100644 index 0000000000..38027de93f --- /dev/null +++ b/cpp/src/qpid/sys/windows/Shlib.cpp @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Shlib.h" +#include "qpid/Exception.h" +#include "qpid/sys/windows/check.h" +#include <windows.h> + +namespace qpid { +namespace sys { + +void Shlib::load(const char* name) { + HMODULE h = LoadLibrary(name); + if (h == NULL) { + throw QPID_WINDOWS_ERROR(GetLastError()); + } + handle = static_cast<void*>(h); +} + +void Shlib::unload() { + if (handle) { + if (FreeLibrary(static_cast<HMODULE>(handle)) == 0) { + throw QPID_WINDOWS_ERROR(GetLastError()); + } + handle = 0; + } +} + +void* Shlib::getSymbol(const char* name) { + void* sym = GetProcAddress(static_cast<HMODULE>(handle), name); + if (sym == NULL) + throw QPID_WINDOWS_ERROR(GetLastError()); + return sym; +} + +}} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/windows/Socket.cpp b/cpp/src/qpid/sys/windows/Socket.cpp new file mode 100755 index 0000000000..11fb8b4133 --- /dev/null +++ b/cpp/src/qpid/sys/windows/Socket.cpp @@ -0,0 +1,343 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Socket.h" +#include "qpid/sys/SocketAddress.h" +#include "qpid/sys/windows/IoHandlePrivate.h" +#include "qpid/sys/windows/check.h" +#include "qpid/sys/Time.h" + +#include <cstdlib> +#include <string.h> + +#include <winsock2.h> + +#include <boost/format.hpp> +#include <boost/lexical_cast.hpp> + +// Need to initialize WinSock. Ideally, this would be a singleton or embedded +// in some one-time initialization function. I tried boost singleton and could +// not get it to compile (and others located in google had the same problem). +// So, this simple static with an interlocked increment will do for known +// use cases at this time. Since this will only shut down winsock at process +// termination, there may be some problems with client programs that also +// expect to load and unload winsock, but we'll see... +// If someone does get an easy-to-use singleton sometime, converting to it +// may be preferable. + +namespace { + +static LONG volatile initialized = 0; + +class WinSockSetup { + // : public boost::details::pool::singleton_default<WinSockSetup> { + +public: + WinSockSetup() { + LONG timesEntered = InterlockedIncrement(&initialized); + if (timesEntered > 1) + return; + err = 0; + WORD wVersionRequested; + WSADATA wsaData; + + /* Request WinSock 2.2 */ + wVersionRequested = MAKEWORD(2, 2); + err = WSAStartup(wVersionRequested, &wsaData); + } + + ~WinSockSetup() { + WSACleanup(); + } + +public: + int error(void) const { return err; } + +protected: + DWORD err; +}; + +static WinSockSetup setup; + +} /* namespace */ + +namespace qpid { +namespace sys { + +namespace { + +std::string getName(SOCKET fd, bool local, bool includeService = false) +{ + sockaddr_in name; // big enough for any socket address + socklen_t namelen = sizeof(name); + if (local) { + QPID_WINSOCK_CHECK(::getsockname(fd, (sockaddr*)&name, &namelen)); + } else { + QPID_WINSOCK_CHECK(::getpeername(fd, (sockaddr*)&name, &namelen)); + } + + char servName[NI_MAXSERV]; + char dispName[NI_MAXHOST]; + if (includeService) { + if (int rc = ::getnameinfo((sockaddr*)&name, namelen, + dispName, sizeof(dispName), + servName, sizeof(servName), + NI_NUMERICHOST | NI_NUMERICSERV) != 0) + throw qpid::Exception(QPID_MSG(gai_strerror(rc))); + return std::string(dispName) + ":" + std::string(servName); + } else { + if (int rc = ::getnameinfo((sockaddr*)&name, namelen, + dispName, sizeof(dispName), + 0, 0, + NI_NUMERICHOST) != 0) + throw qpid::Exception(QPID_MSG(gai_strerror(rc))); + return dispName; + } +} + +std::string getService(SOCKET fd, bool local) +{ + sockaddr_in name; // big enough for any socket address + socklen_t namelen = sizeof(name); + + if (local) { + QPID_WINSOCK_CHECK(::getsockname(fd, (sockaddr*)&name, &namelen)); + } else { + QPID_WINSOCK_CHECK(::getpeername(fd, (sockaddr*)&name, &namelen)); + } + + char servName[NI_MAXSERV]; + if (int rc = ::getnameinfo((sockaddr*)&name, namelen, + 0, 0, + servName, sizeof(servName), + NI_NUMERICHOST | NI_NUMERICSERV) != 0) + throw qpid::Exception(QPID_MSG(gai_strerror(rc))); + return servName; +} +} // namespace + +Socket::Socket() : + IOHandle(new IOHandlePrivate), + nonblocking(false), + nodelay(false) +{ + SOCKET& socket = impl->fd; + if (socket != INVALID_SOCKET) Socket::close(); + SOCKET s = ::socket (PF_INET, SOCK_STREAM, 0); + if (s == INVALID_SOCKET) throw QPID_WINDOWS_ERROR(WSAGetLastError()); + socket = s; +} + +Socket::Socket(IOHandlePrivate* h) : + IOHandle(h), + nonblocking(false), + nodelay(false) +{} + +void +Socket::createSocket(const SocketAddress& sa) const +{ + SOCKET& socket = impl->fd; + if (socket != INVALID_SOCKET) Socket::close(); + + SOCKET s = ::socket (getAddrInfo(sa).ai_family, + getAddrInfo(sa).ai_socktype, + 0); + if (s == INVALID_SOCKET) throw QPID_WINDOWS_ERROR(WSAGetLastError()); + socket = s; + + try { + if (nonblocking) setNonblocking(); + if (nodelay) setTcpNoDelay(); + } catch (std::exception&) { + closesocket(s); + socket = INVALID_SOCKET; + throw; + } +} + +void Socket::setTimeout(const Duration& interval) const +{ + const SOCKET& socket = impl->fd; + int64_t nanosecs = interval; + nanosecs /= (1000 * 1000); // nsecs -> usec -> msec + int msec = 0; + if (nanosecs > std::numeric_limits<int>::max()) + msec = std::numeric_limits<int>::max(); + else + msec = static_cast<int>(nanosecs); + setsockopt(socket, SOL_SOCKET, SO_SNDTIMEO, (char *)&msec, sizeof(msec)); + setsockopt(socket, SOL_SOCKET, SO_RCVTIMEO, (char *)&msec, sizeof(msec)); +} + +void Socket::setNonblocking() const { + u_long nonblock = 1; + QPID_WINSOCK_CHECK(ioctlsocket(impl->fd, FIONBIO, &nonblock)); +} + +void Socket::connect(const std::string& host, uint16_t port) const +{ + SocketAddress sa(host, boost::lexical_cast<std::string>(port)); + connect(sa); +} + +void +Socket::connect(const SocketAddress& addr) const +{ + const SOCKET& socket = impl->fd; + const addrinfo *addrs = &(getAddrInfo(addr)); + int error = 0; + WSASetLastError(0); + while (addrs != 0) { + if ((::connect(socket, addrs->ai_addr, addrs->ai_addrlen) == 0) || + (WSAGetLastError() == WSAEWOULDBLOCK)) + break; + // Error... save this error code and see if there are other address + // to try before throwing the exception. + error = WSAGetLastError(); + addrs = addrs->ai_next; + } + if (error) + throw qpid::Exception(QPID_MSG(strError(error) << ": " << connectname)); +} + +void +Socket::close() const +{ + SOCKET& socket = impl->fd; + if (socket == INVALID_SOCKET) return; + QPID_WINSOCK_CHECK(closesocket(socket)); + socket = INVALID_SOCKET; +} + + +int Socket::write(const void *buf, size_t count) const +{ + const SOCKET& socket = impl->fd; + int sent = ::send(socket, (const char *)buf, count, 0); + if (sent == SOCKET_ERROR) + return -1; + return sent; +} + +int Socket::read(void *buf, size_t count) const +{ + const SOCKET& socket = impl->fd; + int received = ::recv(socket, (char *)buf, count, 0); + if (received == SOCKET_ERROR) + return -1; + return received; +} + +int Socket::listen(uint16_t port, int backlog) const +{ + const SOCKET& socket = impl->fd; + BOOL yes=1; + QPID_WINSOCK_CHECK(setsockopt(socket, SOL_SOCKET, SO_REUSEADDR, (char *)&yes, sizeof(yes))); + struct sockaddr_in name; + memset(&name, 0, sizeof(name)); + name.sin_family = AF_INET; + name.sin_port = htons(port); + name.sin_addr.s_addr = 0; + if (::bind(socket, (struct sockaddr*)&name, sizeof(name)) == SOCKET_ERROR) + throw Exception(QPID_MSG("Can't bind to port " << port << ": " << strError(WSAGetLastError()))); + if (::listen(socket, backlog) == SOCKET_ERROR) + throw Exception(QPID_MSG("Can't listen on port " << port << ": " << strError(WSAGetLastError()))); + + socklen_t namelen = sizeof(name); + QPID_WINSOCK_CHECK(::getsockname(socket, (struct sockaddr*)&name, &namelen)); + return ntohs(name.sin_port); +} + +Socket* Socket::accept() const +{ + SOCKET afd = ::accept(impl->fd, 0, 0); + if (afd != INVALID_SOCKET) + return new Socket(new IOHandlePrivate(afd)); + else if (WSAGetLastError() == EAGAIN) + return 0; + else throw QPID_WINDOWS_ERROR(WSAGetLastError()); +} + +std::string Socket::getSockname() const +{ + return getName(impl->fd, true); +} + +std::string Socket::getPeername() const +{ + return getName(impl->fd, false); +} + +std::string Socket::getPeerAddress() const +{ + if (!connectname.empty()) + return std::string (connectname); + return getName(impl->fd, false, true); +} + +std::string Socket::getLocalAddress() const +{ + return getName(impl->fd, true, true); +} + +uint16_t Socket::getLocalPort() const +{ + return atoi(getService(impl->fd, true).c_str()); +} + +uint16_t Socket::getRemotePort() const +{ + return atoi(getService(impl->fd, true).c_str()); +} + +int Socket::getError() const +{ + int result; + socklen_t rSize = sizeof (result); + + QPID_WINSOCK_CHECK(::getsockopt(impl->fd, SOL_SOCKET, SO_ERROR, (char *)&result, &rSize)); + return result; +} + +void Socket::setTcpNoDelay() const +{ + int flag = 1; + int result = setsockopt(impl->fd, + IPPROTO_TCP, + TCP_NODELAY, + (char *)&flag, + sizeof(flag)); + QPID_WINSOCK_CHECK(result); + nodelay = true; +} + +inline IOHandlePrivate* IOHandlePrivate::getImpl(const qpid::sys::IOHandle &h) +{ + return h.impl; +} + +SOCKET toSocketHandle(const Socket& s) +{ + return IOHandlePrivate::getImpl(s)->fd; +} + +}} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/windows/SocketAddress.cpp b/cpp/src/qpid/sys/windows/SocketAddress.cpp new file mode 100644 index 0000000000..a3e03c9be8 --- /dev/null +++ b/cpp/src/qpid/sys/windows/SocketAddress.cpp @@ -0,0 +1,70 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/SocketAddress.h" + +#include "qpid/sys/windows/check.h" + +#include <ws2tcpip.h> +#include <string.h> + +namespace qpid { +namespace sys { + +SocketAddress::SocketAddress(const std::string& host0, const std::string& port0) : + host(host0), + port(port0), + addrInfo(0) +{ + ::addrinfo hints; + ::memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_INET; // In order to allow AF_INET6 we'd have to change createTcp() as well + hints.ai_socktype = SOCK_STREAM; + + const char* node = 0; + if (host.empty()) { + hints.ai_flags |= AI_PASSIVE; + } else { + node = host.c_str(); + } + const char* service = port.empty() ? "0" : port.c_str(); + + int n = ::getaddrinfo(node, service, &hints, &addrInfo); + if (n != 0) + throw Exception(QPID_MSG("Cannot resolve " << host << ": " << ::gai_strerror(n))); +} + +SocketAddress::~SocketAddress() +{ + ::freeaddrinfo(addrInfo); +} + +std::string SocketAddress::asString() const +{ + return host + ":" + port; +} + +const ::addrinfo& getAddrInfo(const SocketAddress& sa) +{ + return *sa.addrInfo; +} + +}} diff --git a/cpp/src/qpid/sys/Thread.h b/cpp/src/qpid/sys/windows/StrError.cpp index 3188e45341..9c1bfcd79c 100644..100755 --- a/cpp/src/qpid/sys/Thread.h +++ b/cpp/src/qpid/sys/windows/StrError.cpp @@ -1,6 +1,3 @@ -#ifndef _sys_Thread_h -#define _sys_Thread_h - /* * * Licensed to the Apache Software Foundation (ASF) under one @@ -21,34 +18,30 @@ * under the License. * */ -#include <boost/shared_ptr.hpp> + +#include "qpid/sys/StrError.h" +#include <string> +#include <string.h> +#include <windows.h> namespace qpid { namespace sys { -class Runnable; -class ThreadPrivate; - -class Thread -{ - boost::shared_ptr<ThreadPrivate> impl; - - public: - Thread(); - explicit Thread(qpid::sys::Runnable*); - explicit Thread(qpid::sys::Runnable&); - - void join(); - - unsigned long id(); - - static Thread current(); - - /** ID of current thread for logging. - * Workaround for broken Thread::current() in APR - */ - static unsigned long logId() { return current().id(); } -}; +std::string strError(int err) { + const size_t bufsize = 512; + char buf[bufsize]; + if (0 == FormatMessage (FORMAT_MESSAGE_MAX_WIDTH_MASK + | FORMAT_MESSAGE_FROM_SYSTEM, + 0, + err, + 0, // Default language + buf, + bufsize, + 0)) + { + strerror_s (buf, bufsize, err); + } + return std::string(buf); +} }} -#endif /*!_sys_Thread_h*/ diff --git a/cpp/src/qpid/sys/windows/SystemInfo.cpp b/cpp/src/qpid/sys/windows/SystemInfo.cpp new file mode 100755 index 0000000000..ea53fc199c --- /dev/null +++ b/cpp/src/qpid/sys/windows/SystemInfo.cpp @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +/* GetNativeSystemInfo call requires _WIN32_WINNT 0x0501 or higher */ +#ifndef _WIN32_WINNT +# define _WIN32_WINNT 0x0501 +#endif + +#include "qpid/sys/IntegerTypes.h" +#include "qpid/sys/SystemInfo.h" + +#include <winsock2.h> +#include <ws2tcpip.h> +#include <windows.h> +#include <tlhelp32.h> + +#ifndef HOST_NAME_MAX +# define HOST_NAME_MAX 256 +#endif + +namespace qpid { +namespace sys { + +long SystemInfo::concurrency() { + SYSTEM_INFO sys_info; + ::GetSystemInfo (&sys_info); + long activeProcessors = 0; + DWORD_PTR mask = sys_info.dwActiveProcessorMask; + while (mask != 0) { + if (mask & 1) + ++activeProcessors; + mask >>= 1; + } + return activeProcessors; +} + +bool SystemInfo::getLocalHostname (TcpAddress &address) { + char name[HOST_NAME_MAX]; + if (::gethostname(name, sizeof(name)) != 0) { + errno = WSAGetLastError(); + return false; + } + address.host = name; + return true; +} + +void SystemInfo::getLocalIpAddresses (uint16_t port, + std::vector<Address> &addrList) { + enum { MAX_URL_INTERFACES = 100 }; + static const std::string LOCALHOST("127.0.0.1"); + + SOCKET s = socket (PF_INET, SOCK_STREAM, 0); + if (s != INVALID_SOCKET) { + INTERFACE_INFO interfaces[MAX_URL_INTERFACES]; + DWORD filledBytes = 0; + WSAIoctl (s, + SIO_GET_INTERFACE_LIST, + 0, + 0, + interfaces, + sizeof (interfaces), + &filledBytes, + 0, + 0); + unsigned int interfaceCount = filledBytes / sizeof (INTERFACE_INFO); + for (unsigned int i = 0; i < interfaceCount; ++i) { + if (interfaces[i].iiFlags & IFF_UP) { + std::string addr(inet_ntoa(interfaces[i].iiAddress.AddressIn.sin_addr)); + if (addr != LOCALHOST) + addrList.push_back(TcpAddress(addr, port)); + } + } + closesocket (s); + } +} + +void SystemInfo::getSystemId (std::string &osName, + std::string &nodeName, + std::string &release, + std::string &version, + std::string &machine) +{ + osName = "Microsoft Windows"; + + char node[MAX_COMPUTERNAME_LENGTH + 1]; + DWORD nodelen = MAX_COMPUTERNAME_LENGTH + 1; + GetComputerName (node, &nodelen); + nodeName = node; + + OSVERSIONINFOEX vinfo; + vinfo.dwOSVersionInfoSize = sizeof(vinfo); + GetVersionEx ((OSVERSIONINFO *)&vinfo); + + SYSTEM_INFO sinfo; + GetNativeSystemInfo(&sinfo); + + switch(vinfo.dwMajorVersion) { + case 5: + switch(vinfo.dwMinorVersion) { + case 0: + release ="2000"; + break; + case 1: + release = "XP"; + break; + case 2: + if (sinfo.wProcessorArchitecture == PROCESSOR_ARCHITECTURE_AMD64 || + sinfo.wProcessorArchitecture == PROCESSOR_ARCHITECTURE_IA64) + release = "XP-64"; + else + release = "Server 2003"; + break; + default: + release = "Windows"; + } + break; + case 6: + if (vinfo.wProductType == VER_NT_SERVER) + release = "Server 2008"; + else + release = "Vista"; + break; + default: + release = "Microsoft Windows"; + } + version = vinfo.szCSDVersion; + + switch(sinfo.wProcessorArchitecture) { + case PROCESSOR_ARCHITECTURE_AMD64: + machine = "x86-64"; + break; + case PROCESSOR_ARCHITECTURE_IA64: + machine = "IA64"; + break; + case PROCESSOR_ARCHITECTURE_INTEL: + machine = "x86"; + break; + default: + machine = "unknown"; + break; + } +} + +uint32_t SystemInfo::getProcessId() +{ + return static_cast<uint32_t>(::GetCurrentProcessId()); +} + +uint32_t SystemInfo::getParentProcessId() +{ + // Only want info for the current process, so ask for something specific. + // The module info won't be used here but it keeps the snapshot limited to + // the current process so a search through all processes is not needed. + HANDLE snap = CreateToolhelp32Snapshot(TH32CS_SNAPMODULE, 0); + if (snap == INVALID_HANDLE_VALUE) + return 0; + PROCESSENTRY32 entry; + entry.dwSize = sizeof(entry); + if (!Process32First(snap, &entry)) + entry.th32ParentProcessID = 0; + CloseHandle(snap); + return static_cast<uint32_t>(entry.th32ParentProcessID); +} + +std::string SystemInfo::getProcessName() +{ + std::string name; + + // Only want info for the current process, so ask for something specific. + // The module info won't be used here but it keeps the snapshot limited to + // the current process so a search through all processes is not needed. + HANDLE snap = CreateToolhelp32Snapshot(TH32CS_SNAPMODULE, 0); + if (snap == INVALID_HANDLE_VALUE) + return name; + PROCESSENTRY32 entry; + entry.dwSize = sizeof(entry); + if (!Process32First(snap, &entry)) + entry.szExeFile[0] = '\0'; + CloseHandle(snap); + name = entry.szExeFile; + return name; +} + +}} // namespace qpid::sys diff --git a/cpp/src/qpid/sys/windows/Thread.cpp b/cpp/src/qpid/sys/windows/Thread.cpp new file mode 100755 index 0000000000..fed82e4d54 --- /dev/null +++ b/cpp/src/qpid/sys/windows/Thread.cpp @@ -0,0 +1,88 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Thread.h" +#include "qpid/sys/Runnable.h" +#include "qpid/sys/windows/check.h" + +#include <process.h> +#include <windows.h> + +namespace { +unsigned __stdcall runRunnable(void* p) +{ + static_cast<qpid::sys::Runnable*>(p)->run(); + _endthreadex(0); + return 0; +} +} + +namespace qpid { +namespace sys { + +class ThreadPrivate { + friend class Thread; + + HANDLE threadHandle; + unsigned threadId; + + ThreadPrivate(Runnable* runnable) { + uintptr_t h = _beginthreadex(0, + 0, + runRunnable, + runnable, + 0, + &threadId); + QPID_WINDOWS_CHECK_CRT_NZ(h); + threadHandle = reinterpret_cast<HANDLE>(h); + } + + ThreadPrivate() + : threadHandle(GetCurrentThread()), threadId(GetCurrentThreadId()) {} +}; + +Thread::Thread() {} + +Thread::Thread(Runnable* runnable) : impl(new ThreadPrivate(runnable)) {} + +Thread::Thread(Runnable& runnable) : impl(new ThreadPrivate(&runnable)) {} + +void Thread::join() { + if (impl) { + DWORD status = WaitForSingleObject (impl->threadHandle, INFINITE); + QPID_WINDOWS_CHECK_NOT(status, WAIT_FAILED); + CloseHandle (impl->threadHandle); + impl->threadHandle = 0; + } +} + +unsigned long Thread::id() { + return impl ? impl->threadId : 0; +} + +/* static */ +Thread Thread::current() { + Thread t; + t.impl.reset(new ThreadPrivate()); + return t; +} + +}} /* qpid::sys */ diff --git a/cpp/src/qpid/sys/windows/Time.cpp b/cpp/src/qpid/sys/windows/Time.cpp new file mode 100644 index 0000000000..1d7b94e8d7 --- /dev/null +++ b/cpp/src/qpid/sys/windows/Time.cpp @@ -0,0 +1,103 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Time.h" +#include <ostream> +#include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/thread/thread_time.hpp> +#include <windows.h> + +using namespace boost::posix_time; + +namespace qpid { +namespace sys { + +AbsTime::AbsTime(const AbsTime& t, const Duration& d) { + if (d == Duration::max()) { + timepoint = ptime(max_date_time); + } + else { + time_duration td = microseconds(d.nanosecs / 1000); + timepoint = t.timepoint + td; + } +} + +AbsTime AbsTime::FarFuture() { + AbsTime ff; + ptime maxd(max_date_time); + ff.timepoint = maxd; + return ff; +} + +AbsTime AbsTime::now() { + AbsTime time_now; + time_now.timepoint = boost::get_system_time(); + return time_now; +} + +Duration::Duration(const AbsTime& time0) : nanosecs(0) { + time_period p(ptime(min_date_time), time0.timepoint); + nanosecs = p.length().total_nanoseconds(); +} + +Duration::Duration(const AbsTime& start, const AbsTime& finish) { + time_duration d = finish.timepoint - start.timepoint; + nanosecs = d.total_nanoseconds(); +} + +std::ostream& operator<<(std::ostream& o, const Duration& d) { + return o << int64_t(d) << "ns"; +} + +std::ostream& operator<<(std::ostream& o, const AbsTime& t) { + std::string time_string = to_simple_string(t.timepoint); + return o << time_string; +} + + +void toPtime(ptime& pt, const AbsTime& t) { + pt = t.getPrivate(); +} + +void sleep(int secs) { + ::Sleep(secs * 1000); +} + +void usleep(uint64_t usecs) { + DWORD msecs = usecs / 1000; + if (msecs == 0) + msecs = 1; + ::Sleep(msecs); +} + +void outputFormattedNow(std::ostream& o) { + ::time_t rawtime; + ::tm timeinfo; + char time_string[100]; + + ::time( &rawtime ); + ::localtime_s(&timeinfo, &rawtime); + ::strftime(time_string, 100, + "%Y-%m-%d %H:%M:%S", + &timeinfo); + o << time_string << " "; +} +}} diff --git a/cpp/src/qpid/sys/windows/uuid.cpp b/cpp/src/qpid/sys/windows/uuid.cpp new file mode 100644 index 0000000000..6ff3a3cb8a --- /dev/null +++ b/cpp/src/qpid/sys/windows/uuid.cpp @@ -0,0 +1,59 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include <Rpc.h> +#ifdef uuid_t /* Done in rpcdce.h */ +# undef uuid_t +#endif + +#include "qpid/sys/windows/uuid.h" + +#include <string.h> + +void uuid_clear (uuid_t uu) { + UuidCreateNil (reinterpret_cast<UUID*>(uu)); +} + +void uuid_copy (uuid_t dst, const uuid_t src) { + memcpy (dst, src, qpid::sys::UuidSize); +} + +void uuid_generate (uuid_t out) { + UuidCreate (reinterpret_cast<UUID*>(out)); +} + +int uuid_is_null (const uuid_t uu) { + RPC_STATUS unused; + return UuidIsNil ((UUID*)uu, &unused); +} + +int uuid_parse (const char *in, uuid_t uu) { + return UuidFromString ((unsigned char*)in, (UUID*)uu) == RPC_S_OK ? 0 : -1; +} + +void uuid_unparse (const uuid_t uu, char *out) { + unsigned char *formatted; + if (UuidToString((UUID*)uu, &formatted) == RPC_S_OK) { + strncpy_s (out, 36+1, (char*)formatted, _TRUNCATE); + RpcStringFree(&formatted); + } +} + diff --git a/cpp/src/qpid/sys/IOHandle.h b/cpp/src/qpid/sys/windows/uuid.h index d06512da58..c79abe95c6 100644 --- a/cpp/src/qpid/sys/IOHandle.h +++ b/cpp/src/qpid/sys/windows/uuid.h @@ -1,5 +1,5 @@ -#ifndef _sys_IOHandle_h -#define _sys_IOHandle_h +#ifndef _sys_windows_uuid_h +#define _sys_windows_uuid_h /* * @@ -22,24 +22,17 @@ * */ -namespace qpid { -namespace sys { +#include "qpid/CommonImportExport.h" +#include <qpid/sys/IntegerTypes.h> -/** - * This is a class intended to abstract the Unix concept of file descriptor or the Windows concept of HANDLE - */ -class PollerHandle; -class IOHandlePrivate; -class IOHandle { - friend class PollerHandle; - -protected: - IOHandlePrivate* const impl; - - IOHandle(IOHandlePrivate*); - virtual ~IOHandle(); -}; +namespace qpid { namespace sys { const size_t UuidSize = 16; }} +typedef uint8_t uuid_t[qpid::sys::UuidSize]; -}} +QPID_COMMON_EXTERN void uuid_clear (uuid_t uu); +QPID_COMMON_EXTERN void uuid_copy (uuid_t dst, const uuid_t src); +QPID_COMMON_EXTERN void uuid_generate (uuid_t out); +QPID_COMMON_EXTERN int uuid_is_null (const uuid_t uu); // Returns 1 if null, else 0 +QPID_COMMON_EXTERN int uuid_parse (const char *in, uuid_t uu); // Returns 0 on success, else -1 +QPID_COMMON_EXTERN void uuid_unparse (const uuid_t uu, char *out); -#endif // _sys_IOHandle_h +#endif /*!_sys_windows_uuid_h*/ |