QPID-5750: [linearstore] Broker fails with JERR__AIO: AIO error. (AIO write operation failed: Invalid argument (-22)

git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk@1594215 13f79535-47bb-0310-9956-ffa450edef68

11 files changed, 142 insertions, 25 deletions

diff --git a/qpid/cpp/src/qpid/linearstore/ISSUES b/qpid/cpp/src/qpid/linearstore/ISSUES
index 528ff1edbe..ad7fa4c0d6 100644
--- a/qpid/cpp/src/qpid/linearstore/ISSUES
+++ b/qpid/cpp/src/qpid/linearstore/ISSUES
@@ -45,8 +45,9 @@ Current/pending:
                    * Recovery/reading of message content
                    * Empty file pool status and management
    5464 -        [linearstore] Incompletely created journal files accumulate in EFP
-#  -    1078142  [linearstore] qpidd closes connection with (distributed) transactional client while checking previous transaction, broker signals error (closed by error: Queue Ve0-2: async_dequeue() failed: exception 0x0103 wmgr::get_events() threw JERR__AIO: AIO error)
+#  5750 1078142  [linearstore] qpidd closes connection with (distributed) transactional client while checking previous transaction, broker signals error (closed by error: Queue Ve0-2: async_dequeue() failed: exception 0x0103 wmgr::get_events() threw JERR__AIO: AIO error)
                    * jexception 0x0103 wmgr::get_events() threw JERR__AIO: AIO error. (AIO write operation failed: Invalid argument (-22) [pg=0 size=8192 offset=4096 fh=22])
+   -    1088944  [Linearstore] store does not return all files to EFP after purging big queue
    -    1078937  [linearstore] Installation and tests for new store analysis tool qpid-qls-analyze
 
 Fixed/closed (in commit order):
@@ -115,6 +116,9 @@ NO-JIRA -        Added missing Apache copyright/license text
    5651       -  [C++ broker] segfault in qpid::linearstore::journal::jdir::clear_dir when declaring durable queue
                    svn r.1582730 2014-03-28 Proposed fix by Pavel Moravec
                    * Bug introduced by r.1578899.
+   5661       -  [linearstore] Set default cmake build to exclude linearstore
+                   svn r.1584379 2014-04-03 Proposed solution.
+                   * Run ccmake, select BUILD_LINEARSTORE to change its value to ON to build.
 
 Ordered checkin list:
 =====================
@@ -147,6 +151,7 @@ no.   svn r  Q-JIRA     RHBZ       Date
 23. 1578899    5362        - 2014-03-18
 24. 1582730    5651        - 2014-03-28
 25. 1583778    5362        - 2014-04-01
+26. 1584379    5661        - 2014-04-03
 
 See above sections for details on these checkins.
 
diff --git a/qpid/cpp/src/qpid/linearstore/journal/JournalFile.cpp b/qpid/cpp/src/qpid/linearstore/journal/JournalFile.cpp
index fc6ced4fd2..f4bf88647e 100644
--- a/qpid/cpp/src/qpid/linearstore/journal/JournalFile.cpp
+++ b/qpid/cpp/src/qpid/linearstore/journal/JournalFile.cpp
@@ -170,6 +170,11 @@ void JournalFile::asyncFileHeaderWrite(io_context_t ioContextPtr,
                     queueName_.data());
     const std::size_t wr_size = QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB * 1024;
     aio::prep_pwrite(aioControlBlockPtr_, fileHandle_, (void*)fileHeaderBasePtr_, wr_size, 0UL);
+    if (!aio::is_aligned(aioControlBlockPtr_->u.c.buf, QLS_AIO_ALIGN_BOUNDARY_BYTES)) {
+        std::ostringstream oss;
+        oss << "AIO operation on misaligned buffer: iocb->u.c.buf=" << aioControlBlockPtr_->u.c.buf << std::endl;
+        throw jexception(jerrno::JERR__AIO, oss.str(), "JournalFile", "asyncFileHeaderWrite");
+    }
     if (aio::submit(ioContextPtr, 1, &aioControlBlockPtr_) < 0) {
         std::ostringstream oss;
         oss << "queue=\"" << queueName_ << "\" fid=0x" << std::hex <<  fileSeqNum_ << " wr_size=0x" << wr_size << " foffs=0x0";
@@ -186,6 +191,11 @@ void JournalFile::asyncPageWrite(io_context_t ioContextPtr,
     const std::size_t wr_size = dataSize_dblks * QLS_DBLK_SIZE_BYTES;
     const uint64_t foffs = submittedDblkCount_.get() * QLS_DBLK_SIZE_BYTES;
     aio::prep_pwrite_2(aioControlBlockPtr, fileHandle_, data, wr_size, foffs);
+    if (!aio::is_aligned(aioControlBlockPtr->u.c.buf, QLS_AIO_ALIGN_BOUNDARY_BYTES)) {
+        std::ostringstream oss;
+        oss << "AIO operation on misaligned buffer: iocb->u.c.buf=" << aioControlBlockPtr->u.c.buf << std::endl;
+        throw jexception(jerrno::JERR__AIO, oss.str(), "JournalFile", "asyncPageWrite");
+    }
     pmgr::page_cb* pcbp = (pmgr::page_cb*)(aioControlBlockPtr->data); // This page's control block (pcb)
     pcbp->_wdblks = dataSize_dblks;
     pcbp->_jfp = this;
diff --git a/qpid/cpp/src/qpid/linearstore/journal/RecoveryManager.cpp b/qpid/cpp/src/qpid/linearstore/journal/RecoveryManager.cpp
index 59b3d1ced5..940048b464 100644
--- a/qpid/cpp/src/qpid/linearstore/journal/RecoveryManager.cpp
+++ b/qpid/cpp/src/qpid/linearstore/journal/RecoveryManager.cpp
@@ -593,7 +593,18 @@ bool RecoveryManager::getNextRecordHeader()
 
     bool hdr_ok = false;
     uint64_t file_id = currentJournalFileItr_->second->journalFilePtr_->getFileSeqNum();
-    std::streampos file_pos = inFileStream_.tellg();
+    std::streampos file_pos = 0;
+    if (inFileStream_.is_open()) {
+        inFileStream_.clear();
+        file_pos = inFileStream_.tellg();
+    }
+    if (file_pos == std::streampos(-1)) {
+        std::ostringstream oss;
+        oss << "tellg() failure: fail=" << (inFileStream_.fail()?"T":"F") << " bad=" << (inFileStream_.bad()?"T":"F");
+        oss << " eof=" << (inFileStream_.eof()?"T":"F") << " good=" << (inFileStream_.good()?"T":"F");
+        oss << " rdstate=0x" << std::hex << inFileStream_.rdstate() << std::dec;
+        throw jexception(jerrno::JERR_RCVM_STREAMBAD, oss.str(), "RecoveryManager", "getNextRecordHeader");
+    }
     while (!hdr_ok) {
         if (needNextFile()) {
             if (!getNextFile(true)) {
@@ -606,6 +617,8 @@ bool RecoveryManager::getNextRecordHeader()
         if (file_pos == std::streampos(-1)) {
             std::ostringstream oss;
             oss << "tellg() failure: fail=" << (inFileStream_.fail()?"T":"F") << " bad=" << (inFileStream_.bad()?"T":"F");
+            oss << " eof=" << (inFileStream_.eof()?"T":"F") << " good=" << (inFileStream_.good()?"T":"F");
+            oss << " rdstate=0x" << std::hex << inFileStream_.rdstate() << std::dec;
             throw jexception(jerrno::JERR_RCVM_STREAMBAD, oss.str(), "RecoveryManager", "getNextRecordHeader");
         }
         inFileStream_.read((char*)&h, sizeof(rec_hdr_t));
@@ -627,7 +640,7 @@ bool RecoveryManager::getNextRecordHeader()
             {
 //std::cout << " 0x" << std::hex << file_pos << ".e.0x" << h._rid << std::dec << std::flush; // DEBUG
                 if (::rec_hdr_check(&h, QLS_ENQ_MAGIC, QLS_JRNL_VERSION, currentSerial_) != 0) {
-                    lastRecord(file_id, file_pos);
+                    checkJournalAlignment(file_id, file_pos);
                     return false;
                 }
                 enq_rec er;
@@ -663,7 +676,7 @@ bool RecoveryManager::getNextRecordHeader()
             {
 //std::cout << " 0x" << std::hex << file_pos << ".d.0x" << h._rid << std::dec << std::flush; // DEBUG
                 if (::rec_hdr_check(&h, QLS_DEQ_MAGIC, QLS_JRNL_VERSION, currentSerial_) != 0) {
-                    lastRecord(file_id, file_pos);
+                    checkJournalAlignment(file_id, file_pos);
                     return false;
                 }
                 deq_rec dr;
@@ -697,7 +710,7 @@ bool RecoveryManager::getNextRecordHeader()
             {
 //std::cout << " 0x" << std::hex << file_pos << ".a.0x" << h._rid << std::dec << std::flush; // DEBUG
                 if (::rec_hdr_check(&h, QLS_TXA_MAGIC, QLS_JRNL_VERSION, currentSerial_) != 0) {
-                    lastRecord(file_id, file_pos);
+                    checkJournalAlignment(file_id, file_pos);
                     return false;
                 }
                 txn_rec ar;
@@ -724,7 +737,7 @@ bool RecoveryManager::getNextRecordHeader()
             {
 //std::cout << " 0x" << std::hex << file_pos << ".c.0x" << h._rid << std::dec << std::flush; // DEBUG
                 if (::rec_hdr_check(&h, QLS_TXC_MAGIC, QLS_JRNL_VERSION, currentSerial_) != 0) {
-                    lastRecord(file_id, file_pos);
+                    checkJournalAlignment(file_id, file_pos);
                     return false;
                 }
                 txn_rec cr;
diff --git a/qpid/cpp/src/qpid/linearstore/journal/aio.h b/qpid/cpp/src/qpid/linearstore/journal/aio.h
index 54e3401748..14589e7580 100644
--- a/qpid/cpp/src/qpid/linearstore/journal/aio.h
+++ b/qpid/cpp/src/qpid/linearstore/journal/aio.h
@@ -24,6 +24,7 @@
 
 #include <libaio.h>
 #include <cstring>
+#include <stdint.h>
 
 namespace qpid {
 namespace linearstore {
@@ -39,21 +40,71 @@ typedef io_event aio_event;
 class aio
 {
 public:
+    /*
+     * \brief Initialize an AIO context. Causes kernel resources to be initialized for
+     * AIO operations.
+     *
+     * \param maxevents The maximum number of events to be handled
+     * \param ctxp Pointer to context struct to be initialized
+     */
     static inline int queue_init(int maxevents, io_context_t* ctxp)
     {
         return ::io_queue_init(maxevents, ctxp);
     }
 
+    /*
+     * \brief Release an AIO context. Causes kernel resources previously initialized to
+     * be released.
+     *
+     * \param ctx AIO context struct to be released
+     */
     static inline int queue_release(io_context_t ctx)
     {
         return ::io_queue_release(ctx);
     }
 
+    /*
+     * \brief Submit asynchronous I/O blocks for processing
+     *
+     * The io_submit() system call queues nr I/O request blocks for processing in the AIO context ctx.
+     * The iocbpp argument should be an array of nr AIO control blocks, which will be submitted to context ctx.
+     *
+     * \param ctx AIO context
+     * \param nr Number of AIO operations
+     * \param aios Array of nr pointers to AIO control blocks, one for each AIO operation
+     * \return On success, io_submit() returns the number of iocbs submitted (which may be 0 if nr is zero).
+     *         A negative number indicates an error:
+     *         - -EAGAIN Insufficient resources are available to queue any iocbs.
+     *         - -EBADF  The file descriptor specified in the first iocb is invalid.
+     *         - -EFAULT One of the data structures points to invalid data.
+     *         - -EINVAL The AIO context specified by ctx_id is invalid.  nr is less than 0. The iocb at *iocbpp[0]
+     *                   is not properly initialized, or the operation specified is invalid for the file descriptor
+     *                   in the iocb.
+     */
     static inline int submit(io_context_t ctx, long nr, aio_cb* aios[])
     {
         return ::io_submit(ctx, nr, aios);
     }
 
+    /*
+     * \brief Get list of completed AIO operations
+     *
+     * The io_getevents() system call attempts to read at least min_nr events and up to nr events from the
+     * completion queue of the AIO context specified by ctx_id.  The timeout argument specifies the amount of time
+     * to wait for events, where a NULL timeout waits until at least min_nr events have been seen.  Note that timeout
+     * is relative.
+     *
+     * \param ctx AIO context
+     * \param min_nr Minimum number of events to return, will wait until min_nr events are accumulated or until timeout
+     * \param nr Number of events to return
+     * \param events Pointer to array of aio_event structs, one for each completed event
+     * \param timeout Time to wait for min_nr events; 0 will cause an indefinite wait for min_nr events
+     * \return On success, number of events read: 0 if no events are available, or less than min_nr
+     *         if the timeout has elapsed. A negative number indicates an error:
+     *         - -EFAULT Either events or timeout is an invalid pointer.
+     *         - -EINVAL ctx_id is invalid.  min_nr is out of range or nr is out of range.
+     *         - -EINTR  Interrupted by a signal handler; see signal(7).
+     */
     static inline int getevents(io_context_t ctx, long min_nr, long nr, aio_event* events, timespec* const timeout)
     {
         return ::io_getevents(ctx, min_nr, nr, events, timeout);
@@ -65,7 +116,7 @@ public:
      *
      * \param aiocbp Pointer to the aio_cb struct to be prepared.
      * \param fd File descriptor to be used for read.
-     * \param buf Pointer to buffer in which read data is to be placed.
+     * \param buf Pointer to buffer in which read data is to be placed. MUST BE PAGE_ALIGNED.
      * \param count Number of bytes to read - buffer must be large enough.
      * \param offset Offset within file from which data will be read.
      */
@@ -80,7 +131,7 @@ public:
      *
      * \param aiocbp Pointer to the aio_cb struct to be prepared.
      * \param fd File descriptor to be used for read.
-     * \param buf Pointer to buffer in which read data is to be placed.
+     * \param buf Pointer to buffer in which read data is to be placed. MUST BE PAGE_ALIGNED.
      * \param count Number of bytes to read - buffer must be large enough.
      * \param offset Offset within file from which data will be read.
      */
@@ -101,7 +152,7 @@ public:
      *
      * \param aiocbp Pointer to the aio_cb struct to be prepared.
      * \param fd File descriptor to be used for write.
-     * \param buf Pointer to buffer in which data to be written is located.
+     * \param buf Pointer to buffer in which data to be written is located. MUST BE PAGE_ALIGNED.
      * \param count Number of bytes to write.
      * \param offset Offset within file to which data will be written.
      */
@@ -117,7 +168,7 @@ public:
      *
      * \param aiocbp Pointer to the aio_cb struct to be prepared.
      * \param fd File descriptor to be used for write.
-     * \param buf Pointer to buffer in which data to be written is located.
+     * \param buf Pointer to buffer in which data to be written is located. MUST BE PAGE_ALIGNED.
      * \param count Number of bytes to write.
      * \param offset Offset within file to which data will be written.
      */
@@ -131,6 +182,18 @@ public:
         aiocbp->u.c.nbytes = count;
         aiocbp->u.c.offset = offset;
     }
+
+    /**
+     * \brief Function to check the alignment of memory.
+     *
+     * \param ptr Pointer to be checked
+     * \param byte_count Alignment count (or boundary)
+     * \returns true if ptr is aligned with byte_count, false otherwise
+     */
+    static inline bool is_aligned(const void* ptr, uint64_t byte_count)
+    {
+        return ((uintptr_t)(ptr)) % (byte_count) == 0;
+    }
 };
 
 }}}
diff --git a/qpid/cpp/src/qpid/linearstore/journal/jcntl.cpp b/qpid/cpp/src/qpid/linearstore/journal/jcntl.cpp
index ab367754d5..d0bebf3dc5 100644
--- a/qpid/cpp/src/qpid/linearstore/journal/jcntl.cpp
+++ b/qpid/cpp/src/qpid/linearstore/journal/jcntl.cpp
@@ -92,7 +92,7 @@ jcntl::initialize(EmptyFilePool* efpp,
     _jdir.clear_dir(); // Clear any existing journal files
     _linearFileController.initialize(_jdir.dirname(), efpp, 0ULL);
     _linearFileController.getNextJournalFile();
-    _wmgr.initialize(cbp, wcache_pgsize_sblks, wcache_num_pages, QLS_WMGR_MAXDTOKPP, QLS_WMGR_MAXWAITUS);
+    _wmgr.initialize(cbp, wcache_pgsize_sblks, wcache_num_pages, QLS_WMGR_MAXDTOKPP, QLS_WMGR_MAXWAITUS, 0);
     _init_flag = true;
 }
 
diff --git a/qpid/cpp/src/qpid/linearstore/journal/jerrno.cpp b/qpid/cpp/src/qpid/linearstore/journal/jerrno.cpp
index e176278d87..88fde914f0 100644
--- a/qpid/cpp/src/qpid/linearstore/journal/jerrno.cpp
+++ b/qpid/cpp/src/qpid/linearstore/journal/jerrno.cpp
@@ -85,6 +85,7 @@ const uint32_t jerrno::JERR_WMGR_ENQDISCONT      = 0x0803;
 const uint32_t jerrno::JERR_WMGR_DEQDISCONT      = 0x0804;
 const uint32_t jerrno::JERR_WMGR_DEQRIDNOTENQ    = 0x0805;
 const uint32_t jerrno::JERR_WMGR_BADFH           = 0x0806;
+const uint32_t jerrno::JERR_WMGR_NOTSBLKALIGNED  = 0x0807;
 
 // class RecoveryManager
 const uint32_t jerrno::JERR_RCVM_OPENRD          = 0x0900;
@@ -93,8 +94,7 @@ const uint32_t jerrno::JERR_RCVM_READ            = 0x0902;
 const uint32_t jerrno::JERR_RCVM_WRITE           = 0x0903;
 const uint32_t jerrno::JERR_RCVM_NULLXID         = 0x0904;
 const uint32_t jerrno::JERR_RCVM_NOTDBLKALIGNED  = 0x0905;
-const uint32_t jerrno::JERR_RCVM_NULLFID         = 0x0906;
-
+const uint32_t jerrno::JERR_RCVM_NULLFID         = 0x0907;
 
 // class data_tok
 const uint32_t jerrno::JERR_DTOK_ILLEGALSTATE    = 0x0a00;
@@ -178,6 +178,7 @@ jerrno::__init()
     _err_map[JERR_WMGR_DEQDISCONT] = "JERR_WMGR_DEQDISCONT: Dequeued new dtok when previous dequeue returned partly completed (state DEQ_PART).";
     _err_map[JERR_WMGR_DEQRIDNOTENQ] = "JERR_WMGR_DEQRIDNOTENQ: Dequeue rid is not enqueued.";
     _err_map[JERR_WMGR_BADFH] = "JERR_WMGR_BADFH: Bad file handle.";
+    _err_map[JERR_WMGR_NOTSBLKALIGNED] = "JERR_WMGR_NOTSBLKALIGNED: Offset is not soft block (sblk)-aligned";
 
     // class RecoveryManager
     _err_map[JERR_RCVM_OPENRD] = "JERR_RCVM_OPENRD: Unable to open file for read";
diff --git a/qpid/cpp/src/qpid/linearstore/journal/jerrno.h b/qpid/cpp/src/qpid/linearstore/journal/jerrno.h
index 7c4602b6dd..27b7013ca7 100644
--- a/qpid/cpp/src/qpid/linearstore/journal/jerrno.h
+++ b/qpid/cpp/src/qpid/linearstore/journal/jerrno.h
@@ -103,6 +103,7 @@ namespace journal {
         static const uint32_t JERR_WMGR_DEQDISCONT;     ///< Deq. new dtok when previous part compl.
         static const uint32_t JERR_WMGR_DEQRIDNOTENQ;   ///< Deq. rid not enqueued
         static const uint32_t JERR_WMGR_BADFH;          ///< Bad file handle
+        static const uint32_t JERR_WMGR_NOTSBLKALIGNED; ///< Offset is not soft block (sblk)-aligned
 
         // class RecoveryManager
         static const uint32_t JERR_RCVM_OPENRD;         ///< Unable to open file for read
diff --git a/qpid/cpp/src/qpid/linearstore/journal/pmgr.cpp b/qpid/cpp/src/qpid/linearstore/journal/pmgr.cpp
index 54755dd4ed..764beaa879 100644
--- a/qpid/cpp/src/qpid/linearstore/journal/pmgr.cpp
+++ b/qpid/cpp/src/qpid/linearstore/journal/pmgr.cpp
@@ -30,7 +30,6 @@ pmgr::page_cb::page_cb(uint16_t index):
         _state(UNUSED),
         _frid(0),
         _wdblks(0),
-        _rdblks(0),
         _pdtokl(0),
         _jfp(0),
         _pbuff(0)
diff --git a/qpid/cpp/src/qpid/linearstore/journal/pmgr.h b/qpid/cpp/src/qpid/linearstore/journal/pmgr.h
index 157a6f0566..e618397647 100644
--- a/qpid/cpp/src/qpid/linearstore/journal/pmgr.h
+++ b/qpid/cpp/src/qpid/linearstore/journal/pmgr.h
@@ -66,7 +66,6 @@ public:
         page_state _state;          ///< Status of page
         uint64_t _frid;             ///< First rid in page (used for fhdr init)
         uint32_t _wdblks;           ///< Total number of dblks in page so far
-        uint32_t _rdblks;           ///< Total number of dblks in page
         std::deque<data_tok*>* _pdtokl; ///< Page message tokens list
         JournalFile* _jfp;          ///< Journal file for incrementing compl counts
         void* _pbuff;               ///< Page buffer
diff --git a/qpid/cpp/src/qpid/linearstore/journal/wmgr.cpp b/qpid/cpp/src/qpid/linearstore/journal/wmgr.cpp
index 7c590713f5..7dc8f1d416 100644
--- a/qpid/cpp/src/qpid/linearstore/journal/wmgr.cpp
+++ b/qpid/cpp/src/qpid/linearstore/journal/wmgr.cpp
@@ -79,7 +79,7 @@ wmgr::initialize(aio_callback* const cbp,
                  const uint16_t wcache_num_pages,
                  const uint32_t max_dtokpp,
                  const uint32_t max_iowait_us,
-                 std::size_t eo)
+                 std::size_t end_offset)
 {
     _enq_busy = false;
     _deq_busy = false;
@@ -90,11 +90,16 @@ wmgr::initialize(aio_callback* const cbp,
 
     initialize(cbp, wcache_pgsize_sblks, wcache_num_pages);
 
-    if (eo)
+    if (end_offset)
     {
+        if(!aio::is_aligned((const void*)end_offset, QLS_AIO_ALIGN_BOUNDARY_BYTES)) {
+            std::ostringstream oss;
+            oss << "Recovery using misaligned end_offset (0x" << std::hex << end_offset << std::dec << ")" << std::endl;
+            throw jexception(jerrno::JERR_WMGR_NOTSBLKALIGNED, oss.str(), "wmgr", "initialize");
+        }
         const uint32_t wr_pg_size_dblks = _cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS;
-        uint32_t data_dblks = (eo / QLS_DBLK_SIZE_BYTES) - (QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_DBLKS); // exclude file header
-        _pg_cntr = data_dblks / wr_pg_size_dblks;
+        uint32_t data_dblks = (end_offset / QLS_DBLK_SIZE_BYTES) - (QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_DBLKS); // exclude file header
+        _pg_cntr = data_dblks / wr_pg_size_dblks; // Must be set to get file rotation synchronized (this is determined by value of _pg_cntr)
         _pg_offset_dblks = data_dblks - (_pg_cntr * wr_pg_size_dblks);
     }
 }
@@ -727,7 +732,8 @@ wmgr::get_events(timespec* const timeout,
         if (ret == -EINTR) // Interrupted by signal
             return 0;
         std::ostringstream oss;
-        oss << "io_getevents() failed: " << std::strerror(-ret) << " (" << ret << ")";
+        oss << "io_getevents() failed: " << std::strerror(-ret) << " (" << ret << ") ctx_id=" << _ioctx;
+        oss << " min_nr=" << (flush ? _aio_evt_rem : 1) << " nr=" << _aio_evt_rem;
         throw jexception(jerrno::JERR__AIO, oss.str(), "wmgr", "get_events");
     }
 
@@ -750,16 +756,36 @@ wmgr::get_events(timespec* const timeout,
         long aioret = (long)_aio_event_arr[i].res;
         if (aioret < 0) {
             std::ostringstream oss;
-            oss << "AIO write operation failed: " << std::strerror(-aioret) << " (" << aioret << ") [";
+            oss << "AIO write operation failed: " << std::strerror(-aioret) << " (" << aioret << ")" << std::endl;
+            oss << "  data=" << _aio_event_arr[i].data << std::endl;
+            oss << "  obj=" << _aio_event_arr[i].obj << std::endl;
+            oss << "  res=" << _aio_event_arr[i].res << std::endl;
+            oss << "  res2=" << _aio_event_arr[i].res2 << std::endl;
+            oss << "  iocb->data=" << aiocbp->data << std::endl;
+            oss << "  iocb->key=" << aiocbp->key << std::endl;
+            oss << "  iocb->aio_lio_opcode=" << aiocbp->aio_lio_opcode << std::endl;
+            oss << "  iocb->aio_reqprio=" << aiocbp->aio_reqprio << std::endl;
+            oss << "  iocb->aio_fildes=" << aiocbp->aio_fildes << std::endl;
+            oss << "  iocb->u.c.buf=" << aiocbp->u.c.buf << std::endl;
+            oss << "  iocb->u.c.nbytes=0x" << std::hex <<  aiocbp->u.c.nbytes << std::dec << " (" << aiocbp->u.c.nbytes << ")" << std::endl;
+            oss << "  iocb->u.c.offset=0x" << std::hex << aiocbp->u.c.offset << std::dec << " (" << aiocbp->u.c.offset << ")" << std::endl;
+            oss << "  iocb->u.c.flags=0x" << std::hex << aiocbp->u.c.flags << std::dec << " (" << aiocbp->u.c.flags << ")" << std::endl;
+            oss << "  iocb->u.c.resfd=" << aiocbp->u.c.resfd << std::endl;
             if (pcbp) {
-                oss << "pg=" << pcbp->_index;
+                oss << "  Page Control Block: (iocb->data):" << std::endl;
+                oss << "    pcb.index=" << pcbp->_index << std::endl;
+                oss << "    pcb.state=" << pcbp->_state << " (" << pmgr::page_state_str(pcbp->_state) << ")" << std::endl;
+                oss << "    pcb.frid=0x" << std::hex << pcbp->_frid << std::dec << std::endl;
+                oss << "    pcb.wdblks=0x" << std::hex << pcbp->_wdblks << std::dec << std::endl;
+                oss << "    pcb.pdtokl.size=" << pcbp->_pdtokl->size() << std::endl;
+                oss << "    pcb.pbuff=" << pcbp->_pbuff << std::endl;
+                oss << "    JournalFile (pcb.jfp):" << std::endl;
+                oss << pcbp->_jfp->status_str(6) << std::endl;
             } else {
                 file_hdr_t* fhp = (file_hdr_t*)aiocbp->u.c.buf;
                 oss << "fnum=" << fhp->_file_number;
                 oss << " qname=" << std::string((char*)fhp + sizeof(file_hdr_t), fhp->_queue_name_len);
             }
-            oss << " size=" << aiocbp->u.c.nbytes;
-            oss << " offset=" << aiocbp->u.c.offset << " fh=" << aiocbp->aio_fildes << "]";
             throw jexception(jerrno::JERR__AIO, oss.str(), "wmgr", "get_events");
         }
         if (pcbp) // Page writes have pcb
diff --git a/qpid/cpp/src/qpid/linearstore/journal/wmgr.h b/qpid/cpp/src/qpid/linearstore/journal/wmgr.h
index 8eaa2364ad..99da20bab9 100644
--- a/qpid/cpp/src/qpid/linearstore/journal/wmgr.h
+++ b/qpid/cpp/src/qpid/linearstore/journal/wmgr.h
@@ -96,7 +96,7 @@ public:
                     const uint16_t wcache_num_pages,
                     const uint32_t max_dtokpp,
                     const uint32_t max_iowait_us,
-                    std::size_t eo = 0);
+                    std::size_t end_offset);
     iores enqueue(const void* const data_buff,
                   const std::size_t tot_data_len,
                   const std::size_t this_data_len,