diff options
| author | Kim van der Riet <kpvdr@apache.org> | 2013-10-22 19:09:56 +0000 |
|---|---|---|
| committer | Kim van der Riet <kpvdr@apache.org> | 2013-10-22 19:09:56 +0000 |
| commit | 7a31b8b5ee92000d4aaeca99f1cea8fbc56820d6 (patch) | |
| tree | a0b74b2f98e07d92396f2623b7e1184790f63093 /qpid/cpp | |
| parent | 6105357d06968d24a74c378a857776de460524d9 (diff) | |
| download | qpid-python-7a31b8b5ee92000d4aaeca99f1cea8fbc56820d6.tar.gz | |
QPID-4984: WIP: Copy of work over from branch to trunk. Non-tx works with some known bugs, tx not yet operational.
git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk@1534736 13f79535-47bb-0310-9956-ffa450edef68
Diffstat (limited to 'qpid/cpp')
90 files changed, 15942 insertions, 0 deletions
diff --git a/qpid/cpp/src/CMakeLists.txt b/qpid/cpp/src/CMakeLists.txt index eafa45ea7c..c07fb14a9b 100644 --- a/qpid/cpp/src/CMakeLists.txt +++ b/qpid/cpp/src/CMakeLists.txt @@ -227,6 +227,7 @@ execute_process(COMMAND ${RUBY_EXECUTABLE} -I ${rgen_dir} ${rgen_dir}/generate $ ${CMAKE_CURRENT_SOURCE_DIR}/qpid/acl/management-schema.xml ${CMAKE_CURRENT_SOURCE_DIR}/qpid/ha/management-schema.xml ${CMAKE_CURRENT_SOURCE_DIR}/qpid/legacystore/management-schema.xml + ${CMAKE_CURRENT_SOURCE_DIR}/qpid/linearstore/management-schema.xml ) set(mgen_dir ${qpid-cpp_SOURCE_DIR}/managementgen) set(regen_mgmt OFF) @@ -1509,6 +1510,10 @@ endif (NOT WIN32) # Legacy store # include (legacystore.cmake) +# +# Linear store +# +include (linearstore.cmake) # Now create the config file from all the info learned above. configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake @@ -1516,6 +1521,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake add_subdirectory(qpid/store) add_subdirectory(tests) add_subdirectory(tests/legacystore) +add_subdirectory(tests/linearstore) # Support for pkg-config diff --git a/qpid/cpp/src/linearstore.cmake b/qpid/cpp/src/linearstore.cmake new file mode 100644 index 0000000000..6365905c57 --- /dev/null +++ b/qpid/cpp/src/linearstore.cmake @@ -0,0 +1,174 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +# Legacy store library CMake fragment, to be included in CMakeLists.txt +# + +if (DEFINED linearstore_force) + set (linearstore_default ${linearstore_force}) +else (DEFINED linearstore_force) + set (linearstore_default OFF) + if (UNIX) + # + # Find required BerkelyDB + # + include (finddb.cmake) + if (DB_FOUND) + # + # find libaio + # + CHECK_LIBRARY_EXISTS (aio io_queue_init "" HAVE_AIO) + CHECK_INCLUDE_FILES (libaio.h HAVE_AIO_H) + if (HAVE_AIO AND HAVE_AIO_H) + # + # find libuuid + # + CHECK_LIBRARY_EXISTS (uuid uuid_compare "" HAVE_UUID) + CHECK_INCLUDE_FILES(uuid/uuid.h HAVE_UUID_H) + IF (HAVE_UUID AND HAVE_UUID_H) + # + # allow linearstore to be built + # + set (linearstore_default ON) + ENDIF (HAVE_UUID AND HAVE_UUID_H) + endif (HAVE_AIO AND HAVE_AIO_H) + endif (DB_FOUND) + endif (UNIX) +endif (DEFINED linearstore_force) + +option(BUILD_LINEARSTORE "Build linearstore persistent store" ${linearstore_default}) + +if (BUILD_LINEARSTORE) + if (NOT UNIX) + message(FATAL_ERROR "Linearstore produced only on Unix platforms") + endif (NOT UNIX) + if (NOT DB_FOUND) + message(FATAL_ERROR "Linearstore requires BerkeleyDB which is absent.") + endif (NOT DB_FOUND) + if (NOT HAVE_AIO) + message(FATAL_ERROR "Linearstore requires libaio which is absent.") + endif (NOT HAVE_AIO) + if (NOT HAVE_AIO_H) + message(FATAL_ERROR "Linearstore requires libaio.h which is absent.") + endif (NOT HAVE_AIO_H) + if (NOT HAVE_UUID) + message(FATAL_ERROR "Linearstore requires uuid which is absent.") + endif (NOT HAVE_UUID) + if (NOT HAVE_UUID_H) + message(FATAL_ERROR "Linearstore requires uuid.h which is absent.") + endif (NOT HAVE_UUID_H) + + # Journal source files + set (linear_jrnl_SOURCES + qpid/linearstore/jrnl/data_tok.cpp + qpid/linearstore/jrnl/deq_rec.cpp + qpid/linearstore/jrnl/EmptyFilePool.cpp + qpid/linearstore/jrnl/EmptyFilePoolManager.cpp + qpid/linearstore/jrnl/EmptyFilePoolPartition.cpp + qpid/linearstore/jrnl/enq_map.cpp + qpid/linearstore/jrnl/enq_rec.cpp + qpid/linearstore/jrnl/jcntl.cpp + qpid/linearstore/jrnl/jdir.cpp + qpid/linearstore/jrnl/jerrno.cpp + qpid/linearstore/jrnl/jexception.cpp + qpid/linearstore/jrnl/JournalFile.cpp + qpid/linearstore/jrnl/JournalLog.cpp + qpid/linearstore/jrnl/jrec.cpp + qpid/linearstore/jrnl/LinearFileController.cpp + qpid/linearstore/jrnl/pmgr.cpp + qpid/linearstore/jrnl/RecoveryManager.cpp + qpid/linearstore/jrnl/time_ns.cpp + qpid/linearstore/jrnl/txn_map.cpp + qpid/linearstore/jrnl/txn_rec.cpp + qpid/linearstore/jrnl/wmgr.cpp + ) + + # linearstore source files + set (linear_store_SOURCES + qpid/linearstore/StorePlugin.cpp + qpid/linearstore/BindingDbt.cpp + qpid/linearstore/BufferValue.cpp + qpid/linearstore/DataTokenImpl.cpp + qpid/linearstore/IdDbt.cpp + qpid/linearstore/IdSequence.cpp + qpid/linearstore/JournalImpl.cpp + qpid/linearstore/MessageStoreImpl.cpp + qpid/linearstore/PreparedTransaction.cpp + qpid/linearstore/JournalLogImpl.cpp + qpid/linearstore/TxnCtxt.cpp + ) + + set (util_SOURCES + qpid/linearstore/jrnl/utils/deq_hdr.c + qpid/linearstore/jrnl/utils/enq_hdr.c + qpid/linearstore/jrnl/utils/file_hdr.c + qpid/linearstore/jrnl/utils/rec_hdr.c + qpid/linearstore/jrnl/utils/rec_tail.c + qpid/linearstore/jrnl/utils/txn_hdr.c + ) + + # linearstore include directories + get_property(dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES) + set (linear_include_DIRECTORIES + ${dirs} + ${CMAKE_CURRENT_SOURCE_DIR}/qpid/linearstore + ) + + if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/db-inc.h) + message(STATUS "Including BDB from ${DB_INCLUDE_DIR}/db_cxx.h") + file(WRITE + ${CMAKE_CURRENT_BINARY_DIR}/db-inc.h + "#include <${DB_INCLUDE_DIR}/db_cxx.h>\n") + endif() + + add_library (linearstoreutils SHARED + ${util_SOURCES} + ) + + target_link_libraries (linearstoreutils + rt + ) + + add_library (linearstore MODULE + ${linear_jrnl_SOURCES} + ${linear_store_SOURCES} + ${linear_qmf_SOURCES} + ) + + set_target_properties (linearstore PROPERTIES + PREFIX "" + COMPILE_DEFINITIONS _IN_QPID_BROKER + OUTPUT_NAME linearstore + INCLUDE_DIRECTORIES "${linear_include_DIRECTORIES}" + ) + + target_link_libraries (linearstore + aio + uuid + qpidcommon qpidtypes qpidbroker linearstoreutils + ${DB_LIBRARY} + ) + +install(TARGETS linearstore + DESTINATION ${QPIDD_MODULE_DIR} + COMPONENT ${QPID_COMPONENT_BROKER}) + +else (BUILD_LINEARSTORE) + message(STATUS "Linearstore is excluded from build.") +endif (BUILD_LINEARSTORE) diff --git a/qpid/cpp/src/qpid/linearstore/BindingDbt.cpp b/qpid/cpp/src/qpid/linearstore/BindingDbt.cpp new file mode 100644 index 0000000000..47738cce39 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/BindingDbt.cpp @@ -0,0 +1,50 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/BindingDbt.h" + +namespace qpid { +namespace linearstore { + +BindingDbt::BindingDbt(const qpid::broker::PersistableExchange& e, const qpid::broker::PersistableQueue& q, const std::string& k, const qpid::framing::FieldTable& a) + : data(new char[encodedSize(e, q, k, a)]), + buffer(data, encodedSize(e, q, k, a)) +{ + buffer.putLongLong(q.getPersistenceId()); + buffer.putShortString(q.getName()); + buffer.putShortString(k); + buffer.put(a); + + set_data(data); + set_size(encodedSize(e, q, k, a)); +} + +BindingDbt::~BindingDbt() +{ + delete [] data; +} + +uint32_t BindingDbt::encodedSize(const qpid::broker::PersistableExchange& /*not used*/, const qpid::broker::PersistableQueue& q, const std::string& k, const qpid::framing::FieldTable& a) +{ + return 8 /*queue id*/ + q.getName().size() + 1 + k.size() + 1 + a.encodedSize(); +} + +}} diff --git a/qpid/cpp/src/qpid/linearstore/BindingDbt.h b/qpid/cpp/src/qpid/linearstore/BindingDbt.h new file mode 100644 index 0000000000..7297ea7c7c --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/BindingDbt.h @@ -0,0 +1,56 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_BINDINGDBT_H +#define QPID_LEGACYSTORE_BINDINGDBT_H + +#include "db-inc.h" +#include "qpid/broker/PersistableExchange.h" +#include "qpid/broker/PersistableQueue.h" +#include "qpid/framing/Buffer.h" +#include "qpid/framing/FieldTable.h" + +namespace qpid{ +namespace linearstore{ + +class BindingDbt : public Dbt +{ + char* data; + qpid::framing::Buffer buffer; + + static uint32_t encodedSize(const qpid::broker::PersistableExchange& e, + const qpid::broker::PersistableQueue& q, + const std::string& k, + const qpid::framing::FieldTable& a); + +public: + BindingDbt(const qpid::broker::PersistableExchange& e, + const qpid::broker::PersistableQueue& q, + const std::string& k, + const qpid::framing::FieldTable& a); + + virtual ~BindingDbt(); + +}; + +}} + +#endif // ifndef QPID_LEGACYSTORE_BINDINGDBT_H diff --git a/qpid/cpp/src/qpid/linearstore/BufferValue.cpp b/qpid/cpp/src/qpid/linearstore/BufferValue.cpp new file mode 100644 index 0000000000..5115055375 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/BufferValue.cpp @@ -0,0 +1,56 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/BufferValue.h" + +namespace qpid { +namespace linearstore { + + + +BufferValue::BufferValue(uint32_t size, uint64_t offset) + : data(new char[size]), + buffer(data, size) +{ + set_data(data); + set_size(size); + set_flags(DB_DBT_USERMEM | DB_DBT_PARTIAL); + set_doff(offset); + set_dlen(size); + set_ulen(size); +} + +BufferValue::BufferValue(const qpid::broker::Persistable& p) + : data(new char[p.encodedSize()]), + buffer(data, p.encodedSize()) +{ + p.encode(buffer); + + set_data(data); + set_size(p.encodedSize()); +} + +BufferValue::~BufferValue() +{ + delete [] data; +} + +}} diff --git a/qpid/cpp/src/qpid/linearstore/BufferValue.h b/qpid/cpp/src/qpid/linearstore/BufferValue.h new file mode 100644 index 0000000000..5af1ac6a43 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/BufferValue.h @@ -0,0 +1,46 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_BUFFERVALUE_H +#define QPID_LEGACYSTORE_BUFFERVALUE_H + +#include "db-inc.h" +#include "qpid/broker/Persistable.h" +#include "qpid/framing/Buffer.h" + +namespace qpid{ +namespace linearstore{ + +class BufferValue : public Dbt +{ + char* data; + +public: + qpid::framing::Buffer buffer; + + BufferValue(uint32_t size, uint64_t offset); + BufferValue(const qpid::broker::Persistable& p); + virtual ~BufferValue(); +}; + +}} + +#endif // ifndef QPID_LEGACYSTORE_BUFFERVALUE_H diff --git a/qpid/cpp/src/qpid/linearstore/Cursor.h b/qpid/cpp/src/qpid/linearstore/Cursor.h new file mode 100644 index 0000000000..c7e7f34faa --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/Cursor.h @@ -0,0 +1,50 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_CURSOR_H +#define QPID_LEGACYSTORE_CURSOR_H + +#include <boost/shared_ptr.hpp> +#include "db-inc.h" + +namespace qpid{ +namespace linearstore{ + +class Cursor +{ + Dbc* cursor; +public: + typedef boost::shared_ptr<Db> db_ptr; + + Cursor() : cursor(0) {} + virtual ~Cursor() { if(cursor) cursor->close(); } + + void open(db_ptr db, DbTxn* txn, uint32_t flags = 0) { db->cursor(txn, &cursor, flags); } + void close() { if(cursor) cursor->close(); cursor = 0; } + Dbc* get() { return cursor; } + Dbc* operator->() { return cursor; } + bool next(Dbt& key, Dbt& value) { return cursor->get(&key, &value, DB_NEXT) == 0; } + bool current(Dbt& key, Dbt& value) { return cursor->get(&key, &value, DB_CURRENT) == 0; } +}; + +}} + +#endif // ifndef QPID_LEGACYSTORE_CURSOR_H diff --git a/qpid/cpp/src/qpid/linearstore/DataTokenImpl.cpp b/qpid/cpp/src/qpid/linearstore/DataTokenImpl.cpp new file mode 100644 index 0000000000..0b1f3d7941 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/DataTokenImpl.cpp @@ -0,0 +1,28 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/DataTokenImpl.h" + +using namespace qpid::linearstore; + +DataTokenImpl::DataTokenImpl():data_tok() {} + +DataTokenImpl::~DataTokenImpl() {} diff --git a/qpid/cpp/src/qpid/linearstore/DataTokenImpl.h b/qpid/cpp/src/qpid/linearstore/DataTokenImpl.h new file mode 100644 index 0000000000..17de01d237 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/DataTokenImpl.h @@ -0,0 +1,47 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_DATATOKENIMPL_H +#define QPID_LEGACYSTORE_DATATOKENIMPL_H + +#include "qpid/linearstore/jrnl/data_tok.h" +#include "qpid/broker/PersistableMessage.h" +#include <boost/intrusive_ptr.hpp> + +namespace qpid{ +namespace linearstore{ + +class DataTokenImpl : public qpid::qls_jrnl::data_tok, public qpid::RefCounted +{ + private: + boost::intrusive_ptr<qpid::broker::PersistableMessage> sourceMsg; + public: + DataTokenImpl(); + virtual ~DataTokenImpl(); + + inline boost::intrusive_ptr<qpid::broker::PersistableMessage>& getSourceMessage() { return sourceMsg; } + inline void setSourceMessage(const boost::intrusive_ptr<qpid::broker::PersistableMessage>& msg) { sourceMsg = msg; } +}; + +} // namespace msgstore +} // namespace mrg + +#endif // ifndef QPID_LEGACYSTORE_DATATOKENIMPL_H diff --git a/qpid/cpp/src/qpid/linearstore/IdDbt.cpp b/qpid/cpp/src/qpid/linearstore/IdDbt.cpp new file mode 100644 index 0000000000..d427085bbe --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/IdDbt.cpp @@ -0,0 +1,42 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/IdDbt.h" + +using namespace qpid::linearstore; + +IdDbt::IdDbt() : id(0) +{ + init(); +} + +IdDbt::IdDbt(uint64_t _id) : id(_id) +{ + init(); +} + +void IdDbt::init() +{ + set_data(&id); + set_size(sizeof(uint64_t)); + set_ulen(sizeof(uint64_t)); + set_flags(DB_DBT_USERMEM); +} diff --git a/qpid/cpp/src/qpid/linearstore/IdDbt.h b/qpid/cpp/src/qpid/linearstore/IdDbt.h new file mode 100644 index 0000000000..f8bb0647db --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/IdDbt.h @@ -0,0 +1,42 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_IDDBT_H +#define QPID_LEGACYSTORE_IDDBT_H + +#include "db-inc.h" + +namespace qpid{ +namespace linearstore{ + +class IdDbt : public Dbt +{ + void init(); +public: + uint64_t id; + + IdDbt(uint64_t id); + IdDbt(); +}; + +}} + +#endif // ifndef QPID_LEGACYSTORE_IDDBT_H diff --git a/qpid/cpp/src/qpid/linearstore/IdSequence.cpp b/qpid/cpp/src/qpid/linearstore/IdSequence.cpp new file mode 100644 index 0000000000..4d3172ffe9 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/IdSequence.cpp @@ -0,0 +1,40 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/IdSequence.h" + +using namespace qpid::linearstore; +using qpid::sys::Mutex; + +IdSequence::IdSequence() : id(1) {} + +uint64_t IdSequence::next() +{ + Mutex::ScopedLock guard(lock); + if (!id) id++; // avoid 0 when folding around + return id++; +} + +void IdSequence::reset(uint64_t value) +{ + //deliberately not threadsafe, used only on recovery + id = value; +} diff --git a/qpid/cpp/src/qpid/linearstore/IdSequence.h b/qpid/cpp/src/qpid/linearstore/IdSequence.h new file mode 100644 index 0000000000..11b31a2813 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/IdSequence.h @@ -0,0 +1,43 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_IDSEQUENCE_H +#define QPID_LEGACYSTORE_IDSEQUENCE_H + +#include "qpid/framing/amqp_types.h" +#include "qpid/sys/Mutex.h" + +namespace qpid{ +namespace linearstore{ + +class IdSequence +{ + qpid::sys::Mutex lock; + uint64_t id; +public: + IdSequence(); + uint64_t next(); + void reset(uint64_t value); +}; + +}} + +#endif // ifndef QPID_LEGACYSTORE_IDSEQUENCE_H diff --git a/qpid/cpp/src/qpid/linearstore/JournalImpl.cpp b/qpid/cpp/src/qpid/linearstore/JournalImpl.cpp new file mode 100644 index 0000000000..412a4922ca --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/JournalImpl.cpp @@ -0,0 +1,653 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/JournalImpl.h" + +#include "qpid/linearstore/JournalLogImpl.h" +#include "qpid/linearstore/jrnl/jerrno.h" +#include "qpid/linearstore/jrnl/jexception.h" +#include "qpid/linearstore/jrnl/EmptyFilePool.h" +#include "qpid/linearstore/StoreException.h" +#include "qpid/log/Statement.h" +#include "qpid/management/ManagementAgent.h" +#include "qpid/sys/Monitor.h" +#include "qpid/sys/Timer.h" + +#include "qmf/org/apache/qpid/linearstore/EventCreated.h" +#include "qmf/org/apache/qpid/linearstore/EventEnqThresholdExceeded.h" +#include "qmf/org/apache/qpid/linearstore/EventFull.h" +#include "qmf/org/apache/qpid/linearstore/EventRecovered.h" + +using namespace qpid::qls_jrnl; +using namespace qpid::linearstore; +using qpid::management::ManagementAgent; +namespace _qmf = qmf::org::apache::qpid::linearstore; + +InactivityFireEvent::InactivityFireEvent(JournalImpl* p, const qpid::sys::Duration timeout): + qpid::sys::TimerTask(timeout, "JournalInactive:"+p->id()), _parent(p) {} + +void InactivityFireEvent::fire() { qpid::sys::Mutex::ScopedLock sl(_ife_lock); if (_parent) _parent->flushFire(); } + +GetEventsFireEvent::GetEventsFireEvent(JournalImpl* p, const qpid::sys::Duration timeout): + qpid::sys::TimerTask(timeout, "JournalGetEvents:"+p->id()), _parent(p) {} + +void GetEventsFireEvent::fire() { qpid::sys::Mutex::ScopedLock sl(_gefe_lock); if (_parent) _parent->getEventsFire(); } + +JournalImpl::JournalImpl(qpid::sys::Timer& timer_, + const std::string& journalId, + const std::string& journalDirectory, + JournalLogImpl& journalLogRef, + const qpid::sys::Duration getEventsTimeout, + const qpid::sys::Duration flushTimeout, + qpid::management::ManagementAgent* a, + DeleteCallback onDelete): + jcntl(journalId, journalDirectory, journalLogRef), + timer(timer_), + _journalLogRef(journalLogRef), + getEventsTimerSetFlag(false), +// lastReadRid(0), + writeActivityFlag(false), + flushTriggeredFlag(true), +// _xidp(0), +// _datap(0), +// _dlen(0), +// _dtok(), +// _external(false), + deleteCallback(onDelete) +{ + getEventsFireEventsPtr = new GetEventsFireEvent(this, getEventsTimeout); + inactivityFireEventPtr = new InactivityFireEvent(this, flushTimeout); + { + timer.start(); + timer.add(inactivityFireEventPtr); + } + + initManagement(a); + + QLS_LOG2(notice, _jid, "Created"); + std::ostringstream oss; + oss << "Journal directory = \"" << journalDirectory << "\""; + QLS_LOG2(debug, _jid, oss.str()); +} + +JournalImpl::~JournalImpl() +{ + if (deleteCallback) deleteCallback(*this); + if (_init_flag && !_stop_flag){ + try { stop(true); } // NOTE: This will *block* until all outstanding disk aio calls are complete! + catch (const jexception& e) { QLS_LOG2(error, _jid, e.what()); } + } + getEventsFireEventsPtr->cancel(); + inactivityFireEventPtr->cancel(); +// free_read_buffers(); + + if (_mgmtObject.get() != 0) { + _mgmtObject->resourceDestroy(); + _mgmtObject.reset(); + } + + QLS_LOG2(notice, _jid, "Destroyed"); +} + +void +JournalImpl::initManagement(qpid::management::ManagementAgent* a) +{ + _agent = a; + if (_agent != 0) + { + _mgmtObject = _qmf::Journal::shared_ptr ( + new _qmf::Journal(_agent, this)); + + _mgmtObject->set_name(_jid); + _mgmtObject->set_directory(_jdir.dirname()); +// _mgmtObject->set_baseFileName(_base_filename); +// _mgmtObject->set_readPageSize(JRNL_RMGR_PAGE_SIZE * JRNL_SBLK_SIZE); +// _mgmtObject->set_readPages(JRNL_RMGR_PAGES); + + // The following will be set on initialize(), but being properties, these must be set to 0 in the meantime + //_mgmtObject->set_initialFileCount(0); + //_mgmtObject->set_dataFileSize(0); + //_mgmtObject->set_currentFileCount(0); + _mgmtObject->set_writePageSize(0); + _mgmtObject->set_writePages(0); + + _agent->addObject(_mgmtObject, 0, true); + } +} + + +void +JournalImpl::initialize(qpid::qls_jrnl::EmptyFilePool* efpp_, + const uint16_t wcache_num_pages, + const uint32_t wcache_pgsize_sblks, + qpid::qls_jrnl::aio_callback* const cbp) +{ +// efpp->createJournal(_jdir); +// QLS_LOG2(notice, _jid, "Initialized"); +// std::ostringstream oss; +//// oss << "Initialize; num_jfiles=" << num_jfiles << " jfsize_sblks=" << jfsize_sblks; +// oss << "Initialize; efpPartitionNumber=" << efpp_->getPartitionNumber(); +// oss << " efpFileSizeKb=" << efpp_->fileSizeKib(); +// oss << " wcache_pgsize_sblks=" << wcache_pgsize_sblks; +// oss << " wcache_num_pages=" << wcache_num_pages; +// QLS_LOG2(debug, _jid, oss.str()); + jcntl::initialize(/*num_jfiles, auto_expand, ae_max_jfiles, jfsize_sblks,*/ efpp_, wcache_num_pages, wcache_pgsize_sblks, cbp); +// QLS_LOG2(debug, _jid, "Initialization complete"); + // TODO: replace for linearstore: _lpmgr +/* + if (_mgmtObject.get() != 0) + { + _mgmtObject->set_initialFileCount(_lpmgr.num_jfiles()); + _mgmtObject->set_autoExpand(_lpmgr.is_ae()); + _mgmtObject->set_currentFileCount(_lpmgr.num_jfiles()); + _mgmtObject->set_maxFileCount(_lpmgr.ae_max_jfiles()); + _mgmtObject->set_dataFileSize(_jfsize_sblks * JRNL_SBLK_SIZE); + _mgmtObject->set_writePageSize(wcache_pgsize_sblks * JRNL_SBLK_SIZE); + _mgmtObject->set_writePages(wcache_num_pages); + } + if (_agent != 0) + _agent->raiseEvent::(qmf::org::apache::qpid::linearstore::EventCreated(_jid, _jfsize_sblks * JRNL_SBLK_SIZE, _lpmgr.num_jfiles()), + qpid::management::ManagementAgent::SEV_NOTE); +*/ +} + +void +JournalImpl::recover(/*const uint16_t num_jfiles, + const bool auto_expand, + const uint16_t ae_max_jfiles, + const uint32_t jfsize_sblks,*/ + boost::shared_ptr<qpid::qls_jrnl::EmptyFilePoolManager> efpm, + const uint16_t wcache_num_pages, + const uint32_t wcache_pgsize_sblks, + qpid::qls_jrnl::aio_callback* const cbp, + boost::ptr_list<PreparedTransaction>* prep_tx_list_ptr, + uint64_t& highest_rid, + uint64_t queue_id) +{ + std::ostringstream oss1; +// oss1 << "Recover; num_jfiles=" << num_jfiles << " jfsize_sblks=" << jfsize_sblks; + oss1 << "Recover;"; + oss1 << " queue_id = 0x" << std::hex << queue_id << std::dec; + oss1 << " wcache_pgsize_sblks=" << wcache_pgsize_sblks; + oss1 << " wcache_num_pages=" << wcache_num_pages; + QLS_LOG2(debug, _jid, oss1.str()); + // TODO: replace for linearstore: _lpmgr +/* + if (_mgmtObject.get() != 0) + { + _mgmtObject->set_initialFileCount(_lpmgr.num_jfiles()); + _mgmtObject->set_autoExpand(_lpmgr.is_ae()); + _mgmtObject->set_currentFileCount(_lpmgr.num_jfiles()); + _mgmtObject->set_maxFileCount(_lpmgr.ae_max_jfiles()); + _mgmtObject->set_dataFileSize(_jfsize_sblks * JRNL_SBLK_SIZE); + _mgmtObject->set_writePageSize(wcache_pgsize_sblks * JRNL_SBLK_SIZE); + _mgmtObject->set_writePages(wcache_num_pages); + } +*/ + + if (prep_tx_list_ptr) { + // Create list of prepared xids + std::vector<std::string> prep_xid_list; + for (PreparedTransaction::list::iterator i = prep_tx_list_ptr->begin(); i != prep_tx_list_ptr->end(); i++) { + prep_xid_list.push_back(i->xid); + } + + jcntl::recover(/*num_jfiles, auto_expand, ae_max_jfiles, jfsize_sblks,*/efpm.get(), wcache_num_pages, wcache_pgsize_sblks, + cbp, &prep_xid_list, highest_rid); + } else { + jcntl::recover(/*num_jfiles, auto_expand, ae_max_jfiles, jfsize_sblks,*/efpm.get(), wcache_num_pages, wcache_pgsize_sblks, + cbp, 0, highest_rid); + } + + // Populate PreparedTransaction lists from _tmap + if (prep_tx_list_ptr) + { + for (PreparedTransaction::list::iterator i = prep_tx_list_ptr->begin(); i != prep_tx_list_ptr->end(); i++) { + txn_data_list tdl = _tmap.get_tdata_list(i->xid); // tdl will be empty if xid not found + for (tdl_itr tdl_itr = tdl.begin(); tdl_itr < tdl.end(); tdl_itr++) { + if (tdl_itr->_enq_flag) { // enqueue op + i->enqueues->add(queue_id, tdl_itr->_rid); + } else { // dequeue op + i->dequeues->add(queue_id, tdl_itr->_drid); + } + } + } + } + std::ostringstream oss2; + oss2 << "Recover phase 1 complete; highest rid found = 0x" << std::hex << highest_rid; + oss2 << std::dec << "; emap.size=" << _emap.size() << "; tmap.size=" << _tmap.size(); + oss2 << "; journal now read-only."; + QLS_LOG2(debug, _jid, oss2.str()); + + if (_mgmtObject.get() != 0) + { + _mgmtObject->inc_recordDepth(_emap.size()); + _mgmtObject->inc_enqueues(_emap.size()); + _mgmtObject->inc_txn(_tmap.size()); + _mgmtObject->inc_txnEnqueues(_tmap.enq_cnt()); + _mgmtObject->inc_txnDequeues(_tmap.deq_cnt()); + } +} + +void +JournalImpl::recover_complete() +{ + jcntl::recover_complete(); + QLS_LOG2(debug, _jid, "Recover phase 2 complete; journal now writable."); + // TODO: replace for linearstore: _lpmgr +/* + if (_agent != 0) + _agent->raiseEvent(qmf::org::apache::qpid::linearstore::EventRecovered(_jid, _jfsize_sblks * JRNL_SBLK_SIZE, _lpmgr.num_jfiles(), + _emap.size(), _tmap.size(), _tmap.enq_cnt(), _tmap.deq_cnt()), qpid::management::ManagementAgent::SEV_NOTE); +*/ +} + +//#define MAX_AIO_SLEEPS 1000000 // tot: ~10 sec +//#define AIO_SLEEP_TIME_US 10 // 0.01 ms +// Return true if content is recovered from store; false if content is external and must be recovered from an external store. +// Throw exception for all errors. +/* +bool +JournalImpl::loadMsgContent(uint64_t rid, std::string& data, size_t length, size_t offset) +{ + qpid::sys::Mutex::ScopedLock sl(_read_lock); + if (_dtok.rid() != rid) + { + // Free any previous msg + free_read_buffers(); + + // Last read encountered out-of-order rids, check if this rid is in that list + bool oooFlag = false; + for (std::vector<uint64_t>::const_iterator i=oooRidList.begin(); i!=oooRidList.end() && !oooFlag; i++) { + if (*i == rid) { + oooFlag = true; + } + } + + // TODO: This is a brutal approach - very inefficient and slow. Rather introduce a system of remembering + // jumpover points and allow the read to jump back to the first known jumpover point - but this needs + // a mechanism in rrfc to accomplish it. Also helpful is a struct containing a journal address - a + // combination of lid/offset. + // NOTE: The second part of the if stmt (rid < lastReadRid) is required to handle browsing. + if (oooFlag || rid < lastReadRid) { + _rmgr.invalidate(); + oooRidList.clear(); + } + _dlen = 0; + _dtok.reset(); + _dtok.set_wstate(DataTokenImpl::ENQ); + _dtok.set_rid(0); + _external = false; + size_t xlen = 0; + bool transient = false; + bool done = false; + bool rid_found = false; + while (!done) { + iores res = read_data_record(&_datap, _dlen, &_xidp, xlen, transient, _external, &_dtok); + switch (res) { + case qpid::qls_jrnl::RHM_IORES_SUCCESS: + if (_dtok.rid() != rid) { + // Check if this is an out-of-order rid that may impact next read + if (_dtok.rid() > rid) + oooRidList.push_back(_dtok.rid()); + free_read_buffers(); + // Reset data token for next read + _dlen = 0; + _dtok.reset(); + _dtok.set_wstate(DataTokenImpl::ENQ); + _dtok.set_rid(0); + } else { + rid_found = _dtok.rid() == rid; + lastReadRid = rid; + done = true; + } + break; + case qpid::qls_jrnl::RHM_IORES_PAGE_AIOWAIT: + if (get_wr_events(&_aio_cmpl_timeout) == qpid::qls_jrnl::jerrno::AIO_TIMEOUT) { + std::stringstream ss; + ss << "read_data_record() returned " << qpid::qls_jrnl::iores_str(res); + ss << "; timed out waiting for page to be processed."; + throw jexception(qpid::qls_jrnl::jerrno::JERR__TIMEOUT, ss.str().c_str(), "JournalImpl", + "loadMsgContent"); + } + break; + default: + std::stringstream ss; + ss << "read_data_record() returned " << qpid::qls_jrnl::iores_str(res); + throw jexception(qpid::qls_jrnl::jerrno::JERR__UNEXPRESPONSE, ss.str().c_str(), "JournalImpl", + "loadMsgContent"); + } + } + if (!rid_found) { + std::stringstream ss; + ss << "read_data_record() was unable to find rid 0x" << std::hex << rid << std::dec; + ss << " (" << rid << "); last rid found was 0x" << std::hex << _dtok.rid() << std::dec; + ss << " (" << _dtok.rid() << ")"; + throw jexception(qpid::qls_jrnl::jerrno::JERR__RECNFOUND, ss.str().c_str(), "JournalImpl", "loadMsgContent"); + } + } + + if (_external) return false; + + uint32_t hdr_offs = qpid::framing::Buffer(static_cast<char*>(_datap), sizeof(uint32_t)).getLong() + sizeof(uint32_t); + if (hdr_offs + offset + length > _dlen) { + data.append((const char*)_datap + hdr_offs + offset, _dlen - hdr_offs - offset); + } else { + data.append((const char*)_datap + hdr_offs + offset, length); + } + return true; +} +*/ + +void +JournalImpl::enqueue_data_record(const void* const data_buff, const size_t tot_data_len, + const size_t this_data_len, data_tok* dtokp, const bool transient) +{ + handleIoResult(jcntl::enqueue_data_record(data_buff, tot_data_len, this_data_len, dtokp, transient)); + + if (_mgmtObject.get() != 0) + { + _mgmtObject->inc_enqueues(); + _mgmtObject->inc_recordDepth(); + } +} + +void +JournalImpl::enqueue_extern_data_record(const size_t tot_data_len, data_tok* dtokp, + const bool transient) +{ + handleIoResult(jcntl::enqueue_extern_data_record(tot_data_len, dtokp, transient)); + + if (_mgmtObject.get() != 0) + { + _mgmtObject->inc_enqueues(); + _mgmtObject->inc_recordDepth(); + } +} + +void +JournalImpl::enqueue_txn_data_record(const void* const data_buff, const size_t tot_data_len, + const size_t this_data_len, data_tok* dtokp, const std::string& xid, const bool transient) +{ + bool txn_incr = _mgmtObject.get() != 0 ? _tmap.in_map(xid) : false; + + handleIoResult(jcntl::enqueue_txn_data_record(data_buff, tot_data_len, this_data_len, dtokp, xid, transient)); + + if (_mgmtObject.get() != 0) + { + if (!txn_incr) // If this xid was not in _tmap, it will be now... + _mgmtObject->inc_txn(); + _mgmtObject->inc_enqueues(); + _mgmtObject->inc_txnEnqueues(); + _mgmtObject->inc_recordDepth(); + } +} + +void +JournalImpl::enqueue_extern_txn_data_record(const size_t tot_data_len, data_tok* dtokp, + const std::string& xid, const bool transient) +{ + bool txn_incr = _mgmtObject.get() != 0 ? _tmap.in_map(xid) : false; + + handleIoResult(jcntl::enqueue_extern_txn_data_record(tot_data_len, dtokp, xid, transient)); + + if (_mgmtObject.get() != 0) + { + if (!txn_incr) // If this xid was not in _tmap, it will be now... + _mgmtObject->inc_txn(); + _mgmtObject->inc_enqueues(); + _mgmtObject->inc_txnEnqueues(); + _mgmtObject->inc_recordDepth(); + } +} + +void +JournalImpl::dequeue_data_record(data_tok* const dtokp, const bool txn_coml_commit) +{ + handleIoResult(jcntl::dequeue_data_record(dtokp, txn_coml_commit)); + + if (_mgmtObject.get() != 0) + { + _mgmtObject->inc_dequeues(); + _mgmtObject->inc_txnDequeues(); + _mgmtObject->dec_recordDepth(); + } +} + +void +JournalImpl::dequeue_txn_data_record(data_tok* const dtokp, const std::string& xid, const bool txn_coml_commit) +{ + bool txn_incr = _mgmtObject.get() != 0 ? _tmap.in_map(xid) : false; + + handleIoResult(jcntl::dequeue_txn_data_record(dtokp, xid, txn_coml_commit)); + + if (_mgmtObject.get() != 0) + { + if (!txn_incr) // If this xid was not in _tmap, it will be now... + _mgmtObject->inc_txn(); + _mgmtObject->inc_dequeues(); + _mgmtObject->inc_txnDequeues(); + _mgmtObject->dec_recordDepth(); + } +} + +void +JournalImpl::txn_abort(data_tok* const dtokp, const std::string& xid) +{ + handleIoResult(jcntl::txn_abort(dtokp, xid)); + + if (_mgmtObject.get() != 0) + { + _mgmtObject->dec_txn(); + _mgmtObject->inc_txnAborts(); + } +} + +void +JournalImpl::txn_commit(data_tok* const dtokp, const std::string& xid) +{ + handleIoResult(jcntl::txn_commit(dtokp, xid)); + + if (_mgmtObject.get() != 0) + { + _mgmtObject->dec_txn(); + _mgmtObject->inc_txnCommits(); + } +} + +void +JournalImpl::stop(bool block_till_aio_cmpl) +{ + InactivityFireEvent* ifep = dynamic_cast<InactivityFireEvent*>(inactivityFireEventPtr.get()); + assert(ifep); // dynamic_cast can return null if the cast fails + ifep->cancel(); + jcntl::stop(block_till_aio_cmpl); + + if (_mgmtObject.get() != 0) { + _mgmtObject->resourceDestroy(); + _mgmtObject.reset(); + } +} + +iores +JournalImpl::flush(const bool block_till_aio_cmpl) +{ + const iores res = jcntl::flush(block_till_aio_cmpl); + { + qpid::sys::Mutex::ScopedLock sl(_getf_lock); + if (_wmgr.get_aio_evt_rem() && !getEventsTimerSetFlag) { setGetEventTimer(); } + } + return res; +} + +/* +void +JournalImpl::log(qpid::qls_jrnl::log_level ll, const std::string& log_stmt) const +{ + log(ll, log_stmt.c_str()); +} + +void +JournalImpl::log(qpid::qls_jrnl::log_level ll, const char* const log_stmt) const +{ + switch (ll) + { + case LOG_TRACE: QPID_LOG(trace, "QLS Journal \"" << _jid << "\": " << log_stmt); break; + case LOG_DEBUG: QPID_LOG(debug, "QLS Journal \"" << _jid << "\": " << log_stmt); break; + case LOG_INFO: QPID_LOG(info, "QLS Journal \"" << _jid << "\": " << log_stmt); break; + case LOG_NOTICE: QPID_LOG(notice, "QLS Journal \"" << _jid << "\": " << log_stmt); break; + case LOG_WARN: QPID_LOG(warning, "QLS Journal \"" << _jid << "\": " << log_stmt); break; + case LOG_ERROR: QPID_LOG(error, "QLS Journal \"" << _jid << "\": " << log_stmt); break; + case LOG_CRITICAL: QPID_LOG(critical, "QLS Journal \"" << _jid << "\": " << log_stmt); break; + } +} +*/ + +void +JournalImpl::getEventsFire() +{ + qpid::sys::Mutex::ScopedLock sl(_getf_lock); + getEventsTimerSetFlag = false; + if (_wmgr.get_aio_evt_rem()) { jcntl::get_wr_events(0); } + if (_wmgr.get_aio_evt_rem()) { setGetEventTimer(); } +} + +void +JournalImpl::flushFire() +{ + if (writeActivityFlag) { + writeActivityFlag = false; + flushTriggeredFlag = false; + } else { + if (!flushTriggeredFlag) { + flush(); + flushTriggeredFlag = true; + } + } + inactivityFireEventPtr->setupNextFire(); + { + timer.add(inactivityFireEventPtr); + } +} + +void +JournalImpl::wr_aio_cb(std::vector<data_tok*>& dtokl) +{ + for (std::vector<data_tok*>::const_iterator i=dtokl.begin(); i!=dtokl.end(); i++) + { + DataTokenImpl* dtokp = static_cast<DataTokenImpl*>(*i); + if (/*!is_stopped() &&*/ dtokp->getSourceMessage()) + { + switch (dtokp->wstate()) + { + case data_tok::ENQ: +//std::cout << "<<<>>> JournalImpl::wr_aio_cb() ENQ dtokp rid=0x" << std::hex << dtokp->rid() << std::dec << std::endl << std::flush; // DEBUG + dtokp->getSourceMessage()->enqueueComplete(); + break; + case data_tok::DEQ: +//std::cout << "<<<>>> JournalImpl::wr_aio_cb() DEQ dtokp rid=0x" << std::hex << dtokp->rid() << std::dec << std::endl << std::flush; // DEBUG +/* Don't need to signal until we have a way to ack completion of dequeue in AMQP + dtokp->getSourceMessage()->dequeueComplete(); + if ( dtokp->getSourceMessage()->isDequeueComplete() ) // clear id after last dequeue + dtokp->getSourceMessage()->setPersistenceId(0); +*/ + break; + default: ; + } + } + dtokp->release(); + } +} + +void +JournalImpl::rd_aio_cb(std::vector<uint16_t>& /*pil*/) +{} + +/* +void +JournalImpl::free_read_buffers() +{ + if (_xidp) { + ::free(_xidp); + _xidp = 0; + _datap = 0; + } else if (_datap) { + ::free(_datap); + _datap = 0; + } +} +*/ + +void +JournalImpl::createStore() { + +} + +void +JournalImpl::handleIoResult(const iores r) +{ + writeActivityFlag = true; + switch (r) + { + case qpid::qls_jrnl::RHM_IORES_SUCCESS: + return; +/* + case qpid::qls_jrnl::RHM_IORES_FULL: + { + std::ostringstream oss; + oss << "Journal full on queue \"" << _jid << "\"."; + QLS_LOG2(critical, _jid, "Journal full."); + if (_agent != 0) + _agent->raiseEvent(qmf::org::apache::qpid::linearstore::EventFull(_jid, "Journal full"), qpid::management::ManagementAgent::SEV_ERROR); + THROW_STORE_FULL_EXCEPTION(oss.str()); + } +*/ + default: + { + std::ostringstream oss; + oss << "Unexpected I/O response (" << qpid::qls_jrnl::iores_str(r) << ")."; + QLS_LOG2(error, _jid, oss.str()); + THROW_STORE_FULL_EXCEPTION(oss.str()); + } + } +} + +qpid::management::Manageable::status_t JournalImpl::ManagementMethod (uint32_t /*methodId*/, + qpid::management::Args& /*args*/, + std::string& /*text*/) +{ + Manageable::status_t status = Manageable::STATUS_UNKNOWN_METHOD; + +/* + switch (methodId) + { + case _qmf::Journal::METHOD_EXPAND : + //_qmf::ArgsJournalExpand& eArgs = (_qmf::ArgsJournalExpand&) args; + + // Implement "expand" using eArgs.i_by (expand-by argument) + + status = Manageable::STATUS_NOT_IMPLEMENTED; + break; + } +*/ + + return status; +} diff --git a/qpid/cpp/src/qpid/linearstore/JournalImpl.h b/qpid/cpp/src/qpid/linearstore/JournalImpl.h new file mode 100644 index 0000000000..fd2433803c --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/JournalImpl.h @@ -0,0 +1,244 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LINEARSTORE_JOURNALIMPL_H +#define QPID_LINEARSTORE_JOURNALIMPL_H + +#include <set> +#include "qpid/linearstore/jrnl/enums.h" +#include "qpid/linearstore/jrnl/EmptyFilePoolTypes.h" +#include "qpid/linearstore/jrnl/jcntl.h" +#include "qpid/linearstore/DataTokenImpl.h" +#include "qpid/linearstore/PreparedTransaction.h" +#include "qpid/broker/PersistableQueue.h" +#include "qpid/sys/Timer.h" +#include "qpid/sys/Time.h" +#include <boost/ptr_container/ptr_list.hpp> +#include <boost/intrusive_ptr.hpp> +#include "qpid/management/Manageable.h" +#include "qmf/org/apache/qpid/linearstore/Journal.h" + +namespace qpid{ + +namespace sys { +class Timer; +} +namespace qls_jrnl { +class EmptyFilePool; +} + +namespace linearstore{ + +class JournalImpl; +class JournalLogImpl; + +class InactivityFireEvent : public qpid::sys::TimerTask +{ + JournalImpl* _parent; + qpid::sys::Mutex _ife_lock; + + public: + InactivityFireEvent(JournalImpl* p, const qpid::sys::Duration timeout); + virtual ~InactivityFireEvent() {} + void fire(); + inline void cancel() { qpid::sys::Mutex::ScopedLock sl(_ife_lock); _parent = 0; } +}; + +class GetEventsFireEvent : public qpid::sys::TimerTask +{ + JournalImpl* _parent; + qpid::sys::Mutex _gefe_lock; + + public: + GetEventsFireEvent(JournalImpl* p, const qpid::sys::Duration timeout); + virtual ~GetEventsFireEvent() {} + void fire(); + inline void cancel() { qpid::sys::Mutex::ScopedLock sl(_gefe_lock); _parent = 0; } +}; + +class JournalImpl : public qpid::broker::ExternalQueueStore, public qpid::qls_jrnl::jcntl, public qpid::qls_jrnl::aio_callback +{ + public: + typedef boost::function<void (JournalImpl&)> DeleteCallback; + + protected: + qpid::sys::Timer& timer; + JournalLogImpl& _journalLogRef; + bool getEventsTimerSetFlag; + boost::intrusive_ptr<qpid::sys::TimerTask> getEventsFireEventsPtr; + qpid::sys::Mutex _getf_lock; + qpid::sys::Mutex _read_lock; + + bool writeActivityFlag; + bool flushTriggeredFlag; + boost::intrusive_ptr<qpid::sys::TimerTask> inactivityFireEventPtr; + + qpid::management::ManagementAgent* _agent; + qmf::org::apache::qpid::linearstore::Journal::shared_ptr _mgmtObject; + DeleteCallback deleteCallback; + + public: + + JournalImpl(qpid::sys::Timer& timer, + const std::string& journalId, + const std::string& journalDirectory, + JournalLogImpl& journalLogRef, + const qpid::sys::Duration getEventsTimeout, + const qpid::sys::Duration flushTimeout, + qpid::management::ManagementAgent* agent, + DeleteCallback deleteCallback=DeleteCallback() ); + + virtual ~JournalImpl(); + + void initManagement(qpid::management::ManagementAgent* agent); + + void initialize(qpid::qls_jrnl::EmptyFilePool* efp, + const uint16_t wcache_num_pages, + const uint32_t wcache_pgsize_sblks, + qpid::qls_jrnl::aio_callback* const cbp); + + inline void initialize(qpid::qls_jrnl::EmptyFilePool* efpp, + const uint16_t wcache_num_pages, + const uint32_t wcache_pgsize_sblks) { + initialize(efpp, wcache_num_pages, wcache_pgsize_sblks, this); + } + + void recover(boost::shared_ptr<qpid::qls_jrnl::EmptyFilePoolManager> efpm, + const uint16_t wcache_num_pages, + const uint32_t wcache_pgsize_sblks, + qpid::qls_jrnl::aio_callback* const cbp, + boost::ptr_list<PreparedTransaction>* prep_tx_list_ptr, + uint64_t& highest_rid, + uint64_t queue_id); + + inline void recover(boost::shared_ptr<qpid::qls_jrnl::EmptyFilePoolManager> efpm, + const uint16_t wcache_num_pages, + const uint32_t wcache_pgsize_sblks, + boost::ptr_list<PreparedTransaction>* prep_tx_list_ptr, + uint64_t& highest_rid, + uint64_t queue_id) { + recover(efpm, wcache_num_pages, wcache_pgsize_sblks, this, prep_tx_list_ptr, highest_rid, queue_id); + } + + void recover_complete(); + + // Temporary fn to read and save last msg read from journal so it can be assigned + // in chunks. To be replaced when coding to do this direct from the journal is ready. + // Returns true if the record is extern, false if local. +// bool loadMsgContent(uint64_t rid, std::string& data, size_t length, size_t offset = 0); + + // Overrides for write inactivity timer + void enqueue_data_record(const void* const data_buff, const size_t tot_data_len, + const size_t this_data_len, qpid::qls_jrnl::data_tok* dtokp, + const bool transient = false); + + void enqueue_extern_data_record(const size_t tot_data_len, qpid::qls_jrnl::data_tok* dtokp, + const bool transient = false); + + void enqueue_txn_data_record(const void* const data_buff, const size_t tot_data_len, + const size_t this_data_len, qpid::qls_jrnl::data_tok* dtokp, const std::string& xid, + const bool transient = false); + + void enqueue_extern_txn_data_record(const size_t tot_data_len, qpid::qls_jrnl::data_tok* dtokp, + const std::string& xid, const bool transient = false); + + void dequeue_data_record(qpid::qls_jrnl::data_tok* const dtokp, const bool txn_coml_commit = false); + + void dequeue_txn_data_record(qpid::qls_jrnl::data_tok* const dtokp, const std::string& xid, const bool txn_coml_commit = false); + + void txn_abort(qpid::qls_jrnl::data_tok* const dtokp, const std::string& xid); + + void txn_commit(qpid::qls_jrnl::data_tok* const dtokp, const std::string& xid); + + void stop(bool block_till_aio_cmpl = false); + + // Overrides for get_events timer + qpid::qls_jrnl::iores flush(const bool block_till_aio_cmpl = false); + + // TimerTask callback + void getEventsFire(); + void flushFire(); + + // AIO callbacks + virtual void wr_aio_cb(std::vector<qpid::qls_jrnl::data_tok*>& dtokl); + virtual void rd_aio_cb(std::vector<uint16_t>& pil); + + qpid::management::ManagementObject::shared_ptr GetManagementObject (void) const + { return _mgmtObject; } + + qpid::management::Manageable::status_t ManagementMethod (uint32_t, + qpid::management::Args&, + std::string&); + + void resetDeleteCallback() { deleteCallback = DeleteCallback(); } + + protected: +// void free_read_buffers(); + void createStore(); + + inline void setGetEventTimer() + { + getEventsFireEventsPtr->setupNextFire(); + timer.add(getEventsFireEventsPtr); + getEventsTimerSetFlag = true; + } + void handleIoResult(const qpid::qls_jrnl::iores r); + + // Management instrumentation callbacks overridden from jcntl + inline void instr_incr_outstanding_aio_cnt() { + if (_mgmtObject.get() != 0) _mgmtObject->inc_outstandingAIOs(); + } + inline void instr_decr_outstanding_aio_cnt() { + if (_mgmtObject.get() != 0) _mgmtObject->dec_outstandingAIOs(); + } + +}; // class JournalImpl + +class TplJournalImpl : public JournalImpl +{ + public: + TplJournalImpl(qpid::sys::Timer& timer, + const std::string& journalId, + const std::string& journalDirectory, + JournalLogImpl& journalLogRef, + const qpid::sys::Duration getEventsTimeout, + const qpid::sys::Duration flushTimeout, + qpid::management::ManagementAgent* agent) : + JournalImpl(timer, journalId, journalDirectory, journalLogRef, getEventsTimeout, flushTimeout, agent) + {} + + virtual ~TplJournalImpl() {} + +/* + // Special version of read_data_record that ignores transactions - needed when reading the TPL + inline qpid::qls_jrnl::iores read_data_record(void** const datapp, std::size_t& dsize, + void** const xidpp, std::size_t& xidsize, bool& transient, bool& external, + qpid::qls_jrnl::data_tok* const dtokp) { + return JournalImpl::read_data_record(datapp, dsize, xidpp, xidsize, transient, external, dtokp, true); + } + inline void read_reset() { _rmgr.invalidate(); } +*/ +}; // class TplJournalImpl + +} // namespace msgstore +} // namespace mrg + +#endif // ifndef QPID_LINEARSTORE_JOURNALIMPL_H diff --git a/qpid/cpp/src/qpid/linearstore/JournalLogImpl.cpp b/qpid/cpp/src/qpid/linearstore/JournalLogImpl.cpp new file mode 100644 index 0000000000..b7935b1509 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/JournalLogImpl.cpp @@ -0,0 +1,59 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/JournalLogImpl.h" + +namespace qpid { +namespace linearstore { + +JournalLogImpl::JournalLogImpl(const qpid::qls_jrnl::JournalLog::log_level_t logLevelThreshold) : qpid::qls_jrnl::JournalLog(logLevelThreshold) {} +JournalLogImpl::~JournalLogImpl() {} + +void +JournalLogImpl::log(const qpid::qls_jrnl::JournalLog::log_level_t level, + const std::string& log_stmt) const { + switch (level) { + case LOG_CRITICAL: QPID_LOG(critical, "Linear Store: " << log_stmt); break; + case LOG_ERROR: QPID_LOG(error, "Linear Store: " << log_stmt); break; + case LOG_WARN: QPID_LOG(warning, "Linear Store: " << log_stmt); break; + case LOG_NOTICE: QPID_LOG(notice, "Linear Store: " << log_stmt); break; + case LOG_INFO: QPID_LOG(info, "Linear Store: " << log_stmt); break; + case LOG_DEBUG: QPID_LOG(debug, "Linear Store: " << log_stmt); break; + default: QPID_LOG(trace, "Linear Store: " << log_stmt); + } +} + +void +JournalLogImpl::log(const qpid::qls_jrnl::JournalLog::log_level_t level, + const std::string& jid, + const std::string& log_stmt) const { + switch (level) { + case LOG_CRITICAL: QPID_LOG(critical, "Linear Store: Journal \'" << jid << "\":" << log_stmt); break; + case LOG_ERROR: QPID_LOG(error, "Linear Store: Journal \'" << jid << "\":" << log_stmt); break; + case LOG_WARN: QPID_LOG(warning, "Linear Store: Journal \'" << jid << "\":" << log_stmt); break; + case LOG_NOTICE: QPID_LOG(notice, "Linear Store: Journal \'" << jid << "\":" << log_stmt); break; + case LOG_INFO: QPID_LOG(info, "Linear Store: Journal \'" << jid << "\":" << log_stmt); break; + case LOG_DEBUG: QPID_LOG(debug, "Linear Store: Journal \'" << jid << "\":" << log_stmt); break; + default: QPID_LOG(trace, "Linear Store: Journal \'" << jid << "\":" << log_stmt); + } +} + +}} // namespace qpid::linearstore diff --git a/qpid/cpp/src/qpid/linearstore/JournalLogImpl.h b/qpid/cpp/src/qpid/linearstore/JournalLogImpl.h new file mode 100644 index 0000000000..a43af23ac6 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/JournalLogImpl.h @@ -0,0 +1,48 @@ + /* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_LOG_H +#define QPID_LEGACYSTORE_LOG_H + +#include "qpid/linearstore/jrnl/JournalLog.h" +#include "qpid/log/Statement.h" + +#define QLS_LOG(level, msg) QPID_LOG(level, "Linear Store: " << msg) +#define QLS_LOG2(level, queue, msg) QPID_LOG(level, "Linear Store: Journal \'" << queue << "\":" << msg) + +namespace qpid { +namespace linearstore { + +class JournalLogImpl : public qpid::qls_jrnl::JournalLog +{ +public: + JournalLogImpl(const qpid::qls_jrnl::JournalLog::log_level_t logLevelThreshold); + virtual ~JournalLogImpl(); + virtual void log(const qpid::qls_jrnl::JournalLog::log_level_t logLevel, + const std::string& logStatement) const; + virtual void log(const qpid::qls_jrnl::JournalLog::log_level_t logLevel, + const std::string& journalId, + const std::string& logStatement) const; +}; + +}} // namespace qpid::linearstore + +#endif // QPID_LEGACYSTORE_LOG_H diff --git a/qpid/cpp/src/qpid/linearstore/MessageStoreImpl.cpp b/qpid/cpp/src/qpid/linearstore/MessageStoreImpl.cpp new file mode 100644 index 0000000000..cbc395d61a --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/MessageStoreImpl.cpp @@ -0,0 +1,1640 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/MessageStoreImpl.h" + +#include "db-inc.h" +#include "qpid/broker/QueueSettings.h" +#include "qpid/linearstore/BindingDbt.h" +#include "qpid/linearstore/BufferValue.h" +#include "qpid/linearstore/IdDbt.h" +#include "qpid/linearstore/jrnl/EmptyFilePoolManager.h" +#include "qpid/linearstore/jrnl/txn_map.h" +#include "qpid/framing/FieldValue.h" +#include "qmf/org/apache/qpid/linearstore/Package.h" +#include "qpid/linearstore/StoreException.h" +#include <dirent.h> + + +#define MAX_AIO_SLEEPS 100000 // tot: ~1 sec +#define AIO_SLEEP_TIME_US 10 // 0.01 ms + +namespace _qmf = qmf::org::apache::qpid::linearstore; + +namespace qpid{ +namespace linearstore{ + + +const std::string MessageStoreImpl::storeTopLevelDir("qls"); // Sets the top-level store dir name + +// FIXME aconway 2010-03-09: was 10 +qpid::sys::Duration MessageStoreImpl::defJournalGetEventsTimeout(1 * qpid::sys::TIME_MSEC); // 10ms +qpid::sys::Duration MessageStoreImpl::defJournalFlushTimeout(500 * qpid::sys::TIME_MSEC); // 0.5s +qpid::sys::Mutex TxnCtxt::globalSerialiser; + +MessageStoreImpl::TplRecoverStruct::TplRecoverStruct(const uint64_t rid_, + const bool deq_flag_, + const bool commit_flag_, + const bool tpc_flag_) : + rid(rid_), + deq_flag(deq_flag_), + commit_flag(commit_flag_), + tpc_flag(tpc_flag_) +{} + +MessageStoreImpl::MessageStoreImpl(qpid::broker::Broker* broker_, const char* envpath_) : + defaultEfpPartitionNumber(0), + defaultEfpFileSize_kib(0), + truncateFlag(false), + wCachePgSizeSblks(0), + wCacheNumPages(0), + tplWCachePgSizeSblks(0), + tplWCacheNumPages(0), + highestRid(0), + isInit(false), + envPath(envpath_), + broker(broker_), + jrnlLog(qpid::qls_jrnl::JournalLog::LOG_NOTICE), + mgmtObject(), + agent(0) +{} + +uint32_t MessageStoreImpl::chkJrnlWrPageCacheSize(const uint32_t param_, const std::string& paramName_) +{ + uint32_t p = param_; + + if (p == 0) { + // For zero value, use default + p = QLS_WMGR_DEF_PAGE_SIZE_KIB; + QLS_LOG(warning, "parameter " << paramName_ << " (" << param_ << ") must be a power of 2 between 1 and 128; changing this parameter to default value (" << p << ")"); + } else if ( p > 128 || (p & (p-1)) ) { + // For any positive value that is not a power of 2, use closest value + if (p < 6) p = 4; + else if (p < 12) p = 8; + else if (p < 24) p = 16; + else if (p < 48) p = 32; + else if (p < 96) p = 64; + else p = 128; + QLS_LOG(warning, "parameter " << paramName_ << " (" << param_ << ") must be a power of 2 between 1 and 128; changing this parameter to closest allowable value (" << p << ")"); + } + return p; +} + +uint16_t MessageStoreImpl::getJrnlWrNumPages(const uint32_t wrPageSizeKib_) +{ + uint32_t wrPageSizeSblks = wrPageSizeKib_ / QLS_SBLK_SIZE_KIB; // convert from KiB to number sblks + uint32_t defTotWCacheSizeSblks = QLS_WMGR_DEF_PAGE_SIZE_SBLKS * QLS_WMGR_DEF_PAGES; + switch (wrPageSizeKib_) + { + case 1: + case 2: + case 4: + // 256 KiB total cache + return defTotWCacheSizeSblks / wrPageSizeSblks / 4; + case 8: + case 16: + // 512 KiB total cache + return defTotWCacheSizeSblks / wrPageSizeSblks / 2; + default: // 32, 64, 128 + // 1 MiB total cache + return defTotWCacheSizeSblks / wrPageSizeSblks; + } +} + +qpid::qls_jrnl::efpPartitionNumber_t MessageStoreImpl::chkEfpPartition(const qpid::qls_jrnl::efpPartitionNumber_t partition_, + const std::string& /*paramName_*/) { + // TODO: check against list of existing partitions, throw if not found + return partition_; +} + +qpid::qls_jrnl::efpDataSize_kib_t MessageStoreImpl::chkEfpFileSizeKiB(const qpid::qls_jrnl::efpDataSize_kib_t efpFileSizeKib_, + const std::string& paramName_) { + uint8_t rem = efpFileSizeKib_ % uint64_t(QLS_SBLK_SIZE_KIB); + if (rem != 0) { + uint64_t newVal = efpFileSizeKib_ - rem; + if (rem >= (QLS_SBLK_SIZE_KIB / 2)) + newVal += QLS_SBLK_SIZE_KIB; + QLS_LOG(warning, "Parameter " << paramName_ << " (" << efpFileSizeKib_ << ") must be a multiple of " << + QLS_SBLK_SIZE_KIB << "; changing this parameter to the closest allowable value (" << + newVal << ")"); + return newVal; + } + return efpFileSizeKib_; + + // TODO: check against list of existing pools in the given partition +} + +void MessageStoreImpl::initManagement () +{ + if (broker != 0) { + agent = broker->getManagementAgent(); + if (agent != 0) { + _qmf::Package packageInitializer(agent); + mgmtObject = _qmf::Store::shared_ptr ( + new _qmf::Store(agent, this, broker)); + + mgmtObject->set_location(storeDir); + mgmtObject->set_tplIsInitialized(false); + mgmtObject->set_tplDirectory(getTplBaseDir()); + mgmtObject->set_tplWritePageSize(tplWCachePgSizeSblks * QLS_SBLK_SIZE_BYTES); + mgmtObject->set_tplWritePages(tplWCacheNumPages); + + agent->addObject(mgmtObject, 0, true); + + // Initialize all existing queues (ie those recovered before management was initialized) + for (JournalListMapItr i=journalList.begin(); i!=journalList.end(); i++) { + i->second->initManagement(agent); + } + } + } +} + +bool MessageStoreImpl::init(const qpid::Options* options_) +{ + // Extract and check options + const StoreOptions* opts = static_cast<const StoreOptions*>(options_); + qpid::qls_jrnl::efpPartitionNumber_t efpPartition = chkEfpPartition(opts->efpPartition, "efp-partition"); + qpid::qls_jrnl::efpDataSize_kib_t efpFilePoolSize_kib = chkEfpFileSizeKiB(opts->efpFileSizeKib, "efp-file-size"); + uint32_t jrnlWrCachePageSizeKib = chkJrnlWrPageCacheSize(opts->wCachePageSizeKib, "wcache-page-size"); + uint32_t tplJrnlWrCachePageSizeKib = chkJrnlWrPageCacheSize(opts->tplWCachePageSizeKib, "tpl-wcache-page-size"); + + // Pass option values to init() + return init(opts->storeDir, efpPartition, efpFilePoolSize_kib, opts->truncateFlag, jrnlWrCachePageSizeKib, tplJrnlWrCachePageSizeKib); +} + +// These params, taken from options, are assumed to be correct and verified +bool MessageStoreImpl::init(const std::string& storeDir_, + qpid::qls_jrnl::efpPartitionNumber_t efpPartition_, + qpid::qls_jrnl::efpDataSize_kib_t efpFileSize_kib_, + const bool truncateFlag_, + uint32_t wCachePageSizeKib_, + uint32_t tplWCachePageSizeKib_) +{ + if (isInit) return true; + + // Set geometry members (converting to correct units where req'd) + defaultEfpPartitionNumber = efpPartition_; + defaultEfpFileSize_kib = efpFileSize_kib_; + wCachePgSizeSblks = wCachePageSizeKib_ / QLS_SBLK_SIZE_KIB; // convert from KiB to number sblks + wCacheNumPages = getJrnlWrNumPages(wCachePageSizeKib_); + tplWCachePgSizeSblks = tplWCachePageSizeKib_ / QLS_SBLK_SIZE_KIB; // convert from KiB to number sblks + tplWCacheNumPages = getJrnlWrNumPages(tplWCachePageSizeKib_); + if (storeDir_.size()>0) storeDir = storeDir_; + + if (truncateFlag_) + truncateInit(); + else + init(); + + QLS_LOG(notice, "Store module initialized; store-dir=" << storeDir_); + QLS_LOG(info, "> Default EFP partition: " << defaultEfpPartitionNumber); + QLS_LOG(info, "> Default EFP file size: " << defaultEfpFileSize_kib << " (KiB)"); + QLS_LOG(info, "> Default write cache page size: " << wCachePageSizeKib_ << " (KiB)"); + QLS_LOG(info, "> Default number of write cache pages: " << wCacheNumPages); + QLS_LOG(info, "> TPL write cache page size: " << tplWCachePageSizeKib_ << " (KiB)"); + QLS_LOG(info, "> TPL number of write cache pages: " << tplWCacheNumPages); + QLS_LOG(info, "> EFP partition: " << defaultEfpPartitionNumber); + QLS_LOG(info, "> EFP file size pool: " << defaultEfpFileSize_kib << " (KiB)"); + + return isInit; +} + +void MessageStoreImpl::init() +{ + const int retryMax = 3; + int bdbRetryCnt = 0; + do { + if (bdbRetryCnt++ > 0) + { + closeDbs(); + ::usleep(1000000); // 1 sec delay + QLS_LOG(error, "Previoius BDB store initialization failed, retrying (" << bdbRetryCnt << " of " << retryMax << ")..."); + } + + try { + qpid::qls_jrnl::jdir::create_dir(getBdbBaseDir()); + + dbenv.reset(new DbEnv(0)); + dbenv->set_errpfx("linearstore"); + dbenv->set_lg_regionmax(256000); // default = 65000 + dbenv->open(getBdbBaseDir().c_str(), DB_THREAD | DB_CREATE | DB_INIT_TXN | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_USE_ENVIRON | DB_RECOVER, 0); + + // Databases are constructed here instead of the constructor so that the DB_RECOVER flag can be used + // against the database environment. Recover can only be performed if no databases have been created + // against the environment at the time of recovery, as recovery invalidates the environment. + queueDb.reset(new Db(dbenv.get(), 0)); + dbs.push_back(queueDb); + configDb.reset(new Db(dbenv.get(), 0)); + dbs.push_back(configDb); + exchangeDb.reset(new Db(dbenv.get(), 0)); + dbs.push_back(exchangeDb); + mappingDb.reset(new Db(dbenv.get(), 0)); + dbs.push_back(mappingDb); + bindingDb.reset(new Db(dbenv.get(), 0)); + dbs.push_back(bindingDb); + generalDb.reset(new Db(dbenv.get(), 0)); + dbs.push_back(generalDb); + + TxnCtxt txn; + txn.begin(dbenv.get(), false); + try { + open(queueDb, txn.get(), "queues.db", false); + open(configDb, txn.get(), "config.db", false); + open(exchangeDb, txn.get(), "exchanges.db", false); + open(mappingDb, txn.get(), "mappings.db", true); + open(bindingDb, txn.get(), "bindings.db", true); + open(generalDb, txn.get(), "general.db", false); + txn.commit(); + } catch (...) { txn.abort(); throw; } + // NOTE: during normal initialization, agent == 0 because the store is initialized before the management infrastructure. + // However during a truncated initialization in a cluster, agent != 0. We always pass 0 as the agent for the + // TplStore to keep things consistent in a cluster. See https://bugzilla.redhat.com/show_bug.cgi?id=681026 + tplStorePtr.reset(new TplJournalImpl(broker->getTimer(), "TplStore", getTplBaseDir(), jrnlLog, defJournalGetEventsTimeout, defJournalFlushTimeout, 0)); + isInit = true; + } catch (const DbException& e) { + if (e.get_errno() == DB_VERSION_MISMATCH) + { + QLS_LOG(error, "Database environment mismatch: This version of db4 does not match that which created the store database.: " << e.what()); + THROW_STORE_EXCEPTION_2("Database environment mismatch: This version of db4 does not match that which created the store database. " + "(If recovery is not important, delete the contents of the store directory. Otherwise, try upgrading the database using " + "db_upgrade or using db_recover - but the db4-utils package must also be installed to use these utilities.)", e); + } + QLS_LOG(error, "BDB exception occurred while initializing store: " << e.what()); + if (bdbRetryCnt >= retryMax) + THROW_STORE_EXCEPTION_2("BDB exception occurred while initializing store", e); + } catch (const StoreException&) { + throw; + } catch (const qpid::qls_jrnl::jexception& e) { + QLS_LOG(error, "Journal Exception occurred while initializing store: " << e); + THROW_STORE_EXCEPTION_2("Journal Exception occurred while initializing store", e.what()); + } catch (...) { + QLS_LOG(error, "Unknown exception occurred while initializing store."); + throw; + } + } while (!isInit); + + efpMgr.reset(new qpid::qls_jrnl::EmptyFilePoolManager(getStoreTopLevelDir(), jrnlLog)); + efpMgr->findEfpPartitions(); +} + +void MessageStoreImpl::finalize() +{ + if (tplStorePtr.get() && tplStorePtr->is_ready()) tplStorePtr->stop(true); + { + qpid::sys::Mutex::ScopedLock sl(journalListLock); + for (JournalListMapItr i = journalList.begin(); i != journalList.end(); i++) + { + JournalImpl* jQueue = i->second; + jQueue->resetDeleteCallback(); + if (jQueue->is_ready()) jQueue->stop(true); + } + } + + if (mgmtObject.get() != 0) { + mgmtObject->resourceDestroy(); + mgmtObject.reset(); + } +} + +void MessageStoreImpl::truncateInit() +{ + if (isInit) { + { + qpid::sys::Mutex::ScopedLock sl(journalListLock); + if (journalList.size()) { // check no queues exist + std::ostringstream oss; + oss << "truncateInit() called with " << journalList.size() << " queues still in existence"; + THROW_STORE_EXCEPTION(oss.str()); + } + } + closeDbs(); + dbs.clear(); + if (tplStorePtr->is_ready()) tplStorePtr->stop(true); + dbenv->close(0); + isInit = false; + } + qpid::qls_jrnl::jdir::delete_dir(getBdbBaseDir()); + qpid::qls_jrnl::jdir::delete_dir(getJrnlBaseDir()); + qpid::qls_jrnl::jdir::delete_dir(getTplBaseDir()); + QLS_LOG(notice, "Store directory " << getStoreTopLevelDir() << " was truncated."); + init(); +} + +void MessageStoreImpl::chkTplStoreInit() +{ + // Prevent multiple threads from late-initializing the TPL + qpid::sys::Mutex::ScopedLock sl(tplInitLock); + if (!tplStorePtr->is_ready()) { + qpid::qls_jrnl::jdir::create_dir(getTplBaseDir()); + tplStorePtr->initialize(/*tplNumJrnlFiles, false, 0, tplJrnlFsizeSblks,*/ getEmptyFilePool(defaultEfpPartitionNumber, defaultEfpFileSize_kib), tplWCacheNumPages, tplWCachePgSizeSblks); + if (mgmtObject.get() != 0) mgmtObject->set_tplIsInitialized(true); + } +} + +void MessageStoreImpl::open(db_ptr db_, + DbTxn* txn_, + const char* file_, + bool dupKey_) +{ + if(dupKey_) db_->set_flags(DB_DUPSORT); + db_->open(txn_, file_, 0, DB_BTREE, DB_CREATE | DB_THREAD, 0); +} + +void MessageStoreImpl::closeDbs() +{ + for (std::list<db_ptr >::iterator i = dbs.begin(); i != dbs.end(); i++) { + (*i)->close(0); + } + dbs.clear(); +} + +MessageStoreImpl::~MessageStoreImpl() +{ + finalize(); + try { + closeDbs(); + } catch (const DbException& e) { + QLS_LOG(error, "Error closing BDB databases: " << e.what()); + } catch (const qpid::qls_jrnl::jexception& e) { + QLS_LOG(error, "Error: " << e.what()); + } catch (const std::exception& e) { + QLS_LOG(error, "Error: " << e.what()); + } catch (...) { + QLS_LOG(error, "Unknown error in MessageStoreImpl::~MessageStoreImpl()"); + } + + if (mgmtObject.get() != 0) { + mgmtObject->resourceDestroy(); + mgmtObject.reset(); + } +} + +void MessageStoreImpl::create(qpid::broker::PersistableQueue& queue_, + const qpid::framing::FieldTable& args_) +{ + QLS_LOG(info, "*** MessageStoreImpl::create() queue=\"" << queue_.getName() << "\""); // DEBUG + checkInit(); + if (queue_.getPersistenceId()) { + THROW_STORE_EXCEPTION("Queue already created: " + queue_.getName()); + } + JournalImpl* jQueue = 0; + + if (queue_.getName().size() == 0) + { + QLS_LOG(error, "Cannot create store for empty (null) queue name - ignoring and attempting to continue."); + return; + } + + jQueue = new JournalImpl(broker->getTimer(), queue_.getName(), getJrnlDir(queue_.getName()), jrnlLog, + defJournalGetEventsTimeout, defJournalFlushTimeout, agent, + boost::bind(&MessageStoreImpl::journalDeleted, this, _1)); + { + qpid::sys::Mutex::ScopedLock sl(journalListLock); + journalList[queue_.getName()]=jQueue; + } + + queue_.setExternalQueueStore(dynamic_cast<qpid::broker::ExternalQueueStore*>(jQueue)); + try { + jQueue->initialize(getEmptyFilePool(args_), wCacheNumPages, wCachePgSizeSblks); + } catch (const qpid::qls_jrnl::jexception& e) { + THROW_STORE_EXCEPTION(std::string("Queue ") + queue_.getName() + ": create() failed: " + e.what()); + } + try { + if (!create(queueDb, queueIdSequence, queue_)) { + THROW_STORE_EXCEPTION("Queue already exists: " + queue_.getName()); + } + } catch (const DbException& e) { + THROW_STORE_EXCEPTION_2("Error creating queue named " + queue_.getName(), e); + } +} + +qpid::qls_jrnl::EmptyFilePool* +MessageStoreImpl::getEmptyFilePool(const qpid::qls_jrnl::efpPartitionNumber_t efpPartitionNumber_, + const qpid::qls_jrnl::efpDataSize_kib_t efpFileSizeKib_) { + qpid::qls_jrnl::EmptyFilePool* efpp = efpMgr->getEmptyFilePool(efpPartitionNumber_, efpFileSizeKib_); + if (efpp == 0) { + std::ostringstream oss; + oss << "Partition=" << efpPartitionNumber_ << "; EfpFileSize=" << efpFileSizeKib_ << " KiB"; + throw qpid::qls_jrnl::jexception(qpid::qls_jrnl::jerrno::JERR_EFP_NOEFP, oss.str(), "MessageStoreImpl", "getEmptyFilePool"); + } + return efpp; +} + +qpid::qls_jrnl::EmptyFilePool* +MessageStoreImpl::getEmptyFilePool(const qpid::framing::FieldTable& args_) { + qpid::framing::FieldTable::ValuePtr value; + qpid::qls_jrnl::efpPartitionNumber_t localEfpPartition = defaultEfpPartitionNumber; + value = args_.get("qpid.efp_partition"); + if (value.get() != 0 && !value->empty() && value->convertsTo<int>()) { + localEfpPartition = chkEfpPartition((uint32_t)value->get<int>(), "qpid.efp_partition"); + } + + qpid::qls_jrnl::efpDataSize_kib_t localEfpFileSizeKib = defaultEfpFileSize_kib; + value = args_.get("qpid.efp_file_size"); + if (value.get() != 0 && !value->empty() && value->convertsTo<int>()) { + localEfpFileSizeKib = chkEfpFileSizeKiB((uint32_t)value->get<int>(),"qpid.efp_file_size" ); + } + return getEmptyFilePool(localEfpPartition, localEfpFileSizeKib); +} + +void MessageStoreImpl::destroy(qpid::broker::PersistableQueue& queue_) +{ + QLS_LOG(info, "*** MessageStoreImpl::destroy() queue=\"" << queue_.getName() << "\""); + checkInit(); + destroy(queueDb, queue_); + deleteBindingsForQueue(queue_); + qpid::broker::ExternalQueueStore* eqs = queue_.getExternalQueueStore(); + if (eqs) { + JournalImpl* jQueue = static_cast<JournalImpl*>(eqs); + jQueue->delete_jrnl_files(); + queue_.setExternalQueueStore(0); // will delete the journal if exists + { + qpid::sys::Mutex::ScopedLock sl(journalListLock); + journalList.erase(queue_.getName()); + } + } +} + +void MessageStoreImpl::create(const qpid::broker::PersistableExchange& exchange_, + const qpid::framing::FieldTable& /*args_*/) +{ + checkInit(); + if (exchange_.getPersistenceId()) { + THROW_STORE_EXCEPTION("Exchange already created: " + exchange_.getName()); + } + try { + if (!create(exchangeDb, exchangeIdSequence, exchange_)) { + THROW_STORE_EXCEPTION("Exchange already exists: " + exchange_.getName()); + } + } catch (const DbException& e) { + THROW_STORE_EXCEPTION_2("Error creating exchange named " + exchange_.getName(), e); + } +} + +void MessageStoreImpl::destroy(const qpid::broker::PersistableExchange& exchange) +{ + checkInit(); + destroy(exchangeDb, exchange); + //need to also delete bindings + IdDbt key(exchange.getPersistenceId()); + bindingDb->del(0, &key, DB_AUTO_COMMIT); +} + +void MessageStoreImpl::create(const qpid::broker::PersistableConfig& general_) +{ + checkInit(); + if (general_.getPersistenceId()) { + THROW_STORE_EXCEPTION("General configuration item already created"); + } + try { + if (!create(generalDb, generalIdSequence, general_)) { + THROW_STORE_EXCEPTION("General configuration already exists"); + } + } catch (const DbException& e) { + THROW_STORE_EXCEPTION_2("Error creating general configuration", e); + } +} + +void MessageStoreImpl::destroy(const qpid::broker::PersistableConfig& general_) +{ + checkInit(); + destroy(generalDb, general_); +} + +bool MessageStoreImpl::create(db_ptr db_, + IdSequence& seq_, + const qpid::broker::Persistable& p_) +{ + uint64_t id (seq_.next()); + Dbt key(&id, sizeof(id)); + BufferValue value (p_); + + int status; + TxnCtxt txn; + txn.begin(dbenv.get(), true); + try { + status = db_->put(txn.get(), &key, &value, DB_NOOVERWRITE); + txn.commit(); + } catch (...) { + txn.abort(); + throw; + } + if (status == DB_KEYEXIST) { + return false; + } else { + p_.setPersistenceId(id); + return true; + } +} + +void MessageStoreImpl::destroy(db_ptr db, const qpid::broker::Persistable& p_) +{ + qpid::sys::Mutex::ScopedLock sl(bdbLock); + IdDbt key(p_.getPersistenceId()); + db->del(0, &key, DB_AUTO_COMMIT); +} + + +void MessageStoreImpl::bind(const qpid::broker::PersistableExchange& e_, + const qpid::broker::PersistableQueue& q_, + const std::string& k_, + const qpid::framing::FieldTable& a_) +{ + checkInit(); + IdDbt key(e_.getPersistenceId()); + BindingDbt value(e_, q_, k_, a_); + TxnCtxt txn; + txn.begin(dbenv.get(), true); + try { + put(bindingDb, txn.get(), key, value); + txn.commit(); + } catch (...) { + txn.abort(); + throw; + } +} + +void MessageStoreImpl::unbind(const qpid::broker::PersistableExchange& e_, + const qpid::broker::PersistableQueue& q_, + const std::string& k_, + const qpid::framing::FieldTable& /*a_*/) +{ + checkInit(); + deleteBinding(e_, q_, k_); +} + +void MessageStoreImpl::recover(qpid::broker::RecoveryManager& registry_) +{ + checkInit(); + txn_list prepared; + recoverLockedMappings(prepared); + + queue_index queues;//id->queue + exchange_index exchanges;//id->exchange + message_index messages;//id->message + + TxnCtxt txn; + txn.begin(dbenv.get(), false); + try { + //read all queues, calls recoversMessages + recoverQueues(txn, registry_, queues, prepared, messages); + + //recover exchange & bindings: + recoverExchanges(txn, registry_, exchanges); + recoverBindings(txn, exchanges, queues); + + //recover general-purpose configuration + recoverGeneral(txn, registry_); + + txn.commit(); + } catch (const DbException& e) { + txn.abort(); + THROW_STORE_EXCEPTION_2("Error on recovery", e); + } catch (...) { + txn.abort(); + throw; + } + + //recover transactions: + for (txn_list::iterator i = prepared.begin(); i != prepared.end(); i++) { + const PreparedTransaction pt = *i; + if (mgmtObject.get() != 0) { + mgmtObject->inc_tplTransactionDepth(); + mgmtObject->inc_tplTxnPrepares(); + } + + std::string xid = pt.xid; + + // Restore data token state in TxnCtxt + TplRecoverMapCitr citr = tplRecoverMap.find(xid); + if (citr == tplRecoverMap.end()) THROW_STORE_EXCEPTION("XID not found in tplRecoverMap"); + + // If a record is found that is dequeued but not committed/aborted from tplStore, then a complete() call + // was interrupted part way through committing/aborting the impacted queues. Complete this process. + bool incomplTplTxnFlag = citr->second.deq_flag; + + if (citr->second.tpc_flag) { + // Dtx (2PC) transaction + TPCTxnCtxt* tpcc = new TPCTxnCtxt(xid, &messageIdSequence); + std::auto_ptr<qpid::broker::TPCTransactionContext> txn(tpcc); + tpcc->recoverDtok(citr->second.rid, xid); + tpcc->prepare(tplStorePtr.get()); + + qpid::broker::RecoverableTransaction::shared_ptr dtx; + if (!incomplTplTxnFlag) dtx = registry_.recoverTransaction(xid, txn); + if (pt.enqueues.get()) { + for (LockedMappings::iterator j = pt.enqueues->begin(); j != pt.enqueues->end(); j++) { + tpcc->addXidRecord(queues[j->first]->getExternalQueueStore()); + if (!incomplTplTxnFlag) dtx->enqueue(queues[j->first], messages[j->second]); + } + } + if (pt.dequeues.get()) { + for (LockedMappings::iterator j = pt.dequeues->begin(); j != pt.dequeues->end(); j++) { + tpcc->addXidRecord(queues[j->first]->getExternalQueueStore()); + if (!incomplTplTxnFlag) dtx->dequeue(queues[j->first], messages[j->second]); + } + } + + if (incomplTplTxnFlag) { + tpcc->complete(citr->second.commit_flag); + } + } else { + // Local (1PC) transaction + boost::shared_ptr<TxnCtxt> opcc(new TxnCtxt(xid, &messageIdSequence)); + opcc->recoverDtok(citr->second.rid, xid); + opcc->prepare(tplStorePtr.get()); + + if (pt.enqueues.get()) { + for (LockedMappings::iterator j = pt.enqueues->begin(); j != pt.enqueues->end(); j++) { + opcc->addXidRecord(queues[j->first]->getExternalQueueStore()); + } + } + if (pt.dequeues.get()) { + for (LockedMappings::iterator j = pt.dequeues->begin(); j != pt.dequeues->end(); j++) { + opcc->addXidRecord(queues[j->first]->getExternalQueueStore()); + } + } + if (incomplTplTxnFlag) { + opcc->complete(citr->second.commit_flag); + } else { + completed(*opcc.get(), citr->second.commit_flag); + } + } + } + registry_.recoveryComplete(); +} + +void MessageStoreImpl::recoverQueues(TxnCtxt& txn, + qpid::broker::RecoveryManager& registry, + queue_index& queue_index, + txn_list& prepared, + message_index& messages) +{ + QLS_LOG(info, "*** MessageStoreImpl::recoverQueues()"); + Cursor queues; + queues.open(queueDb, txn.get()); + + uint64_t maxQueueId(1); + + IdDbt key; + Dbt value; + //read all queues + while (queues.next(key, value)) { + qpid::framing::Buffer buffer(reinterpret_cast<char*>(value.get_data()), value.get_size()); + //create a Queue instance + qpid::broker::RecoverableQueue::shared_ptr queue = registry.recoverQueue(buffer); + //set the persistenceId and update max as required + queue->setPersistenceId(key.id); + + const std::string queueName = queue->getName().c_str(); + JournalImpl* jQueue = 0; + if (queueName.size() == 0) + { + QLS_LOG(error, "Cannot recover empty (null) queue name - ignoring and attempting to continue."); + break; + } + jQueue = new JournalImpl(broker->getTimer(), queueName, getJrnlDir(queueName),jrnlLog, + defJournalGetEventsTimeout, defJournalFlushTimeout, agent, + boost::bind(&MessageStoreImpl::journalDeleted, this, _1)); + { + qpid::sys::Mutex::ScopedLock sl(journalListLock); + journalList[queueName] = jQueue; + } + queue->setExternalQueueStore(dynamic_cast<qpid::broker::ExternalQueueStore*>(jQueue)); + + try + { + long rcnt = 0L; // recovered msg count + long idcnt = 0L; // in-doubt msg count + uint64_t thisHighestRid = 0ULL; + jQueue->recover(boost::dynamic_pointer_cast<qpid::qls_jrnl::EmptyFilePoolManager>(efpMgr), wCacheNumPages, wCachePgSizeSblks, &prepared, thisHighestRid, key.id); + + // Check for changes to queue store settings qpid.file_count and qpid.file_size resulting + // from recovery of a store that has had its size changed externally by the resize utility. + // If so, update the queue store settings so that QMF queries will reflect the new values. +/* + const qpid::framing::FieldTable& storeargs = queue->getSettings().storeSettings; + qpid::framing::FieldTable::ValuePtr value; + value = storeargs.get("qpid.file_count"); + if (value.get() != 0 && !value->empty() && value->convertsTo<int>() && (uint16_t)value->get<int>() != jQueue->num_jfiles()) { + queue->addArgument("qpid.file_count", jQueue->num_jfiles()); + } + value = storeargs.get("qpid.file_size"); + if (value.get() != 0 && !value->empty() && value->convertsTo<int>() && (uint32_t)value->get<int>() != jQueue->jfsize_sblks()/JRNL_RMGR_PAGE_SIZE) { + queue->addArgument("qpid.file_size", jQueue->jfsize_sblks()/JRNL_RMGR_PAGE_SIZE); + } +*/ + + if (highestRid == 0ULL) + highestRid = thisHighestRid; + else if (thisHighestRid - highestRid < 0x8000000000000000ULL) // RFC 1982 comparison for unsigned 64-bit + highestRid = thisHighestRid; + recoverMessages(txn, registry, queue, prepared, messages, rcnt, idcnt); + QLS_LOG(info, "Recovered queue \"" << queueName << "\": " << rcnt << " messages recovered; " << idcnt << " messages in-doubt."); + jQueue->recover_complete(); // start journal. + } catch (const qpid::qls_jrnl::jexception& e) { + THROW_STORE_EXCEPTION(std::string("Queue ") + queueName + ": recoverQueues() failed: " + e.what()); + } + //read all messages: done on a per queue basis if using Journal + + queue_index[key.id] = queue; + maxQueueId = std::max(key.id, maxQueueId); + } + + // NOTE: highestRid is set by both recoverQueues() and recoverTplStore() as + // the messageIdSequence is used for both queue journals and the tpl journal. + messageIdSequence.reset(highestRid + 1); + QLS_LOG(info, "Most recent persistence id found: 0x" << std::hex << highestRid << std::dec); + + queueIdSequence.reset(maxQueueId + 1); +} + + +void MessageStoreImpl::recoverExchanges(TxnCtxt& txn_, + qpid::broker::RecoveryManager& registry_, + exchange_index& index_) +{ + //TODO: this is a copy&paste from recoverQueues - refactor! + Cursor exchanges; + exchanges.open(exchangeDb, txn_.get()); + + uint64_t maxExchangeId(1); + IdDbt key; + Dbt value; + //read all exchanges + while (exchanges.next(key, value)) { + qpid::framing::Buffer buffer(reinterpret_cast<char*>(value.get_data()), value.get_size()); + //create a Exchange instance + qpid::broker::RecoverableExchange::shared_ptr exchange = registry_.recoverExchange(buffer); + if (exchange) { + //set the persistenceId and update max as required + exchange->setPersistenceId(key.id); + index_[key.id] = exchange; + QLS_LOG(info, "Recovered exchange \"" << exchange->getName() << '"'); + } + maxExchangeId = std::max(key.id, maxExchangeId); + } + exchangeIdSequence.reset(maxExchangeId + 1); +} + +void MessageStoreImpl::recoverBindings(TxnCtxt& txn_, + exchange_index& exchanges_, + queue_index& queues_) +{ + Cursor bindings; + bindings.open(bindingDb, txn_.get()); + + IdDbt key; + Dbt value; + while (bindings.next(key, value)) { + qpid::framing::Buffer buffer(reinterpret_cast<char*>(value.get_data()), value.get_size()); + if (buffer.available() < 8) { + QLS_LOG(error, "Not enough data for binding: " << buffer.available()); + THROW_STORE_EXCEPTION("Not enough data for binding"); + } + uint64_t queueId = buffer.getLongLong(); + std::string queueName; + std::string routingkey; + qpid::framing::FieldTable args; + buffer.getShortString(queueName); + buffer.getShortString(routingkey); + buffer.get(args); + exchange_index::iterator exchange = exchanges_.find(key.id); + queue_index::iterator queue = queues_.find(queueId); + if (exchange != exchanges_.end() && queue != queues_.end()) { + //could use the recoverable queue here rather than the name... + exchange->second->bind(queueName, routingkey, args); + QLS_LOG(info, "Recovered binding exchange=" << exchange->second->getName() + << " key=" << routingkey + << " queue=" << queueName); + } else { + //stale binding, delete it + QLS_LOG(warning, "Deleting stale binding"); + bindings->del(0); + } + } +} + +void MessageStoreImpl::recoverGeneral(TxnCtxt& txn_, + qpid::broker::RecoveryManager& registry_) +{ + Cursor items; + items.open(generalDb, txn_.get()); + + uint64_t maxGeneralId(1); + IdDbt key; + Dbt value; + //read all items + while (items.next(key, value)) { + qpid::framing::Buffer buffer(reinterpret_cast<char*>(value.get_data()), value.get_size()); + //create instance + qpid::broker::RecoverableConfig::shared_ptr config = registry_.recoverConfig(buffer); + //set the persistenceId and update max as required + config->setPersistenceId(key.id); + maxGeneralId = std::max(key.id, maxGeneralId); + } + generalIdSequence.reset(maxGeneralId + 1); +} + +void MessageStoreImpl::recoverMessages(TxnCtxt& /*txn*/, + qpid::broker::RecoveryManager& recovery, + qpid::broker::RecoverableQueue::shared_ptr& queue, + txn_list& prepared, + message_index& messages, + long& rcnt, + long& idcnt) +{ + QLS_LOG(info, "*** MessageStoreImpl::recoverMessages() queue=\"" << queue->getName() << "\""); + size_t preambleLength = sizeof(uint32_t)/*header size*/; + + JournalImpl* jc = static_cast<JournalImpl*>(queue->getExternalQueueStore()); + DataTokenImpl dtok; + size_t readSize = 0; + unsigned msg_count = 0; + + // TODO: This optimization to skip reading if there are no enqueued messages to read + // breaks the python system test in phase 6 with "Exception: Cannot write lock file" + // Figure out what is breaking. + //bool read = jc->get_enq_cnt() > 0; + bool read = true; + + void* dbuff = NULL; size_t dbuffSize = 0; + void* xidbuff = NULL; size_t xidbuffSize = 0; + bool transientFlag = false; + bool externalFlag = false; + + dtok.set_wstate(DataTokenImpl::ENQ); + + // Read the message from the Journal. + try { + unsigned aio_sleep_cnt = 0; + while (read) { + qpid::qls_jrnl::iores res = jc->read_data_record(&dbuff, dbuffSize, &xidbuff, xidbuffSize, transientFlag, externalFlag, &dtok); + readSize = dtok.dsize(); + + switch (res) + { + case qpid::qls_jrnl::RHM_IORES_SUCCESS: { + msg_count++; + qpid::broker::RecoverableMessage::shared_ptr msg; + char* data = (char*)dbuff; + + unsigned headerSize; + if (externalFlag) { + msg = getExternMessage(recovery, dtok.rid(), headerSize); // large message external to jrnl + } else { + headerSize = qpid::framing::Buffer(data, preambleLength).getLong(); + qpid::framing::Buffer headerBuff(data+ preambleLength, headerSize); /// do we want read size or header size ???? + msg = recovery.recoverMessage(headerBuff); + } + msg->setPersistenceId(dtok.rid()); + // At some future point if delivery attempts are stored, then this call would + // become optional depending on that information. + msg->setRedelivered(); + // Reset the TTL for the recovered message + msg->computeExpiration(broker->getExpiryPolicy()); + + uint32_t contentOffset = headerSize + preambleLength; + uint64_t contentSize = readSize - contentOffset; + if (msg->loadContent(contentSize) && !externalFlag) { + //now read the content + qpid::framing::Buffer contentBuff(data + contentOffset, contentSize); + msg->decodeContent(contentBuff); + } + + PreparedTransaction::list::iterator i = PreparedTransaction::getLockedPreparedTransaction(prepared, queue->getPersistenceId(), dtok.rid()); + if (i == prepared.end()) { // not in prepared list + rcnt++; + queue->recover(msg); + } else { + uint64_t rid = dtok.rid(); + std::string xid(i->xid); + TplRecoverMapCitr citr = tplRecoverMap.find(xid); + if (citr == tplRecoverMap.end()) THROW_STORE_EXCEPTION("XID not found in tplRecoverMap"); + + // deq present in prepared list: this xid is part of incomplete txn commit/abort + // or this is a 1PC txn that must be rolled forward + if (citr->second.deq_flag || !citr->second.tpc_flag) { + if (jc->is_enqueued(rid, true)) { + // Enqueue is non-tx, dequeue tx + assert(jc->is_locked(rid)); // This record MUST be locked by a txn dequeue + if (!citr->second.commit_flag) { + rcnt++; + queue->recover(msg); // recover message in abort case only + } + } else { + // Enqueue and/or dequeue tx + qpid::qls_jrnl::txn_map& tmap = jc->get_txn_map(); + qpid::qls_jrnl::txn_data_list txnList = tmap.get_tdata_list(xid); // txnList will be empty if xid not found + bool enq = false; + bool deq = false; + for (qpid::qls_jrnl::tdl_itr j = txnList.begin(); j<txnList.end(); j++) { + if (j->_enq_flag && j->_rid == rid) enq = true; + else if (!j->_enq_flag && j->_drid == rid) deq = true; + } + if (enq && !deq && citr->second.commit_flag) { + rcnt++; + queue->recover(msg); // recover txn message in commit case only + } + } + } else { + idcnt++; + messages[rid] = msg; + } + } + + dtok.reset(); + dtok.set_wstate(DataTokenImpl::ENQ); + + if (xidbuff) + ::free(xidbuff); + else if (dbuff) + ::free(dbuff); + aio_sleep_cnt = 0; + break; + } + case qpid::qls_jrnl::RHM_IORES_PAGE_AIOWAIT: + if (++aio_sleep_cnt > MAX_AIO_SLEEPS) + THROW_STORE_EXCEPTION("Timeout waiting for AIO in MessageStoreImpl::recoverMessages()"); + ::usleep(AIO_SLEEP_TIME_US); + break; + case qpid::qls_jrnl::RHM_IORES_EMPTY: + read = false; + break; // done with all messages. (add call in jrnl to test that _emap is empty.) + default: + std::ostringstream oss; + oss << "recoverMessages(): Queue: " << queue->getName() << ": Unexpected return from journal read: " << qpid::qls_jrnl::iores_str(res); + THROW_STORE_EXCEPTION(oss.str()); + } // switch + } // while + } catch (const qpid::qls_jrnl::jexception& e) { + THROW_STORE_EXCEPTION(std::string("Queue ") + queue->getName() + ": recoverMessages() failed: " + e.what()); + } +} + +qpid::broker::RecoverableMessage::shared_ptr MessageStoreImpl::getExternMessage(qpid::broker::RecoveryManager& /*recovery*/, + uint64_t /*messageId*/, + unsigned& /*headerSize*/) +{ + throw qpid::qls_jrnl::jexception(qpid::qls_jrnl::jerrno::JERR__NOTIMPL, "MessageStoreImpl", "getExternMessage"); +} + +int MessageStoreImpl::enqueueMessage(TxnCtxt& txn_, + IdDbt& msgId_, + qpid::broker::RecoverableMessage::shared_ptr& msg_, + queue_index& index_, + txn_list& prepared_, + message_index& messages_) +{ + Cursor mappings; + mappings.open(mappingDb, txn_.get()); + + IdDbt value; + + int count(0); + for (int status = mappings->get(&msgId_, &value, DB_SET); status == 0; status = mappings->get(&msgId_, &value, DB_NEXT_DUP)) { + if (index_.find(value.id) == index_.end()) { + QLS_LOG(warning, "Recovered message for queue that no longer exists"); + mappings->del(0); + } else { + qpid::broker::RecoverableQueue::shared_ptr queue = index_[value.id]; + if (PreparedTransaction::isLocked(prepared_, value.id, msgId_.id)) { + messages_[msgId_.id] = msg_; + } else { + queue->recover(msg_); + } + count++; + } + } + mappings.close(); + return count; +} + +void MessageStoreImpl::readTplStore() +{ + QLS_LOG(info, "*** MessageStoreImpl::readTplStore()"); + tplRecoverMap.clear(); + qpid::qls_jrnl::txn_map& tmap = tplStorePtr->get_txn_map(); + DataTokenImpl dtok; + void* dbuff = NULL; size_t dbuffSize = 0; + void* xidbuff = NULL; size_t xidbuffSize = 0; + bool transientFlag = false; + bool externalFlag = false; + bool done = false; + try { + unsigned aio_sleep_cnt = 0; + while (!done) { + dtok.reset(); + dtok.set_wstate(DataTokenImpl::ENQ); + qpid::qls_jrnl::iores res = tplStorePtr->read_data_record(&dbuff, dbuffSize, &xidbuff, xidbuffSize, transientFlag, externalFlag, &dtok); + switch (res) { + case qpid::qls_jrnl::RHM_IORES_SUCCESS: { + // Every TPL record contains both data and an XID + assert(dbuffSize>0); + assert(xidbuffSize>0); + std::string xid(static_cast<const char*>(xidbuff), xidbuffSize); + bool is2PC = *(static_cast<char*>(dbuff)) != 0; + + // Check transaction details; add to recover map + qpid::qls_jrnl::txn_data_list txnList = tmap.get_tdata_list(xid); // txnList will be empty if xid not found + if (!txnList.empty()) { // xid found in tmap + unsigned enqCnt = 0; + unsigned deqCnt = 0; + uint64_t rid = 0; + + // Assume commit (roll forward) in cases where only prepare has been called - ie only enqueue record exists. + // Note: will apply to both 1PC and 2PC transactions. + bool commitFlag = true; + + for (qpid::qls_jrnl::tdl_itr j = txnList.begin(); j<txnList.end(); j++) { + if (j->_enq_flag) { + rid = j->_rid; + enqCnt++; + } else { + commitFlag = j->_commit_flag; + deqCnt++; + } + } + assert(enqCnt == 1); + assert(deqCnt <= 1); + tplRecoverMap.insert(TplRecoverMapPair(xid, TplRecoverStruct(rid, deqCnt == 1, commitFlag, is2PC))); + } + + ::free(xidbuff); + aio_sleep_cnt = 0; + break; + } + case qpid::qls_jrnl::RHM_IORES_PAGE_AIOWAIT: + if (++aio_sleep_cnt > MAX_AIO_SLEEPS) + THROW_STORE_EXCEPTION("Timeout waiting for AIO in MessageStoreImpl::recoverTplStore()"); + ::usleep(AIO_SLEEP_TIME_US); + break; + case qpid::qls_jrnl::RHM_IORES_EMPTY: + done = true; + break; // done with all messages. (add call in jrnl to test that _emap is empty.) + default: + std::ostringstream oss; + oss << "readTplStore(): Unexpected result from journal read: " << qpid::qls_jrnl::iores_str(res); + THROW_STORE_EXCEPTION(oss.str()); + } // switch + } + } catch (const qpid::qls_jrnl::jexception& e) { + THROW_STORE_EXCEPTION(std::string("TPL recoverTplStore() failed: ") + e.what()); + } +} + +void MessageStoreImpl::recoverTplStore() +{ + QLS_LOG(info, "*** MessageStoreImpl::recoverTplStore()"); +/* + if (qpid::qls_jrnl::jdir::exists(tplStorePtr->jrnl_dir() + tplStorePtr->base_filename() + ".jinf")) { + uint64_t thisHighestRid = 0ULL; + tplStorePtr->recover(tplNumJrnlFiles, false, 0, tplJrnlFsizeSblks, tplWCachePgSizeSblks, tplWCacheNumPages, 0, thisHighestRid, 0); + if (highestRid == 0ULL) + highestRid = thisHighestRid; + else if (thisHighestRid - highestRid < 0x8000000000000000ULL) // RFC 1982 comparison for unsigned 64-bit + highestRid = thisHighestRid; + + // Load tplRecoverMap by reading the TPL store + readTplStore(); + + tplStorePtr->recover_complete(); // start journal. + } +*/ +} + +void MessageStoreImpl::recoverLockedMappings(txn_list& txns) +{ + QLS_LOG(info, "*** MessageStoreImpl::recoverLockedMappings()"); + if (!tplStorePtr->is_ready()) + recoverTplStore(); + + // Abort unprepared xids and populate the locked maps + for (TplRecoverMapCitr i = tplRecoverMap.begin(); i != tplRecoverMap.end(); i++) { + LockedMappings::shared_ptr enq_ptr; + enq_ptr.reset(new LockedMappings); + LockedMappings::shared_ptr deq_ptr; + deq_ptr.reset(new LockedMappings); + txns.push_back(new PreparedTransaction(i->first, enq_ptr, deq_ptr)); + } +} + +void MessageStoreImpl::collectPreparedXids(std::set<std::string>& /*xids*/) +{ + QLS_LOG(info, "*** MessageStoreImpl::collectPreparedXids()"); +/* + if (tplStorePtr->is_ready()) { + tplStorePtr->read_reset(); + readTplStore(); + } else { + recoverTplStore(); + } + for (TplRecoverMapCitr i = tplRecoverMap.begin(); i != tplRecoverMap.end(); i++) { + // Discard all txns that are to be rolled forward/back and 1PC transactions + if (!i->second.deq_flag && i->second.tpc_flag) + xids.insert(i->first); + } +*/ +} + +void MessageStoreImpl::stage(const boost::intrusive_ptr<qpid::broker::PersistableMessage>& /*msg*/) +{ + throw qpid::qls_jrnl::jexception(qpid::qls_jrnl::jerrno::JERR__NOTIMPL, "MessageStoreImpl", "stage"); +} + +void MessageStoreImpl::destroy(qpid::broker::PersistableMessage& /*msg*/) +{ + throw qpid::qls_jrnl::jexception(qpid::qls_jrnl::jerrno::JERR__NOTIMPL, "MessageStoreImpl", "destroy"); +} + +void MessageStoreImpl::appendContent(const boost::intrusive_ptr<const qpid::broker::PersistableMessage>& /*msg*/, + const std::string& /*data*/) +{ + throw qpid::qls_jrnl::jexception(qpid::qls_jrnl::jerrno::JERR__NOTIMPL, "MessageStoreImpl", "appendContent"); +} + +void MessageStoreImpl::loadContent(const qpid::broker::PersistableQueue& /*queue*/, + const boost::intrusive_ptr<const qpid::broker::PersistableMessage>& /*msg*/, + std::string& /*data*/, + uint64_t /*offset*/, + uint32_t /*length*/) +{ + throw qpid::qls_jrnl::jexception(qpid::qls_jrnl::jerrno::JERR__NOTIMPL, "MessageStoreImpl", "loadContent"); +/* + checkInit(); + uint64_t messageId (msg->getPersistenceId()); + + if (messageId != 0) { + try { + JournalImpl* jc = static_cast<JournalImpl*>(queue.getExternalQueueStore()); + if (jc && jc->is_enqueued(messageId) ) { + if (!jc->loadMsgContent(messageId, data, length, offset)) { + std::ostringstream oss; + oss << "Queue " << queue.getName() << ": loadContent() failed: Message " << messageId << " is extern"; + THROW_STORE_EXCEPTION(oss.str()); + } + } else { + std::ostringstream oss; + oss << "Queue " << queue.getName() << ": loadContent() failed: Message " << messageId << " not enqueued"; + THROW_STORE_EXCEPTION(oss.str()); + } + } catch (const qpid::qls_jrnl::jexception& e) { + THROW_STORE_EXCEPTION(std::string("Queue ") + queue.getName() + ": loadContent() failed: " + e.what()); + } + } else { + THROW_STORE_EXCEPTION("Cannot load content. Message not known to store!"); + } +*/ +} + +void MessageStoreImpl::flush(const qpid::broker::PersistableQueue& queue_) +{ +// QLS_LOG(info, "*** MessageStoreImpl::flush() queue=\"" << queue_.getName() << "\""); + if (queue_.getExternalQueueStore() == 0) return; + checkInit(); + std::string qn = queue_.getName(); + try { + JournalImpl* jc = static_cast<JournalImpl*>(queue_.getExternalQueueStore()); + if (jc) { + // TODO: check if this result should be used... + /*mrg::journal::iores res =*/ jc->flush(); + } + } catch (const qpid::qls_jrnl::jexception& e) { + THROW_STORE_EXCEPTION(std::string("Queue ") + qn + ": flush() failed: " + e.what() ); + } +} + +void MessageStoreImpl::enqueue(qpid::broker::TransactionContext* ctxt_, + const boost::intrusive_ptr<qpid::broker::PersistableMessage>& msg_, + const qpid::broker::PersistableQueue& queue_) +{ + //QLS_LOG(info, "*** MessageStoreImpl::enqueue() queue=\"" << queue_.getName() << "\""); + checkInit(); + uint64_t queueId (queue_.getPersistenceId()); + if (queueId == 0) { + THROW_STORE_EXCEPTION("Queue not created: " + queue_.getName()); + } + + TxnCtxt implicit; + TxnCtxt* txn = 0; + if (ctxt_) { + txn = check(ctxt_); + } else { + txn = &implicit; + } + + if (msg_->getPersistenceId() == 0) { + msg_->setPersistenceId(messageIdSequence.next()); + } + store(&queue_, txn, msg_); + + // add queue* to the txn map.. + if (ctxt_) txn->addXidRecord(queue_.getExternalQueueStore()); +} + +uint64_t MessageStoreImpl::msgEncode(std::vector<char>& buff_, + const boost::intrusive_ptr<qpid::broker::PersistableMessage>& message_) +{ + uint32_t headerSize = message_->encodedHeaderSize(); + uint64_t size = message_->encodedSize() + sizeof(uint32_t); + try { buff_ = std::vector<char>(size); } // long + headers + content + catch (const std::exception& e) { + std::ostringstream oss; + oss << "Unable to allocate memory for encoding message; requested size: " << size << "; error: " << e.what(); + THROW_STORE_EXCEPTION(oss.str()); + } + qpid::framing::Buffer buffer(&buff_[0],size); + buffer.putLong(headerSize); + message_->encode(buffer); + return size; +} + +void MessageStoreImpl::store(const qpid::broker::PersistableQueue* queue_, + TxnCtxt* txn_, + const boost::intrusive_ptr<qpid::broker::PersistableMessage>& message_) +{ + //QLS_LOG(info, "*** MessageStoreImpl::store() queue=\"" << queue_->getName() << "\""); + std::vector<char> buff; + uint64_t size = msgEncode(buff, message_); + + try { + if (queue_) { + boost::intrusive_ptr<DataTokenImpl> dtokp(new DataTokenImpl); + dtokp->addRef(); + dtokp->setSourceMessage(message_); + dtokp->set_external_rid(true); + dtokp->set_rid(message_->getPersistenceId()); // set the messageID into the Journal header (record-id) + + JournalImpl* jc = static_cast<JournalImpl*>(queue_->getExternalQueueStore()); + if (txn_->getXid().empty()) { + jc->enqueue_data_record(&buff[0], size, size, dtokp.get(), !message_->isPersistent()); + } else { + jc->enqueue_txn_data_record(&buff[0], size, size, dtokp.get(), txn_->getXid(), !message_->isPersistent()); + } + } else { + THROW_STORE_EXCEPTION(std::string("MessageStoreImpl::store() failed: queue NULL.")); + } + } catch (const qpid::qls_jrnl::jexception& e) { + THROW_STORE_EXCEPTION(std::string("Queue ") + queue_->getName() + ": MessageStoreImpl::store() failed: " + + e.what()); + } +} + +void MessageStoreImpl::dequeue(qpid::broker::TransactionContext* ctxt_, + const boost::intrusive_ptr<qpid::broker::PersistableMessage>& msg_, + const qpid::broker::PersistableQueue& queue_) +{ + //QLS_LOG(info, "*** MessageStoreImpl::dequeue() queue=\"" << queue_.getName() << "\""); + checkInit(); + uint64_t queueId (queue_.getPersistenceId()); + uint64_t messageId (msg_->getPersistenceId()); + if (queueId == 0) { + THROW_STORE_EXCEPTION("Queue \"" + queue_.getName() + "\" has null queue Id (has not been created)"); + } + if (messageId == 0) { + THROW_STORE_EXCEPTION("Queue \"" + queue_.getName() + "\": Dequeuing message with null persistence Id."); + } + + TxnCtxt implicit; + TxnCtxt* txn = 0; + if (ctxt_) { + txn = check(ctxt_); + } else { + txn = &implicit; + } + + // add queue* to the txn map.. + if (ctxt_) txn->addXidRecord(queue_.getExternalQueueStore()); + async_dequeue(ctxt_, msg_, queue_); + msg_->dequeueComplete(); +} + +void MessageStoreImpl::async_dequeue(qpid::broker::TransactionContext* ctxt_, + const boost::intrusive_ptr<qpid::broker::PersistableMessage>& msg_, + const qpid::broker::PersistableQueue& queue_) +{ + //QLS_LOG(info, "*** MessageStoreImpl::async_dequeue() queue=\"" << queue_.getName() << "\""); + boost::intrusive_ptr<DataTokenImpl> ddtokp(new DataTokenImpl); + ddtokp->setSourceMessage(msg_); + ddtokp->set_external_rid(true); + ddtokp->set_rid(messageIdSequence.next()); + ddtokp->set_dequeue_rid(msg_->getPersistenceId()); + ddtokp->set_wstate(DataTokenImpl::ENQ); + std::string tid; + if (ctxt_) { + TxnCtxt* txn = check(ctxt_); + tid = txn->getXid(); + } + // Manually increase the ref count, as raw pointers are used beyond this point + ddtokp->addRef(); + try { + JournalImpl* jc = static_cast<JournalImpl*>(queue_.getExternalQueueStore()); + if (tid.empty()) { + jc->dequeue_data_record(ddtokp.get()); + } else { + jc->dequeue_txn_data_record(ddtokp.get(), tid); + } + } catch (const qpid::qls_jrnl::jexception& e) { + ddtokp->release(); + THROW_STORE_EXCEPTION(std::string("Queue ") + queue_.getName() + ": async_dequeue() failed: " + e.what()); + } +} + +uint32_t MessageStoreImpl::outstandingQueueAIO(const qpid::broker::PersistableQueue& /*queue_*/) +{ +/* checkInit();*/ + return 0; +} + +void MessageStoreImpl::completed(TxnCtxt& txn_, + bool commit_) +{ + try { + chkTplStoreInit(); // Late initialize (if needed) + + // Nothing to do if not prepared + if (txn_.getDtok()->is_enqueued()) { + txn_.incrDtokRef(); + DataTokenImpl* dtokp = txn_.getDtok(); + dtokp->set_dequeue_rid(dtokp->rid()); + dtokp->set_rid(messageIdSequence.next()); + tplStorePtr->dequeue_txn_data_record(txn_.getDtok(), txn_.getXid(), commit_); + } + txn_.complete(commit_); + if (mgmtObject.get() != 0) { + mgmtObject->dec_tplTransactionDepth(); + if (commit_) + mgmtObject->inc_tplTxnCommits(); + else + mgmtObject->inc_tplTxnAborts(); + } + } catch (const std::exception& e) { + QLS_LOG(error, "Error completing xid " << txn_.getXid() << ": " << e.what()); + throw; + } +} + +std::auto_ptr<qpid::broker::TransactionContext> MessageStoreImpl::begin() +{ + checkInit(); + // pass sequence number for c/a + return std::auto_ptr<qpid::broker::TransactionContext>(new TxnCtxt(&messageIdSequence)); +} + +std::auto_ptr<qpid::broker::TPCTransactionContext> MessageStoreImpl::begin(const std::string& xid_) +{ + checkInit(); + IdSequence* jtx = &messageIdSequence; + // pass sequence number for c/a + return std::auto_ptr<qpid::broker::TPCTransactionContext>(new TPCTxnCtxt(xid_, jtx)); +} + +void MessageStoreImpl::prepare(qpid::broker::TPCTransactionContext& ctxt_) +{ + checkInit(); + TxnCtxt* txn = dynamic_cast<TxnCtxt*>(&ctxt_); + if(!txn) throw qpid::broker::InvalidTransactionContextException(); + localPrepare(txn); +} + +void MessageStoreImpl::localPrepare(TxnCtxt* ctxt_) +{ + try { + chkTplStoreInit(); // Late initialize (if needed) + + // This sync is required to ensure multi-queue atomicity - ie all txn data + // must hit the disk on *all* queues before the TPL prepare (enq) is written. + ctxt_->sync(); + + ctxt_->incrDtokRef(); + DataTokenImpl* dtokp = ctxt_->getDtok(); + dtokp->set_external_rid(true); + dtokp->set_rid(messageIdSequence.next()); + char tpcFlag = static_cast<char>(ctxt_->isTPC()); + tplStorePtr->enqueue_txn_data_record(&tpcFlag, sizeof(char), sizeof(char), dtokp, ctxt_->getXid(), false); + ctxt_->prepare(tplStorePtr.get()); + // make sure all the data is written to disk before returning + ctxt_->sync(); + if (mgmtObject.get() != 0) { + mgmtObject->inc_tplTransactionDepth(); + mgmtObject->inc_tplTxnPrepares(); + } + } catch (const std::exception& e) { + QLS_LOG(error, "Error preparing xid " << ctxt_->getXid() << ": " << e.what()); + throw; + } +} + +void MessageStoreImpl::commit(qpid::broker::TransactionContext& ctxt_) +{ + checkInit(); + TxnCtxt* txn(check(&ctxt_)); + if (!txn->isTPC()) { + if (txn->impactedQueuesEmpty()) return; + localPrepare(dynamic_cast<TxnCtxt*>(txn)); + } + completed(*dynamic_cast<TxnCtxt*>(txn), true); +} + +void MessageStoreImpl::abort(qpid::broker::TransactionContext& ctxt_) +{ + checkInit(); + TxnCtxt* txn(check(&ctxt_)); + if (!txn->isTPC()) { + if (txn->impactedQueuesEmpty()) return; + localPrepare(dynamic_cast<TxnCtxt*>(txn)); + } + completed(*dynamic_cast<TxnCtxt*>(txn), false); +} + +TxnCtxt* MessageStoreImpl::check(qpid::broker::TransactionContext* ctxt_) +{ + TxnCtxt* txn = dynamic_cast<TxnCtxt*>(ctxt_); + if(!txn) throw qpid::broker::InvalidTransactionContextException(); + return txn; +} + +void MessageStoreImpl::put(db_ptr db_, + DbTxn* txn_, + Dbt& key_, + Dbt& value_) +{ + try { + int status = db_->put(txn_, &key_, &value_, DB_NODUPDATA); + if (status == DB_KEYEXIST) { + THROW_STORE_EXCEPTION("duplicate data"); + } else if (status) { + THROW_STORE_EXCEPTION(DbEnv::strerror(status)); + } + } catch (const DbException& e) { + THROW_STORE_EXCEPTION(e.what()); + } +} + +void MessageStoreImpl::deleteBindingsForQueue(const qpid::broker::PersistableQueue& queue_) +{ + TxnCtxt txn; + txn.begin(dbenv.get(), true); + try { + { + Cursor bindings; + bindings.open(bindingDb, txn.get()); + + IdDbt key; + Dbt value; + while (bindings.next(key, value)) { + qpid::framing::Buffer buffer(reinterpret_cast<char*>(value.get_data()), value.get_size()); + if (buffer.available() < 8) { + THROW_STORE_EXCEPTION("Not enough data for binding"); + } + uint64_t queueId = buffer.getLongLong(); + if (queue_.getPersistenceId() == queueId) { + bindings->del(0); + QLS_LOG(debug, "Deleting binding for " << queue_.getName() << " " << key.id << "->" << queueId); + } + } + } + txn.commit(); + } catch (const std::exception& e) { + txn.abort(); + THROW_STORE_EXCEPTION_2("Error deleting bindings", e.what()); + } catch (...) { + txn.abort(); + throw; + } + QLS_LOG(debug, "Deleted all bindings for " << queue_.getName() << ":" << queue_.getPersistenceId()); +} + +void MessageStoreImpl::deleteBinding(const qpid::broker::PersistableExchange& exchange_, + const qpid::broker::PersistableQueue& queue_, + const std::string& bkey_) +{ + TxnCtxt txn; + txn.begin(dbenv.get(), true); + try { + { + Cursor bindings; + bindings.open(bindingDb, txn.get()); + + IdDbt key(exchange_.getPersistenceId()); + Dbt value; + + for (int status = bindings->get(&key, &value, DB_SET); status == 0; status = bindings->get(&key, &value, DB_NEXT_DUP)) { + qpid::framing::Buffer buffer(reinterpret_cast<char*>(value.get_data()), value.get_size()); + if (buffer.available() < 8) { + THROW_STORE_EXCEPTION("Not enough data for binding"); + } + uint64_t queueId = buffer.getLongLong(); + if (queue_.getPersistenceId() == queueId) { + std::string q; + std::string k; + buffer.getShortString(q); + buffer.getShortString(k); + if (bkey_ == k) { + bindings->del(0); + QLS_LOG(debug, "Deleting binding for " << queue_.getName() << " " << key.id << "->" << queueId); + } + } + } + } + txn.commit(); + } catch (const std::exception& e) { + txn.abort(); + THROW_STORE_EXCEPTION_2("Error deleting bindings", e.what()); + } catch (...) { + txn.abort(); + throw; + } +} + +std::string MessageStoreImpl::getStoreTopLevelDir() { + std::ostringstream dir; + dir << storeDir << "/" << storeTopLevelDir; + return dir.str(); +} + + +std::string MessageStoreImpl::getJrnlBaseDir() +{ + std::ostringstream dir; + dir << storeDir << "/" << storeTopLevelDir << "/jrnl/" ; + return dir.str(); +} + +std::string MessageStoreImpl::getBdbBaseDir() +{ + std::ostringstream dir; + dir << storeDir << "/" << storeTopLevelDir << "/dat/" ; + return dir.str(); +} + +std::string MessageStoreImpl::getTplBaseDir() +{ + std::ostringstream dir; + dir << storeDir << "/" << storeTopLevelDir << "/tpl/" ; + return dir.str(); +} + +std::string MessageStoreImpl::getJrnlDir(const std::string& queueName_) +{ + std::ostringstream oss; + oss << getJrnlBaseDir() << queueName_; + return oss.str(); +} + +std::string MessageStoreImpl::getStoreDir() const { return storeDir; } + +void MessageStoreImpl::journalDeleted(JournalImpl& j_) { + qpid::sys::Mutex::ScopedLock sl(journalListLock); + journalList.erase(j_.id()); +} + +MessageStoreImpl::StoreOptions::StoreOptions(const std::string& name_) : + qpid::Options(name_), + truncateFlag(defTruncateFlag), + wCachePageSizeKib(defWCachePageSizeKib), + tplWCachePageSizeKib(defTplWCachePageSizeKib), + efpPartition(defEfpPartition), + efpFileSizeKib(defEfpFileSizeKib) +{ + addOptions() + ("store-dir", qpid::optValue(storeDir, "DIR"), + "Store directory location for persistence (instead of using --data-dir value). " + "Required if --no-data-dir is also used.") + ("truncate", qpid::optValue(truncateFlag, "yes|no"), + "If yes|true|1, will truncate the store (discard any existing records). If no|false|0, will preserve " + "the existing store files for recovery.") + ("wcache-page-size", qpid::optValue(wCachePageSizeKib, "N"), + "Size of the pages in the write page cache in KiB. " + "Allowable values - powers of 2: 1, 2, 4, ... , 128. " + "Lower values decrease latency at the expense of throughput.") + ("tpl-wcache-page-size", qpid::optValue(tplWCachePageSizeKib, "N"), + "Size of the pages in the transaction prepared list write page cache in KiB. " + "Allowable values - powers of 2: 1, 2, 4, ... , 128. " + "Lower values decrease latency at the expense of throughput.") + ("efp-partition", qpid::optValue(efpPartition, "N"), + "Empty File Pool partition to use for finding empty journal files") + ("efp-file-size", qpid::optValue(efpFileSizeKib, "N"), + "Empty File Pool file size in KiB to use for journal files. Must be a multiple of 4 KiB.") + ; +} + +}} diff --git a/qpid/cpp/src/qpid/linearstore/MessageStoreImpl.h b/qpid/cpp/src/qpid/linearstore/MessageStoreImpl.h new file mode 100644 index 0000000000..d1ce7a2b8a --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/MessageStoreImpl.h @@ -0,0 +1,368 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_MESSAGESTOREIMPL_H +#define QPID_LEGACYSTORE_MESSAGESTOREIMPL_H + +#include <iomanip> +#include <string> + +#include "db-inc.h" +#include "qpid/linearstore/Cursor.h" +#include "qpid/linearstore/IdDbt.h" +#include "qpid/linearstore/IdSequence.h" +#include "qpid/linearstore/JournalImpl.h" +#include "qpid/linearstore/JournalLogImpl.h" +#include "qpid/linearstore/jrnl/jcfg.h" +#include "qpid/linearstore/jrnl/EmptyFilePoolTypes.h" +#include "qpid/linearstore/PreparedTransaction.h" +#include "qpid/broker/Broker.h" +#include "qpid/broker/MessageStore.h" +#include "qpid/management/Manageable.h" +#include "qmf/org/apache/qpid/linearstore/Store.h" +#include "qpid/linearstore/TxnCtxt.h" + +// Assume DB_VERSION_MAJOR == 4 +#if (DB_VERSION_MINOR == 2) +#include <errno.h> +#define DB_BUFFER_SMALL ENOMEM +#endif + +namespace qpid { namespace sys { +class Timer; +}} + +namespace qpid{ +namespace qls_jrnl { +class EmptyFilePoolManager; +} +namespace linearstore{ + +/** + * An implementation of the MessageStore interface based on Berkeley DB + */ +class MessageStoreImpl : public qpid::broker::MessageStore, public qpid::management::Manageable +{ + public: + typedef boost::shared_ptr<Db> db_ptr; + typedef boost::shared_ptr<DbEnv> dbEnv_ptr; + + struct StoreOptions : public qpid::Options { + StoreOptions(const std::string& name="Linear Store Options"); + std::string clusterName; + std::string storeDir; + bool truncateFlag; + uint32_t wCachePageSizeKib; + uint32_t tplWCachePageSizeKib; + uint16_t efpPartition; + uint64_t efpFileSizeKib; + }; + + protected: + typedef std::map<uint64_t, qpid::broker::RecoverableQueue::shared_ptr> queue_index; + typedef std::map<uint64_t, qpid::broker::RecoverableExchange::shared_ptr> exchange_index; + typedef std::map<uint64_t, qpid::broker::RecoverableMessage::shared_ptr> message_index; + + typedef LockedMappings::map txn_lock_map; + typedef boost::ptr_list<PreparedTransaction> txn_list; + + // Structs for Transaction Recover List (TPL) recover state + struct TplRecoverStruct { + uint64_t rid; // rid of TPL record + bool deq_flag; + bool commit_flag; + bool tpc_flag; + TplRecoverStruct(const uint64_t _rid, const bool _deq_flag, const bool _commit_flag, const bool _tpc_flag); + }; + typedef TplRecoverStruct TplRecover; + typedef std::pair<std::string, TplRecover> TplRecoverMapPair; + typedef std::map<std::string, TplRecover> TplRecoverMap; + typedef TplRecoverMap::const_iterator TplRecoverMapCitr; + + typedef std::map<std::string, JournalImpl*> JournalListMap; + typedef JournalListMap::iterator JournalListMapItr; + + // Default store settings + static const bool defTruncateFlag = false; + static const uint32_t defWCachePageSizeKib = QLS_WMGR_DEF_PAGE_SIZE_KIB; + static const uint32_t defTplWCachePageSizeKib = defWCachePageSizeKib / 8; + static const uint16_t defEfpPartition = 1; + static const uint64_t defEfpFileSizeKib = 512 * QLS_SBLK_SIZE_KIB; + static const std::string storeTopLevelDir; + + static qpid::sys::Duration defJournalGetEventsTimeout; + static qpid::sys::Duration defJournalFlushTimeout; + + std::list<db_ptr> dbs; + dbEnv_ptr dbenv; + db_ptr queueDb; + db_ptr configDb; + db_ptr exchangeDb; + db_ptr mappingDb; + db_ptr bindingDb; + db_ptr generalDb; + + // Pointer to Transaction Prepared List (TPL) journal instance + boost::shared_ptr<TplJournalImpl> tplStorePtr; + TplRecoverMap tplRecoverMap; + qpid::sys::Mutex tplInitLock; + JournalListMap journalList; + qpid::sys::Mutex journalListLock; + qpid::sys::Mutex bdbLock; + + IdSequence queueIdSequence; + IdSequence exchangeIdSequence; + IdSequence generalIdSequence; + IdSequence messageIdSequence; + std::string storeDir; + qpid::qls_jrnl::efpPartitionNumber_t defaultEfpPartitionNumber; + qpid::qls_jrnl::efpDataSize_kib_t defaultEfpFileSize_kib; + bool truncateFlag; + uint32_t wCachePgSizeSblks; + uint16_t wCacheNumPages; + uint32_t tplWCachePgSizeSblks; + uint16_t tplWCacheNumPages; + uint64_t highestRid; + bool isInit; + const char* envPath; + qpid::broker::Broker* broker; + JournalLogImpl jrnlLog; + boost::shared_ptr<qpid::qls_jrnl::EmptyFilePoolManager> efpMgr; + + qmf::org::apache::qpid::linearstore::Store::shared_ptr mgmtObject; + qpid::management::ManagementAgent* agent; + + + // Parameter validation and calculation + static uint32_t chkJrnlWrPageCacheSize(const uint32_t param, + const std::string& paramName/*, + const uint16_t jrnlFsizePgs*/); + static uint16_t getJrnlWrNumPages(const uint32_t wrPageSizeKiB); + static qpid::qls_jrnl::efpPartitionNumber_t chkEfpPartition(const qpid::qls_jrnl::efpPartitionNumber_t partition, + const std::string& paramName); + static qpid::qls_jrnl::efpDataSize_kib_t chkEfpFileSizeKiB(const qpid::qls_jrnl::efpDataSize_kib_t efpFileSizeKiB, + const std::string& paramName); + + void init(); + + void recoverQueues(TxnCtxt& txn, + qpid::broker::RecoveryManager& recovery, + queue_index& index, + txn_list& locked, + message_index& messages); + void recoverMessages(TxnCtxt& txn, + qpid::broker::RecoveryManager& recovery, + queue_index& index, + txn_list& locked, + message_index& prepared); + void recoverMessages(TxnCtxt& txn, + qpid::broker::RecoveryManager& recovery, + qpid::broker::RecoverableQueue::shared_ptr& queue, + txn_list& locked, + message_index& prepared, + long& rcnt, + long& idcnt); + qpid::broker::RecoverableMessage::shared_ptr getExternMessage(qpid::broker::RecoveryManager& recovery, + uint64_t mId, + unsigned& headerSize); + void recoverExchanges(TxnCtxt& txn, + qpid::broker::RecoveryManager& recovery, + exchange_index& index); + void recoverBindings(TxnCtxt& txn, + exchange_index& exchanges, + queue_index& queues); + void recoverGeneral(TxnCtxt& txn, + qpid::broker::RecoveryManager& recovery); + int enqueueMessage(TxnCtxt& txn, + IdDbt& msgId, + qpid::broker::RecoverableMessage::shared_ptr& msg, + queue_index& index, + txn_list& locked, + message_index& prepared); + void readTplStore(); + void recoverTplStore(); + void recoverLockedMappings(txn_list& txns); + TxnCtxt* check(qpid::broker::TransactionContext* ctxt); + uint64_t msgEncode(std::vector<char>& buff, const boost::intrusive_ptr<qpid::broker::PersistableMessage>& message); + void store(const qpid::broker::PersistableQueue* queue, + TxnCtxt* txn, + const boost::intrusive_ptr<qpid::broker::PersistableMessage>& message); + void async_dequeue(qpid::broker::TransactionContext* ctxt, + const boost::intrusive_ptr<qpid::broker::PersistableMessage>& msg, + const qpid::broker::PersistableQueue& queue); + void destroy(db_ptr db, + const qpid::broker::Persistable& p); + bool create(db_ptr db, + IdSequence& seq, + const qpid::broker::Persistable& p); + void completed(TxnCtxt& txn, + bool commit); + void deleteBindingsForQueue(const qpid::broker::PersistableQueue& queue); + void deleteBinding(const qpid::broker::PersistableExchange& exchange, + const qpid::broker::PersistableQueue& queue, + const std::string& key); + + void put(db_ptr db, + DbTxn* txn, + Dbt& key, + Dbt& value); + void open(db_ptr db, + DbTxn* txn, + const char* file, + bool dupKey); + void closeDbs(); + + // journal functions + void createJrnlQueue(const qpid::broker::PersistableQueue& queue); + std::string getJrnlDir(const std::string& queueName); + qpid::qls_jrnl::EmptyFilePool* getEmptyFilePool(const qpid::qls_jrnl::efpPartitionNumber_t p, const qpid::qls_jrnl::efpDataSize_kib_t s); + qpid::qls_jrnl::EmptyFilePool* getEmptyFilePool(const qpid::framing::FieldTable& args); + std::string getStoreTopLevelDir(); + std::string getJrnlBaseDir(); + std::string getBdbBaseDir(); + std::string getTplBaseDir(); + inline void checkInit() { + // TODO: change the default dir to ~/.qpidd + if (!isInit) { init("/tmp"); isInit = true; } + } + void chkTplStoreInit(); + + // debug aid for printing XIDs that may contain non-printable chars + static std::string xid2str(const std::string xid) { + std::ostringstream oss; + oss << std::hex << std::setfill('0'); + for (unsigned i=0; i<xid.size(); i++) { + if (isprint(xid[i])) + oss << xid[i]; + else + oss << "/" << std::setw(2) << (int)((char)xid[i]); + } + return oss.str(); + } + + public: + typedef boost::shared_ptr<MessageStoreImpl> shared_ptr; + + MessageStoreImpl(qpid::broker::Broker* broker, const char* envpath = 0); + + virtual ~MessageStoreImpl(); + + bool init(const qpid::Options* options); + + bool init(const std::string& dir, + qpid::qls_jrnl::efpPartitionNumber_t efpPartition = defEfpPartition, + qpid::qls_jrnl::efpDataSize_kib_t efpFileSizeKib = defEfpFileSizeKib, + const bool truncateFlag = false, + uint32_t wCachePageSize = defWCachePageSizeKib, + uint32_t tplWCachePageSize = defTplWCachePageSizeKib); + + void truncateInit(); + + void initManagement (); + + void finalize(); + + // --- Implementation of qpid::broker::MessageStore --- + + void create(qpid::broker::PersistableQueue& queue, + const qpid::framing::FieldTable& args); + + void destroy(qpid::broker::PersistableQueue& queue); + + void create(const qpid::broker::PersistableExchange& queue, + const qpid::framing::FieldTable& args); + + void destroy(const qpid::broker::PersistableExchange& queue); + + void bind(const qpid::broker::PersistableExchange& exchange, + const qpid::broker::PersistableQueue& queue, + const std::string& key, + const qpid::framing::FieldTable& args); + + void unbind(const qpid::broker::PersistableExchange& exchange, + const qpid::broker::PersistableQueue& queue, + const std::string& key, + const qpid::framing::FieldTable& args); + + void create(const qpid::broker::PersistableConfig& config); + + void destroy(const qpid::broker::PersistableConfig& config); + + void recover(qpid::broker::RecoveryManager& queues); + + void stage(const boost::intrusive_ptr<qpid::broker::PersistableMessage>& msg); + + void destroy(qpid::broker::PersistableMessage& msg); + + void appendContent(const boost::intrusive_ptr<const qpid::broker::PersistableMessage>& msg, + const std::string& data); + + void loadContent(const qpid::broker::PersistableQueue& queue, + const boost::intrusive_ptr<const qpid::broker::PersistableMessage>& msg, + std::string& data, + uint64_t offset, + uint32_t length); + + void enqueue(qpid::broker::TransactionContext* ctxt, + const boost::intrusive_ptr<qpid::broker::PersistableMessage>& msg, + const qpid::broker::PersistableQueue& queue); + + void dequeue(qpid::broker::TransactionContext* ctxt, + const boost::intrusive_ptr<qpid::broker::PersistableMessage>& msg, + const qpid::broker::PersistableQueue& queue); + + void flush(const qpid::broker::PersistableQueue& queue); + + uint32_t outstandingQueueAIO(const qpid::broker::PersistableQueue& queue); + + void collectPreparedXids(std::set<std::string>& xids); + + std::auto_ptr<qpid::broker::TransactionContext> begin(); + + std::auto_ptr<qpid::broker::TPCTransactionContext> begin(const std::string& xid); + + void prepare(qpid::broker::TPCTransactionContext& ctxt); + + void localPrepare(TxnCtxt* ctxt); + + void commit(qpid::broker::TransactionContext& ctxt); + + void abort(qpid::broker::TransactionContext& ctxt); + + // --- Implementation of qpid::management::Managable --- + + qpid::management::ManagementObject::shared_ptr GetManagementObject (void) const + { return mgmtObject; } + + inline qpid::management::Manageable::status_t ManagementMethod (uint32_t, qpid::management::Args&, std::string&) + { return qpid::management::Manageable::STATUS_OK; } + + std::string getStoreDir() const; + + private: + void journalDeleted(JournalImpl&); + +}; // class MessageStoreImpl + +} // namespace msgstore +} // namespace mrg + +#endif // ifndef QPID_LEGACYSTORE_MESSAGESTOREIMPL_H diff --git a/qpid/cpp/src/qpid/linearstore/PreparedTransaction.cpp b/qpid/cpp/src/qpid/linearstore/PreparedTransaction.cpp new file mode 100644 index 0000000000..a353a339e3 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/PreparedTransaction.cpp @@ -0,0 +1,81 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/PreparedTransaction.h" +#include <algorithm> + +using namespace qpid::linearstore; +using std::string; + +void LockedMappings::add(queue_id queue, message_id message) +{ + locked.push_back(std::make_pair(queue, message)); +} + +bool LockedMappings::isLocked(queue_id queue, message_id message) +{ + idpair op( std::make_pair(queue, message) ); + return find(locked.begin(), locked.end(), op) != locked.end(); +} + +void LockedMappings::add(LockedMappings::map& map, std::string& key, queue_id queue, message_id message) +{ + LockedMappings::map::iterator i = map.find(key); + if (i == map.end()) { + LockedMappings::shared_ptr ptr(new LockedMappings()); + i = map.insert(std::make_pair(key, ptr)).first; + } + i->second->add(queue, message); +} + +bool PreparedTransaction::isLocked(queue_id queue, message_id message) +{ + return (enqueues.get() && enqueues->isLocked(queue, message)) + || (dequeues.get() && dequeues->isLocked(queue, message)); +} + + +bool PreparedTransaction::isLocked(PreparedTransaction::list& txns, queue_id queue, message_id message) +{ + for (PreparedTransaction::list::iterator i = txns.begin(); i != txns.end(); i++) { + if (i->isLocked(queue, message)) { + return true; + } + } + return false; +} + +PreparedTransaction::list::iterator PreparedTransaction::getLockedPreparedTransaction(PreparedTransaction::list& txns, queue_id queue, message_id message) +{ + for (PreparedTransaction::list::iterator i = txns.begin(); i != txns.end(); i++) { + if (i->isLocked(queue, message)) { + return i; + } + } + return txns.end(); +} + +PreparedTransaction::PreparedTransaction(const std::string& _xid, + LockedMappings::shared_ptr _enqueues, + LockedMappings::shared_ptr _dequeues) + + : xid(_xid), enqueues(_enqueues), dequeues(_dequeues) {} + diff --git a/qpid/cpp/src/qpid/linearstore/PreparedTransaction.h b/qpid/cpp/src/qpid/linearstore/PreparedTransaction.h new file mode 100644 index 0000000000..9888342579 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/PreparedTransaction.h @@ -0,0 +1,75 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_PREPAREDTRANSACTION_H +#define QPID_LEGACYSTORE_PREPAREDTRANSACTION_H + +#include <list> +#include <map> +#include <set> +#include <stdint.h> +#include <string> +#include <boost/shared_ptr.hpp> +#include <boost/ptr_container/ptr_list.hpp> + +namespace qpid{ +namespace linearstore{ + +typedef uint64_t queue_id; +typedef uint64_t message_id; + +class LockedMappings +{ +public: + typedef boost::shared_ptr<LockedMappings> shared_ptr; + typedef std::map<std::string, shared_ptr> map; + typedef std::pair<queue_id, message_id> idpair; + typedef std::list<idpair>::iterator iterator; + + void add(queue_id queue, message_id message); + bool isLocked(queue_id queue, message_id message); + std::size_t size() { return locked.size(); } + iterator begin() { return locked.begin(); } + iterator end() { return locked.end(); } + + static void add(LockedMappings::map& map, std::string& key, queue_id queue, message_id message); + +private: + std::list<idpair> locked; +}; + +struct PreparedTransaction +{ + typedef boost::ptr_list<PreparedTransaction> list; + + const std::string xid; + const LockedMappings::shared_ptr enqueues; + const LockedMappings::shared_ptr dequeues; + + PreparedTransaction(const std::string& xid, LockedMappings::shared_ptr enqueues, LockedMappings::shared_ptr dequeues); + bool isLocked(queue_id queue, message_id message); + static bool isLocked(PreparedTransaction::list& txns, queue_id queue, message_id message); + static PreparedTransaction::list::iterator getLockedPreparedTransaction(PreparedTransaction::list& txns, queue_id queue, message_id message); +}; + +}} + +#endif // ifndef QPID_LEGACYSTORE_PREPAREDTRANSACTION_H diff --git a/qpid/cpp/src/qpid/linearstore/StoreException.h b/qpid/cpp/src/qpid/linearstore/StoreException.h new file mode 100644 index 0000000000..9accfd0bd9 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/StoreException.h @@ -0,0 +1,56 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_STOREEXCEPTION_H +#define QPID_LEGACYSTORE_STOREEXCEPTION_H + +#include "qpid/linearstore/IdDbt.h" +#include <boost/format.hpp> + +namespace qpid{ +namespace linearstore{ + +class StoreException : public std::exception +{ + std::string text; +public: + StoreException(const std::string& _text) : text(_text) {} + StoreException(const std::string& _text, const DbException& cause) : text(_text + ": " + cause.what()) {} + virtual ~StoreException() throw() {} + virtual const char* what() const throw() { return text.c_str(); } +}; + +class StoreFullException : public StoreException +{ +public: + StoreFullException(const std::string& _text) : StoreException(_text) {} + StoreFullException(const std::string& _text, const DbException& cause) : StoreException(_text, cause) {} + virtual ~StoreFullException() throw() {} + +}; + +#define THROW_STORE_EXCEPTION(MESSAGE) throw StoreException(boost::str(boost::format("%s (%s:%d)") % (MESSAGE) % __FILE__ % __LINE__)) +#define THROW_STORE_EXCEPTION_2(MESSAGE, EXCEPTION) throw StoreException(boost::str(boost::format("%s (%s:%d)") % (MESSAGE) % __FILE__ % __LINE__), EXCEPTION) +#define THROW_STORE_FULL_EXCEPTION(MESSAGE) throw StoreFullException(boost::str(boost::format("%s (%s:%d)") % (MESSAGE) % __FILE__ % __LINE__)) + +}} + +#endif // ifndef QPID_LEGACYSTORE_STOREEXCEPTION_H diff --git a/qpid/cpp/src/qpid/linearstore/StorePlugin.cpp b/qpid/cpp/src/qpid/linearstore/StorePlugin.cpp new file mode 100644 index 0000000000..9826240068 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/StorePlugin.cpp @@ -0,0 +1,81 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/broker/Broker.h" +#include "qpid/Plugin.h" +#include "qpid/Options.h" +#include "qpid/DataDir.h" +#include "qpid/linearstore/JournalLogImpl.h" +#include "qpid/linearstore/MessageStoreImpl.h" + +using qpid::linearstore::MessageStoreImpl; + +namespace qpid { +namespace broker { + +using namespace std; + +struct StorePlugin : public Plugin { + + MessageStoreImpl::StoreOptions options; + boost::shared_ptr<MessageStoreImpl> store; + + Options* getOptions() { return &options; } + + void earlyInitialize (Plugin::Target& target) + { + Broker* broker = dynamic_cast<Broker*>(&target); + if (!broker) return; + store.reset(new MessageStoreImpl(broker)); + DataDir& dataDir = broker->getDataDir (); + if (options.storeDir.empty ()) + { + if (!dataDir.isEnabled ()) + throw Exception ("linearstore: If broker option --data-dir is blank or --no-data-dir is specified, linearstore option --store-dir must be present."); + + options.storeDir = dataDir.getPath (); + } + store->init(&options); + boost::shared_ptr<qpid::broker::MessageStore> brokerStore(store); + broker->setStore(brokerStore); + target.addFinalizer(boost::bind(&StorePlugin::finalize, this)); + } + + void initialize(Plugin::Target& target) + { + Broker* broker = dynamic_cast<Broker*>(&target); + if (!broker) return; + if (!store) return; + QLS_LOG(info, "Enabling management instrumentation."); + store->initManagement(); + } + + void finalize() + { + store.reset(); + } + + const char* id() {return "StorePlugin";} +}; + +static StorePlugin instance; // Static initialization. + +}} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/linearstore/TxnCtxt.cpp b/qpid/cpp/src/qpid/linearstore/TxnCtxt.cpp new file mode 100644 index 0000000000..022d9c13e2 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/TxnCtxt.cpp @@ -0,0 +1,184 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/TxnCtxt.h" + +#include <sstream> + +#include "qpid/linearstore/jrnl/jexception.h" +#include "qpid/linearstore/StoreException.h" + +namespace qpid{ +namespace linearstore{ + +void TxnCtxt::completeTxn(bool commit) { + sync(); + for (ipqItr i = impactedQueues.begin(); i != impactedQueues.end(); i++) { + commitTxn(static_cast<JournalImpl*>(*i), commit); + } + impactedQueues.clear(); + if (preparedXidStorePtr) + commitTxn(preparedXidStorePtr, commit); +} + +void TxnCtxt::commitTxn(JournalImpl* jc, bool commit) { + if (jc && loggedtx) { /* if using journal */ + boost::intrusive_ptr<DataTokenImpl> dtokp(new DataTokenImpl); + dtokp->addRef(); + dtokp->set_external_rid(true); + dtokp->set_rid(loggedtx->next()); + try { + if (commit) { + jc->txn_commit(dtokp.get(), getXid()); + sync(); + } else { + jc->txn_abort(dtokp.get(), getXid()); + } + } catch (const qpid::qls_jrnl::jexception& e) { + THROW_STORE_EXCEPTION(std::string("Error commit") + e.what()); + } + } +} + +// static +uuid_t TxnCtxt::uuid; + +// static +IdSequence TxnCtxt::uuidSeq; + +// static +bool TxnCtxt::staticInit = TxnCtxt::setUuid(); + +// static +bool TxnCtxt::setUuid() { + ::uuid_generate(uuid); + return true; +} + +TxnCtxt::TxnCtxt(IdSequence* _loggedtx) : loggedtx(_loggedtx), dtokp(new DataTokenImpl), preparedXidStorePtr(0), txn(0) { + if (loggedtx) { +// // Human-readable tid: 53 bytes +// // uuit_t is a char[16] +// tid.reserve(53); +// uint64_t* u1 = (uint64_t*)uuid; +// uint64_t* u2 = (uint64_t*)(uuid + sizeof(uint64_t)); +// std::stringstream s; +// s << "tid:" << std::hex << std::setfill('0') << std::setw(16) << uuidSeq.next() << ":" << std::setw(16) << *u1 << std::setw(16) << *u2; +// tid.assign(s.str()); + + // Binary tid: 24 bytes + tid.reserve(24); + uint64_t c = uuidSeq.next(); + tid.append((char*)&c, sizeof(c)); + tid.append((char*)&uuid, sizeof(uuid)); + } +} + +TxnCtxt::TxnCtxt(std::string _tid, IdSequence* _loggedtx) : loggedtx(_loggedtx), dtokp(new DataTokenImpl), preparedXidStorePtr(0), tid(_tid), txn(0) {} + +TxnCtxt::~TxnCtxt() { abort(); } + +void TxnCtxt::sync() { + if (loggedtx) { + try { + for (ipqItr i = impactedQueues.begin(); i != impactedQueues.end(); i++) + jrnl_flush(static_cast<JournalImpl*>(*i)); + if (preparedXidStorePtr) + jrnl_flush(preparedXidStorePtr); + for (ipqItr i = impactedQueues.begin(); i != impactedQueues.end(); i++) + jrnl_sync(static_cast<JournalImpl*>(*i), &qpid::qls_jrnl::jcntl::_aio_cmpl_timeout); + if (preparedXidStorePtr) + jrnl_sync(preparedXidStorePtr, &qpid::qls_jrnl::jcntl::_aio_cmpl_timeout); + } catch (const qpid::qls_jrnl::jexception& e) { + THROW_STORE_EXCEPTION(std::string("Error during txn sync: ") + e.what()); + } + } +} + +void TxnCtxt::jrnl_flush(JournalImpl* jc) { + if (jc && !(jc->is_txn_synced(getXid()))) + jc->flush(); +} + +void TxnCtxt::jrnl_sync(JournalImpl* jc, timespec* timeout) { + if (!jc || jc->is_txn_synced(getXid())) + return; + while (jc->get_wr_aio_evt_rem()) { + if (jc->get_wr_events(timeout) == qpid::qls_jrnl::jerrno::AIO_TIMEOUT && timeout) + THROW_STORE_EXCEPTION(std::string("Error: timeout waiting for TxnCtxt::jrnl_sync()")); + } +} + +void TxnCtxt::begin(DbEnv* env, bool sync) { + int err; + try { err = env->txn_begin(0, &txn, 0); } + catch (const DbException&) { txn = 0; throw; } + if (err != 0) { + std::ostringstream oss; + oss << "Error: Env::txn_begin() returned error code: " << err; + THROW_STORE_EXCEPTION(oss.str()); + } + if (sync) + globalHolder = AutoScopedLock(new qpid::sys::Mutex::ScopedLock(globalSerialiser)); +} + +void TxnCtxt::commit() { + if (txn) { + txn->commit(0); + txn = 0; + globalHolder.reset(); + } +} + +void TxnCtxt::abort(){ + if (txn) { + txn->abort(); + txn = 0; + globalHolder.reset(); + } +} + +DbTxn* TxnCtxt::get() { return txn; } + +bool TxnCtxt::isTPC() { return false; } + +const std::string& TxnCtxt::getXid() { return tid; } + +void TxnCtxt::addXidRecord(qpid::broker::ExternalQueueStore* queue) { impactedQueues.insert(queue); } + +void TxnCtxt::complete(bool commit) { completeTxn(commit); } + +bool TxnCtxt::impactedQueuesEmpty() { return impactedQueues.empty(); } + +DataTokenImpl* TxnCtxt::getDtok() { return dtokp.get(); } + +void TxnCtxt::incrDtokRef() { dtokp->addRef(); } + +void TxnCtxt::recoverDtok(const uint64_t rid, const std::string xid) { + dtokp->set_rid(rid); + dtokp->set_wstate(DataTokenImpl::ENQ); + dtokp->set_xid(xid); + dtokp->set_external_rid(true); +} + +TPCTxnCtxt::TPCTxnCtxt(const std::string& _xid, IdSequence* _loggedtx) : TxnCtxt(_loggedtx), xid(_xid) {} + +}} diff --git a/qpid/cpp/src/qpid/linearstore/TxnCtxt.h b/qpid/cpp/src/qpid/linearstore/TxnCtxt.h new file mode 100644 index 0000000000..4c009d7a94 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/TxnCtxt.h @@ -0,0 +1,117 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_TXNCTXT_H +#define QPID_LEGACYSTORE_TXNCTXT_H + +#include "db-inc.h" +#include <memory> +#include <set> +#include <string> + +#include "qpid/linearstore/DataTokenImpl.h" +#include "qpid/linearstore/IdSequence.h" +#include "qpid/linearstore/JournalImpl.h" +#include "qpid/broker/PersistableQueue.h" +#include "qpid/broker/TransactionalStore.h" +#include "qpid/sys/Mutex.h" +#include "qpid/sys/uuid.h" + +#include <boost/intrusive_ptr.hpp> + +namespace qpid{ +namespace linearstore{ + +class TxnCtxt : public qpid::broker::TransactionContext +{ + protected: + static qpid::sys::Mutex globalSerialiser; + + static uuid_t uuid; + static IdSequence uuidSeq; + static bool staticInit; + static bool setUuid(); + + typedef std::set<qpid::broker::ExternalQueueStore*> ipqdef; + typedef ipqdef::iterator ipqItr; + typedef std::auto_ptr<qpid::sys::Mutex::ScopedLock> AutoScopedLock; + + ipqdef impactedQueues; // list of Queues used in the txn + IdSequence* loggedtx; + boost::intrusive_ptr<DataTokenImpl> dtokp; + AutoScopedLock globalHolder; + JournalImpl* preparedXidStorePtr; + + /** + * local txn id, if non XA. + */ + std::string tid; + DbTxn* txn; + + virtual void completeTxn(bool commit); + void commitTxn(JournalImpl* jc, bool commit); + void jrnl_flush(JournalImpl* jc); + void jrnl_sync(JournalImpl* jc, timespec* timeout); + + public: + TxnCtxt(IdSequence* _loggedtx=NULL); + TxnCtxt(std::string _tid, IdSequence* _loggedtx); + virtual ~TxnCtxt(); + + /** + * Call to make sure all the data for this txn is written to safe store + * + *@return if the data successfully synced. + */ + void sync(); + void begin(DbEnv* env, bool sync = false); + void commit(); + void abort(); + DbTxn* get(); + virtual bool isTPC(); + virtual const std::string& getXid(); + + void addXidRecord(qpid::broker::ExternalQueueStore* queue); + inline void prepare(JournalImpl* _preparedXidStorePtr) { preparedXidStorePtr = _preparedXidStorePtr; } + void complete(bool commit); + bool impactedQueuesEmpty(); + DataTokenImpl* getDtok(); + void incrDtokRef(); + void recoverDtok(const uint64_t rid, const std::string xid); +}; + + +class TPCTxnCtxt : public TxnCtxt, public qpid::broker::TPCTransactionContext +{ + protected: + const std::string xid; + + public: + TPCTxnCtxt(const std::string& _xid, IdSequence* _loggedtx); + inline virtual bool isTPC() { return true; } + inline virtual const std::string& getXid() { return xid; } +}; + +}} + +#endif // ifndef QPID_LEGACYSTORE_TXNCTXT_H + + diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/AtomicCounter.h b/qpid/cpp/src/qpid/linearstore/jrnl/AtomicCounter.h new file mode 100644 index 0000000000..2f9d406b7c --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/AtomicCounter.h @@ -0,0 +1,125 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LINEARSTORE_ATOMICCOUNTER_H_ +#define QPID_LINEARSTORE_ATOMICCOUNTER_H_ + +#include "qpid/linearstore/jrnl/slock.h" + +namespace qpid { +namespace qls_jrnl { + +template <class T> +class AtomicCounter +{ +private: + T count_; + mutable smutex countMutex; + +public: + AtomicCounter(const T& initValue = T(0)) : count_(initValue) {} + + virtual ~AtomicCounter() {} + + T get() const { + slock l(countMutex); + return count_; + } + + T increment() { + slock l(countMutex); + return ++count_; + } + + T add(const T& a) { + slock l(countMutex); + count_ += a; + return count_; + } + + T addLimit(const T& a, const T& limit, const uint32_t jerr) { + slock l(countMutex); + if (count_ + a > limit) throw jexception(jerr, "AtomicCounter", "addLimit"); + count_ += a; + return count_; + } + + T decrement() { + slock l(countMutex); + return --count_; + } + + T decrementLimit(const T& limit = T(0), const uint32_t jerr = jerrno::JERR__UNDERFLOW) { + slock l(countMutex); + if (count_ < limit + 1) { + throw jexception(jerr, "AtomicCounter", "decrementLimit"); + } + return --count_; + } + + T subtract(const T& s) { + slock l(countMutex); + count_ -= s; + return count_; + } + + T subtractLimit(const T& s, const T& limit = T(0), const uint32_t jerr = jerrno::JERR__UNDERFLOW) { + slock l(countMutex); + if (count_ < limit + s) throw jexception(jerr, "AtomicCounter", "subtractLimit"); + count_ -= s; + return count_; + } + + bool operator==(const T& o) const { + slock l(countMutex); + return count_ == o; + } + + bool operator<(const T& o) const { + slock l(countMutex); + return count_ < o; + } + + bool operator<=(const T& o) const { + slock l(countMutex); + return count_ <= o; + } + + friend T operator-(const T& a, const AtomicCounter& b) { + slock l(b.countMutex); + return a - b.count_; + } + + friend T operator-(const AtomicCounter& a, const T& b) { + slock l(a.countMutex); + return a.count_ - b; + } + + friend T operator-(const AtomicCounter&a, const AtomicCounter& b) { + slock l1(a.countMutex); + slock l2(b.countMutex); + return a.count_ - b.count_; + } +}; + +}} // namespace qpid::qls_jrnl + +#endif // QPID_LINEARSTORE_ATOMICCOUNTER_H_ diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePool.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePool.cpp new file mode 100644 index 0000000000..c78028362a --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePool.cpp @@ -0,0 +1,326 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "EmptyFilePool.h" + +#include <cctype> +#include <fstream> +#include "qpid/linearstore/jrnl/EmptyFilePoolPartition.h" +#include "qpid/linearstore/jrnl/jcfg.h" +#include "qpid/linearstore/jrnl/jdir.h" +#include "qpid/linearstore/jrnl/JournalFile.h" +#include "qpid/linearstore/jrnl/JournalLog.h" +#include "qpid/linearstore/jrnl/slock.h" +#include "qpid/linearstore/jrnl/utils/file_hdr.h" +#include <sys/stat.h> +#include <unistd.h> +#include <uuid/uuid.h> +#include <vector> + +namespace qpid { +namespace qls_jrnl { + +EmptyFilePool::EmptyFilePool(const std::string& efpDirectory, + const EmptyFilePoolPartition* partitionPtr, + JournalLog& journalLogRef) : + efpDirectory_(efpDirectory), + efpDataSize_kib_(fileSizeKbFromDirName(efpDirectory, partitionPtr->getPartitionNumber())), + partitionPtr_(partitionPtr), + journalLogRef_(journalLogRef) +{} + +EmptyFilePool::~EmptyFilePool() {} + +void EmptyFilePool::initialize() { + std::vector<std::string> dirList; + jdir::read_dir(efpDirectory_, dirList, false, true, false, false); + for (std::vector<std::string>::iterator i = dirList.begin(); i != dirList.end(); ++i) { + size_t dotPos = i->rfind("."); + if (dotPos != std::string::npos) { + if (i->substr(dotPos).compare(".jrnl") == 0 && i->length() == 41) { + std::string emptyFile(efpDirectory_ + "/" + (*i)); + if (validateEmptyFile(emptyFile)) { + pushEmptyFile(emptyFile); + } + } + } + } +} + +efpDataSize_kib_t EmptyFilePool::dataSize_kib() const { + return efpDataSize_kib_; +} + +efpFileSize_kib_t EmptyFilePool::fileSize_kib() const { + return efpDataSize_kib_ + (QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB); +} + +efpDataSize_sblks_t EmptyFilePool::dataSize_sblks() const { + return efpDataSize_kib_ / QLS_SBLK_SIZE_KIB; +} + +efpFileSize_sblks_t EmptyFilePool::fileSize_sblks() const { + return (efpDataSize_kib_ / QLS_SBLK_SIZE_KIB) + QLS_JRNL_FHDR_RES_SIZE_SBLKS; +} + +efpFileCount_t EmptyFilePool::numEmptyFiles() const { + slock l(emptyFileListMutex_); + return efpFileCount_t(emptyFileList_.size()); +} + +efpDataSize_kib_t EmptyFilePool::cumFileSize_kib() const { + slock l(emptyFileListMutex_); + return efpDataSize_kib_t(emptyFileList_.size()) * efpDataSize_kib_; +} + +efpPartitionNumber_t EmptyFilePool::getPartitionNumber() const { + return partitionPtr_->getPartitionNumber(); +} + +const EmptyFilePoolPartition* EmptyFilePool::getPartition() const { + return partitionPtr_; +} + +const efpIdentity_t EmptyFilePool::getIdentity() const { + return efpIdentity_t(partitionPtr_->getPartitionNumber(), efpDataSize_kib_); +} + +std::string EmptyFilePool::takeEmptyFile(const std::string& destDirectory) { + std::string emptyFileName = popEmptyFile(); + std::string newFileName = destDirectory + emptyFileName.substr(emptyFileName.rfind('/')); // NOTE: substr() includes leading '/' + if (moveEmptyFile(emptyFileName.c_str(), newFileName.c_str())) { + // Try again with new UUID for file name + newFileName = destDirectory + "/" + getEfpFileName(); + if (moveEmptyFile(emptyFileName.c_str(), newFileName.c_str())) { + pushEmptyFile(emptyFileName); + std::ostringstream oss; + oss << "file=\"" << emptyFileName << "\" dest=\"" << newFileName << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_FMOVE, oss.str(), "EmptyFilePool", "takeEmptyFile"); + } + } + return newFileName; +} + +void EmptyFilePool::returnEmptyFile(const std::string& fqSrcFile) { + std::string emptyFileName(efpDirectory_ + fqSrcFile.substr(fqSrcFile.rfind('/'))); // NOTE: substr() includes leading '/' + if (moveEmptyFile(fqSrcFile.c_str(), emptyFileName.c_str())) { + // Try again with new UUID for file name + emptyFileName = efpDirectory_ + "/" + getEfpFileName(); + if (moveEmptyFile(fqSrcFile.c_str(), emptyFileName.c_str())) { + // Failed twice in a row - delete file + ::unlink(fqSrcFile.c_str()); + return; + } + } + resetEmptyFileHeader(emptyFileName); + pushEmptyFile(emptyFileName); +} + +// --- protected functions --- + +void EmptyFilePool::createEmptyFile() { + ::file_hdr_t fh; + ::file_hdr_create(&fh, QLS_FILE_MAGIC, QLS_JRNL_VERSION, QLS_JRNL_FHDR_RES_SIZE_SBLKS, partitionPtr_->getPartitionNumber(), efpDataSize_kib_); + std::string efpfn = getEfpFileName(); + std::ofstream ofs(efpfn.c_str(), std::ofstream::out | std::ofstream::binary); + if (ofs.good()) { + ofs.write((char*)&fh, sizeof(::file_hdr_t)); + uint64_t rem = ((efpDataSize_kib_ + (QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB)) * 1024) - sizeof(::file_hdr_t); + while (rem--) + ofs.put('\0'); + ofs.close(); + pushEmptyFile(efpfn); +//std::cout << "WARNING: EFP " << efpDirectory << " is empty - created new journal file " << efpfn.substr(efpfn.rfind('/') + 1) << " on the fly" << std::endl; + } else { +//std::cerr << "ERROR: Unable to open file \"" << efpfn << "\"" << std::endl; // DEBUG + } +} + +std::string EmptyFilePool::getEfpFileName() { + uuid_t uuid; + ::uuid_generate(uuid); // NOTE: uuid_generate() is not thread safe + char uuid_str[37]; // 36 char uuid + trailing \0 + ::uuid_unparse(uuid, uuid_str); + std::ostringstream oss; + oss << efpDirectory_ << "/" << uuid_str << QLS_JRNL_FILE_EXTENSION; + return oss.str(); +} + +std::string EmptyFilePool::popEmptyFile() { + std::string emptyFileName; + bool isEmpty = false; + { + slock l(emptyFileListMutex_); + isEmpty = emptyFileList_.empty(); + } + if (isEmpty) { + createEmptyFile(); + } + { + slock l(emptyFileListMutex_); + emptyFileName = emptyFileList_.front(); + emptyFileList_.pop_front(); + } + return emptyFileName; +} + +void EmptyFilePool::pushEmptyFile(const std::string fqFileName) { + slock l(emptyFileListMutex_); + emptyFileList_.push_back(fqFileName); +} + +void EmptyFilePool::resetEmptyFileHeader(const std::string& fqFileName) { + std::fstream fs(fqFileName.c_str(), std::fstream::in | std::fstream::out | std::fstream::binary); + if (fs.good()) { + const std::streamsize buffsize = QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_BYTES; + char buff[buffsize]; + fs.read((char*)buff, buffsize); + std::streampos bytesRead = fs.tellg(); + if (bytesRead == buffsize) { + ::file_hdr_reset((::file_hdr_t*)buff); + ::memset(buff + sizeof(::file_hdr_t), 0, MAX_FILE_HDR_LEN - sizeof(::file_hdr_t)); // set rest of buffer to 0 + fs.seekp(0, std::fstream::beg); + fs.write(buff, buffsize); + std::streampos bytesWritten = fs.tellp(); + if (bytesWritten != buffsize) { +//std::cerr << "ERROR: Unable to write file header of file \"" << fqFileName_ << "\": tried to write " << buffsize << " bytes; wrote " << bytesWritten << " bytes." << std::endl; + } + } else { +//std::cerr << "ERROR: Unable to read file header of file \"" << fqFileName_ << "\": tried to read " << sizeof(::file_hdr_t) << " bytes; read " << bytesRead << " bytes." << std::endl; + } + fs.close(); + } else { +//std::cerr << "ERROR: Unable to open file \"" << fqFileName_ << "\" for reading" << std::endl; // DEBUG + } +} + +bool EmptyFilePool::validateEmptyFile(const std::string& emptyFileName) const { + std::ostringstream oss; + struct stat s; + if (::stat(emptyFileName.c_str(), &s)) + { + oss << "stat: file=\"" << emptyFileName << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_STAT, oss.str(), "EmptyFilePool", "validateEmptyFile"); + } + + // Size matches pool + efpDataSize_kib_t expectedSize = (QLS_SBLK_SIZE_KIB + efpDataSize_kib_) * 1024; + if ((efpDataSize_kib_t)s.st_size != expectedSize) { + oss << "ERROR: File " << emptyFileName << ": Incorrect size: Expected=" << expectedSize << "; actual=" << s.st_size; + journalLogRef_.log(JournalLog::LOG_ERROR, oss.str()); + return false; + } + + // Open file and read header + std::fstream fs(emptyFileName.c_str(), std::fstream::in | std::fstream::out | std::fstream::binary); + if (!fs) { + oss << "ERROR: File " << emptyFileName << ": Unable to open for reading"; + journalLogRef_.log(JournalLog::LOG_ERROR, oss.str()); + return false; + } + const std::streamsize buffsize = QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_BYTES; + char buff[buffsize]; + fs.read((char*)buff, buffsize); + std::streampos bytesRead = fs.tellg(); + if (bytesRead != buffsize) { + oss << "ERROR: Unable to read file header of file \"" << emptyFileName << "\": tried to read " << buffsize << " bytes; read " << bytesRead << " bytes"; + journalLogRef_.log(JournalLog::LOG_ERROR, oss.str()); + fs.close(); + return false; + } + + // Check file header + const bool jrnlMagicError = ((::file_hdr_t*)buff)->_rhdr._magic != QLS_FILE_MAGIC; + const bool jrnlVersionError = ((::file_hdr_t*)buff)->_rhdr._version != QLS_JRNL_VERSION; + const bool jrnlPartitionError = ((::file_hdr_t*)buff)->_efp_partition != partitionPtr_->getPartitionNumber(); + const bool jrnlFileSizeError = ((::file_hdr_t*)buff)->_file_size_kib != efpDataSize_kib_; + if (jrnlMagicError || jrnlVersionError || jrnlPartitionError || jrnlFileSizeError) + { + oss << "ERROR: File " << emptyFileName << ": Invalid file header - mismatched header fields: " << + (jrnlMagicError ? "magic " : "") << + (jrnlVersionError ? "version " : "") << + (jrnlPartitionError ? "partition" : "") << + (jrnlFileSizeError ? "file-size" : ""); + journalLogRef_.log(JournalLog::LOG_ERROR, oss.str()); + fs.close(); + return false; + } + + // Check file header is reset + if (!::is_file_hdr_reset((::file_hdr_t*)buff)) { + ::file_hdr_reset((::file_hdr_t*)buff); + ::memset(buff + sizeof(::file_hdr_t), 0, MAX_FILE_HDR_LEN - sizeof(::file_hdr_t)); // set rest of buffer to 0 + fs.seekp(0, std::fstream::beg); + fs.write(buff, buffsize); + std::streampos bytesWritten = fs.tellp(); + if (bytesWritten != buffsize) { + oss << "ERROR: Unable to write file header of file \"" << emptyFileName << "\": tried to write " << buffsize << " bytes; wrote " << bytesWritten << " bytes"; + journalLogRef_.log(JournalLog::LOG_ERROR, oss.str()); + fs.close(); + return false; + } + oss << "WARNING: File " << emptyFileName << ": File header not reset"; + journalLogRef_.log(JournalLog::LOG_WARN, oss.str()); + } + + // Close file + fs.close(); + return true; +} + +// static +efpDataSize_kib_t EmptyFilePool::fileSizeKbFromDirName(const std::string& dirName, + const efpPartitionNumber_t partitionNumber) { + // Check for dirName format 'NNNk', where NNN is a number, convert NNN into an integer. NNN cannot be 0. + std::string n(dirName.substr(dirName.rfind('/')+1)); + bool valid = true; + for (uint16_t charNum = 0; charNum < n.length(); ++charNum) { + if (charNum < n.length()-1) { + if (!::isdigit((int)n[charNum])) { + valid = false; + break; + } + } else { + valid = n[charNum] == 'k'; + } + } + efpDataSize_kib_t s = ::atol(n.c_str()); + if (!valid || s == 0 || s % QLS_SBLK_SIZE_KIB != 0) { + std::ostringstream oss; + oss << "Partition: " << partitionNumber << "; EFP directory: \'" << n << "\'"; + throw jexception(jerrno::JERR_EFP_BADEFPDIRNAME, oss.str(), "EmptyFilePool", "fileSizeKbFromDirName"); + } + return s; +} + +// static +int EmptyFilePool::moveEmptyFile(const std::string& from, + const std::string& to) { + if (::rename(from.c_str(), to.c_str())) { + if (errno == EEXIST) return errno; // File name exists + std::ostringstream oss; + oss << "file=\"" << from << "\" dest=\"" << to << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_FMOVE, oss.str(), "EmptyFilePool", "returnEmptyFile"); + } + return 0; +} + +}} // namespace qpid::qls_jrnl diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePool.h b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePool.h new file mode 100644 index 0000000000..e4eeeaa042 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePool.h @@ -0,0 +1,95 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_QLS_JRNL_EMPTYFILEPOOL_H_ +#define QPID_QLS_JRNL_EMPTYFILEPOOL_H_ + +namespace qpid { +namespace qls_jrnl { + + class EmptyFilePool; + +}} // namespace qpid::qls_jrnl + +#include <deque> +#include "qpid/linearstore/jrnl/EmptyFilePoolTypes.h" +#include "qpid/linearstore/jrnl/smutex.h" +#include <string> + +namespace qpid { +namespace qls_jrnl { +class EmptyFilePoolPartition; +class jdir; +class JournalFile; +class JournalLog; + +class EmptyFilePool +{ +protected: + typedef std::deque<std::string> emptyFileList_t; + typedef emptyFileList_t::iterator emptyFileListItr_t; + + const std::string efpDirectory_; + const efpDataSize_kib_t efpDataSize_kib_; + const EmptyFilePoolPartition* partitionPtr_; + JournalLog& journalLogRef_; + +private: + emptyFileList_t emptyFileList_; + smutex emptyFileListMutex_; + +public: + EmptyFilePool(const std::string& efpDirectory, + const EmptyFilePoolPartition* partitionPtr, + JournalLog& journalLogRef); + virtual ~EmptyFilePool(); + + void initialize(); + efpDataSize_kib_t dataSize_kib() const; + efpFileSize_kib_t fileSize_kib() const; + efpDataSize_sblks_t dataSize_sblks() const; + efpFileSize_sblks_t fileSize_sblks() const; + efpFileCount_t numEmptyFiles() const; + efpDataSize_kib_t cumFileSize_kib() const; + efpPartitionNumber_t getPartitionNumber() const; + const EmptyFilePoolPartition* getPartition() const; + const efpIdentity_t getIdentity() const; + + std::string takeEmptyFile(const std::string& destDirectory); + void returnEmptyFile(const std::string& srcFile); + +protected: + void createEmptyFile(); + std::string getEfpFileName(); + std::string popEmptyFile(); + void pushEmptyFile(const std::string fqFileName); + void resetEmptyFileHeader(const std::string& fqFileName); + bool validateEmptyFile(const std::string& emptyFileName) const; + + static efpDataSize_kib_t fileSizeKbFromDirName(const std::string& dirName, + const efpPartitionNumber_t partitionNumber); + static int moveEmptyFile(const std::string& fromFqPath, + const std::string& toFqPath); +}; + +}} // namespace qpid::qls_jrnl + +#endif /* QPID_QLS_JRNL_EMPTYFILEPOOL_H_ */ diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolManager.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolManager.cpp new file mode 100644 index 0000000000..a9b3b12b81 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolManager.cpp @@ -0,0 +1,187 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "EmptyFilePoolManager.h" + +#include <dirent.h> +#include "qpid/linearstore/jrnl/EmptyFilePoolPartition.h" +#include "qpid/linearstore/jrnl/jdir.h" +#include "qpid/linearstore/jrnl/slock.h" +#include <vector> + +namespace qpid { +namespace qls_jrnl { + +EmptyFilePoolManager::EmptyFilePoolManager(const std::string& qlsStorePath, + JournalLog& journalLogRef) : + qlsStorePath_(qlsStorePath), + journalLogRef_(journalLogRef) +{} + +EmptyFilePoolManager::~EmptyFilePoolManager() { + slock l(partitionMapMutex_); + for (partitionMapItr_t i = partitionMap_.begin(); i != partitionMap_.end(); ++i) { + delete i->second; + } + partitionMap_.clear(); +} + +void EmptyFilePoolManager::findEfpPartitions() { + //std::cout << "*** Reading " << qlsStorePath << std::endl; // DEBUG + std::vector<std::string> dirList; + jdir::read_dir(qlsStorePath_, dirList, true, false, true, false); + for (std::vector<std::string>::iterator i = dirList.begin(); i != dirList.end(); ++i) { + if ((*i)[0] == 'p' && i->length() == 4) { // Filter: look only at names pNNN + efpPartitionNumber_t pn = ::atoi(i->c_str() + 1); + std::string fullDirPath(qlsStorePath_ + "/" + (*i)); + EmptyFilePoolPartition* efppp = 0; + try { + efppp = new EmptyFilePoolPartition(pn, fullDirPath, journalLogRef_); + { + slock l(partitionMapMutex_); + partitionMap_[pn] = efppp; + } + } catch (const std::exception& e) { + if (efppp != 0) { + delete efppp; + efppp = 0; + } + //std::cerr << "Unable to initialize partition " << pn << " (\'" << fullDirPath << "\'): " << e.what() << std::endl; + } + if (efppp != 0) + efppp->findEmptyFilePools(); + } + } + // TODO: Log results +/* + QLS_LOG(info, "EFP Manager initialization complete"); + std::vector<qpid::qls_jrnl::EmptyFilePoolPartition*> partitionList; + std::vector<qpid::qls_jrnl::EmptyFilePool*> filePoolList; + getEfpPartitions(partitionList); + if (partitionList.size() == 0) { + QLS_LOG(error, "NO EFP PARTITIONS FOUND! No queue creation is possible.") + } else { + QLS_LOG(info, "> EFP Partitions found: " << partitionList.size()); + for (std::vector<qpid::qls_jrnl::EmptyFilePoolPartition*>::const_iterator i=partitionList.begin(); i!= partitionList.end(); ++i) { + filePoolList.clear(); + (*i)->getEmptyFilePools(filePoolList); + QLS_LOG(info, " * Partition " << (*i)->partitionNumber() << " containing " << filePoolList.size() << " pool" << + (filePoolList.size()>1 ? "s" : "") << " at \'" << (*i)->partitionDirectory() << "\'"); + for (std::vector<qpid::qls_jrnl::EmptyFilePool*>::const_iterator j=filePoolList.begin(); j!=filePoolList.end(); ++j) { + QLS_LOG(info, " - EFP \'" << (*j)->dataSize_kib() << "k\' containing " << (*j)->numEmptyFiles() << + " files of size " << (*j)->dataSize_kib() << " KiB totaling " << (*j)->cumFileSize_kib() << " KiB"); + } + } + } +*/ +} + +void EmptyFilePoolManager::getEfpFileSizes(std::vector<efpDataSize_kib_t>& efpFileSizeList, + const efpPartitionNumber_t efpPartitionNumber) const { + if (efpPartitionNumber == 0) { + for (partitionMapConstItr_t i=partitionMap_.begin(); i!=partitionMap_.end(); ++i) { + i->second->getEmptyFilePoolSizes_kib(efpFileSizeList); + } + } else { + partitionMapConstItr_t i = partitionMap_.find(efpPartitionNumber); + if (i != partitionMap_.end()) { + i->second->getEmptyFilePoolSizes_kib(efpFileSizeList); + } + } +} + +EmptyFilePoolPartition* EmptyFilePoolManager::getEfpPartition(const efpPartitionNumber_t partitionNumber) { + partitionMapItr_t i = partitionMap_.find(partitionNumber); + if (i == partitionMap_.end()) + return 0; + else + return i->second; +} + +void EmptyFilePoolManager::getEfpPartitionNumbers(std::vector<efpPartitionNumber_t>& partitionNumberList, + const efpDataSize_kib_t efpDataSize_kib) const { + slock l(partitionMapMutex_); + for (partitionMapConstItr_t i=partitionMap_.begin(); i!=partitionMap_.end(); ++i) { + if (efpDataSize_kib == 0) { + partitionNumberList.push_back(i->first); + } else { + std::vector<efpDataSize_kib_t> efpFileSizeList; + i->second->getEmptyFilePoolSizes_kib(efpFileSizeList); + for (std::vector<efpDataSize_kib_t>::iterator j=efpFileSizeList.begin(); j!=efpFileSizeList.end(); ++j) { + if (*j == efpDataSize_kib) { + partitionNumberList.push_back(i->first); + break; + } + } + } + } +} + +void EmptyFilePoolManager::getEfpPartitions(std::vector<EmptyFilePoolPartition*>& partitionList, + const efpDataSize_kib_t efpDataSize_kib) { + slock l(partitionMapMutex_); + for (partitionMapConstItr_t i=partitionMap_.begin(); i!=partitionMap_.end(); ++i) { + if (efpDataSize_kib == 0) { + partitionList.push_back(i->second); + } else { + std::vector<efpDataSize_kib_t> efpFileSizeList; + i->second->getEmptyFilePoolSizes_kib(efpFileSizeList); + for (std::vector<efpDataSize_kib_t>::iterator j=efpFileSizeList.begin(); j!=efpFileSizeList.end(); ++j) { + if (*j == efpDataSize_kib) { + partitionList.push_back(i->second); + break; + } + } + } + } +} + +EmptyFilePool* EmptyFilePoolManager::getEmptyFilePool(const efpIdentity_t efpIdentity) { + return getEmptyFilePool(efpIdentity.first, efpIdentity.second); +} + +EmptyFilePool* EmptyFilePoolManager::getEmptyFilePool(const efpPartitionNumber_t partitionNumber, + const efpDataSize_kib_t efpDataSize_kib) { + EmptyFilePoolPartition* efppp = getEfpPartition(partitionNumber); + if (efppp != 0) + return efppp->getEmptyFilePool(efpDataSize_kib); + return 0; +} + +void EmptyFilePoolManager::getEmptyFilePools(std::vector<EmptyFilePool*>& emptyFilePoolList, + const efpPartitionNumber_t efpPartitionNumber) { + if (efpPartitionNumber == 0) { + for (partitionMapConstItr_t i=partitionMap_.begin(); i!=partitionMap_.end(); ++i) { + i->second->getEmptyFilePools(emptyFilePoolList); + } + } else { + partitionMapConstItr_t i = partitionMap_.find(efpPartitionNumber); + if (i != partitionMap_.end()) { + i->second->getEmptyFilePools(emptyFilePoolList); + } + } +} + +uint16_t EmptyFilePoolManager::getNumEfpPartitions() const { + return partitionMap_.size(); +} + +}} // namespace qpid::qls_jrnl diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolManager.h b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolManager.h new file mode 100644 index 0000000000..eab3783d90 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolManager.h @@ -0,0 +1,67 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_QLS_JRNL_EMPTYFILEPOOLMANAGER_H_ +#define QPID_QLS_JRNL_EMPTYFILEPOOLMANAGER_H_ + +#include <map> +#include "qpid/linearstore/jrnl/EmptyFilePoolPartition.h" +#include "qpid/linearstore/jrnl/smutex.h" + +namespace qpid { +namespace qls_jrnl { + +class EmptyFilePoolManager +{ +protected: + typedef std::map<efpPartitionNumber_t, EmptyFilePoolPartition*> partitionMap_t; + typedef partitionMap_t::iterator partitionMapItr_t; + typedef partitionMap_t::const_iterator partitionMapConstItr_t; + + std::string qlsStorePath_; + JournalLog& journalLogRef_; + partitionMap_t partitionMap_; + smutex partitionMapMutex_; + +public: + EmptyFilePoolManager(const std::string& qlsStorePath_, + JournalLog& journalLogRef_); + virtual ~EmptyFilePoolManager(); + + void findEfpPartitions(); + void getEfpFileSizes(std::vector<efpDataSize_kib_t>& efpFileSizeList, + const efpPartitionNumber_t efpPartitionNumber = 0) const; + EmptyFilePoolPartition* getEfpPartition(const efpPartitionNumber_t partitionNumber); + void getEfpPartitionNumbers(std::vector<efpPartitionNumber_t>& partitionNumberList, + const efpDataSize_kib_t efpDataSize_kib = 0) const; + void getEfpPartitions(std::vector<EmptyFilePoolPartition*>& partitionList, + const efpDataSize_kib_t efpDataSize_kib = 0); + EmptyFilePool* getEmptyFilePool(const efpIdentity_t efpIdentity); + EmptyFilePool* getEmptyFilePool(const efpPartitionNumber_t partitionNumber, + const efpDataSize_kib_t efpDataSize_kib); + void getEmptyFilePools(std::vector<EmptyFilePool*>& emptyFilePoolList, + const efpPartitionNumber_t efpPartitionNumber = 0); + uint16_t getNumEfpPartitions() const; +}; + +}} // namespace qpid::qls_jrnl + +#endif /* QPID_QLS_JRNL_EMPTYFILEPOOLMANAGER_H_ */ diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolPartition.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolPartition.cpp new file mode 100644 index 0000000000..4d8b697b1d --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolPartition.cpp @@ -0,0 +1,131 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/jrnl/EmptyFilePoolPartition.h" + +#include <dirent.h> +#include "qpid/linearstore/jrnl/jdir.h" +#include "qpid/linearstore/jrnl/jerrno.h" +#include "qpid/linearstore/jrnl/jexception.h" +#include "qpid/linearstore/jrnl/slock.h" + +namespace qpid { +namespace qls_jrnl { + +// static +const std::string EmptyFilePoolPartition::s_efpTopLevelDir_("efp"); // Sets the top-level efp dir within a partition + +EmptyFilePoolPartition::EmptyFilePoolPartition(const efpPartitionNumber_t partitionNum, + const std::string& partitionDir, + JournalLog& journalLogRef) : + partitionNum_(partitionNum), + partitionDir_(partitionDir), + journalLogRef_(journalLogRef) +{ + validatePartitionDir(); +} + +EmptyFilePoolPartition::~EmptyFilePoolPartition() { + slock l(efpMapMutex_); + for (efpMapItr_t i = efpMap_.begin(); i != efpMap_.end(); ++i) { + delete i->second; + } + efpMap_.clear(); +} + +void +EmptyFilePoolPartition::findEmptyFilePools() { + //std::cout << "Reading " << partitionDir << std::endl; // DEBUG + std::vector<std::string> dirList; + jdir::read_dir(partitionDir_, dirList, true, false, false, false); + bool foundEfpDir = false; + for (std::vector<std::string>::iterator i = dirList.begin(); i != dirList.end(); ++i) { + if (i->compare(s_efpTopLevelDir_) == 0) { + foundEfpDir = true; + break; + } + } + if (foundEfpDir) { + std::string efpDir(partitionDir_ + "/" + s_efpTopLevelDir_); + //std::cout << "Reading " << efpDir << std::endl; // DEBUG + dirList.clear(); + jdir::read_dir(efpDir, dirList, true, false, false, true); + for (std::vector<std::string>::iterator i = dirList.begin(); i != dirList.end(); ++i) { + EmptyFilePool* efpp = 0; + try { + efpp = new EmptyFilePool(*i, this, journalLogRef_); + { + slock l(efpMapMutex_); + efpMap_[efpp->dataSize_kib()] = efpp; + } + } + catch (const std::exception& e) { + if (efpp != 0) { + delete efpp; + efpp = 0; + } + //std::cerr << "WARNING: " << e.what() << std::endl; + } + if (efpp != 0) + efpp->initialize(); + } + } +} + +EmptyFilePool* EmptyFilePoolPartition::getEmptyFilePool(const efpDataSize_kib_t efpDataSize_kib) { + efpMapItr_t i = efpMap_.find(efpDataSize_kib); + if (i == efpMap_.end()) + return 0; + return i->second; +} + +void EmptyFilePoolPartition::getEmptyFilePools(std::vector<EmptyFilePool*>& efpList) { + for (efpMapItr_t i=efpMap_.begin(); i!=efpMap_.end(); ++i) { + efpList.push_back(i->second); + } +} + +void EmptyFilePoolPartition::getEmptyFilePoolSizes_kib(std::vector<efpDataSize_kib_t>& efpDataSizesList_kib) const { + for (efpMapConstItr_t i=efpMap_.begin(); i!=efpMap_.end(); ++i) { + efpDataSizesList_kib.push_back(i->first); + } +} + +std::string EmptyFilePoolPartition::getPartitionDirectory() const { + return partitionDir_; +} + +efpPartitionNumber_t EmptyFilePoolPartition::getPartitionNumber() const { + return partitionNum_; +} + +// --- protected functions --- + +void EmptyFilePoolPartition::validatePartitionDir() { + if (!jdir::is_dir(partitionDir_)) { + std::ostringstream ss; + ss << "Invalid partition directory: \'" << partitionDir_ << "\' is not a directory"; + throw jexception(jerrno::JERR_EFP_BADPARTITIONDIR, ss.str(), "EmptyFilePoolPartition", "validatePartitionDir"); + } + // TODO: other validity checks here +} + +}} // namespace qpid::qls_jrnl diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolPartition.h b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolPartition.h new file mode 100644 index 0000000000..5b8ba7b4fa --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolPartition.h @@ -0,0 +1,77 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_QLS_JRNL_EMPTYFILEPOOLPARTITION_H_ +#define QPID_QLS_JRNL_EMPTYFILEPOOLPARTITION_H_ + +namespace qpid { +namespace qls_jrnl { + + class EmptyFilePoolPartition; + +}} // namespace qpid::qls_jrnl + +#include "qpid/linearstore/jrnl/EmptyFilePool.h" +#include "qpid/linearstore/jrnl/EmptyFilePoolTypes.h" +#include "qpid/linearstore/jrnl/smutex.h" +#include <string> +#include <map> +#include <vector> + +namespace qpid { +namespace qls_jrnl { +class JournalLog; + +class EmptyFilePoolPartition +{ +public: + static const std::string s_efpTopLevelDir_; +protected: + typedef std::map<efpDataSize_kib_t, EmptyFilePool*> efpMap_t; + typedef efpMap_t::iterator efpMapItr_t; + typedef efpMap_t::const_iterator efpMapConstItr_t; + + const efpPartitionNumber_t partitionNum_; + const std::string partitionDir_; + JournalLog& journalLogRef_; + efpMap_t efpMap_; + smutex efpMapMutex_; + +public: + EmptyFilePoolPartition(const efpPartitionNumber_t partitionNum, + const std::string& partitionDir, + JournalLog& journalLogRef); + virtual ~EmptyFilePoolPartition(); + + void findEmptyFilePools(); + EmptyFilePool* getEmptyFilePool(const efpDataSize_kib_t efpDataSize_kib); + void getEmptyFilePools(std::vector<EmptyFilePool*>& efpList); + void getEmptyFilePoolSizes_kib(std::vector<efpDataSize_kib_t>& efpDataSizesList) const; + std::string getPartitionDirectory() const; + efpPartitionNumber_t getPartitionNumber() const; + +protected: + void validatePartitionDir(); +}; + +}} // namespace qpid::qls_jrnl + +#endif /* QPID_QLS_JRNL_EMPTYFILEPOOLPARTITION_H_ */ diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolTypes.h b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolTypes.h new file mode 100644 index 0000000000..4c470e4e40 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolTypes.h @@ -0,0 +1,41 @@ + /* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_QLS_JRNL_EMPTYFILEPOOLTYPES_H_ +#define QPID_QLS_JRNL_EMPTYFILEPOOLTYPES_H_ + +#include <stdint.h> +#include <utility> // std::pair + +namespace qpid { +namespace qls_jrnl { + + typedef uint64_t efpDataSize_kib_t; ///< Size of data part of file (excluding file header) in kib + typedef uint64_t efpFileSize_kib_t; ///< Size of file (header + data) in kib + typedef uint32_t efpDataSize_sblks_t; ///< Size of data part of file (excluding file header) in sblks + typedef uint32_t efpFileSize_sblks_t; ///< Size of file (header + data) in sblks + typedef uint32_t efpFileCount_t; ///< Number of files in a partition or pool + typedef uint16_t efpPartitionNumber_t; ///< Number assigned to a partition + typedef std::pair<efpPartitionNumber_t, efpDataSize_kib_t> efpIdentity_t; ///< Unique identity of a pool, consisting of the partition number and data size + +}} // namespace qpid::qls_jrnl + +#endif /* QPID_QLS_JRNL_EMPTYFILEPOOLTYPES_H_ */ diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/JournalFile.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/JournalFile.cpp new file mode 100644 index 0000000000..e02e965823 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/JournalFile.cpp @@ -0,0 +1,287 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/jrnl/JournalFile.h" + +#include <fcntl.h> +#include "qpid/linearstore/jrnl/jcfg.h" +#include "qpid/linearstore/jrnl/jexception.h" +#include "qpid/linearstore/jrnl/pmgr.h" +#include "qpid/linearstore/jrnl/utils/file_hdr.h" +#include <unistd.h> + +namespace qpid { +namespace qls_jrnl { + +JournalFile::JournalFile(const std::string& fqFileName, + const uint64_t fileSeqNum, + const efpDataSize_kib_t efpDataSize_kib) : + fqFileName_(fqFileName), + fileSeqNum_(fileSeqNum), + fileHandle_(-1), + fileCloseFlag_(false), + fileHeaderBasePtr_ (0), + fileHeaderPtr_(0), + aioControlBlockPtr_(0), + fileSize_dblks_(((efpDataSize_kib * 1024) + (QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_BYTES)) / QLS_DBLK_SIZE_BYTES), + enqueuedRecordCount_(0), + submittedDblkCount_(0), + completedDblkCount_(0), + outstandingAioOpsCount_(0) +{} + +JournalFile::~JournalFile() { + finalize(); +} + +void +JournalFile::initialize(const uint32_t completedDblkCount) { + if (::posix_memalign(&fileHeaderBasePtr_, QLS_AIO_ALIGN_BOUNDARY_BYTES, QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB * 1024)) + { + std::ostringstream oss; + oss << "posix_memalign(): blksize=" << QLS_AIO_ALIGN_BOUNDARY_BYTES << " size=" << (QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB * 1024); + oss << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR__MALLOC, oss.str(), "JournalFile", "initialize"); + } + fileHeaderPtr_ = (::file_hdr_t*)fileHeaderBasePtr_; + aioControlBlockPtr_ = new aio_cb; + if (completedDblkCount > 0UL) { + submittedDblkCount_.add(completedDblkCount); + completedDblkCount_.add(completedDblkCount); + } +} + +void +JournalFile::finalize() { + if (fileHeaderBasePtr_ != 0) { + std::free(fileHeaderBasePtr_); + fileHeaderBasePtr_ = 0; + fileHeaderPtr_ = 0; + } + if (aioControlBlockPtr_ != 0) { + delete(aioControlBlockPtr_); + aioControlBlockPtr_ = 0; + } +} + +const std::string JournalFile::getDirectory() const { + return fqFileName_.substr(0, fqFileName_.rfind('/')); +} + +const std::string JournalFile::getFileName() const { + return fqFileName_.substr(fqFileName_.rfind('/')+1); +} + +const std::string JournalFile::getFqFileName() const { + return fqFileName_; +} + +uint64_t JournalFile::getFileSeqNum() const { + return fileSeqNum_; +} + +bool JournalFile::isOpen() const { + return fileHandle_ >= 0; +} + +int JournalFile::open() { + fileHandle_ = ::open(fqFileName_.c_str(), O_WRONLY | O_DIRECT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); // 0644 -rw-r--r-- + if (fileHandle_ < 0) { + std::ostringstream oss; + oss << "file=\"" << fqFileName_ << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JNLF_OPEN, oss.str(), "JournalFile", "open"); + } + return fileHandle_; +} + +void JournalFile::close() { + if (fileHandle_ >= 0) { + if (getOutstandingAioDblks()) { + fileCloseFlag_ = true; // Close later when all outstanding AIOs have returned + } else { + int res = ::close(fileHandle_); + fileHandle_ = -1; + if (res != 0) { + std::ostringstream oss; + oss << "file=\"" << fqFileName_ << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JNLF_CLOSE, oss.str(), "JournalFile", "open"); + } + } + } +} + +void JournalFile::asyncFileHeaderWrite(io_context_t ioContextPtr, + const efpPartitionNumber_t efpPartitionNumber, + const efpDataSize_kib_t efpDataSize_kib, + const uint16_t userFlags, + const uint64_t recordId, + const uint64_t firstRecordOffset, + const std::string queueName) { + ::file_hdr_create(fileHeaderPtr_, QLS_FILE_MAGIC, QLS_JRNL_VERSION, QLS_JRNL_FHDR_RES_SIZE_SBLKS, efpPartitionNumber, efpDataSize_kib); + ::file_hdr_init(fileHeaderBasePtr_, + QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB * 1024, + userFlags, + recordId, + firstRecordOffset, + fileSeqNum_, + queueName.size(), + queueName.data()); + aio::prep_pwrite(aioControlBlockPtr_, + fileHandle_, + (void*)fileHeaderBasePtr_, + QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB * 1024, + 0UL); + if (aio::submit(ioContextPtr, 1, &aioControlBlockPtr_) < 0) + throw jexception(jerrno::JERR__AIO, "JournalFile", "asyncPageWrite"); + addSubmittedDblkCount(QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_DBLKS); + incrOutstandingAioOperationCount(); +} + +void JournalFile::asyncPageWrite(io_context_t ioContextPtr, + aio_cb* aioControlBlockPtr, + void* data, + uint32_t dataSize_dblks) { + aio::prep_pwrite_2(aioControlBlockPtr, + fileHandle_, + data, + dataSize_dblks * QLS_DBLK_SIZE_BYTES, + submittedDblkCount_.get() * QLS_DBLK_SIZE_BYTES); + pmgr::page_cb* pcbp = (pmgr::page_cb*)(aioControlBlockPtr->data); // This page's control block (pcb) + pcbp->_wdblks = dataSize_dblks; + pcbp->_jfp = this; + if (aio::submit(ioContextPtr, 1, &aioControlBlockPtr) < 0) { + throw jexception(jerrno::JERR__AIO, "JournalFile", "asyncPageWrite"); // TODO: complete exception details + } + addSubmittedDblkCount(dataSize_dblks); + incrOutstandingAioOperationCount(); +} + +uint32_t JournalFile::getEnqueuedRecordCount() const { + return enqueuedRecordCount_.get(); +} + +uint32_t JournalFile::incrEnqueuedRecordCount() { + return enqueuedRecordCount_.increment(); +} + +uint32_t JournalFile::addEnqueuedRecordCount(const uint32_t a) { + return enqueuedRecordCount_.add(a); +} + +uint32_t JournalFile::decrEnqueuedRecordCount() { + return enqueuedRecordCount_.decrementLimit(); +} + +uint32_t JournalFile::subtrEnqueuedRecordCount(const uint32_t s) { + return enqueuedRecordCount_.subtractLimit(s); +} + +uint32_t JournalFile::getSubmittedDblkCount() const { + return submittedDblkCount_.get(); +} + +uint32_t JournalFile::addSubmittedDblkCount(const uint32_t a) { + return submittedDblkCount_.addLimit(a, fileSize_dblks_, jerrno::JERR_JNLF_FILEOFFSOVFL); +} + +uint32_t JournalFile::getCompletedDblkCount() const { + return completedDblkCount_.get(); +} + +uint32_t JournalFile::addCompletedDblkCount(const uint32_t a) { + return completedDblkCount_.addLimit(a, submittedDblkCount_.get(), jerrno::JERR_JNLF_CMPLOFFSOVFL); +} + +uint16_t JournalFile::getOutstandingAioOperationCount() const { + return outstandingAioOpsCount_.get(); +} + +uint16_t JournalFile::incrOutstandingAioOperationCount() { + return outstandingAioOpsCount_.increment(); +} + +uint16_t JournalFile::decrOutstandingAioOperationCount() { + uint16_t r = outstandingAioOpsCount_.decrementLimit(); + if (fileCloseFlag_ && outstandingAioOpsCount_ == 0) { // Delayed close + close(); + } + return r; +} + +// --- Status helper functions --- + +bool JournalFile::isEmpty() const { + return submittedDblkCount_ == 0; +} + +bool JournalFile::isDataEmpty() const { + return submittedDblkCount_ <= QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_DBLKS; +} + +u_int32_t JournalFile::dblksRemaining() const { + return fileSize_dblks_ - submittedDblkCount_; +} + +bool JournalFile::isFull() const { + return submittedDblkCount_ == fileSize_dblks_; +} + +bool JournalFile::isFullAndComplete() const { + return completedDblkCount_ == fileSize_dblks_; +} + +u_int32_t JournalFile::getOutstandingAioDblks() const { + return submittedDblkCount_ - completedDblkCount_; +} + +bool JournalFile::getNextFile() const { + return isFull(); +} + +bool JournalFile::isNoEnqueuedRecordsRemaining() const { + return !isDataEmpty() && // Must be written to, not empty + enqueuedRecordCount_ == 0; // No remaining enqueued records +} + +// debug aid +const std::string JournalFile::status_str(const uint8_t indentDepth) const { + std::string indent((size_t)indentDepth, '.'); + std::ostringstream oss; + oss << indent << "JournalFile: fileName=" << getFileName() << std::endl; + oss << indent << " directory=" << getDirectory() << std::endl; + oss << indent << " fileSizeDblks=" << fileSize_dblks_ << std::endl; + oss << indent << " open=" << (isOpen() ? "T" : "F") << std::endl; + oss << indent << " fileHandle=" << fileHandle_ << std::endl; + oss << indent << " enqueuedRecordCount=" << getEnqueuedRecordCount() << std::endl; + oss << indent << " submittedDblkCount=" << getSubmittedDblkCount() << std::endl; + oss << indent << " completedDblkCount=" << getCompletedDblkCount() << std::endl; + oss << indent << " outstandingAioOpsCount=" << getOutstandingAioOperationCount() << std::endl; + oss << indent << " isEmpty()=" << (isEmpty() ? "T" : "F") << std::endl; + oss << indent << " isDataEmpty()=" << (isDataEmpty() ? "T" : "F") << std::endl; + oss << indent << " dblksRemaining()=" << dblksRemaining() << std::endl; + oss << indent << " isFull()=" << (isFull() ? "T" : "F") << std::endl; + oss << indent << " isFullAndComplete()=" << (isFullAndComplete() ? "T" : "F") << std::endl; + oss << indent << " getOutstandingAioDblks()=" << getOutstandingAioDblks() << std::endl; + oss << indent << " getNextFile()=" << (getNextFile() ? "T" : "F") << std::endl; + return oss.str(); +} + +}} // namespace qpid::qls_jrnl diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/JournalFile.h b/qpid/cpp/src/qpid/linearstore/jrnl/JournalFile.h new file mode 100644 index 0000000000..262d3ce147 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/JournalFile.h @@ -0,0 +1,114 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LINEARSTORE_JOURNALFILE_H_ +#define QPID_LINEARSTORE_JOURNALFILE_H_ + +#include "qpid/linearstore/jrnl/aio.h" +#include "qpid/linearstore/jrnl/AtomicCounter.h" +#include "qpid/linearstore/jrnl/EmptyFilePoolTypes.h" +#include <stdint.h> +#include <string> + +class file_hdr_t; + +namespace qpid { +namespace qls_jrnl { + +class JournalFile +{ +protected: + const std::string fqFileName_; + const uint64_t fileSeqNum_; + int fileHandle_; + bool fileCloseFlag_; + void* fileHeaderBasePtr_; + ::file_hdr_t* fileHeaderPtr_; + aio_cb* aioControlBlockPtr_; + uint32_t fileSize_dblks_; ///< File size in data blocks, including file header + + AtomicCounter<uint32_t> enqueuedRecordCount_; ///< Count of enqueued records + AtomicCounter<uint32_t> submittedDblkCount_; ///< Write file count (data blocks) for submitted AIO + AtomicCounter<uint32_t> completedDblkCount_; ///< Write file count (data blocks) for completed AIO + AtomicCounter<uint16_t> outstandingAioOpsCount_; ///< Outstanding AIO operations on this file + +public: + JournalFile(const std::string& fqFileName, + const uint64_t fileSeqNum, + const efpDataSize_kib_t efpDataSize_kib); + virtual ~JournalFile(); + + void initialize(const uint32_t completedDblkCount); + void finalize(); + + const std::string getDirectory() const; + const std::string getFileName() const; + const std::string getFqFileName() const; + uint64_t getFileSeqNum() const; + + bool isOpen() const; + int open(); + void close(); + void asyncFileHeaderWrite(io_context_t ioContextPtr, + const efpPartitionNumber_t efpPartitionNumber, + const efpDataSize_kib_t efpDataSize_kib, + const uint16_t userFlags, + const uint64_t recordId, + const uint64_t firstRecordOffset, + const std::string queueName); + void asyncPageWrite(io_context_t ioContextPtr, + aio_cb* aioControlBlockPtr, + void* data, + uint32_t dataSize_dblks); + + uint32_t getEnqueuedRecordCount() const; + uint32_t incrEnqueuedRecordCount(); + uint32_t addEnqueuedRecordCount(const uint32_t a); + uint32_t decrEnqueuedRecordCount(); + uint32_t subtrEnqueuedRecordCount(const uint32_t s); + + uint32_t getSubmittedDblkCount() const; + uint32_t addSubmittedDblkCount(const uint32_t a); + + uint32_t getCompletedDblkCount() const; + uint32_t addCompletedDblkCount(const uint32_t a); + + uint16_t getOutstandingAioOperationCount() const; + uint16_t incrOutstandingAioOperationCount(); + uint16_t decrOutstandingAioOperationCount(); + + // Status helper functions + bool isEmpty() const; ///< True if no writes of any kind have occurred + bool isDataEmpty() const; ///< True if only file header written, data is still empty + u_int32_t dblksRemaining() const; ///< Dblks remaining until full + bool isFull() const; ///< True if all possible dblks have been submitted (but may not yet have returned from AIO) + bool isFullAndComplete() const; ///< True if all submitted dblks have returned from AIO + u_int32_t getOutstandingAioDblks() const; ///< Dblks still to be written + bool getNextFile() const; ///< True when next file is needed + bool isNoEnqueuedRecordsRemaining() const; ///< True when all enqueued records (or parts) have been dequeued + + // debug aid + const std::string status_str(const uint8_t indentDepth) const; +}; + +}} // namespace qpid::qls_jrnl + +#endif // QPID_LINEARSTORE_JOURNALFILE_H_ diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/JournalLog.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/JournalLog.cpp new file mode 100644 index 0000000000..15222e3a47 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/JournalLog.cpp @@ -0,0 +1,61 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "JournalLog.h" +#include <iostream> + +namespace qpid { +namespace qls_jrnl { + +JournalLog::JournalLog(log_level_t logLevelThreshold) : logLevelThreshold_(logLevelThreshold) {} + +JournalLog::~JournalLog() {} + +void JournalLog::log(const log_level_t logLevel, + const std::string& logStatement) const { + if (logLevel >= logLevelThreshold_) { + std::cerr << log_level_str(logLevel) << ": " << logStatement << std::endl; + } +} + +void JournalLog::log(log_level_t logLevel, + const std::string& journalId, + const std::string& logStatement) const { + if (logLevel >= logLevelThreshold_) { + std::cerr << log_level_str(logLevel) << ": Journal \"" << journalId << "\": " << logStatement << std::endl; + } +} + +const char* JournalLog::log_level_str(log_level_t logLevel) { + switch (logLevel) + { + case LOG_TRACE: return "TRACE"; + case LOG_DEBUG: return "DEBUG"; + case LOG_INFO: return "INFO"; + case LOG_NOTICE: return "NOTICE"; + case LOG_WARN: return "WARN"; + case LOG_ERROR: return "ERROR"; + case LOG_CRITICAL: return "CRITICAL"; + } + return "<log level unknown>"; +} + +}} // namespace qpid::qls_jrnl diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/JournalLog.h b/qpid/cpp/src/qpid/linearstore/jrnl/JournalLog.h new file mode 100644 index 0000000000..b5fcdbc256 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/JournalLog.h @@ -0,0 +1,59 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LINEARSTORE_JOURNALLOG_H_ +#define QPID_LINEARSTORE_JOURNALLOG_H_ + +#include <string> + +namespace qpid { +namespace qls_jrnl { + +class JournalLog +{ +public: + typedef enum _log_level { + LOG_TRACE = 0, + LOG_DEBUG, + LOG_INFO, + LOG_NOTICE, + LOG_WARN, + LOG_ERROR, + LOG_CRITICAL + } log_level_t; + +protected: + const log_level_t logLevelThreshold_; + +public: + JournalLog(log_level_t logLevelThreshold); + virtual ~JournalLog(); + virtual void log(const log_level_t logLevel, + const std::string& logStatement) const; + virtual void log(const log_level_t logLevel, + const std::string& journalId, + const std::string& logStatement) const; + static const char* log_level_str(const log_level_t logLevel); +}; + +}} // namespace qpid::qls_jrnl + +#endif // QPID_LINEARSTORE_JOURNALLOG_H_ diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/LinearFileController.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/LinearFileController.cpp new file mode 100644 index 0000000000..925e6597a9 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/LinearFileController.cpp @@ -0,0 +1,304 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/jrnl/LinearFileController.h" + +#include <fstream> +#include "qpid/linearstore/jrnl/EmptyFilePool.h" +#include "qpid/linearstore/jrnl/jcfg.h" +#include "qpid/linearstore/jrnl/jcntl.h" +#include "qpid/linearstore/jrnl/JournalFile.h" +#include "qpid/linearstore/jrnl/slock.h" +#include "qpid/linearstore/jrnl/utils/file_hdr.h" + +namespace qpid { +namespace qls_jrnl { + +LinearFileController::LinearFileController(jcntl& jcntlRef) : + jcntlRef_(jcntlRef), + emptyFilePoolPtr_(0), + currentJournalFilePtr_(0), + fileSeqCounter_(0), + recordIdCounter_(0) +{} + +LinearFileController::~LinearFileController() {} + +void LinearFileController::initialize(const std::string& journalDirectory, + EmptyFilePool* emptyFilePoolPtr, + uint64_t initialFileNumberVal) { + journalDirectory_.assign(journalDirectory); + emptyFilePoolPtr_ = emptyFilePoolPtr; + fileSeqCounter_ = initialFileNumberVal; +} + +void LinearFileController::finalize() { + while (!journalFileList_.empty()) { + delete journalFileList_.front(); + journalFileList_.pop_front(); + } +} + +void LinearFileController::addJournalFile(const std::string& fileName, + const uint64_t fileNumber, + const uint32_t fileSize_kib, + const uint32_t completedDblkCount) { + if (currentJournalFilePtr_) + currentJournalFilePtr_->close(); + currentJournalFilePtr_ = new JournalFile(fileName, fileNumber, fileSize_kib); + currentJournalFilePtr_->initialize(completedDblkCount); + { + slock l(journalFileListMutex_); + journalFileList_.push_back(currentJournalFilePtr_); + } + currentJournalFilePtr_->open(); +} + +efpDataSize_kib_t LinearFileController::dataSize_kib() const { + return emptyFilePoolPtr_->dataSize_kib(); +} + +efpDataSize_sblks_t LinearFileController::dataSize_sblks() const { + return emptyFilePoolPtr_->dataSize_sblks(); +} + +efpFileSize_kib_t LinearFileController::fileSize_kib() const { + return emptyFilePoolPtr_->fileSize_kib(); +} + +efpFileSize_sblks_t LinearFileController::fileSize_sblks() const { + return emptyFilePoolPtr_->fileSize_sblks(); +} + +uint64_t LinearFileController::getNextRecordId() { + return recordIdCounter_.increment(); +} + +void LinearFileController::pullEmptyFileFromEfp() { + if (currentJournalFilePtr_) + currentJournalFilePtr_->close(); + std::string ef = emptyFilePoolPtr_->takeEmptyFile(journalDirectory_); // Moves file from EFP only, returns new file name +//std::cout << "*** LinearFileController::pullEmptyFileFromEfp() qn=" << jcntlRef.id() << " ef=" << ef << std::endl; // DEBUG + addJournalFile(ef, getNextFileSeqNum(), emptyFilePoolPtr_->dataSize_kib(), 0); +} + +void LinearFileController::purgeFilesToEfp() { + slock l(journalFileListMutex_); + while (journalFileList_.front()->isNoEnqueuedRecordsRemaining()) { + emptyFilePoolPtr_->returnEmptyFile(journalFileList_.front()->getFqFileName()); + delete journalFileList_.front(); + journalFileList_.pop_front(); + } +} + +uint32_t LinearFileController::getEnqueuedRecordCount(const efpFileCount_t fileSeqNumber) { + slock l(journalFileListMutex_); + return find(fileSeqNumber)->getEnqueuedRecordCount(); +} + +uint32_t LinearFileController::incrEnqueuedRecordCount(const efpFileCount_t fileSeqNumber) { + assertCurrentJournalFileValid("incrEnqueuedRecordCount"); + return find(fileSeqNumber)->incrEnqueuedRecordCount(); +} + +uint32_t LinearFileController::decrEnqueuedRecordCount(const efpFileCount_t fileSeqNumber) { + slock l(journalFileListMutex_); + return find(fileSeqNumber)->decrEnqueuedRecordCount(); +} + +uint32_t LinearFileController::addWriteCompletedDblkCount(const efpFileCount_t fileSeqNumber, const uint32_t a) { + slock l(journalFileListMutex_); + return find(fileSeqNumber)->addCompletedDblkCount(a); +} + +uint16_t LinearFileController::decrOutstandingAioOperationCount(const efpFileCount_t fileSeqNumber) { + slock l(journalFileListMutex_); + return find(fileSeqNumber)->decrOutstandingAioOperationCount(); +} + +void LinearFileController::asyncFileHeaderWrite(io_context_t ioContextPtr, + const uint16_t userFlags, + const uint64_t recordId, + const uint64_t firstRecordOffset) { + currentJournalFilePtr_->asyncFileHeaderWrite(ioContextPtr, + emptyFilePoolPtr_->getPartitionNumber(), + emptyFilePoolPtr_->dataSize_kib(), + userFlags, + recordId, + firstRecordOffset, + jcntlRef_.id()); +} + +void LinearFileController::asyncPageWrite(io_context_t ioContextPtr, + aio_cb* aioControlBlockPtr, + void* data, + uint32_t dataSize_dblks) { + assertCurrentJournalFileValid("asyncPageWrite"); + currentJournalFilePtr_->asyncPageWrite(ioContextPtr, aioControlBlockPtr, data, dataSize_dblks); +} + +uint64_t LinearFileController::getCurrentFileSeqNum() const { + assertCurrentJournalFileValid("getCurrentFileSeqNum"); + return currentJournalFilePtr_->getFileSeqNum(); +} + +uint32_t LinearFileController::getEnqueuedRecordCount() const { + assertCurrentJournalFileValid("getEnqueuedRecordCount"); + return currentJournalFilePtr_->getEnqueuedRecordCount(); +} + +uint32_t LinearFileController::incrEnqueuedRecordCount() { + assertCurrentJournalFileValid("incrEnqueuedRecordCount"); + return currentJournalFilePtr_->incrEnqueuedRecordCount(); +} + +uint32_t LinearFileController::addEnqueuedRecordCount(const uint32_t a) { + assertCurrentJournalFileValid("addEnqueuedRecordCount"); + return currentJournalFilePtr_->addEnqueuedRecordCount(a); +} + +uint32_t LinearFileController::decrEnqueuedRecordCount() { + assertCurrentJournalFileValid("decrEnqueuedRecordCount"); + return currentJournalFilePtr_->decrEnqueuedRecordCount(); +} + +uint32_t LinearFileController::subtrEnqueuedRecordCount(const uint32_t s) { + assertCurrentJournalFileValid("subtrEnqueuedRecordCount"); + return currentJournalFilePtr_->subtrEnqueuedRecordCount(s); +} + +uint32_t LinearFileController::getWriteSubmittedDblkCount() const { + assertCurrentJournalFileValid("getWriteSubmittedDblkCount"); + return currentJournalFilePtr_->getSubmittedDblkCount(); +} + +uint32_t LinearFileController::addWriteSubmittedDblkCount(const uint32_t a) { + assertCurrentJournalFileValid("addWriteSubmittedDblkCount"); + return currentJournalFilePtr_->addSubmittedDblkCount(a); +} + +uint32_t LinearFileController::getWriteCompletedDblkCount() const { + assertCurrentJournalFileValid("getWriteCompletedDblkCount"); + return currentJournalFilePtr_->getCompletedDblkCount(); +} + +uint32_t LinearFileController::addWriteCompletedDblkCount(const uint32_t a) { + assertCurrentJournalFileValid("addWriteCompletedDblkCount"); + return currentJournalFilePtr_->addCompletedDblkCount(a); +} + +uint16_t LinearFileController::getOutstandingAioOperationCount() const { + assertCurrentJournalFileValid("getOutstandingAioOperationCount"); + return currentJournalFilePtr_->getOutstandingAioOperationCount(); +} + +uint16_t LinearFileController::incrOutstandingAioOperationCount() { + assertCurrentJournalFileValid("incrOutstandingAioOperationCount"); + return currentJournalFilePtr_->incrOutstandingAioOperationCount(); +} + +uint16_t LinearFileController::decrOutstandingAioOperationCount() { + assertCurrentJournalFileValid("decrOutstandingAioOperationCount"); + return currentJournalFilePtr_->decrOutstandingAioOperationCount(); +} + +bool LinearFileController::isEmpty() const { + assertCurrentJournalFileValid("isEmpty"); + return currentJournalFilePtr_->isEmpty(); +} + +bool LinearFileController::isDataEmpty() const { + assertCurrentJournalFileValid("isDataEmpty"); + return currentJournalFilePtr_->isDataEmpty(); +} + +u_int32_t LinearFileController::dblksRemaining() const { + assertCurrentJournalFileValid("dblksRemaining"); + return currentJournalFilePtr_->dblksRemaining(); +} + +bool LinearFileController::isFull() const { + assertCurrentJournalFileValid("isFull"); + return currentJournalFilePtr_->isFull(); +} + +bool LinearFileController::isFullAndComplete() const { + assertCurrentJournalFileValid("isFullAndComplete"); + return currentJournalFilePtr_->isFullAndComplete(); +} + +u_int32_t LinearFileController::getOutstandingAioDblks() const { + assertCurrentJournalFileValid("getOutstandingAioDblks"); + return currentJournalFilePtr_->getOutstandingAioDblks(); +} + +bool LinearFileController::needNextFile() const { + assertCurrentJournalFileValid("getNextFile"); + return currentJournalFilePtr_->getNextFile(); +} + +const std::string LinearFileController::status(const uint8_t indentDepth) const { + std::string indent((size_t)indentDepth, '.'); + std::ostringstream oss; + oss << indent << "LinearFileController: queue=" << jcntlRef_.id() << std::endl; + oss << indent << " journalDirectory=" << journalDirectory_ << std::endl; + oss << indent << " fileSeqCounter=" << fileSeqCounter_.get() << std::endl; + oss << indent << " recordIdCounter=" << recordIdCounter_.get() << std::endl; + oss << indent << " journalFileList.size=" << journalFileList_.size() << std::endl; + if (checkCurrentJournalFileValid()) { + oss << currentJournalFilePtr_->status_str(indentDepth+2); + } else { + oss << indent << " <No current journal file>" << std::endl; + } + return oss.str(); +} + +// --- protected functions --- + +bool LinearFileController::checkCurrentJournalFileValid() const { + return currentJournalFilePtr_ != 0; +} + +void LinearFileController::assertCurrentJournalFileValid(const char* const functionName) const { + if (!checkCurrentJournalFileValid()) { + throw jexception(jerrno::JERR__NULL, "LinearFileController", functionName); + } +} + +// NOTE: NOT THREAD SAFE - journalFileList is accessed by multiple threads - use under external lock +JournalFile* LinearFileController::find(const efpFileCount_t fileSeqNumber) { + if (currentJournalFilePtr_ != 0 && currentJournalFilePtr_->getFileSeqNum() == fileSeqNumber) + return currentJournalFilePtr_; + for (JournalFileListItr_t i=journalFileList_.begin(); i!=journalFileList_.end(); ++i) { + if ((*i)->getFileSeqNum() == fileSeqNumber) { + return *i; + } + } + std::ostringstream oss; + oss << "fileSeqNumber=" << fileSeqNumber; + throw jexception(jerrno::JERR_LFCR_SEQNUMNOTFOUND, oss.str(), "LinearFileController", "find"); +} + +uint64_t LinearFileController::getNextFileSeqNum() { + return fileSeqCounter_.increment(); +} + +}} // namespace qpid::qls_jrnl diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/LinearFileController.h b/qpid/cpp/src/qpid/linearstore/jrnl/LinearFileController.h new file mode 100644 index 0000000000..7721685196 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/LinearFileController.h @@ -0,0 +1,136 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LINEARSTORE_LINEARFILECONTROLLER_H_ +#define QPID_LINEARSTORE_LINEARFILECONTROLLER_H_ + +#include <deque> +#include "qpid/linearstore/jrnl/AtomicCounter.h" +#include "qpid/linearstore/jrnl/EmptyFilePoolTypes.h" + +// libaio forward declares +typedef struct io_context* io_context_t; +typedef struct iocb aio_cb; + +namespace qpid { +namespace qls_jrnl { + +class EmptyFilePool; +class jcntl; +class JournalFile; + +class LinearFileController +{ +protected: + typedef std::deque<JournalFile*> JournalFileList_t; + typedef JournalFileList_t::iterator JournalFileListItr_t; + + jcntl& jcntlRef_; + std::string journalDirectory_; + EmptyFilePool* emptyFilePoolPtr_; + JournalFile* currentJournalFilePtr_; + AtomicCounter<uint64_t> fileSeqCounter_; + AtomicCounter<uint64_t> recordIdCounter_; + + JournalFileList_t journalFileList_; + smutex journalFileListMutex_; + +public: + LinearFileController(jcntl& jcntlRef); + virtual ~LinearFileController(); + + void initialize(const std::string& journalDirectory, + EmptyFilePool* emptyFilePoolPtr, + uint64_t initialFileNumberVal); + void finalize(); + + void addJournalFile(const std::string& fileName, + const uint64_t fileNumber, + const uint32_t fileSize_kib, + const uint32_t completedDblkCount); + + efpDataSize_kib_t dataSize_kib() const; + efpDataSize_sblks_t dataSize_sblks() const; + efpFileSize_kib_t fileSize_kib() const; + efpFileSize_sblks_t fileSize_sblks() const; + uint64_t getNextRecordId(); + void pullEmptyFileFromEfp(); + void purgeFilesToEfp(); + + // Functions for manipulating counts of non-current JournalFile instances in journalFileList_ + uint32_t getEnqueuedRecordCount(const efpFileCount_t fileSeqNumber); + uint32_t incrEnqueuedRecordCount(const efpFileCount_t fileSeqNumber); + uint32_t decrEnqueuedRecordCount(const efpFileCount_t fileSeqNumber); + uint32_t addWriteCompletedDblkCount(const efpFileCount_t fileSeqNumber, + const uint32_t a); + uint16_t decrOutstandingAioOperationCount(const efpFileCount_t fileSeqNumber); + + // Pass-through functions for JournalFile class + void asyncFileHeaderWrite(io_context_t ioContextPtr, + const uint16_t userFlags, + const uint64_t recordId, + const uint64_t firstRecordOffset); + void asyncPageWrite(io_context_t ioContextPtr, + aio_cb* aioControlBlockPtr, + void* data, + uint32_t dataSize_dblks); + + uint64_t getCurrentFileSeqNum() const; + + uint32_t getEnqueuedRecordCount() const; + uint32_t incrEnqueuedRecordCount(); + uint32_t addEnqueuedRecordCount(const uint32_t a); + uint32_t decrEnqueuedRecordCount(); + uint32_t subtrEnqueuedRecordCount(const uint32_t s); + + uint32_t getWriteSubmittedDblkCount() const; + uint32_t addWriteSubmittedDblkCount(const uint32_t a); + + uint32_t getWriteCompletedDblkCount() const; + uint32_t addWriteCompletedDblkCount(const uint32_t a); + + uint16_t getOutstandingAioOperationCount() const; + uint16_t incrOutstandingAioOperationCount(); + uint16_t decrOutstandingAioOperationCount(); + + bool isEmpty() const; // True if no writes of any kind have occurred + bool isDataEmpty() const; // True if only file header written, data is still empty + u_int32_t dblksRemaining() const; // Dblks remaining until full + bool isFull() const; // True if all possible dblks have been submitted (but may not yet have returned from AIO) + bool isFullAndComplete() const; // True if all submitted dblks have returned from AIO + u_int32_t getOutstandingAioDblks() const; // Dblks still to be written + bool needNextFile() const; // True when next file is needed + + // Debug aid + const std::string status(const uint8_t indentDepth) const; + +protected: + void assertCurrentJournalFileValid(const char* const functionName) const; + bool checkCurrentJournalFileValid() const; + JournalFile* find(const efpFileCount_t fileSeqNumber); + uint64_t getNextFileSeqNum(); +}; + +typedef void (LinearFileController::*lfcAddJournalFileFn)(const std::string&, const uint64_t, const uint32_t, const uint32_t); + +}} // namespace qpid::qls_jrnl + +#endif // QPID_LINEARSTORE_LINEARFILECONTROLLER_H_ diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/RecoveryManager.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/RecoveryManager.cpp new file mode 100644 index 0000000000..1370949908 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/RecoveryManager.cpp @@ -0,0 +1,645 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/jrnl/RecoveryManager.h" + +#include <algorithm> +#include <cstdlib> +#include <iomanip> +#include "qpid/linearstore/jrnl/data_tok.h" +#include "qpid/linearstore/jrnl/deq_rec.h" +#include "qpid/linearstore/jrnl/EmptyFilePoolManager.h" +#include "qpid/linearstore/jrnl/enq_map.h" +#include "qpid/linearstore/jrnl/enq_rec.h" +#include "qpid/linearstore/jrnl/jcfg.h" +#include "qpid/linearstore/jrnl/jdir.h" +#include "qpid/linearstore/jrnl/JournalLog.h" +#include "qpid/linearstore/jrnl/jrec.h" +#include "qpid/linearstore/jrnl/LinearFileController.h" +#include "qpid/linearstore/jrnl/txn_map.h" +#include "qpid/linearstore/jrnl/txn_rec.h" +#include "qpid/linearstore/jrnl/utils/enq_hdr.h" +#include "qpid/linearstore/jrnl/utils/file_hdr.h" +#include <sstream> +#include <string> +#include <vector> + +namespace qpid { +namespace qls_jrnl +{ + +RecoveryManager::RecoveryManager(const std::string& journalDirectory, + const std::string& queuename, + enq_map& enqueueMapRef, + txn_map& transactionMapRef, + JournalLog& journalLogRef) : + journalDirectory_(journalDirectory), + queueName_(queuename), + enqueueMapRef_(enqueueMapRef), + transactionMapRef_(transactionMapRef), + journalLogRef_(journalLogRef), + journalEmptyFlag_(false), + firstRecordOffset_(0), + endOffset_(0), + highestRecordId_(0ULL), + highestFileNumber_(0ULL), + lastFileFullFlag_(false), + fileSize_kib_(0) +{} + +RecoveryManager::~RecoveryManager() {} + +void RecoveryManager::analyzeJournals(const std::vector<std::string>* preparedTransactionListPtr, + EmptyFilePoolManager* emptyFilePoolManager, + EmptyFilePool** emptyFilePoolPtrPtr) { + // Analyze file headers of existing journal files + efpIdentity_t efpIdentity; + analyzeJournalFileHeaders(efpIdentity); + *emptyFilePoolPtrPtr = emptyFilePoolManager->getEmptyFilePool(efpIdentity); + fileSize_kib_ = (*emptyFilePoolPtrPtr)->fileSize_kib(); + // Check for file full condition + lastFileFullFlag_ = endOffset_ == (std::streamoff)(*emptyFilePoolPtrPtr)->fileSize_kib() * 1024; + + // Restore all read and write pointers and transactions + if (!journalEmptyFlag_) { + while (getNextRecordHeader()) { + + } + if (inFileStream_.is_open()) { + inFileStream_.close(); + } + // Remove leading files which have no enqueued records + removeEmptyFiles(*emptyFilePoolPtrPtr); + + // Remove all txns from tmap that are not in the prepared list + if (preparedTransactionListPtr) { + std::vector<std::string> xidList; + transactionMapRef_.xid_list(xidList); + for (std::vector<std::string>::iterator itr = xidList.begin(); itr != xidList.end(); itr++) { + std::vector<std::string>::const_iterator pitr = + std::find(preparedTransactionListPtr->begin(), preparedTransactionListPtr->end(), *itr); + if (pitr == preparedTransactionListPtr->end()) { // not found in prepared list + txn_data_list tdl = transactionMapRef_.get_remove_tdata_list(*itr); // tdl will be empty if xid not found + // Unlock any affected enqueues in emap + for (tdl_itr i=tdl.begin(); i<tdl.end(); i++) { + if (i->_enq_flag) { // enq op - decrement enqueue count + enqueueCountList_[i->_pfid]--; + } else if (enqueueMapRef_.is_enqueued(i->_drid, true)) { // deq op - unlock enq record + int16_t ret = enqueueMapRef_.unlock(i->_drid); + if (ret < enq_map::EMAP_OK) { // fail + // enq_map::unlock()'s only error is enq_map::EMAP_RID_NOT_FOUND + std::ostringstream oss; + oss << std::hex << "_emap.unlock(): drid=0x\"" << i->_drid; + throw jexception(jerrno::JERR_MAP_NOTFOUND, oss.str(), "RecoveryManager", "analyzeJournals"); + } + } + } + } + } + } + enqueueMapRef_.rid_list(recordIdList_); + recordIdListConstItr_ = recordIdList_.begin(); + } +} + +std::streamoff RecoveryManager::getEndOffset() const { + return endOffset_; +} + +uint64_t RecoveryManager::getHighestFileNumber() const { + return highestFileNumber_; +} + +uint64_t RecoveryManager::getHighestRecordId() const { + return highestRecordId_; +} + +bool RecoveryManager::isLastFileFull() const { + return lastFileFullFlag_; +} + +bool RecoveryManager::readNextRemainingRecord(void** const dataPtrPtr, + std::size_t& dataSize, + void** const xidPtrPtr, + std::size_t& xidSize, + bool& transient, + bool& external, + data_tok* const dtokp, + bool /*ignore_pending_txns*/) { + if (!dtokp->is_readable()) { + std::ostringstream oss; + oss << std::hex << std::setfill('0') << "dtok_id=0x" << std::setw(8) << dtokp->id(); + oss << "; dtok_rid=0x" << std::setw(16) << dtokp->rid() << "; dtok_wstate=" << dtokp->wstate_str(); + throw jexception(jerrno::JERR_JCNTL_ENQSTATE, oss.str(), "RecoveryManager", "readNextRemainingRecord"); + } + if (recordIdListConstItr_ == recordIdList_.end()) { + return false; + } + enq_map::emap_data_struct_t eds; + enqueueMapRef_.get_data(*recordIdListConstItr_, eds); + uint64_t fileNumber = eds._pfid; + currentJournalFileConstItr_ = fileNumberNameMap_.find(fileNumber); + getNextFile(false); + + inFileStream_.seekg(eds._file_posn, std::ifstream::beg); + if (!inFileStream_.good()) { + std::ostringstream oss; + oss << "Could not find offset 0x" << std::hex << eds._file_posn << " in file " << getCurrentFileName(); + throw jexception(jerrno::JERR__FILEIO, oss.str(), "RecoveryManager", "readNextRemainingRecord"); + } + ::enq_hdr_t enqueueHeader; + inFileStream_.read((char*)&enqueueHeader, sizeof(::enq_hdr_t)); + if (inFileStream_.gcount() != sizeof(::enq_hdr_t)) { + std::ostringstream oss; + oss << "Could not read enqueue header from file " << getCurrentFileName() << " at offset 0x" << std::hex << eds._file_posn; + throw jexception(jerrno::JERR__FILEIO, oss.str(), "RecoveryManager", "readNextRemainingRecord"); + } + // check flags + transient = ::is_enq_transient(&enqueueHeader); + external = ::is_enq_external(&enqueueHeader); + + // read xid + xidSize = enqueueHeader._xidsize; + *xidPtrPtr = ::malloc(xidSize); + if (*xidPtrPtr == 0) { + std::ostringstream oss; + oss << "xidPtr, size=0x" << std::hex << xidSize; + throw jexception(jerrno::JERR__MALLOC, oss.str(), "RecoveryManager", "readNextRemainingRecord"); + } + readJournalData((char*)*xidPtrPtr, xidSize); + + // read data + dataSize = enqueueHeader._dsize; + *dataPtrPtr = ::malloc(dataSize); + if (*xidPtrPtr == 0) { + std::ostringstream oss; + oss << "dataPtr, size=0x" << std::hex << dataSize; + throw jexception(jerrno::JERR__MALLOC, oss.str(), "RecoveryManager", "readNextRemainingRecord"); + } + readJournalData((char*)*dataPtrPtr, dataSize); + return true; +} + +void RecoveryManager::setLinearFileControllerJournals(lfcAddJournalFileFn fnPtr, + LinearFileController* lfcPtr) { +//std::cout << "****** RecoveryManager::setLinearFileControllerJournals():" << std::endl; // DEBUG + for (fileNumberNameMapConstItr_t i = fileNumberNameMap_.begin(); i != fileNumberNameMap_.end(); ++i) { + uint32_t fileDblkCount = i->first == highestFileNumber_ ? // Is this this last file? + endOffset_ / QLS_DBLK_SIZE_BYTES : // Last file uses _endOffset + fileSize_kib_ * 1024 / QLS_DBLK_SIZE_BYTES; // All others use file size to make them full + (lfcPtr->*fnPtr)(i->second, i->first, fileSize_kib_, fileDblkCount); +//std::cout << " ** f=" << i->second.substr(i->second.rfind('/')+1) << ",fn=" << i->first << ",s=" << _fileSize_kib << ",eo=" << fileDblkCount << "(" << (fileDblkCount * QLS_DBLK_SIZE_BYTES / 1024) << "kiB)" << std::endl; // DEBUG + } +} + +std::string RecoveryManager::toString(const std::string& jid, + bool compact) { + std::ostringstream oss; + if (compact) { + oss << "Recovery journal analysis (jid=\"" << jid << "\"):"; + oss << " jfl=["; + for (fileNumberNameMapConstItr_t i=fileNumberNameMap_.begin(); i!=fileNumberNameMap_.end(); ++i) { + if (i!=fileNumberNameMap_.begin()) oss << " "; + oss << i->first << ":" << i->second.substr(i->second.rfind('/')+1); + } + oss << "] ecl=[ "; + for (enqueueCountListConstItr_t j = enqueueCountList_.begin(); j!=enqueueCountList_.end(); ++j) { + if (j != enqueueCountList_.begin()) oss << " "; + oss << *j; + } + oss << " ] empty=" << (journalEmptyFlag_ ? "T" : "F"); + oss << " fro=0x" << std::hex << firstRecordOffset_ << std::dec << " (" << (firstRecordOffset_/QLS_DBLK_SIZE_BYTES) << " dblks)"; + oss << " eo=0x" << std::hex << endOffset_ << std::dec << " (" << (endOffset_/QLS_DBLK_SIZE_BYTES) << " dblks)"; + oss << " hrid=0x" << std::hex << highestRecordId_ << std::dec; + oss << " hfnum=0x" << std::hex << highestFileNumber_ << std::dec; + oss << " lffull=" << (lastFileFullFlag_ ? "T" : "F"); + } else { + oss << "Recovery journal analysis (jid=\"" << jid << "\"):" << std::endl; + oss << " Number of journal files = " << fileNumberNameMap_.size() << std::endl; + oss << " Journal File List:" << std::endl; + for (fileNumberNameMapConstItr_t i=fileNumberNameMap_.begin(); i!=fileNumberNameMap_.end(); ++i) { + oss << " " << i->first << ": " << i->second.substr(i->second.rfind('/')+1) << std::endl; + } + oss << " Enqueue Counts: [ " << std::endl; + for (enqueueCountListConstItr_t j = enqueueCountList_.begin(); j!=enqueueCountList_.end(); ++j) { + if (j != enqueueCountList_.begin()) oss << ", "; + oss << *j; + } + oss << " ]" << std::endl; + oss << " Journal empty = " << (journalEmptyFlag_ ? "TRUE" : "FALSE") << std::endl; + oss << " First record offset in first file = 0x" << std::hex << firstRecordOffset_ << + std::dec << " (" << (firstRecordOffset_/QLS_DBLK_SIZE_BYTES) << " dblks)" << std::endl; + oss << " End offset = 0x" << std::hex << endOffset_ << std::dec << " (" << + (endOffset_/QLS_DBLK_SIZE_BYTES) << " dblks)" << std::endl; + oss << " Highest rid = 0x" << std::hex << highestRecordId_ << std::dec << std::endl; + oss << " Highest file number = 0x" << std::hex << highestFileNumber_ << std::dec << std::endl; + oss << " Last file full = " << (lastFileFullFlag_ ? "TRUE" : "FALSE") << std::endl; + oss << " Enqueued records (txn & non-txn):" << std::endl; + } + return oss.str(); +} + +// --- protected functions --- + +void RecoveryManager::analyzeJournalFileHeaders(efpIdentity_t& efpIdentity) { + std::string headerQueueName; + ::file_hdr_t fileHeader; + directoryList_t directoryList; + jdir::read_dir(journalDirectory_, directoryList, false, true, false, true); + for (directoryListConstItr_t i = directoryList.begin(); i != directoryList.end(); ++i) { + readJournalFileHeader(*i, fileHeader, headerQueueName); + if (headerQueueName.compare(queueName_) != 0) { + std::ostringstream oss; + oss << "Journal file " << (*i) << " belongs to queue \"" << headerQueueName << "\": ignoring"; + journalLogRef_.log(JournalLog::LOG_WARN, queueName_, oss.str()); + } else { + fileNumberNameMap_[fileHeader._file_number] = *i; + if (fileHeader._file_number > highestFileNumber_) { + highestFileNumber_ = fileHeader._file_number; + } + } + } + efpIdentity.first = fileHeader._efp_partition; + efpIdentity.second = fileHeader._file_size_kib; + enqueueCountList_.resize(fileNumberNameMap_.size(), 0); + currentJournalFileConstItr_ = fileNumberNameMap_.begin(); +} + +void RecoveryManager::checkFileStreamOk(bool checkEof) { + if (inFileStream_.fail() || inFileStream_.bad() || checkEof ? inFileStream_.eof() : false) { + throw jexception("read failure"); // TODO complete exception + } +} + +void RecoveryManager::checkJournalAlignment(const std::streampos recordPosition) { + std::streampos currentPosn = recordPosition; + unsigned sblkOffset = currentPosn % QLS_SBLK_SIZE_BYTES; + if (sblkOffset) + { + std::ostringstream oss1; + oss1 << std::hex << "Bad record alignment found at fid=0x" << getCurrentFileNumber(); + oss1 << " offs=0x" << currentPosn << " (likely journal overwrite boundary); " << std::dec; + oss1 << (QLS_SBLK_SIZE_DBLKS - (sblkOffset/QLS_DBLK_SIZE_BYTES)) << " filler record(s) required."; + journalLogRef_.log(JournalLog::LOG_WARN, queueName_, oss1.str()); + + std::ofstream outFileStream(getCurrentFileName().c_str(), std::ios_base::in | std::ios_base::out | std::ios_base::binary); + if (!outFileStream.good()) { + throw jexception(jerrno::JERR__FILEIO, getCurrentFileName(), "RecoveryManager", "checkJournalAlignment"); + } + outFileStream.seekp(currentPosn); + + // Prepare write buffer containing a single empty record (1 dblk) + void* writeBuffer = std::malloc(QLS_DBLK_SIZE_BYTES); + if (writeBuffer == 0) { + throw jexception(jerrno::JERR__MALLOC, "RecoveryManager", "checkJournalAlignment"); + } + const uint32_t xmagic = QLS_EMPTY_MAGIC; + ::memcpy(writeBuffer, (const void*)&xmagic, sizeof(xmagic)); + ::memset((char*)writeBuffer + sizeof(xmagic), QLS_CLEAN_CHAR, QLS_DBLK_SIZE_BYTES - sizeof(xmagic)); + + // Write as many empty records as are needed to get to sblk boundary + while (currentPosn % QLS_SBLK_SIZE_BYTES) { + outFileStream.write((const char*)writeBuffer, QLS_DBLK_SIZE_BYTES); + if (outFileStream.fail()) { + throw jexception(jerrno::JERR_RCVM_WRITE, "RecoveryManager", "checkJournalAlignment"); + } + std::ostringstream oss2; + oss2 << std::hex << "Recover phase write: Wrote filler record: fid=0x" << getCurrentFileNumber(); + oss2 << " offs=0x" << currentPosn; + journalLogRef_.log(JournalLog::LOG_NOTICE, queueName_, oss2.str()); + currentPosn = outFileStream.tellp(); + } + outFileStream.close(); + std::free(writeBuffer); + journalLogRef_.log(JournalLog::LOG_INFO, queueName_, "Bad record alignment fixed."); + } + endOffset_ = currentPosn; +} + +bool RecoveryManager::decodeRecord(jrec& record, + std::size_t& cumulativeSizeRead, + ::rec_hdr_t& headerRecord, + std::streampos& fileOffset) +{ +// uint16_t start_fid = getCurrentFileNumber(); + std::streampos start_file_offs = fileOffset; + + if (highestRecordId_ == 0) { + highestRecordId_ = headerRecord._rid; + } else if (headerRecord._rid - highestRecordId_ < 0x8000000000000000ULL) { // RFC 1982 comparison for unsigned 64-bit + highestRecordId_ = headerRecord._rid; + } + + bool done = false; + while (!done) { + try { + done = record.rcv_decode(headerRecord, &inFileStream_, cumulativeSizeRead); + } + catch (const jexception& e) { +// TODO - review this logic and tidy up how rd._lfid is assigned. See new jinf.get_end_file() fn. +// Original +// if (e.err_code() != jerrno::JERR_JREC_BADRECTAIL || +// fid != (rd._ffid ? rd._ffid - 1 : _num_jfiles - 1)) throw; +// Tried this, but did not work +// if (e.err_code() != jerrno::JERR_JREC_BADRECTAIL || h._magic != 0) throw; + checkJournalAlignment(start_file_offs); +// rd._lfid = start_fid; + return false; + } + if (!done && !getNextFile(false)) { + checkJournalAlignment(start_file_offs); + return false; + } + } + return true; +} + +std::string RecoveryManager::getCurrentFileName() const { + return currentJournalFileConstItr_->second; +} + +uint64_t RecoveryManager::getCurrentFileNumber() const { + return currentJournalFileConstItr_->first; +} + +bool RecoveryManager::getNextFile(bool jumpToFirstRecordOffsetFlag) { + if (inFileStream_.is_open()) { + if (inFileStream_.eof() || !inFileStream_.good()) + { + inFileStream_.clear(); + endOffset_ = inFileStream_.tellg(); // remember file offset before closing + if (endOffset_ == -1) { throw jexception("tellg() failure"); } // Check for error code -1 TODO: compelete exception + inFileStream_.close(); + if (++currentJournalFileConstItr_ == fileNumberNameMap_.end()) { + return false; + } + } + } + if (!inFileStream_.is_open()) + { + inFileStream_.clear(); // clear eof flag, req'd for older versions of c++ + inFileStream_.open(getCurrentFileName().c_str(), std::ios_base::in | std::ios_base::binary); + if (!inFileStream_.good()) { + throw jexception(jerrno::JERR__FILEIO, getCurrentFileName(), "RecoveryManager", "getNextFile"); + } + + // Read file header +//std::cout << " F" << getCurrentFileNumber() << std::flush; // DEBUG + file_hdr_t fhdr; + inFileStream_.read((char*)&fhdr, sizeof(fhdr)); + checkFileStreamOk(true); + if (fhdr._rhdr._magic == QLS_FILE_MAGIC) { + firstRecordOffset_ = fhdr._fro; + std::streamoff foffs = jumpToFirstRecordOffsetFlag ? firstRecordOffset_ : QLS_SBLK_SIZE_BYTES; + inFileStream_.seekg(foffs); + } else { + inFileStream_.close(); + if (currentJournalFileConstItr_ == fileNumberNameMap_.begin()) { + journalEmptyFlag_ = true; + } + return false; + } + } + return true; +} + +bool RecoveryManager::getNextRecordHeader() +{ + std::size_t cum_size_read = 0; + void* xidp = 0; + rec_hdr_t h; + + bool hdr_ok = false; + std::streampos file_pos; + while (!hdr_ok) { + if (!inFileStream_.is_open()) { + if (!getNextFile(true)) { + return false; + } + } + file_pos = inFileStream_.tellg(); +//std::cout << " 0x" << std::hex << file_pos << std::dec; // DEBUG + inFileStream_.read((char*)&h, sizeof(rec_hdr_t)); + if (inFileStream_.gcount() == sizeof(rec_hdr_t)) { + hdr_ok = true; + } else { + if (!getNextFile(true)) { + return false; + } + } + } + + switch(h._magic) { + case QLS_ENQ_MAGIC: + { +//std::cout << ".e" << std::flush; // DEBUG + enq_rec er; + uint64_t start_fid = getCurrentFileNumber(); // fid may increment in decode() if record folds over file boundary + if (!decodeRecord(er, cum_size_read, h, file_pos)) { + return false; + } + if (!er.is_transient()) { // Ignore transient msgs + enqueueCountList_[start_fid]++; + if (er.xid_size()) { + er.get_xid(&xidp); + if (xidp != 0) { throw jexception("Null xid with non-null xid_size"); } // TODO complete exception + std::string xid((char*)xidp, er.xid_size()); + transactionMapRef_.insert_txn_data(xid, txn_data(h._rid, 0, start_fid, true)); + if (transactionMapRef_.set_aio_compl(xid, h._rid) < txn_map::TMAP_OK) { // fail - xid or rid not found + std::ostringstream oss; + oss << std::hex << "_tmap.set_aio_compl: txn_enq xid=\"" << xid << "\" rid=0x" << h._rid; + throw jexception(jerrno::JERR_MAP_NOTFOUND, oss.str(), "RecoveryManager", "getNextRecordHeader"); + } + std::free(xidp); + } else { + if (enqueueMapRef_.insert_pfid(h._rid, start_fid, file_pos) < enq_map::EMAP_OK) { // fail + // The only error code emap::insert_pfid() returns is enq_map::EMAP_DUP_RID. + std::ostringstream oss; + oss << std::hex << "rid=0x" << h._rid << " _pfid=0x" << start_fid; + throw jexception(jerrno::JERR_MAP_DUPLICATE, oss.str(), "RecoveryManager", "getNextRecordHeader"); + } + } + } + } + break; + case QLS_DEQ_MAGIC: + { +//std::cout << ".d" << std::flush; // DEBUG + deq_rec dr; + uint16_t start_fid = getCurrentFileNumber(); // fid may increment in decode() if record folds over file boundary + if (!decodeRecord(dr, cum_size_read, h, file_pos)) { + return false; + } + if (dr.xid_size()) { + // If the enqueue is part of a pending txn, it will not yet be in emap + enqueueMapRef_.lock(dr.deq_rid()); // ignore not found error + dr.get_xid(&xidp); + if (xidp != 0) { throw jexception("Null xid with non-null xid_size"); } // TODO complete exception + std::string xid((char*)xidp, dr.xid_size()); + transactionMapRef_.insert_txn_data(xid, txn_data(dr.rid(), dr.deq_rid(), start_fid, false, + dr.is_txn_coml_commit())); + if (transactionMapRef_.set_aio_compl(xid, dr.rid()) < txn_map::TMAP_OK) { // fail - xid or rid not found + std::ostringstream oss; + oss << std::hex << "_tmap.set_aio_compl: txn_deq xid=\"" << xid << "\" rid=0x" << dr.rid(); + throw jexception(jerrno::JERR_MAP_NOTFOUND, oss.str(), "RecoveryManager", "getNextRecordHeader"); + } + std::free(xidp); + } else { + uint64_t enq_fid; + if (enqueueMapRef_.get_remove_pfid(dr.deq_rid(), enq_fid, true) == enq_map::EMAP_OK) { // ignore not found error + enqueueCountList_[enq_fid]--; + } + } + } + break; + case QLS_TXA_MAGIC: + { +//std::cout << ".a" << std::flush; // DEBUG + txn_rec ar; + if (!decodeRecord(ar, cum_size_read, h, file_pos)) { + return false; + } + // Delete this txn from tmap, unlock any locked records in emap + ar.get_xid(&xidp); + if (xidp != 0) { + throw jexception("Null xid with non-null xid_size"); // TODO complete exception + } + std::string xid((char*)xidp, ar.xid_size()); + txn_data_list tdl = transactionMapRef_.get_remove_tdata_list(xid); // tdl will be empty if xid not found + for (tdl_itr itr = tdl.begin(); itr != tdl.end(); itr++) { + if (itr->_enq_flag) { + enqueueCountList_[itr->_pfid]--; + } else { + enqueueMapRef_.unlock(itr->_drid); // ignore not found error + } + } + std::free(xidp); + } + break; + case QLS_TXC_MAGIC: + { +//std::cout << ".t" << std::flush; // DEBUG + txn_rec cr; + if (!decodeRecord(cr, cum_size_read, h, file_pos)) { + return false; + } + // Delete this txn from tmap, process records into emap + cr.get_xid(&xidp); + if (xidp != 0) { + throw jexception("Null xid with non-null xid_size"); // TODO complete exception + } + std::string xid((char*)xidp, cr.xid_size()); + txn_data_list tdl = transactionMapRef_.get_remove_tdata_list(xid); // tdl will be empty if xid not found + for (tdl_itr itr = tdl.begin(); itr != tdl.end(); itr++) { + if (itr->_enq_flag) { // txn enqueue + if (enqueueMapRef_.insert_pfid(itr->_rid, itr->_pfid, file_pos) < enq_map::EMAP_OK) { // fail + // The only error code emap::insert_pfid() returns is enq_map::EMAP_DUP_RID. + std::ostringstream oss; + oss << std::hex << "rid=0x" << itr->_rid << " _pfid=0x" << itr->_pfid; + throw jexception(jerrno::JERR_MAP_DUPLICATE, oss.str(), "RecoveryManager", "getNextRecordHeader"); + } + } else { // txn dequeue + uint64_t enq_fid; + if (enqueueMapRef_.get_remove_pfid(itr->_drid, enq_fid, true) == enq_map::EMAP_OK) // ignore not found error + enqueueCountList_[enq_fid]--; + } + } + std::free(xidp); + } + break; + case QLS_EMPTY_MAGIC: + { +//std::cout << ".x" << std::flush; // DEBUG + uint32_t rec_dblks = jrec::size_dblks(sizeof(::rec_hdr_t)); + inFileStream_.ignore(rec_dblks * QLS_DBLK_SIZE_BYTES - sizeof(::rec_hdr_t)); + checkFileStreamOk(false); + if (!getNextFile(false)) { + return false; + } + } + break; + case 0: +//std::cout << ".0" << std::endl << std::flush; // DEBUG + checkJournalAlignment(file_pos); + return false; + default: +//std::cout << ".?" << std::endl << std::flush; // DEBUG + // Stop as this is the overwrite boundary. + checkJournalAlignment(file_pos); + return false; + } + return true; +} + +void RecoveryManager::readJournalData(char* target, + const std::streamsize readSize) { + std::streamoff bytesRead = 0; + while (bytesRead < readSize) { + if (inFileStream_.eof()) { + getNextFile(false); + } + bool readFitsInFile = inFileStream_.tellg() + readSize <= fileSize_kib_ * 1024; + std::streamoff readSize = readFitsInFile ? readSize : (fileSize_kib_ * 1024) - inFileStream_.tellg(); + inFileStream_.read(target + bytesRead, readSize); + if (inFileStream_.gcount() != readSize) { + throw jexception(); // TODO - proper exception + } + bytesRead += readSize; + } +} + +// static private +void RecoveryManager::readJournalFileHeader(const std::string& journalFileName, + ::file_hdr_t& fileHeaderRef, + std::string& queueName) { + const std::size_t headerBlockSize = QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB * 1024; + char buffer[headerBlockSize]; + std::ifstream ifs(journalFileName.c_str(), std::ifstream::in | std::ifstream::binary); + if (!ifs.good()) { + std::ostringstream oss; + oss << "File=" << journalFileName; + throw jexception(jerrno::JERR_RCVM_OPENRD, oss.str(), "RecoveryManager", "readJournalFileHeader"); + } + ifs.read(buffer, headerBlockSize); + if (!ifs) { + std::streamsize s = ifs.gcount(); + ifs.close(); + std::ostringstream oss; + oss << "File=" << journalFileName << "; attempted_read_size=" << headerBlockSize << "; actual_read_size=" << s; + throw jexception(jerrno::JERR_RCVM_READ, oss.str(), "RecoveryManager", "readJournalFileHeader"); + } + ifs.close(); + ::memcpy(&fileHeaderRef, buffer, sizeof(::file_hdr_t)); + queueName.assign(buffer + sizeof(::file_hdr_t), fileHeaderRef._queue_name_len); + +} + +void RecoveryManager::removeEmptyFiles(EmptyFilePool* emptyFilePoolPtr) { + while (enqueueCountList_.front() == 0 && enqueueCountList_.size() > 1) { + fileNumberNameMapItr_t i = fileNumberNameMap_.begin(); +//std::cout << "*** File " << i->first << ": " << i->second << " is empty." << std::endl; + emptyFilePoolPtr->returnEmptyFile(i->second); + fileNumberNameMap_.erase(i); + enqueueCountList_.pop_front(); + } +} + +}} // namespace qpid::qls_jrnl diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/RecoveryManager.h b/qpid/cpp/src/qpid/linearstore/jrnl/RecoveryManager.h new file mode 100644 index 0000000000..91446c6abf --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/RecoveryManager.h @@ -0,0 +1,134 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LINEARSTORE_RECOVERYSTATE_H_ +#define QPID_LINEARSTORE_RECOVERYSTATE_H_ + +#include <deque> +#include <fstream> +#include <map> +#include "qpid/linearstore/jrnl/LinearFileController.h" +#include <stdint.h> +#include <vector> + +struct file_hdr_t; +struct rec_hdr_t; + +namespace qpid { +namespace qls_jrnl { + +class data_tok; +class enq_map; +class EmptyFilePool; +class EmptyFilePoolManager; +class JournalLog; +class jrec; +class txn_map; + +class RecoveryManager +{ +protected: + // Types + typedef std::vector<std::string> directoryList_t; + typedef directoryList_t::const_iterator directoryListConstItr_t; + typedef std::map<uint64_t, std::string> fileNumberNameMap_t; + typedef fileNumberNameMap_t::iterator fileNumberNameMapItr_t; + typedef fileNumberNameMap_t::const_iterator fileNumberNameMapConstItr_t; + typedef std::deque<uint32_t> enqueueCountList_t; + typedef enqueueCountList_t::const_iterator enqueueCountListConstItr_t; + typedef std::vector<uint64_t> recordIdList_t; + typedef recordIdList_t::const_iterator recordIdListConstItr_t; + + // Location and identity + const std::string journalDirectory_; + const std::string queueName_; + enq_map& enqueueMapRef_; + txn_map& transactionMapRef_; + JournalLog& journalLogRef_; + + // Initial journal analysis data + fileNumberNameMap_t fileNumberNameMap_; ///< File number - name map + enqueueCountList_t enqueueCountList_; ///< Number enqueued records found for each file + bool journalEmptyFlag_; ///< Journal data files empty + std::streamoff firstRecordOffset_; ///< First record offset in ffid + std::streamoff endOffset_; ///< End offset (first byte past last record) + uint64_t highestRecordId_; ///< Highest rid found + uint64_t highestFileNumber_; ///< Highest file number found + bool lastFileFullFlag_; ///< Last file is full + + // State for recovery of individual enqueued records + uint32_t fileSize_kib_; + fileNumberNameMapConstItr_t currentJournalFileConstItr_; + std::string currentFileName_; + std::ifstream inFileStream_; + recordIdList_t recordIdList_; + recordIdListConstItr_t recordIdListConstItr_; + +public: + RecoveryManager(const std::string& journalDirectory, + const std::string& queuename, + enq_map& enqueueMapRef, + txn_map& transactionMapRef, + JournalLog& journalLogRef); + virtual ~RecoveryManager(); + + void analyzeJournals(const std::vector<std::string>* preparedTransactionListPtr, + EmptyFilePoolManager* emptyFilePoolManager, + EmptyFilePool** emptyFilePoolPtrPtr); + std::streamoff getEndOffset() const; + uint64_t getHighestFileNumber() const; + uint64_t getHighestRecordId() const; + bool isLastFileFull() const; + bool readNextRemainingRecord(void** const dataPtrPtr, + std::size_t& dataSize, + void** const xidPtrPtr, + std::size_t& xidSize, + bool& transient, + bool& external, + data_tok* const dtokp, + bool ignore_pending_txns); + void setLinearFileControllerJournals(lfcAddJournalFileFn fnPtr, + LinearFileController* lfcPtr); + std::string toString(const std::string& jid, + bool compact = true); +protected: + void analyzeJournalFileHeaders(efpIdentity_t& efpIdentity); + void checkFileStreamOk(bool checkEof); + void checkJournalAlignment(const std::streampos recordPosition); + bool decodeRecord(jrec& record, + std::size_t& cumulativeSizeRead, + ::rec_hdr_t& recordHeader, + std::streampos& fileOffset); + std::string getCurrentFileName() const; + uint64_t getCurrentFileNumber() const; + bool getNextFile(bool jumpToFirstRecordOffsetFlag); + bool getNextRecordHeader(); + void readJournalData(char* target, const std::streamsize size); + void removeEmptyFiles(EmptyFilePool* emptyFilePoolPtr); + + static void readJournalFileHeader(const std::string& journalFileName, + ::file_hdr_t& fileHeaderRef, + std::string& queueName); +}; + +}} // namespace qpid::qls_jrnl + +#endif // QPID_LINEARSTORE_RECOVERYSTATE_H_ diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/aio.h b/qpid/cpp/src/qpid/linearstore/jrnl/aio.h new file mode 100644 index 0000000000..7be5b83930 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/aio.h @@ -0,0 +1,140 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_JRNL_AIO_H +#define QPID_LEGACYSTORE_JRNL_AIO_H + +#include <libaio.h> +#include <cstring> +#include <string.h> + +namespace qpid +{ +namespace qls_jrnl +{ + +typedef iocb aio_cb; +typedef io_event aio_event; + +/** + * \brief This class is a C++ wrapper class for the libaio functions used by the journal. Note that only those + * functions used by the journal are included here. This is not a complete implementation of all libaio functions. + */ +class aio +{ +public: + static inline int queue_init(int maxevents, io_context_t* ctxp) + { + return ::io_queue_init(maxevents, ctxp); + } + + static inline int queue_release(io_context_t ctx) + { + return ::io_queue_release(ctx); + } + + static inline int submit(io_context_t ctx, long nr, aio_cb* aios[]) + { + return ::io_submit(ctx, nr, aios); + } + + static inline int getevents(io_context_t ctx, long min_nr, long nr, aio_event* events, timespec* const timeout) + { + return ::io_getevents(ctx, min_nr, nr, events, timeout); + } + + /** + * \brief This function allows iocbs to be initialized with a pointer that can be re-used. This prepares an + * aio_cb struct for read use. (This is a wrapper for libaio's ::io_prep_pread() function.) + * + * \param aiocbp Pointer to the aio_cb struct to be prepared. + * \param fd File descriptor to be used for read. + * \param buf Pointer to buffer in which read data is to be placed. + * \param count Number of bytes to read - buffer must be large enough. + * \param offset Offset within file from which data will be read. + */ + static inline void prep_pread(aio_cb* aiocbp, int fd, void* buf, std::size_t count, int64_t offset) + { + ::io_prep_pread(aiocbp, fd, buf, count, offset); + } + + /** + * \brief Special version of libaio's io_prep_pread() which preserves the value of the data pointer. This allows + * iocbs to be initialized with a pointer that can be re-used. This prepares a aio_cb struct for read use. + * + * \param aiocbp Pointer to the aio_cb struct to be prepared. + * \param fd File descriptor to be used for read. + * \param buf Pointer to buffer in which read data is to be placed. + * \param count Number of bytes to read - buffer must be large enough. + * \param offset Offset within file from which data will be read. + */ + static inline void prep_pread_2(aio_cb* aiocbp, int fd, void* buf, std::size_t count, int64_t offset) + { + std::memset((void*) ((char*) aiocbp + sizeof(void*)), 0, sizeof(aio_cb) - sizeof(void*)); + aiocbp->aio_fildes = fd; + aiocbp->aio_lio_opcode = IO_CMD_PREAD; + aiocbp->aio_reqprio = 0; + aiocbp->u.c.buf = buf; + aiocbp->u.c.nbytes = count; + aiocbp->u.c.offset = offset; + } + + /** + * \brief This function allows iocbs to be initialized with a pointer that can be re-used. This function prepares + * an aio_cb struct for write use. (This is a wrapper for libaio's ::io_prep_pwrite() function.) + * + * \param aiocbp Pointer to the aio_cb struct to be prepared. + * \param fd File descriptor to be used for write. + * \param buf Pointer to buffer in which data to be written is located. + * \param count Number of bytes to write. + * \param offset Offset within file to which data will be written. + */ + static inline void prep_pwrite(aio_cb* aiocbp, int fd, void* buf, std::size_t count, int64_t offset) + { + ::io_prep_pwrite(aiocbp, fd, buf, count, offset); + } + + /** + * \brief Special version of libaio's io_prep_pwrite() which preserves the value of the data pointer. This allows + * iocbs to be initialized with a pointer that can be re-used. This function prepares an aio_cb struct for write + * use. + * + * \param aiocbp Pointer to the aio_cb struct to be prepared. + * \param fd File descriptor to be used for write. + * \param buf Pointer to buffer in which data to be written is located. + * \param count Number of bytes to write. + * \param offset Offset within file to which data will be written. + */ + static inline void prep_pwrite_2(aio_cb* aiocbp, int fd, void* buf, std::size_t count, int64_t offset) + { + std::memset((void*) ((char*) aiocbp + sizeof(void*)), 0, sizeof(aio_cb) - sizeof(void*)); + aiocbp->aio_fildes = fd; + aiocbp->aio_lio_opcode = IO_CMD_PWRITE; + aiocbp->aio_reqprio = 0; + aiocbp->u.c.buf = buf; + aiocbp->u.c.nbytes = count; + aiocbp->u.c.offset = offset; + } +}; + +}} + +#endif // ifndef QPID_LEGACYSTORE_JRNL_AIO_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/aio_callback.h b/qpid/cpp/src/qpid/linearstore/jrnl/aio_callback.h new file mode 100644 index 0000000000..975d114f1f --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/aio_callback.h @@ -0,0 +1,45 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_JRNL_AIO_CALLBACK_H +#define QPID_LEGACYSTORE_JRNL_AIO_CALLBACK_H + +#include <stdint.h> +#include <vector> + +namespace qpid +{ +namespace qls_jrnl +{ + + class data_tok; + + class aio_callback + { + public: + virtual ~aio_callback() {} + virtual void wr_aio_cb(std::vector<data_tok*>& dtokl) = 0; + virtual void rd_aio_cb(std::vector<uint16_t>& pil) = 0; + }; + +}} + +#endif // ifndef QPID_LEGACYSTORE_JRNL_AIO_CALLBACK_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/cvar.h b/qpid/cpp/src/qpid/linearstore/jrnl/cvar.h new file mode 100644 index 0000000000..c22dcb8c1c --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/cvar.h @@ -0,0 +1,76 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_JRNL_CVAR_H +#define QPID_LEGACYSTORE_JRNL_CVAR_H + +#include <cstring> +#include "qpid/linearstore/jrnl/jerrno.h" +#include "qpid/linearstore/jrnl/jexception.h" +#include "qpid/linearstore/jrnl/smutex.h" +#include "qpid/linearstore/jrnl/time_ns.h" +#include <pthread.h> +#include <sstream> + +namespace qpid +{ +namespace qls_jrnl +{ + + // Ultra-simple thread condition variable class + class cvar + { + private: + const smutex& _sm; + pthread_cond_t _c; + public: + inline cvar(const smutex& sm) : _sm(sm) { ::pthread_cond_init(&_c, 0); } + inline ~cvar() { ::pthread_cond_destroy(&_c); } + inline void wait() + { + PTHREAD_CHK(::pthread_cond_wait(&_c, _sm.get()), "::pthread_cond_wait", "cvar", "wait"); + } + inline void timedwait(timespec& ts) + { + PTHREAD_CHK(::pthread_cond_timedwait(&_c, _sm.get(), &ts), "::pthread_cond_timedwait", "cvar", "timedwait"); + } + inline bool waitintvl(const long intvl_ns) + { + time_ns t; t.now(); t+=intvl_ns; + int ret = ::pthread_cond_timedwait(&_c, _sm.get(), &t); + if (ret == ETIMEDOUT) + return true; + PTHREAD_CHK(ret, "::pthread_cond_timedwait", "cvar", "waitintvl"); + return false; + } + inline void signal() + { + PTHREAD_CHK(::pthread_cond_signal(&_c), "::pthread_cond_signal", "cvar", "notify"); + } + inline void broadcast() + { + PTHREAD_CHK(::pthread_cond_broadcast(&_c), "::pthread_cond_broadcast", "cvar", "broadcast"); + } + }; + +}} + +#endif // ifndef QPID_LEGACYSTORE_JRNL_CVAR_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/data_tok.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/data_tok.cpp new file mode 100644 index 0000000000..ab2a763f17 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/data_tok.cpp @@ -0,0 +1,188 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/jrnl/data_tok.h" + +#include <iomanip> +#include "qpid/linearstore/jrnl/jerrno.h" +#include "qpid/linearstore/jrnl/jexception.h" +#include "qpid/linearstore/jrnl/slock.h" +#include <sstream> + +namespace qpid +{ +namespace qls_jrnl +{ + +// Static members + +uint64_t data_tok::_cnt = 0; +smutex data_tok::_mutex; + +data_tok::data_tok(): + _wstate(NONE), +// _rstate(UNREAD), + _dsize(0), + _dblks_written(0), +// _dblks_read(0), + _pg_cnt(0), + _fid(0), + _rid(0), + _xid(), + _dequeue_rid(0), + _external_rid(false) +{ + slock s(_mutex); + _icnt = _cnt++; +} + +data_tok::~data_tok() {} + +const char* +data_tok::wstate_str() const +{ + return wstate_str(_wstate); +} + +const char* +data_tok::wstate_str(write_state wstate) +{ + switch (wstate) + { + case NONE: + return "NONE"; + case ENQ_CACHED: + return "ENQ_CACHED"; + case ENQ_PART: + return "ENQ_PART"; + case ENQ_SUBM: + return "ENQ_SUBM"; + case ENQ: + return "ENQ"; + case DEQ_CACHED: + return "DEQ_CACHED"; + case DEQ_PART: + return "DEQ_PART"; + case DEQ_SUBM: + return "DEQ_SUBM"; + case DEQ: + return "DEQ"; + case ABORT_CACHED: + return "ABORT_CACHED"; + case ABORT_PART: + return "ABORT_PART"; + case ABORT_SUBM: + return "ABORT_SUBM"; + case ABORTED: + return "ABORTED"; + case COMMIT_CACHED: + return "COMMIT_CACHED"; + case COMMIT_PART: + return "COMMIT_PART"; + case COMMIT_SUBM: + return "COMMIT_SUBM"; + case COMMITTED: + return "COMMITTED"; + } + // Not using default: forces compiler to ensure all cases are covered. + return "<wstate unknown>"; +} + +/* +const char* +data_tok::rstate_str() const +{ + return rstate_str(_rstate); +} +*/ + +/* +const char* +data_tok::rstate_str(read_state rstate) +{ + switch (rstate) + { + case NONE: + return "NONE"; + case READ_PART: + return "READ_PART"; + case SKIP_PART: + return "SKIP_PART"; + case READ: + return "READ"; + // Not using default: forces compiler to ensure all cases are covered. + } + return "<rstate unknown>"; +} +*/ + +/* +void +data_tok::set_rstate(const read_state rstate) +{ + if (_wstate != ENQ && rstate != UNREAD) + { + std::ostringstream oss; + oss << "Attempted to change read state to " << rstate_str(rstate); + oss << " while write state is not enqueued (wstate ENQ); wstate=" << wstate_str() << "."; + throw jexception(jerrno::JERR_DTOK_ILLEGALSTATE, oss.str(), "data_tok", + "set_rstate"); + } + _rstate = rstate; +} +*/ + +void +data_tok::reset() +{ + _wstate = NONE; +// _rstate = UNREAD; + _dsize = 0; + _dblks_written = 0; +// _dblks_read = 0; + _pg_cnt = 0; + _fid = 0; + _rid = 0; + _xid.clear(); +} + +// debug aid +std::string +data_tok::status_str() const +{ + std::ostringstream oss; + oss << std::hex << std::setfill('0'); + oss << "dtok id=0x" << _icnt << "; ws=" << wstate_str()/* << "; rs=" << rstate_str()*/; + oss << "; fid=0x" << _fid << "; rid=0x" << _rid << "; xid="; + for (unsigned i=0; i<_xid.size(); i++) + { + if (isprint(_xid[i])) + oss << _xid[i]; + else + oss << "/" << std::setw(2) << (int)((char)_xid[i]); + } + oss << "; drid=0x" << _dequeue_rid << " extrid=" << (_external_rid?"T":"F"); + oss << "; ds=0x" << _dsize << "; dw=0x" << _dblks_written/* << "; dr=0x" << _dblks_read*/; + oss << "; pc=0x" << _pg_cnt; + return oss.str(); +} + +}} diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/data_tok.h b/qpid/cpp/src/qpid/linearstore/jrnl/data_tok.h new file mode 100644 index 0000000000..39dc1c4a81 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/data_tok.h @@ -0,0 +1,160 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_JRNL_DATA_TOK_H +#define QPID_LEGACYSTORE_JRNL_DATA_TOK_H + +namespace qpid +{ +namespace qls_jrnl +{ +class data_tok; +}} + +#include <cassert> +#include <cstddef> +#include "qpid/linearstore/jrnl/smutex.h" +#include <pthread.h> +#include <string> + +namespace qpid +{ +namespace qls_jrnl +{ + + /** + * \class data_tok + * \brief Data block token (data_tok) used to track wstate of a data block through asynchronous + * I/O process + */ + class data_tok + { + public: + // TODO: Fix this, separate write state from operation + // ie: wstate = NONE, CACHED, PART, SUBM, COMPL + // op = ENQUEUE, DEQUEUE, ABORT, COMMIT + enum write_state + { + NONE, ///< Data block not sent to journal + ENQ_CACHED, ///< Data block enqueue written to page cache + ENQ_PART, ///< Data block part-submitted to AIO, waiting for page buffer to free up + ENQ_SUBM, ///< Data block enqueue submitted to AIO + ENQ, ///< Data block enqueue AIO write complete (enqueue complete) + DEQ_CACHED, ///< Data block dequeue written to page cache + DEQ_PART, ///< Data block part-submitted to AIO, waiting for page buffer to free up + DEQ_SUBM, ///< Data block dequeue submitted to AIO + DEQ, ///< Data block dequeue AIO write complete (dequeue complete) + ABORT_CACHED, + ABORT_PART, + ABORT_SUBM, + ABORTED, + COMMIT_CACHED, + COMMIT_PART, + COMMIT_SUBM, + COMMITTED + }; + +/* + enum read_state + { + UNREAD, ///< Data block not read + READ_PART, ///< Data block is part-read; waiting for page buffer to fill + SKIP_PART, ///< Prev. dequeued dblock is part-skipped; waiting for page buffer to fill + READ ///< Data block is fully read + }; +*/ + + protected: + static smutex _mutex; + static uint64_t _cnt; + uint64_t _icnt; + write_state _wstate; ///< Enqueued / dequeued state of data +// read_state _rstate; ///< Read state of data + std::size_t _dsize; ///< Data size in bytes + uint32_t _dblks_written; ///< Data blocks read/written +// uint32_t _dblks_read; ///< Data blocks read/written + uint32_t _pg_cnt; ///< Page counter - incr for each page containing part of data + uint64_t _fid; ///< FID containing header of enqueue record + uint64_t _rid; ///< RID of data set by enqueue operation + std::string _xid; ///< XID set by enqueue operation + uint64_t _dequeue_rid; ///< RID of data set by dequeue operation + bool _external_rid; ///< Flag to indicate external setting of rid + + public: + data_tok(); + virtual ~data_tok(); + + inline uint64_t id() const { return _icnt; } + inline write_state wstate() const { return _wstate; } + const char* wstate_str() const; + static const char* wstate_str(write_state wstate); +// inline read_state rstate() const { return _rstate; } +// const char* rstate_str() const; +// static const char* rstate_str(read_state rstate); + inline bool is_writable() const { return _wstate == NONE || _wstate == ENQ_PART; } + inline bool is_enqueued() const { return _wstate == ENQ; } + inline bool is_readable() const { return _wstate == ENQ; } +// inline bool is_read() const { return _rstate == READ; } + inline bool is_dequeueable() const { return _wstate == ENQ || _wstate == DEQ_PART; } + inline void set_wstate(const write_state wstate) { _wstate = wstate; } +// void set_rstate(const read_state rstate); + inline std::size_t dsize() const { return _dsize; } + inline void set_dsize(std::size_t dsize) { _dsize = dsize; } + + inline uint32_t dblocks_written() const { return _dblks_written; } + inline void incr_dblocks_written(uint32_t dblks_written) + { _dblks_written += dblks_written; } + inline void set_dblocks_written(uint32_t dblks_written) { _dblks_written = dblks_written; } + +// inline uint32_t dblocks_read() const { return _dblks_read; } +// inline void incr_dblocks_read(uint32_t dblks_read) { _dblks_read += dblks_read; } +// inline void set_dblocks_read(uint32_t dblks_read) { _dblks_read = dblks_read; } + + inline uint32_t pg_cnt() const { return _pg_cnt; } + inline uint32_t incr_pg_cnt() { return ++_pg_cnt; } + inline uint32_t decr_pg_cnt() { assert(_pg_cnt != 0); return --_pg_cnt; } + + inline uint64_t fid() const { return _fid; } + inline void set_fid(const uint64_t fid) { _fid = fid; } + inline uint64_t rid() const { return _rid; } + inline void set_rid(const uint64_t rid) { _rid = rid; } + inline uint64_t dequeue_rid() const {return _dequeue_rid; } + inline void set_dequeue_rid(const uint64_t rid) { _dequeue_rid = rid; } + inline bool external_rid() const { return _external_rid; } + inline void set_external_rid(const bool external_rid) { _external_rid = external_rid; } + + inline bool has_xid() const { return !_xid.empty(); } + inline const std::string& xid() const { return _xid; } + inline void clear_xid() { _xid.clear(); } + inline void set_xid(const std::string& xid) { _xid.assign(xid); } + inline void set_xid(const void* xidp, const std::size_t xid_len) + { _xid.assign((const char*)xidp, xid_len); } + + void reset(); + + // debug aid + std::string status_str() const; + }; + +} // namespace qls_jrnl +} // namespace jrnl + +#endif // ifndef QPID_LEGACYSTORE_JRNL_DATA_TOK_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/deq_rec.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/deq_rec.cpp new file mode 100644 index 0000000000..5b251b7908 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/deq_rec.cpp @@ -0,0 +1,464 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +/** + * \file deq_rec.cpp + * + * Qpid asynchronous store plugin library + * + * This file contains the code for the mrg::journal::deq_rec (journal dequeue + * record) class. See comments in file deq_rec.h for details. + * + * \author Kim van der Riet + */ + +#include "qpid/linearstore/jrnl/deq_rec.h" +#include "qpid/linearstore/jrnl/utils/deq_hdr.h" +#include "qpid/linearstore/jrnl/utils/rec_tail.h" + +#include <cassert> +#include <cerrno> +#include <cstdlib> +#include <cstring> +#include <iomanip> +#include "qpid/linearstore/jrnl/jerrno.h" +#include "qpid/linearstore/jrnl/jexception.h" +#include <sstream> + +namespace qpid +{ +namespace qls_jrnl +{ + +deq_rec::deq_rec(): +// _deq_hdr(QLS_DEQ_MAGIC, QLS_JRNL_VERSION, 0, 0, 0, false), + _xidp(0), + _buff(0) +// _deq_tail(_deq_hdr) +{ + ::deq_hdr_init(&_deq_hdr, QLS_DEQ_MAGIC, QLS_JRNL_VERSION, 0, 0, 0, 0); + ::rec_tail_copy(&_deq_tail, &_deq_hdr._rhdr, 0); +} + +deq_rec::deq_rec(const uint64_t rid, const uint64_t drid, const void* const xidp, + const std::size_t xidlen, const bool txn_coml_commit): +// _deq_hdr(QLS_DEQ_MAGIC, QLS_JRNL_VERSION, rid, drid, xidlen, owi, txn_coml_commit), + _xidp(xidp), + _buff(0) +// _deq_tail(_deq_hdr) +{ + ::deq_hdr_init(&_deq_hdr, QLS_DEQ_MAGIC, QLS_JRNL_VERSION, 0, rid, drid, xidlen); + ::rec_tail_copy(&_deq_tail, &_deq_hdr._rhdr, 0); + ::set_txn_coml_commit(&_deq_hdr, txn_coml_commit); +} + +deq_rec::~deq_rec() +{ + clean(); +} + +void +deq_rec::reset() +{ + _deq_hdr._rhdr._rid = 0; +// _deq_hdr.set_owi(false); + ::set_txn_coml_commit(&_deq_hdr, false); + _deq_hdr._deq_rid = 0; + _deq_hdr._xidsize = 0; + _deq_tail._checksum = 0; + _deq_tail._rid = 0; + _xidp = 0; + _buff = 0; +} + +void +deq_rec::reset(const uint64_t rid, const uint64_t drid, const void* const xidp, + const std::size_t xidlen, const bool txn_coml_commit) +{ + _deq_hdr._rhdr._rid = rid; + ::set_txn_coml_commit(&_deq_hdr, txn_coml_commit); + _deq_hdr._deq_rid = drid; + _deq_hdr._xidsize = xidlen; + _deq_tail._rid = rid; + _xidp = xidp; + _buff = 0; +} + +uint32_t +deq_rec::encode(void* wptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks) +{ + assert(wptr != 0); + assert(max_size_dblks > 0); + if (_xidp == 0) + assert(_deq_hdr._xidsize == 0); + + std::size_t rec_offs = rec_offs_dblks * QLS_DBLK_SIZE_BYTES; + std::size_t rem = max_size_dblks * QLS_DBLK_SIZE_BYTES; + std::size_t wr_cnt = 0; + if (rec_offs_dblks) // Continuation of split dequeue record (over 2 or more pages) + { + if (size_dblks(rec_size()) - rec_offs_dblks > max_size_dblks) // Further split required + { + rec_offs -= sizeof(_deq_hdr); + std::size_t wsize = _deq_hdr._xidsize > rec_offs ? _deq_hdr._xidsize - rec_offs : 0; + std::size_t wsize2 = wsize; + if (wsize) + { + if (wsize > rem) + wsize = rem; + std::memcpy(wptr, (const char*)_xidp + rec_offs, wsize); + wr_cnt += wsize; + rem -= wsize; + } + rec_offs -= _deq_hdr._xidsize - wsize2; + if (rem) + { + wsize = sizeof(_deq_tail) > rec_offs ? sizeof(_deq_tail) - rec_offs : 0; + wsize2 = wsize; + if (wsize) + { + if (wsize > rem) + wsize = rem; + std::memcpy((char*)wptr + wr_cnt, (char*)&_deq_tail + rec_offs, wsize); + wr_cnt += wsize; + rem -= wsize; + } + rec_offs -= sizeof(_deq_tail) - wsize2; + } + assert(rem == 0); + assert(rec_offs == 0); + } + else // No further split required + { + rec_offs -= sizeof(_deq_hdr); + std::size_t wsize = _deq_hdr._xidsize > rec_offs ? _deq_hdr._xidsize - rec_offs : 0; + if (wsize) + { + std::memcpy(wptr, (const char*)_xidp + rec_offs, wsize); + wr_cnt += wsize; + } + rec_offs -= _deq_hdr._xidsize - wsize; + wsize = sizeof(_deq_tail) > rec_offs ? sizeof(_deq_tail) - rec_offs : 0; + if (wsize) + { + std::memcpy((char*)wptr + wr_cnt, (char*)&_deq_tail + rec_offs, wsize); + wr_cnt += wsize; +#ifdef QLS_CLEAN + std::size_t rec_offs = rec_offs_dblks * QLS_DBLK_SIZE_BYTES; + std::size_t dblk_rec_size = size_dblks(rec_size() - rec_offs) * QLS_DBLK_SIZE_BYTES; + std::memset((char*)wptr + wr_cnt, QLS_CLEAN_CHAR, dblk_rec_size - wr_cnt); +#endif + } + rec_offs -= sizeof(_deq_tail) - wsize; + assert(rec_offs == 0); + } + } + else // Start at beginning of data record + { + // Assumption: the header will always fit into the first dblk + std::memcpy(wptr, (void*)&_deq_hdr, sizeof(_deq_hdr)); + wr_cnt = sizeof(_deq_hdr); + if (size_dblks(rec_size()) > max_size_dblks) // Split required - can only occur with xid + { + std::size_t wsize; + rem -= sizeof(_deq_hdr); + if (rem) + { + wsize = rem >= _deq_hdr._xidsize ? _deq_hdr._xidsize : rem; + std::memcpy((char*)wptr + wr_cnt, _xidp, wsize); + wr_cnt += wsize; + rem -= wsize; + } + if (rem) + { + wsize = rem >= sizeof(_deq_tail) ? sizeof(_deq_tail) : rem; + std::memcpy((char*)wptr + wr_cnt, (void*)&_deq_tail, wsize); + wr_cnt += wsize; + rem -= wsize; + } + assert(rem == 0); + } + else // No split required + { + if (_deq_hdr._xidsize) + { + std::memcpy((char*)wptr + wr_cnt, _xidp, _deq_hdr._xidsize); + wr_cnt += _deq_hdr._xidsize; + std::memcpy((char*)wptr + wr_cnt, (void*)&_deq_tail, sizeof(_deq_tail)); + wr_cnt += sizeof(_deq_tail); + } +#ifdef QLS_CLEAN + std::size_t dblk_rec_size = size_dblks(rec_size()) * QLS_DBLK_SIZE_BYTES; + std::memset((char*)wptr + wr_cnt, QLS_CLEAN_CHAR, dblk_rec_size - wr_cnt); +#endif + } + } + return size_dblks(wr_cnt); +} + +uint32_t +deq_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks) +{ + assert(rptr != 0); + assert(max_size_dblks > 0); + + std::size_t rd_cnt = 0; + if (rec_offs_dblks) // Continuation of record on new page + { + const uint32_t hdr_xid_dblks = size_dblks(sizeof(deq_hdr_t) + _deq_hdr._xidsize); + const uint32_t hdr_xid_tail_dblks = size_dblks(sizeof(deq_hdr_t) + _deq_hdr._xidsize + + sizeof(rec_tail_t)); + const std::size_t rec_offs = rec_offs_dblks * QLS_DBLK_SIZE_BYTES; + + if (hdr_xid_tail_dblks - rec_offs_dblks <= max_size_dblks) + { + // Remainder of xid fits within this page + if (rec_offs - sizeof(deq_hdr_t) < _deq_hdr._xidsize) + { + // Part of xid still outstanding, copy remainder of xid and tail + const std::size_t xid_offs = rec_offs - sizeof(deq_hdr_t); + const std::size_t xid_rem = _deq_hdr._xidsize - xid_offs; + std::memcpy((char*)_buff + xid_offs, rptr, xid_rem); + rd_cnt = xid_rem; + std::memcpy((void*)&_deq_tail, ((char*)rptr + rd_cnt), sizeof(_deq_tail)); + chk_tail(); + rd_cnt += sizeof(_deq_tail); + } + else + { + // Tail or part of tail only outstanding, complete tail + const std::size_t tail_offs = rec_offs - sizeof(deq_hdr_t) - _deq_hdr._xidsize; + const std::size_t tail_rem = sizeof(rec_tail_t) - tail_offs; + std::memcpy((char*)&_deq_tail + tail_offs, rptr, tail_rem); + chk_tail(); + rd_cnt = tail_rem; + } + } + else if (hdr_xid_dblks - rec_offs_dblks <= max_size_dblks) + { + // Remainder of xid fits within this page, tail split + const std::size_t xid_offs = rec_offs - sizeof(deq_hdr_t); + const std::size_t xid_rem = _deq_hdr._xidsize - xid_offs; + std::memcpy((char*)_buff + xid_offs, rptr, xid_rem); + rd_cnt += xid_rem; + const std::size_t tail_rem = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt; + if (tail_rem) + { + std::memcpy((void*)&_deq_tail, ((char*)rptr + xid_rem), tail_rem); + rd_cnt += tail_rem; + } + } + else + { + // Remainder of xid split + const std::size_t xid_cp_size = (max_size_dblks * QLS_DBLK_SIZE_BYTES); + std::memcpy((char*)_buff + rec_offs - sizeof(deq_hdr_t), rptr, xid_cp_size); + rd_cnt += xid_cp_size; + } + } + else // Start of record + { + // Get and check header + //_deq_hdr.hdr_copy(h); + ::rec_hdr_copy(&_deq_hdr._rhdr, &h); + rd_cnt = sizeof(rec_hdr_t); + _deq_hdr._deq_rid = *(uint64_t*)((char*)rptr + rd_cnt); + rd_cnt += sizeof(uint64_t); + _deq_hdr._xidsize = *(std::size_t*)((char*)rptr + rd_cnt); + rd_cnt = sizeof(deq_hdr_t); + chk_hdr(); + if (_deq_hdr._xidsize) + { + _buff = std::malloc(_deq_hdr._xidsize); + MALLOC_CHK(_buff, "_buff", "deq_rec", "decode"); + const uint32_t hdr_xid_dblks = size_dblks(sizeof(deq_hdr_t) + _deq_hdr._xidsize); + const uint32_t hdr_xid_tail_dblks = size_dblks(sizeof(deq_hdr_t) + _deq_hdr._xidsize + + sizeof(rec_tail_t)); + + // Check if record (header + xid + tail) fits within this page, we can check the + // tail before the expense of copying data to memory + if (hdr_xid_tail_dblks <= max_size_dblks) + { + // Entire header, xid and tail fits within this page + std::memcpy(_buff, (char*)rptr + rd_cnt, _deq_hdr._xidsize); + rd_cnt += _deq_hdr._xidsize; + std::memcpy((void*)&_deq_tail, (char*)rptr + rd_cnt, sizeof(_deq_tail)); + rd_cnt += sizeof(_deq_tail); + chk_tail(); + } + else if (hdr_xid_dblks <= max_size_dblks) + { + // Entire header and xid fit within this page, tail split + std::memcpy(_buff, (char*)rptr + rd_cnt, _deq_hdr._xidsize); + rd_cnt += _deq_hdr._xidsize; + const std::size_t tail_rem = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt; + if (tail_rem) + { + std::memcpy((void*)&_deq_tail, (char*)rptr + rd_cnt, tail_rem); + rd_cnt += tail_rem; + } + } + else + { + // Header fits within this page, xid split + const std::size_t xid_cp_size = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt; + std::memcpy(_buff, (char*)rptr + rd_cnt, xid_cp_size); + rd_cnt += xid_cp_size; + } + } + } + return size_dblks(rd_cnt); +} + +bool +deq_rec::rcv_decode(rec_hdr_t h, std::ifstream* ifsp, std::size_t& rec_offs) +{ + if (rec_offs == 0) + { + //_deq_hdr.hdr_copy(h); + ::rec_hdr_copy(&_deq_hdr._rhdr, &h); + ifsp->read((char*)&_deq_hdr._deq_rid, sizeof(uint64_t)); + ifsp->read((char*)&_deq_hdr._xidsize, sizeof(std::size_t)); +#if defined(JRNL_32_BIT) + ifsp->ignore(sizeof(uint32_t)); // _filler0 +#endif + rec_offs = sizeof(_deq_hdr); + // Read header, allocate (if req'd) for xid + if (_deq_hdr._xidsize) + { + _buff = std::malloc(_deq_hdr._xidsize); + MALLOC_CHK(_buff, "_buff", "enq_rec", "rcv_decode"); + } + } + if (rec_offs < sizeof(_deq_hdr) + _deq_hdr._xidsize) + { + // Read xid (or continue reading xid) + std::size_t offs = rec_offs - sizeof(_deq_hdr); + ifsp->read((char*)_buff + offs, _deq_hdr._xidsize - offs); + std::size_t size_read = ifsp->gcount(); + rec_offs += size_read; + if (size_read < _deq_hdr._xidsize - offs) + { + assert(ifsp->eof()); + // As we may have read past eof, turn off fail bit + ifsp->clear(ifsp->rdstate()&(~std::ifstream::failbit)); + assert(!ifsp->fail() && !ifsp->bad()); + return false; + } + } + if (rec_offs < sizeof(_deq_hdr) + + (_deq_hdr._xidsize ? _deq_hdr._xidsize + sizeof(rec_tail_t) : 0)) + { + // Read tail (or continue reading tail) + std::size_t offs = rec_offs - sizeof(_deq_hdr) - _deq_hdr._xidsize; + ifsp->read((char*)&_deq_tail + offs, sizeof(rec_tail_t) - offs); + std::size_t size_read = ifsp->gcount(); + rec_offs += size_read; + if (size_read < sizeof(rec_tail_t) - offs) + { + assert(ifsp->eof()); + // As we may have read past eof, turn off fail bit + ifsp->clear(ifsp->rdstate()&(~std::ifstream::failbit)); + assert(!ifsp->fail() && !ifsp->bad()); + return false; + } + } + ifsp->ignore(rec_size_dblks() * QLS_DBLK_SIZE_BYTES - rec_size()); + if (_deq_hdr._xidsize) + chk_tail(); // Throws if tail invalid or record incomplete + assert(!ifsp->fail() && !ifsp->bad()); + return true; +} + +std::size_t +deq_rec::get_xid(void** const xidpp) +{ + if (!_buff) + { + *xidpp = 0; + return 0; + } + *xidpp = _buff; + return _deq_hdr._xidsize; +} + +std::string& +deq_rec::str(std::string& str) const +{ + std::ostringstream oss; + oss << "deq_rec: m=" << _deq_hdr._rhdr._magic; + oss << " v=" << (int)_deq_hdr._rhdr._version; + oss << " rid=" << _deq_hdr._rhdr._rid; + oss << " drid=" << _deq_hdr._deq_rid; + if (_xidp) + oss << " xid=\"" << _xidp << "\""; + str.append(oss.str()); + return str; +} + +std::size_t +deq_rec::xid_size() const +{ + return _deq_hdr._xidsize; +} + +std::size_t +deq_rec::rec_size() const +{ + return sizeof(deq_hdr_t) + (_deq_hdr._xidsize ? _deq_hdr._xidsize + sizeof(rec_tail_t) : 0); +} + +void +deq_rec::chk_hdr() const +{ + jrec::chk_hdr(_deq_hdr._rhdr); + if (_deq_hdr._rhdr._magic != QLS_DEQ_MAGIC) + { + std::ostringstream oss; + oss << std::hex << std::setfill('0'); + oss << "deq magic: rid=0x" << std::setw(16) << _deq_hdr._rhdr._rid; + oss << ": expected=0x" << std::setw(8) << QLS_DEQ_MAGIC; + oss << " read=0x" << std::setw(2) << (int)_deq_hdr._rhdr._magic; + throw jexception(jerrno::JERR_JREC_BADRECHDR, oss.str(), "deq_rec", "chk_hdr"); + } +} + +void +deq_rec::chk_hdr(uint64_t rid) const +{ + chk_hdr(); + jrec::chk_rid(_deq_hdr._rhdr, rid); +} + +void +deq_rec::chk_tail() const +{ + jrec::chk_tail(_deq_tail, _deq_hdr._rhdr); +} + +void +deq_rec::clean() +{ + // clean up allocated memory here +} + +} // namespace journal +} // namespace mrg diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/deq_rec.h b/qpid/cpp/src/qpid/linearstore/jrnl/deq_rec.h new file mode 100644 index 0000000000..12ba6af855 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/deq_rec.h @@ -0,0 +1,91 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_JRNL_DEQ_REQ_H +#define QPID_LEGACYSTORE_JRNL_DEQ_REQ_H + +namespace qpid +{ +namespace qls_jrnl +{ +class deq_rec; +}} + +#include <cstddef> +#include "qpid/linearstore/jrnl/utils/deq_hdr.h" +#include "qpid/linearstore/jrnl/jrec.h" + +namespace qpid +{ +namespace qls_jrnl +{ + + /** + * \class deq_rec + * \brief Class to handle a single journal dequeue record. + */ + class deq_rec : public jrec + { + private: + deq_hdr_t _deq_hdr; ///< Dequeue header + const void* _xidp; ///< xid pointer for encoding (writing to disk) + void* _buff; ///< Pointer to buffer to receive data read from disk + rec_tail_t _deq_tail; ///< Record tail, only encoded if XID is present + + public: + // constructor used for read operations and xid will have memory allocated + deq_rec(); + // constructor used for write operations, where xid already exists + deq_rec(const uint64_t rid, const uint64_t drid, const void* const xidp, + const std::size_t xidlen, const bool txn_coml_commit); + virtual ~deq_rec(); + + // Prepare instance for use in reading data from journal + void reset(); + // Prepare instance for use in writing data to journal + void reset(const uint64_t rid, const uint64_t drid, const void* const xidp, + const std::size_t xidlen, const bool txn_coml_commit); + uint32_t encode(void* wptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks); + uint32_t decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, + uint32_t max_size_dblks); + // Decode used for recover + bool rcv_decode(rec_hdr_t h, std::ifstream* ifsp, std::size_t& rec_offs); + + inline bool is_txn_coml_commit() const { return ::is_txn_coml_commit(&_deq_hdr); } + inline uint64_t rid() const { return _deq_hdr._rhdr._rid; } + inline uint64_t deq_rid() const { return _deq_hdr._deq_rid; } + std::size_t get_xid(void** const xidpp); + std::string& str(std::string& str) const; + inline std::size_t data_size() const { return 0; } // This record never carries data + std::size_t xid_size() const; + std::size_t rec_size() const; + + private: + virtual void chk_hdr() const; + virtual void chk_hdr(uint64_t rid) const; + virtual void chk_tail() const; + virtual void clean(); + }; // class deq_rec + +} // namespace journal +} // namespace mrg + +#endif // ifndef QPID_LEGACYSTORE_JRNL_DEQ_REQ_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/enq_map.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/enq_map.cpp new file mode 100644 index 0000000000..3bc49502c8 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/enq_map.cpp @@ -0,0 +1,186 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/jrnl/enq_map.h" + +#include <iomanip> +#include "qpid/linearstore/jrnl/jerrno.h" +#include "qpid/linearstore/jrnl/slock.h" +#include <sstream> + + +namespace qpid +{ +namespace qls_jrnl +{ + +// static return/error codes +int16_t enq_map::EMAP_DUP_RID = -3; +int16_t enq_map::EMAP_LOCKED = -2; +int16_t enq_map::EMAP_RID_NOT_FOUND = -1; +int16_t enq_map::EMAP_OK = 0; +int16_t enq_map::EMAP_FALSE = 0; +int16_t enq_map::EMAP_TRUE = 1; + +enq_map::enq_map(): + _map(){} + +enq_map::~enq_map() {} + + +short +enq_map::insert_pfid(const uint64_t rid, const uint64_t pfid, const std::streampos file_posn) +{ + return insert_pfid(rid, pfid, file_posn, false); +} + +short +enq_map::insert_pfid(const uint64_t rid, const uint64_t pfid, const std::streampos file_posn, const bool locked) +{ + std::pair<emap_itr, bool> ret; + emap_data_struct_t rec(pfid, file_posn, locked); + { + slock s(_mutex); + ret = _map.insert(emap_param(rid, rec)); + } + if (ret.second == false) + return EMAP_DUP_RID; + return EMAP_OK; +} + +short +enq_map::get_pfid(const uint64_t rid, uint64_t& pfid) +{ + slock s(_mutex); + emap_itr itr = _map.find(rid); + if (itr == _map.end()) // not found in map + return EMAP_RID_NOT_FOUND; + if (itr->second._lock) + return EMAP_LOCKED; + pfid = itr->second._pfid; + return EMAP_OK; +} + +short +enq_map::get_remove_pfid(const uint64_t rid, uint64_t& pfid, const bool txn_flag) +{ + slock s(_mutex); + emap_itr itr = _map.find(rid); + if (itr == _map.end()) // not found in map + return EMAP_RID_NOT_FOUND; + if (itr->second._lock && !txn_flag) // locked, but not a commit/abort + return EMAP_LOCKED; + pfid = itr->second._pfid; + _map.erase(itr); + return EMAP_OK; +} + +short +enq_map::get_file_posn(const uint64_t rid, std::streampos& file_posn) { + slock s(_mutex); + emap_itr itr = _map.find(rid); + if (itr == _map.end()) // not found in map + return EMAP_RID_NOT_FOUND; + if (itr->second._lock) + return EMAP_LOCKED; + file_posn = itr->second._file_posn; + return EMAP_OK; +} + +short +enq_map::get_data(const uint64_t rid, emap_data_struct_t& eds) { + slock s(_mutex); + emap_itr itr = _map.find(rid); + if (itr == _map.end()) // not found in map + return EMAP_RID_NOT_FOUND; + eds._pfid = itr->second._pfid; + eds._file_posn = itr->second._file_posn; + eds._lock = itr->second._lock; + return EMAP_OK; +} + +bool +enq_map::is_enqueued(const uint64_t rid, bool ignore_lock) +{ + slock s(_mutex); + emap_itr itr = _map.find(rid); + if (itr == _map.end()) // not found in map + return false; + if (!ignore_lock && itr->second._lock) // locked + return false; + return true; +} + +short +enq_map::lock(const uint64_t rid) +{ + slock s(_mutex); + emap_itr itr = _map.find(rid); + if (itr == _map.end()) // not found in map + return EMAP_RID_NOT_FOUND; + itr->second._lock = true; + return EMAP_OK; +} + +short +enq_map::unlock(const uint64_t rid) +{ + slock s(_mutex); + emap_itr itr = _map.find(rid); + if (itr == _map.end()) // not found in map + return EMAP_RID_NOT_FOUND; + itr->second._lock = false; + return EMAP_OK; +} + +short +enq_map::is_locked(const uint64_t rid) +{ + slock s(_mutex); + emap_itr itr = _map.find(rid); + if (itr == _map.end()) // not found in map + return EMAP_RID_NOT_FOUND; + return itr->second._lock ? EMAP_TRUE : EMAP_FALSE; +} + +void +enq_map::rid_list(std::vector<uint64_t>& rv) +{ + rv.clear(); + { + slock s(_mutex); + for (emap_itr itr = _map.begin(); itr != _map.end(); itr++) + rv.push_back(itr->first); + } +} + +void +enq_map::pfid_list(std::vector<uint64_t>& fv) +{ + fv.clear(); + { + slock s(_mutex); + for (emap_itr itr = _map.begin(); itr != _map.end(); itr++) + fv.push_back(itr->second._pfid); + } +} + +}} diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/enq_map.h b/qpid/cpp/src/qpid/linearstore/jrnl/enq_map.h new file mode 100644 index 0000000000..659e3bd052 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/enq_map.h @@ -0,0 +1,112 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_JRNL_ENQ_MAP_H +#define QPID_LEGACYSTORE_JRNL_ENQ_MAP_H + +namespace qpid +{ +namespace qls_jrnl +{ +class enq_map; +}} + +#include "qpid/linearstore/jrnl/jexception.h" +#include "qpid/linearstore/jrnl/smutex.h" +#include <map> +#include <pthread.h> +#include <vector> + +namespace qpid +{ +namespace qls_jrnl +{ + + /** + * \class enq_map + * \brief Class for storing the physical file id (pfid) and a transaction locked flag for each enqueued + * data block using the record id (rid) as a key. This is the primary mechanism for + * deterimining the enqueue low water mark: if a pfid exists in this map, then there is + * at least one still-enqueued record in that file. (The transaction map must also be + * clear, however.) + * + * Map rids against pfid and lock status. As records are enqueued, they are added to this + * map, and as they are dequeued, they are removed. An enqueue is locked when a transactional + * dequeue is pending that has been neither committed nor aborted. + * <pre> + * key data + * + * rid1 --- [ pfid, txn_lock ] + * rid2 --- [ pfid, txn_lock ] + * rid3 --- [ pfid, txn_lock ] + * ... + * </pre> + */ + class enq_map + { + public: + // return/error codes + static short EMAP_DUP_RID; + static short EMAP_LOCKED; + static short EMAP_RID_NOT_FOUND; + static short EMAP_OK; + static short EMAP_FALSE; + static short EMAP_TRUE; + + typedef struct emap_data_struct_t { + uint64_t _pfid; + std::streampos _file_posn; + bool _lock; + emap_data_struct_t() : _pfid(0), _file_posn(0), _lock(false) {} + emap_data_struct_t(const uint64_t pfid, const std::streampos file_posn, const bool lock) : _pfid(pfid), _file_posn(file_posn), _lock(lock) {} + } emqp_data_struct_t; + typedef std::pair<uint64_t, emap_data_struct_t> emap_param; + typedef std::map<uint64_t, emap_data_struct_t> emap; + typedef emap::iterator emap_itr; + + private: + emap _map; + smutex _mutex; + + public: + enq_map(); + virtual ~enq_map(); + + short insert_pfid(const uint64_t rid, const uint64_t pfid, const std::streampos file_posn); // 0=ok; -3=duplicate rid; + short insert_pfid(const uint64_t rid, const uint64_t pfid, const std::streampos file_posn, const bool locked); // 0=ok; -3=duplicate rid; + short get_pfid(const uint64_t rid, uint64_t& pfid); // >=0=pfid; -1=rid not found; -2=locked + short get_remove_pfid(const uint64_t rid, uint64_t& pfid, const bool txn_flag = false); // >=0=pfid; -1=rid not found; -2=locked + short get_file_posn(const uint64_t rid, std::streampos& file_posn); // -1=rid not found; -2=locked + short get_data(const uint64_t rid, emap_data_struct_t& eds); + bool is_enqueued(const uint64_t rid, bool ignore_lock = false); + short lock(const uint64_t rid); // 0=ok; -1=rid not found + short unlock(const uint64_t rid); // 0=ok; -1=rid not found + short is_locked(const uint64_t rid); // 1=true; 0=false; -1=rid not found + inline void clear() { _map.clear(); } + inline bool empty() const { return _map.empty(); } + inline uint32_t size() const { return uint32_t(_map.size()); } + void rid_list(std::vector<uint64_t>& rv); + void pfid_list(std::vector<uint64_t>& fv); + }; + +}} + +#endif // ifndef QPID_LEGACYSTORE_JRNL_ENQ_MAP_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/enq_rec.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/enq_rec.cpp new file mode 100644 index 0000000000..e968320ac6 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/enq_rec.cpp @@ -0,0 +1,621 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/jrnl/enq_rec.h" + +#include <cassert> +#include <cerrno> +#include <cstdlib> +#include <cstring> +#include <iomanip> +#include "qpid/linearstore/jrnl/jerrno.h" +#include "qpid/linearstore/jrnl/jexception.h" +#include <sstream> + +namespace qpid +{ +namespace qls_jrnl +{ + +// Constructor used for read operations, where buf contains preallocated space to receive data. +enq_rec::enq_rec(): + jrec(), // superclass + //_enq_hdr(QLS_ENQ_MAGIC, QLS_JRNL_VERSION, 0, 0, 0, false, false), + _xidp(0), + _data(0), + _buff(0) + //_enq_tail(_enq_hdr) +{ + ::enq_hdr_init(&_enq_hdr, QLS_ENQ_MAGIC, QLS_JRNL_VERSION, 0, 0, 0, false); + ::rec_tail_copy(&_enq_tail, &_enq_hdr._rhdr, 0); +} + +// Constructor used for transactional write operations, where dbuf contains data to be written. +enq_rec::enq_rec(const uint64_t rid, const void* const dbuf, const std::size_t dlen, + const void* const xidp, const std::size_t xidlen, const bool transient): + jrec(), // superclass + //_enq_hdr(QLS_ENQ_MAGIC, QLS_JRNL_VERSION, rid, xidlen, dlen, owi, transient), + _xidp(xidp), + _data(dbuf), + _buff(0) + //_enq_tail(_enq_hdr) +{ + ::enq_hdr_init(&_enq_hdr, QLS_ENQ_MAGIC, QLS_JRNL_VERSION, 0, rid, xidlen, dlen); + ::rec_tail_copy(&_enq_tail, &_enq_hdr._rhdr, 0); + ::set_enq_transient(&_enq_hdr, transient); +} + +enq_rec::~enq_rec() +{ + clean(); +} + +// Prepare instance for use in reading data from journal, where buf contains preallocated space +// to receive data. +void +enq_rec::reset() +{ + _enq_hdr._rhdr._rid = 0; + ::set_enq_transient(&_enq_hdr, false); + _enq_hdr._xidsize = 0; + _enq_hdr._dsize = 0; + _xidp = 0; + _data = 0; + _buff = 0; + _enq_tail._rid = 0; +} + +// Prepare instance for use in writing transactional data to journal, where dbuf contains data to +// be written. +void +enq_rec::reset(const uint64_t rid, const void* const dbuf, const std::size_t dlen, + const void* const xidp, const std::size_t xidlen, const bool transient, + const bool external) +{ + _enq_hdr._rhdr._rid = rid; + ::set_enq_transient(&_enq_hdr, transient); + ::set_enq_external(&_enq_hdr, external); + _enq_hdr._xidsize = xidlen; + _enq_hdr._dsize = dlen; + _xidp = xidp; + _data = dbuf; + _buff = 0; + _enq_tail._rid = rid; +} + +uint32_t +enq_rec::encode(void* wptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks) +{ + assert(wptr != 0); + assert(max_size_dblks > 0); + if (_xidp == 0) + assert(_enq_hdr._xidsize == 0); + + std::size_t rec_offs = rec_offs_dblks * QLS_DBLK_SIZE_BYTES; + std::size_t rem = max_size_dblks * QLS_DBLK_SIZE_BYTES; + std::size_t wr_cnt = 0; + if (rec_offs_dblks) // Continuation of split data record (over 2 or more pages) + { + if (size_dblks(rec_size()) - rec_offs_dblks > max_size_dblks) // Further split required + { + rec_offs -= sizeof(_enq_hdr); + std::size_t wsize = _enq_hdr._xidsize > rec_offs ? _enq_hdr._xidsize - rec_offs : 0; + std::size_t wsize2 = wsize; + if (wsize) + { + if (wsize > rem) + wsize = rem; + std::memcpy(wptr, (const char*)_xidp + rec_offs, wsize); + wr_cnt = wsize; + rem -= wsize; + } + rec_offs -= _enq_hdr._xidsize - wsize2; + if (rem && !::is_enq_external(&_enq_hdr)) + { + wsize = _enq_hdr._dsize > rec_offs ? _enq_hdr._dsize - rec_offs : 0; + wsize2 = wsize; + if (wsize) + { + if (wsize > rem) + wsize = rem; + std::memcpy((char*)wptr + wr_cnt, (const char*)_data + rec_offs, wsize); + wr_cnt += wsize; + rem -= wsize; + } + rec_offs -= _enq_hdr._dsize - wsize2; + } + if (rem) + { + wsize = sizeof(_enq_tail) > rec_offs ? sizeof(_enq_tail) - rec_offs : 0; + wsize2 = wsize; + if (wsize) + { + if (wsize > rem) + wsize = rem; + std::memcpy((char*)wptr + wr_cnt, (char*)&_enq_tail + rec_offs, wsize); + wr_cnt += wsize; + rem -= wsize; + } + rec_offs -= sizeof(_enq_tail) - wsize2; + } + assert(rem == 0); + assert(rec_offs == 0); + } + else // No further split required + { + rec_offs -= sizeof(_enq_hdr); + std::size_t wsize = _enq_hdr._xidsize > rec_offs ? _enq_hdr._xidsize - rec_offs : 0; + if (wsize) + { + std::memcpy(wptr, (const char*)_xidp + rec_offs, wsize); + wr_cnt += wsize; + } + rec_offs -= _enq_hdr._xidsize - wsize; + wsize = _enq_hdr._dsize > rec_offs ? _enq_hdr._dsize - rec_offs : 0; + if (wsize && !::is_enq_external(&_enq_hdr)) + { + std::memcpy((char*)wptr + wr_cnt, (const char*)_data + rec_offs, wsize); + wr_cnt += wsize; + } + rec_offs -= _enq_hdr._dsize - wsize; + wsize = sizeof(_enq_tail) > rec_offs ? sizeof(_enq_tail) - rec_offs : 0; + if (wsize) + { + std::memcpy((char*)wptr + wr_cnt, (char*)&_enq_tail + rec_offs, wsize); + wr_cnt += wsize; +#ifdef QLS_CLEAN + std::size_t rec_offs = rec_offs_dblks * QLS_DBLK_SIZE_BYTES; + std::size_t dblk_rec_size = size_dblks(rec_size() - rec_offs) * QLS_DBLK_SIZE_BYTES; + std::memset((char*)wptr + wr_cnt, QLS_CLEAN_CHAR, dblk_rec_size - wr_cnt); +#endif + } + rec_offs -= sizeof(_enq_tail) - wsize; + assert(rec_offs == 0); + } + } + else // Start at beginning of data record + { + // Assumption: the header will always fit into the first dblk + std::memcpy(wptr, (void*)&_enq_hdr, sizeof(_enq_hdr)); + wr_cnt = sizeof(_enq_hdr); + if (size_dblks(rec_size()) > max_size_dblks) // Split required + { + std::size_t wsize; + rem -= sizeof(_enq_hdr); + if (rem) + { + wsize = rem >= _enq_hdr._xidsize ? _enq_hdr._xidsize : rem; + std::memcpy((char*)wptr + wr_cnt, _xidp, wsize); + wr_cnt += wsize; + rem -= wsize; + } + if (rem && !::is_enq_external(&_enq_hdr)) + { + wsize = rem >= _enq_hdr._dsize ? _enq_hdr._dsize : rem; + std::memcpy((char*)wptr + wr_cnt, _data, wsize); + wr_cnt += wsize; + rem -= wsize; + } + if (rem) + { + wsize = rem >= sizeof(_enq_tail) ? sizeof(_enq_tail) : rem; + std::memcpy((char*)wptr + wr_cnt, (void*)&_enq_tail, wsize); + wr_cnt += wsize; + rem -= wsize; + } + assert(rem == 0); + } + else // No split required + { + if (_enq_hdr._xidsize) + { + std::memcpy((char*)wptr + wr_cnt, _xidp, _enq_hdr._xidsize); + wr_cnt += _enq_hdr._xidsize; + } + if (!::is_enq_external(&_enq_hdr)) + { + std::memcpy((char*)wptr + wr_cnt, _data, _enq_hdr._dsize); + wr_cnt += _enq_hdr._dsize; + } + std::memcpy((char*)wptr + wr_cnt, (void*)&_enq_tail, sizeof(_enq_tail)); + wr_cnt += sizeof(_enq_tail); +#ifdef QLS_CLEAN + std::size_t dblk_rec_size = size_dblks(rec_size()) * QLS_DBLK_SIZE_BYTES; + std::memset((char*)wptr + wr_cnt, QLS_CLEAN_CHAR, dblk_rec_size - wr_cnt); +#endif + } + } + return size_dblks(wr_cnt); +} + +uint32_t +enq_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks) +{ + assert(rptr != 0); + assert(max_size_dblks > 0); + + std::size_t rd_cnt = 0; + if (rec_offs_dblks) // Continuation of record on new page + { + const uint32_t hdr_xid_data_size = sizeof(enq_hdr_t) + _enq_hdr._xidsize + + (::is_enq_external(&_enq_hdr) ? 0 : _enq_hdr._dsize); + const uint32_t hdr_xid_data_tail_size = hdr_xid_data_size + sizeof(rec_tail_t); + const uint32_t hdr_data_dblks = size_dblks(hdr_xid_data_size); + const uint32_t hdr_tail_dblks = size_dblks(hdr_xid_data_tail_size); + const std::size_t rec_offs = rec_offs_dblks * QLS_DBLK_SIZE_BYTES; + const std::size_t offs = rec_offs - sizeof(enq_hdr_t); + + if (hdr_tail_dblks - rec_offs_dblks <= max_size_dblks) + { + // Remainder of record fits within this page + if (offs < _enq_hdr._xidsize) + { + // some XID still outstanding, copy remainder of XID, data and tail + const std::size_t rem = _enq_hdr._xidsize + _enq_hdr._dsize - offs; + std::memcpy((char*)_buff + offs, rptr, rem); + rd_cnt += rem; + std::memcpy((void*)&_enq_tail, ((char*)rptr + rd_cnt), sizeof(_enq_tail)); + chk_tail(); + rd_cnt += sizeof(_enq_tail); + } + else if (offs < _enq_hdr._xidsize + _enq_hdr._dsize && !::is_enq_external(&_enq_hdr)) + { + // some data still outstanding, copy remainder of data and tail + const std::size_t data_offs = offs - _enq_hdr._xidsize; + const std::size_t data_rem = _enq_hdr._dsize - data_offs; + std::memcpy((char*)_buff + offs, rptr, data_rem); + rd_cnt += data_rem; + std::memcpy((void*)&_enq_tail, ((char*)rptr + rd_cnt), sizeof(_enq_tail)); + chk_tail(); + rd_cnt += sizeof(_enq_tail); + } + else + { + // Tail or part of tail only outstanding, complete tail + const std::size_t tail_offs = rec_offs - sizeof(enq_hdr_t) - _enq_hdr._xidsize - + _enq_hdr._dsize; + const std::size_t tail_rem = sizeof(rec_tail_t) - tail_offs; + std::memcpy((char*)&_enq_tail + tail_offs, rptr, tail_rem); + chk_tail(); + rd_cnt = tail_rem; + } + } + else if (hdr_data_dblks - rec_offs_dblks <= max_size_dblks) + { + // Remainder of xid & data fits within this page; tail split + + /* + * TODO: This section needs revision. Since it is known that the end of the page falls within the + * tail record, it is only necessary to write from the current offset to the end of the page under + * all circumstances. The multiple if/else combinations may be eliminated, as well as one memcpy() + * operation. + * + * Also note that Coverity has detected a possible memory overwrite in this block. It occurs if + * both the following two if() stmsts (numbered) are false. With rd_cnt = 0, this would result in + * the value of tail_rem > sizeof(tail_rec). Practically, this could only happen if the start and + * end of a page both fall within the same tail record, in which case the tail would have to be + * (much!) larger. However, the logic here does not account for this possibility. + * + * If the optimization above is undertaken, this code would probably be removed. + */ + if (offs < _enq_hdr._xidsize) // 1 + { + // some XID still outstanding, copy remainder of XID and data + const std::size_t rem = _enq_hdr._xidsize + _enq_hdr._dsize - offs; + std::memcpy((char*)_buff + offs, rptr, rem); + rd_cnt += rem; + } + else if (offs < _enq_hdr._xidsize + _enq_hdr._dsize && !::is_enq_external(&_enq_hdr)) // 2 + { + // some data still outstanding, copy remainder of data + const std::size_t data_offs = offs - _enq_hdr._xidsize; + const std::size_t data_rem = _enq_hdr._dsize - data_offs; + std::memcpy((char*)_buff + offs, rptr, data_rem); + rd_cnt += data_rem; + } + const std::size_t tail_rem = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt; + if (tail_rem) + { + std::memcpy((void*)&_enq_tail, ((char*)rptr + rd_cnt), tail_rem); + rd_cnt += tail_rem; + } + } + else + { + // Since xid and data are contiguous, both fit within current page - copy whole page + const std::size_t data_cp_size = (max_size_dblks * QLS_DBLK_SIZE_BYTES); + std::memcpy((char*)_buff + offs, rptr, data_cp_size); + rd_cnt += data_cp_size; + } + } + else // Start of record + { + // Get and check header + //_enq_hdr.hdr_copy(h); + ::rec_hdr_copy(&_enq_hdr._rhdr, &h); + rd_cnt = sizeof(rec_hdr_t); + _enq_hdr._xidsize = *(std::size_t*)((char*)rptr + rd_cnt); + rd_cnt += sizeof(std::size_t); +#if defined(JRNL_32_BIT) + rd_cnt += sizeof(uint32_t); // Filler 0 +#endif + _enq_hdr._dsize = *(std::size_t*)((char*)rptr + rd_cnt); + rd_cnt = sizeof(enq_hdr_t); + chk_hdr(); + if (_enq_hdr._xidsize + (::is_enq_external(&_enq_hdr) ? 0 : _enq_hdr._dsize)) + { + _buff = std::malloc(_enq_hdr._xidsize + (::is_enq_external(&_enq_hdr) ? 0 : _enq_hdr._dsize)); + MALLOC_CHK(_buff, "_buff", "enq_rec", "decode"); + + const uint32_t hdr_xid_size = sizeof(enq_hdr_t) + _enq_hdr._xidsize; + const uint32_t hdr_xid_data_size = hdr_xid_size + (::is_enq_external(&_enq_hdr) ? 0 : _enq_hdr._dsize); + const uint32_t hdr_xid_data_tail_size = hdr_xid_data_size + sizeof(rec_tail_t); + const uint32_t hdr_xid_dblks = size_dblks(hdr_xid_size); + const uint32_t hdr_data_dblks = size_dblks(hdr_xid_data_size); + const uint32_t hdr_tail_dblks = size_dblks(hdr_xid_data_tail_size); + // Check if record (header + data + tail) fits within this page, we can check the + // tail before the expense of copying data to memory + if (hdr_tail_dblks <= max_size_dblks) + { + // Header, xid, data and tail fits within this page + if (_enq_hdr._xidsize) + { + std::memcpy(_buff, (char*)rptr + rd_cnt, _enq_hdr._xidsize); + rd_cnt += _enq_hdr._xidsize; + } + if (_enq_hdr._dsize && !::is_enq_external(&_enq_hdr)) + { + std::memcpy((char*)_buff + _enq_hdr._xidsize, (char*)rptr + rd_cnt, + _enq_hdr._dsize); + rd_cnt += _enq_hdr._dsize; + } + std::memcpy((void*)&_enq_tail, (char*)rptr + rd_cnt, sizeof(_enq_tail)); + chk_tail(); + rd_cnt += sizeof(_enq_tail); + } + else if (hdr_data_dblks <= max_size_dblks) + { + // Header, xid and data fit within this page, tail split or separated + if (_enq_hdr._xidsize) + { + std::memcpy(_buff, (char*)rptr + rd_cnt, _enq_hdr._xidsize); + rd_cnt += _enq_hdr._xidsize; + } + if (_enq_hdr._dsize && !::is_enq_external(&_enq_hdr)) + { + std::memcpy((char*)_buff + _enq_hdr._xidsize, (char*)rptr + rd_cnt, + _enq_hdr._dsize); + rd_cnt += _enq_hdr._dsize; + } + const std::size_t tail_rem = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt; + if (tail_rem) + { + std::memcpy((void*)&_enq_tail, (char*)rptr + rd_cnt, tail_rem); + rd_cnt += tail_rem; + } + } + else if (hdr_xid_dblks <= max_size_dblks) + { + // Header and xid fits within this page, data split or separated + if (_enq_hdr._xidsize) + { + std::memcpy(_buff, (char*)rptr + rd_cnt, _enq_hdr._xidsize); + rd_cnt += _enq_hdr._xidsize; + } + if (_enq_hdr._dsize && !::is_enq_external(&_enq_hdr)) + { + const std::size_t data_cp_size = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt; + std::memcpy((char*)_buff + _enq_hdr._xidsize, (char*)rptr + rd_cnt, data_cp_size); + rd_cnt += data_cp_size; + } + } + else + { + // Header fits within this page, xid split or separated + const std::size_t data_cp_size = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt; + std::memcpy(_buff, (char*)rptr + rd_cnt, data_cp_size); + rd_cnt += data_cp_size; + } + } + } + return size_dblks(rd_cnt); +} + +bool +enq_rec::rcv_decode(rec_hdr_t h, std::ifstream* ifsp, std::size_t& rec_offs) +{ + if (rec_offs == 0) + { + // Read header, allocate (if req'd) for xid + //_enq_hdr.hdr_copy(h); + ::rec_hdr_copy(&_enq_hdr._rhdr, &h); + ifsp->read((char*)&_enq_hdr._xidsize, sizeof(std::size_t)); +#if defined(JRNL_32_BIT) + ifsp->ignore(sizeof(uint32_t)); // _filler0 +#endif + ifsp->read((char*)&_enq_hdr._dsize, sizeof(std::size_t)); +#if defined(JRNL_32_BIT) + ifsp->ignore(sizeof(uint32_t)); // _filler1 +#endif + rec_offs = sizeof(_enq_hdr); + if (_enq_hdr._xidsize) + { + _buff = std::malloc(_enq_hdr._xidsize); + MALLOC_CHK(_buff, "_buff", "enq_rec", "rcv_decode"); + } + } + if (rec_offs < sizeof(_enq_hdr) + _enq_hdr._xidsize) + { + // Read xid (or continue reading xid) + std::size_t offs = rec_offs - sizeof(_enq_hdr); + ifsp->read((char*)_buff + offs, _enq_hdr._xidsize - offs); + std::size_t size_read = ifsp->gcount(); + rec_offs += size_read; + if (size_read < _enq_hdr._xidsize - offs) + { + assert(ifsp->eof()); + // As we may have read past eof, turn off fail bit + ifsp->clear(ifsp->rdstate()&(~std::ifstream::failbit)); + assert(!ifsp->fail() && !ifsp->bad()); + return false; + } + } + if (!::is_enq_external(&_enq_hdr)) + { + if (rec_offs < sizeof(_enq_hdr) + _enq_hdr._xidsize + _enq_hdr._dsize) + { + // Ignore data (or continue ignoring data) + std::size_t offs = rec_offs - sizeof(_enq_hdr) - _enq_hdr._xidsize; + ifsp->ignore(_enq_hdr._dsize - offs); + std::size_t size_read = ifsp->gcount(); + rec_offs += size_read; + if (size_read < _enq_hdr._dsize - offs) + { + assert(ifsp->eof()); + // As we may have read past eof, turn off fail bit + ifsp->clear(ifsp->rdstate()&(~std::ifstream::failbit)); + assert(!ifsp->fail() && !ifsp->bad()); + return false; + } + } + } + if (rec_offs < sizeof(_enq_hdr) + _enq_hdr._xidsize + + (::is_enq_external(&_enq_hdr) ? 0 : _enq_hdr._dsize) + sizeof(rec_tail_t)) + { + // Read tail (or continue reading tail) + std::size_t offs = rec_offs - sizeof(_enq_hdr) - _enq_hdr._xidsize; + if (!::is_enq_external(&_enq_hdr)) + offs -= _enq_hdr._dsize; + ifsp->read((char*)&_enq_tail + offs, sizeof(rec_tail_t) - offs); + std::size_t size_read = ifsp->gcount(); + rec_offs += size_read; + if (size_read < sizeof(rec_tail_t) - offs) + { + assert(ifsp->eof()); + // As we may have read past eof, turn off fail bit + ifsp->clear(ifsp->rdstate()&(~std::ifstream::failbit)); + assert(!ifsp->fail() && !ifsp->bad()); + return false; + } + } + ifsp->ignore(rec_size_dblks() * QLS_DBLK_SIZE_BYTES - rec_size()); + chk_tail(); // Throws if tail invalid or record incomplete + assert(!ifsp->fail() && !ifsp->bad()); + return true; +} + +std::size_t +enq_rec::get_xid(void** const xidpp) +{ + if (!_buff || !_enq_hdr._xidsize) + { + *xidpp = 0; + return 0; + } + *xidpp = _buff; + return _enq_hdr._xidsize; +} + +std::size_t +enq_rec::get_data(void** const datapp) +{ + if (!_buff) + { + *datapp = 0; + return 0; + } + if (::is_enq_external(&_enq_hdr)) + *datapp = 0; + else + *datapp = (void*)((char*)_buff + _enq_hdr._xidsize); + return _enq_hdr._dsize; +} + +std::string& +enq_rec::str(std::string& str) const +{ + std::ostringstream oss; + oss << "enq_rec: m=" << _enq_hdr._rhdr._magic; + oss << " v=" << (int)_enq_hdr._rhdr._version; + oss << " rid=" << _enq_hdr._rhdr._rid; + if (_xidp) + oss << " xid=\"" << _xidp << "\""; + oss << " len=" << _enq_hdr._dsize; + str.append(oss.str()); + return str; +} + +std::size_t +enq_rec::rec_size() const +{ + return rec_size(_enq_hdr._xidsize, _enq_hdr._dsize, ::is_enq_external(&_enq_hdr)); +} + +std::size_t +enq_rec::rec_size(const std::size_t xidsize, const std::size_t dsize, const bool external) +{ + if (external) + return sizeof(enq_hdr_t) + xidsize + sizeof(rec_tail_t); + return sizeof(enq_hdr_t) + xidsize + dsize + sizeof(rec_tail_t); +} + +void +enq_rec::set_rid(const uint64_t rid) +{ + _enq_hdr._rhdr._rid = rid; + _enq_tail._rid = rid; +} + +void +enq_rec::chk_hdr() const +{ + jrec::chk_hdr(_enq_hdr._rhdr); + if (_enq_hdr._rhdr._magic != QLS_ENQ_MAGIC) + { + std::ostringstream oss; + oss << std::hex << std::setfill('0'); + oss << "enq magic: rid=0x" << std::setw(16) << _enq_hdr._rhdr._rid; + oss << ": expected=0x" << std::setw(8) << QLS_ENQ_MAGIC; + oss << " read=0x" << std::setw(2) << (int)_enq_hdr._rhdr._magic; + throw jexception(jerrno::JERR_JREC_BADRECHDR, oss.str(), "enq_rec", "chk_hdr"); + } +} + +void +enq_rec::chk_hdr(uint64_t rid) const +{ + chk_hdr(); + jrec::chk_rid(_enq_hdr._rhdr, rid); +} + +void +enq_rec::chk_tail() const +{ + jrec::chk_tail(_enq_tail, _enq_hdr._rhdr); +} + +void +enq_rec::clean() +{ + // clean up allocated memory here +} + +}} diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/enq_rec.h b/qpid/cpp/src/qpid/linearstore/jrnl/enq_rec.h new file mode 100644 index 0000000000..b3502d9ad6 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/enq_rec.h @@ -0,0 +1,104 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_JRNL_ENQ_REC_H +#define QPID_LEGACYSTORE_JRNL_ENQ_REC_H + +namespace qpid +{ +namespace qls_jrnl +{ +class enq_rec; +}} + +#include <cstddef> +#include "qpid/linearstore/jrnl/utils/enq_hdr.h" +#include "qpid/linearstore/jrnl/jrec.h" + +namespace qpid +{ +namespace qls_jrnl +{ + + /** + * \class enq_rec + * \brief Class to handle a single journal enqueue record. + */ + class enq_rec : public jrec + { + private: + enq_hdr_t _enq_hdr; + const void* _xidp; ///< xid pointer for encoding (for writing to disk) + const void* _data; ///< Pointer to data to be written to disk + void* _buff; ///< Pointer to buffer to receive data read from disk + rec_tail_t _enq_tail; + + public: + /** + * \brief Constructor used for read operations. + */ + enq_rec(); + + /** + * \brief Constructor used for write operations, where mbuf contains data to be written. + */ + enq_rec(const uint64_t rid, const void* const dbuf, const std::size_t dlen, + const void* const xidp, const std::size_t xidlen, const bool transient); + + /** + * \brief Destructor + */ + virtual ~enq_rec(); + + // Prepare instance for use in reading data from journal, xid and data will be allocated + void reset(); + // Prepare instance for use in writing data to journal + void reset(const uint64_t rid, const void* const dbuf, const std::size_t dlen, + const void* const xidp, const std::size_t xidlen, const bool transient, + const bool external); + + uint32_t encode(void* wptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks); + uint32_t decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks); + // Decode used for recover + bool rcv_decode(rec_hdr_t h, std::ifstream* ifsp, std::size_t& rec_offs); + + std::size_t get_xid(void** const xidpp); + std::size_t get_data(void** const datapp); + inline bool is_transient() const { return ::is_enq_transient(&_enq_hdr); } + inline bool is_external() const { return ::is_enq_external(&_enq_hdr); } + std::string& str(std::string& str) const; + inline std::size_t data_size() const { return _enq_hdr._dsize; } + inline std::size_t xid_size() const { return _enq_hdr._xidsize; } + std::size_t rec_size() const; + static std::size_t rec_size(const std::size_t xidsize, const std::size_t dsize, const bool external); + inline uint64_t rid() const { return _enq_hdr._rhdr._rid; } + void set_rid(const uint64_t rid); + + private: + void chk_hdr() const; + void chk_hdr(uint64_t rid) const; + void chk_tail() const; + virtual void clean(); + }; // class enq_rec + +}} + +#endif // ifndef QPID_LEGACYSTORE_JRNL_ENQ_REC_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/enums.h b/qpid/cpp/src/qpid/linearstore/jrnl/enums.h new file mode 100644 index 0000000000..511a2a41ab --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/enums.h @@ -0,0 +1,99 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LINEARSTORE_JRNL_ENUMS_H +#define QPID_LINEARSTORE_JRNL_ENUMS_H + +namespace qpid +{ +namespace qls_jrnl +{ + + // TODO: Change this to flags, as multiple of these conditions may exist simultaneously + /** + * \brief Enumeration of possible return states from journal read and write operations. + */ + enum _iores + { + RHM_IORES_SUCCESS = 0, ///< Success: IO operation completed noramlly. + RHM_IORES_PAGE_AIOWAIT, ///< IO operation suspended - next page is waiting for AIO. + RHM_IORES_FILE_AIOWAIT, ///< IO operation suspended - next file is waiting for AIO. + RHM_IORES_EMPTY, ///< During read operations, nothing further is available to read. +// RHM_IORES_RCINVALID, ///< Read page cache is invalid (ie obsolete or uninitialized) +// RHM_IORES_ENQCAPTHRESH, ///< Enqueue capacity threshold (limit) reached. +// RHM_IORES_FULL, ///< During write operations, the journal files are full. +// RHM_IORES_BUSY, ///< Another blocking operation is in progress. + RHM_IORES_TXPENDING ///< Operation blocked by pending transaction. +// RHM_IORES_NOTIMPL ///< Function is not implemented. + }; + typedef _iores iores; + + static inline const char* iores_str(iores res) + { + switch (res) + { + case RHM_IORES_SUCCESS: return "RHM_IORES_SUCCESS"; + case RHM_IORES_PAGE_AIOWAIT: return "RHM_IORES_PAGE_AIOWAIT"; + case RHM_IORES_FILE_AIOWAIT: return "RHM_IORES_FILE_AIOWAIT"; + case RHM_IORES_EMPTY: return "RHM_IORES_EMPTY"; +// case RHM_IORES_RCINVALID: return "RHM_IORES_RCINVALID"; +// case RHM_IORES_ENQCAPTHRESH: return "RHM_IORES_ENQCAPTHRESH"; +// case RHM_IORES_FULL: return "RHM_IORES_FULL"; +// case RHM_IORES_BUSY: return "RHM_IORES_BUSY"; + case RHM_IORES_TXPENDING: return "RHM_IORES_TXPENDING"; +// case RHM_IORES_NOTIMPL: return "RHM_IORES_NOTIMPL"; + } + return "<iores unknown>"; + } + +/* + enum _log_level + { + LOG_TRACE = 0, + LOG_DEBUG, + LOG_INFO, + LOG_NOTICE, + LOG_WARN, + LOG_ERROR, + LOG_CRITICAL + }; + typedef _log_level log_level_t; + + static inline const char* log_level_str(log_level_t ll) + { + switch (ll) + { + case LOG_TRACE: return "TRACE"; + case LOG_DEBUG: return "DEBUG"; + case LOG_INFO: return "INFO"; + case LOG_NOTICE: return "NOTICE"; + case LOG_WARN: return "WARN"; + case LOG_ERROR: return "ERROR"; + case LOG_CRITICAL: return "CRITICAL"; + } + return "<log level unknown>"; + } +*/ + + +}} + +#endif // ifndef QPID_LINEARSTORE_JRNL_ENUMS_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/jcfg.h b/qpid/cpp/src/qpid/linearstore/jrnl/jcfg.h new file mode 100644 index 0000000000..432887e577 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/jcfg.h @@ -0,0 +1,58 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_JRNL_JCFG_H +#define QPID_LEGACYSTORE_JRNL_JCFG_H + +#define QLS_SBLK_SIZE_BYTES 4096 /**< Disk softblock size in bytes, should match size used on disk media */ +#define QLS_AIO_ALIGN_BOUNDARY_BYTES QLS_SBLK_SIZE_BYTES /** Memory alignment boundary used for DMA */ +/** +* <b>Rule:</b> Data block size (QLS_DBLK_SIZE_BYTES) MUST be a power of 2 AND +* a power of 2 factor of the disk softblock size (QLS_SBLK_SIZE_BYTES): +* <pre> +* n * QLS_DBLK_SIZE_BYTES == QLS_SBLK_SIZE_BYTES (n = 1,2,4,8...) +* </pre> +*/ +#define QLS_DBLK_SIZE_BYTES 128 /**< Data block size in bytes (CANNOT BE LESS THAN 32!) */ +#define QLS_SBLK_SIZE_DBLKS (QLS_SBLK_SIZE_BYTES / QLS_DBLK_SIZE_BYTES) /**< Disk softblock size in multiples of QLS_DBLK_SIZE_BYTES */ +#define QLS_SBLK_SIZE_KIB (QLS_SBLK_SIZE_BYTES / 1024) /**< Disk softblock size in KiB */ + +#define QLS_WMGR_DEF_PAGE_SIZE_KIB 32 /**< Journal write page size in KiB (default) */ +#define QLS_WMGR_DEF_PAGE_SIZE_SBLKS (QLS_WMGR_DEF_PAGE_SIZE_KIB / QLS_SBLK_SIZE_KIB) /**< Journal write page size in softblocks (default) */ +#define QLS_WMGR_DEF_PAGES 32 /**< Number of pages to use in wmgr (default) */ + +#define QLS_WMGR_MAXDTOKPP 1024 /**< Max. dtoks (data blocks) per page in wmgr */ +#define QLS_WMGR_MAXWAITUS 100 /**< Max. wait time (us) before submitting AIO */ + +#define QLS_JRNL_FILE_EXTENSION ".jrnl" /**< Extension for journal data files */ +#define QLS_TXA_MAGIC 0x61534c51 /**< ("RHMa" in little endian) Magic for dtx abort hdrs */ +#define QLS_TXC_MAGIC 0x63534c51 /**< ("RHMc" in little endian) Magic for dtx commit hdrs */ +#define QLS_DEQ_MAGIC 0x64534c51 /**< ("QLSd" in little endian) Magic for deq rec hdrs */ +#define QLS_ENQ_MAGIC 0x65534c51 /**< ("QLSe" in little endian) Magic for enq rec hdrs */ +#define QLS_FILE_MAGIC 0x66534c51 /**< ("QLSf" in little endian) Magic for file hdrs */ +#define QLS_EMPTY_MAGIC 0x78534c51 /**< ("QLSx" in little endian) Magic for empty dblk */ +#define QLS_JRNL_VERSION 2 /**< Version (of file layout) */ +#define QLS_JRNL_FHDR_RES_SIZE_SBLKS 1 /**< Journal file header reserved size in sblks (as defined by QLS_SBLK_SIZE_BYTES) */ + +#define QLS_CLEAN /**< If defined, writes QLS_CLEAN_CHAR to all filled areas on disk */ +#define QLS_CLEAN_CHAR 0xff /**< Char used to clear empty space on disk */ + +#endif /* ifndef QPID_LEGACYSTORE_JRNL_JCFG_H */ diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/jcntl.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/jcntl.cpp new file mode 100644 index 0000000000..e8873cddcd --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/jcntl.cpp @@ -0,0 +1,549 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/jrnl/jcntl.h" + +#include <algorithm> +#include <cassert> +#include <cerrno> +#include <cstdlib> +#include <cstring> +#include <fstream> +#include <iomanip> +#include <iostream> +#include <qpid/linearstore/jrnl/EmptyFilePool.h> +#include <qpid/linearstore/jrnl/EmptyFilePoolManager.h> +#include "qpid/linearstore/jrnl/jerrno.h" +#include "qpid/linearstore/jrnl/JournalLog.h" +#include "qpid/linearstore/jrnl/utils/enq_hdr.h" +#include <limits> +#include <sstream> +#include <unistd.h> + +namespace qpid +{ +namespace qls_jrnl +{ + +#define AIO_CMPL_TIMEOUT_SEC 5 +#define AIO_CMPL_TIMEOUT_NSEC 0 +#define FINAL_AIO_CMPL_TIMEOUT_SEC 15 +#define FINAL_AIO_CMPL_TIMEOUT_NSEC 0 + +// Static +timespec jcntl::_aio_cmpl_timeout; ///< Timeout for blocking libaio returns +timespec jcntl::_final_aio_cmpl_timeout; ///< Timeout for blocking libaio returns when stopping or finalizing +bool jcntl::_init = init_statics(); +bool jcntl::init_statics() +{ + _aio_cmpl_timeout.tv_sec = AIO_CMPL_TIMEOUT_SEC; + _aio_cmpl_timeout.tv_nsec = AIO_CMPL_TIMEOUT_NSEC; + _final_aio_cmpl_timeout.tv_sec = FINAL_AIO_CMPL_TIMEOUT_SEC; + _final_aio_cmpl_timeout.tv_nsec = FINAL_AIO_CMPL_TIMEOUT_NSEC; + return true; +} + + +// Functions + +jcntl::jcntl(const std::string& jid, + const std::string& jdir, + JournalLog& jrnl_log): + _jid(jid), + _jdir(jdir), + _init_flag(false), + _stop_flag(false), + _readonly_flag(false), + _jrnl_log(jrnl_log), + _linearFileController(*this), + _emptyFilePoolPtr(0), + _emap(), + _tmap(), + _wmgr(this, _emap, _tmap, _linearFileController), + _recoveryManager(_jdir.dirname(), _jid, _emap, _tmap, jrnl_log) +{} + +jcntl::~jcntl() +{ + if (_init_flag && !_stop_flag) + try { stop(true); } + catch (const jexception& e) { std::cerr << e << std::endl; } + _linearFileController.finalize(); +} + +void +jcntl::initialize(EmptyFilePool* efpp, + const uint16_t wcache_num_pages, + const uint32_t wcache_pgsize_sblks, + aio_callback* const cbp) +{ + _init_flag = false; + _stop_flag = false; + _readonly_flag = false; + + _emap.clear(); + _tmap.clear(); + + _linearFileController.finalize(); + +// _lpmgr.finalize(); + + // Set new file geometry parameters +// assert(num_jfiles >= JRNL_MIN_NUM_FILES); +// assert(num_jfiles <= JRNL_MAX_NUM_FILES); +// _emap.set_num_jfiles(num_jfiles); +// _tmap.set_num_jfiles(num_jfiles); + +// assert(jfsize_sblks >= JRNL_MIN_FILE_SIZE); +// assert(jfsize_sblks <= JRNL_MAX_FILE_SIZE); +// _jfsize_sblks = jfsize_sblks; + + // Clear any existing journal files + _jdir.clear_dir(); +// _lpmgr.initialize(num_jfiles, ae, ae_max_jfiles, this, &new_fcntl); // Creates new journal files + + _linearFileController.initialize(_jdir.dirname(), efpp, 0ULL); + _linearFileController.pullEmptyFileFromEfp(); +// std::cout << _linearFileController.status(2); +// _wrfc.initialize(_jfsize_sblks); +// _rrfc.initialize(); +// _rrfc.set_findex(0); +// _rmgr.initialize(cbp); + _wmgr.initialize(cbp, wcache_pgsize_sblks, wcache_num_pages, QLS_WMGR_MAXDTOKPP, QLS_WMGR_MAXWAITUS); + + // Write info file (<basename>.jinf) to disk +// write_infofile(); + + _init_flag = true; +} + +void +jcntl::recover(EmptyFilePoolManager* efpmp, + const uint16_t wcache_num_pages, + const uint32_t wcache_pgsize_sblks, + aio_callback* const cbp, + const std::vector<std::string>* prep_txn_list_ptr, + uint64_t& highest_rid) +{ + _init_flag = false; + _stop_flag = false; + _readonly_flag = false; + + _emap.clear(); + _tmap.clear(); + + _linearFileController.finalize(); + +// _lpmgr.finalize(); + +// assert(num_jfiles >= JRNL_MIN_NUM_FILES); +// assert(num_jfiles <= JRNL_MAX_NUM_FILES); +// assert(jfsize_sblks >= JRNL_MIN_FILE_SIZE); +// assert(jfsize_sblks <= JRNL_MAX_FILE_SIZE); +// _jfsize_sblks = jfsize_sblks; + + // Verify journal dir and journal files + _jdir.verify_dir(); +// _rcvdat.reset(num_jfiles/*, ae, ae_max_jfiles*/); + +// rcvr_janalyze(prep_txn_list_ptr, efpm); + efpIdentity_t efpIdentity; + _recoveryManager.analyzeJournals(prep_txn_list_ptr, efpmp, &_emptyFilePoolPtr); + + highest_rid = _recoveryManager.getHighestRecordId(); +// if (_rcvdat._jfull) +// throw jexception(jerrno::JERR_JCNTL_RECOVERJFULL, "jcntl", "recover"); + _jrnl_log.log(/*LOG_DEBUG*/JournalLog::LOG_INFO, _jid, _recoveryManager.toString(_jid, true)); + +// _lpmgr.recover(_rcvdat, this, &new_fcntl); + + _linearFileController.initialize(_jdir.dirname(), _emptyFilePoolPtr, _recoveryManager.getHighestFileNumber()); +// _linearFileController.setFileNumberCounter(_recoveryManager.getHighestFileNumber()); + _recoveryManager.setLinearFileControllerJournals(&qpid::qls_jrnl::LinearFileController::addJournalFile, &_linearFileController); +// _wrfc.initialize(_jfsize_sblks, &_rcvdat); +// _rrfc.initialize(); +// _rrfc.set_findex(_rcvdat.ffid()); +// _rmgr.initialize(cbp); + _wmgr.initialize(cbp, wcache_pgsize_sblks, wcache_num_pages, QLS_WMGR_MAXDTOKPP, QLS_WMGR_MAXWAITUS, + (_recoveryManager.isLastFileFull() ? 0 : _recoveryManager.getEndOffset())); + + _readonly_flag = true; + _init_flag = true; +} + +void +jcntl::recover_complete() +{ + if (!_readonly_flag) + throw jexception(jerrno::JERR_JCNTL_NOTRECOVERED, "jcntl", "recover_complete"); +// for (uint16_t i=0; i<_lpmgr.num_jfiles(); i++) +// _lpmgr.get_fcntlp(i)->reset(&_rcvdat); +// _wrfc.initialize(_jfsize_sblks, &_rcvdat); +// _rrfc.initialize(); +// _rrfc.set_findex(_rcvdat.ffid()); +// _rmgr.recover_complete(); + _readonly_flag = false; +} + +void +jcntl::delete_jrnl_files() +{ + stop(true); // wait for AIO to complete + _linearFileController.purgeFilesToEfp(); + _jdir.delete_dir(); +} + + +iores +jcntl::enqueue_data_record(const void* const data_buff, + const std::size_t tot_data_len, + const std::size_t this_data_len, + data_tok* dtokp, + const bool transient) +{ + iores r; + check_wstatus("enqueue_data_record"); + { + slock s(_wr_mutex); + while (handle_aio_wait(_wmgr.enqueue(data_buff, tot_data_len, this_data_len, dtokp, 0, 0, transient, false), r, + dtokp)) ; + } + return r; +} + +iores +jcntl::enqueue_extern_data_record(const std::size_t tot_data_len, + data_tok* dtokp, + const bool transient) +{ + iores r; + check_wstatus("enqueue_extern_data_record"); + { + slock s(_wr_mutex); + while (handle_aio_wait(_wmgr.enqueue(0, tot_data_len, 0, dtokp, 0, 0, transient, true), r, dtokp)) ; + } + return r; +} + +iores +jcntl::enqueue_txn_data_record(const void* const data_buff, + const std::size_t tot_data_len, + const std::size_t this_data_len, + data_tok* dtokp, + const std::string& xid, + const bool transient) +{ + iores r; + check_wstatus("enqueue_tx_data_record"); + { + slock s(_wr_mutex); + while (handle_aio_wait(_wmgr.enqueue(data_buff, tot_data_len, this_data_len, dtokp, xid.data(), xid.size(), + transient, false), r, dtokp)) ; + } + return r; +} + +iores +jcntl::enqueue_extern_txn_data_record(const std::size_t tot_data_len, + data_tok* dtokp, + const std::string& xid, + const bool transient) +{ + iores r; + check_wstatus("enqueue_extern_txn_data_record"); + { + slock s(_wr_mutex); + while (handle_aio_wait(_wmgr.enqueue(0, tot_data_len, 0, dtokp, xid.data(), xid.size(), transient, true), r, + dtokp)) ; + } + return r; +} + +iores +jcntl::read_data_record(void** const datapp, + std::size_t& dsize, + void** const xidpp, + std::size_t& xidsize, + bool& transient, + bool& external, + data_tok* const dtokp, + bool ignore_pending_txns) +{ + check_rstatus("read_data"); + if (_recoveryManager.readNextRemainingRecord(datapp, dsize, xidpp, xidsize, transient, external, dtokp, ignore_pending_txns)) + return RHM_IORES_SUCCESS; + return RHM_IORES_EMPTY; +/* + if (!dtokp->is_readable()) { + std::ostringstream oss; + oss << std::hex << std::setfill('0'); + oss << "dtok_id=0x" << std::setw(8) << dtokp->id(); + oss << "; dtok_rid=0x" << std::setw(16) << dtokp->rid(); + oss << "; dtok_wstate=" << dtokp->wstate_str(); + throw jexception(jerrno::JERR_JCNTL_ENQSTATE, oss.str(), "jcntl", "read_data_record"); + } + std::vector<uint64_t> ridl; + _emap.rid_list(ridl); + enq_map::emap_data_struct_t eds; + for (std::vector<uint64_t>::const_iterator i=ridl.begin(); i!=ridl.end(); ++i) { + short res = _emap.get_data(*i, eds); + if (res == enq_map::EMAP_OK) { + std::ifstream ifs(_recoveryManager._fm[eds._pfid].c_str(), std::ifstream::in | std::ifstream::binary); + if (!ifs.good()) { + std::ostringstream oss; + oss << "rid=" << (*i) << " pfid=" << eds._pfid << " file=" << _recoveryManager._fm[eds._pfid] << " file_posn=" << eds._file_posn; + throw jexception(jerrno::JERR_RCVM_OPENRD, oss.str(), "jcntl", "read_data_record"); + } + ifs.seekg(eds._file_posn, std::ifstream::beg); + ::enq_hdr_t eh; + ifs.read((char*)&eh, sizeof(::enq_hdr_t)); + if (!::validate_enq_hdr(&eh, QLS_ENQ_MAGIC, QLS_JRNL_VERSION, *i)) { + std::ostringstream oss; + oss << "rid=" << (*i) << " pfid=" << eds._pfid << " file=" << _recoveryManager._fm[eds._pfid] << " file_posn=" << eds._file_posn; + throw jexception(jerrno::JERR_JCNTL_INVALIDENQHDR, oss.str(), "jcntl", "read_data_record"); + } + dsize = eh._dsize; + xidsize = eh._xidsize; + transient = ::is_enq_transient(&eh); + external = ::is_enq_external(&eh); + if (xidsize) { + *xidpp = ::malloc(xidsize); + ifs.read((char*)(*xidpp), xidsize); + } else { + *xidpp = 0; + } + if (dsize) { + *datapp = ::malloc(dsize); + ifs.read((char*)(*datapp), dsize); + } else { + *datapp = 0; + } + } + } +*/ +/* + check_rstatus("read_data"); + iores res = _rmgr.read(datapp, dsize, xidpp, xidsize, transient, external, dtokp, ignore_pending_txns); + if (res == RHM_IORES_RCINVALID) + { + get_wr_events(0); // check for outstanding write events + iores sres = _rmgr.synchronize(); // flushes all outstanding read events + if (sres != RHM_IORES_SUCCESS) + return sres; + // TODO: Does linear store need this? +// _rmgr.wait_for_validity(&_aio_cmpl_timeout, true); // throw if timeout occurs + res = _rmgr.read(datapp, dsize, xidpp, xidsize, transient, external, dtokp, ignore_pending_txns); + } + return res; +*/ + return RHM_IORES_SUCCESS; +} + +iores +jcntl::dequeue_data_record(data_tok* const dtokp, + const bool txn_coml_commit) +{ + iores r; + check_wstatus("dequeue_data"); + { + slock s(_wr_mutex); + while (handle_aio_wait(_wmgr.dequeue(dtokp, 0, 0, txn_coml_commit), r, dtokp)) ; + } + return r; +} + +iores +jcntl::dequeue_txn_data_record(data_tok* const dtokp, + const std::string& xid, + const bool txn_coml_commit) +{ + iores r; + check_wstatus("dequeue_data"); + { + slock s(_wr_mutex); + while (handle_aio_wait(_wmgr.dequeue(dtokp, xid.data(), xid.size(), txn_coml_commit), r, dtokp)) ; + } + return r; +} + +iores +jcntl::txn_abort(data_tok* const dtokp, + const std::string& xid) +{ + iores r; + check_wstatus("txn_abort"); + { + slock s(_wr_mutex); + while (handle_aio_wait(_wmgr.abort(dtokp, xid.data(), xid.size()), r, dtokp)) ; + } + return r; +} + +iores +jcntl::txn_commit(data_tok* const dtokp, + const std::string& xid) +{ + iores r; + check_wstatus("txn_commit"); + { + slock s(_wr_mutex); + while (handle_aio_wait(_wmgr.commit(dtokp, xid.data(), xid.size()), r, dtokp)) ; + } + return r; +} + +bool +jcntl::is_txn_synced(const std::string& xid) +{ + slock s(_wr_mutex); + bool res = _wmgr.is_txn_synced(xid); + return res; +} + +int32_t +jcntl::get_wr_events(timespec* const timeout) +{ + stlock t(_wr_mutex); + if (!t.locked()) + return jerrno::LOCK_TAKEN; + return _wmgr.get_events(timeout, false); +} + +void +jcntl::stop(const bool block_till_aio_cmpl) +{ + if (_readonly_flag) + check_rstatus("stop"); + else + check_wstatus("stop"); + _stop_flag = true; + if (!_readonly_flag) + flush(block_till_aio_cmpl); + _linearFileController.finalize(); +} + +LinearFileController& +jcntl::getLinearFileControllerRef() { + return _linearFileController; +} + +iores +jcntl::flush(const bool block_till_aio_cmpl) +{ + if (!_init_flag) + return RHM_IORES_SUCCESS; + if (_readonly_flag) + throw jexception(jerrno::JERR_JCNTL_READONLY, "jcntl", "flush"); + iores res; + { + slock s(_wr_mutex); + res = _wmgr.flush(); + } + if (block_till_aio_cmpl) + aio_cmpl_wait(); + return res; +} + +// Protected/Private functions + +void +jcntl::check_wstatus(const char* fn_name) const +{ + if (!_init_flag) + throw jexception(jerrno::JERR__NINIT, "jcntl", fn_name); + if (_readonly_flag) + throw jexception(jerrno::JERR_JCNTL_READONLY, "jcntl", fn_name); + if (_stop_flag) + throw jexception(jerrno::JERR_JCNTL_STOPPED, "jcntl", fn_name); +} + +void +jcntl::check_rstatus(const char* fn_name) const +{ + if (!_init_flag) + throw jexception(jerrno::JERR__NINIT, "jcntl", fn_name); + if (_stop_flag) + throw jexception(jerrno::JERR_JCNTL_STOPPED, "jcntl", fn_name); +} + + +void +jcntl::aio_cmpl_wait() +{ + //while (_wmgr.get_aio_evt_rem()) + while (true) + { + uint32_t aer; + { + slock s(_wr_mutex); + aer = _wmgr.get_aio_evt_rem(); + } + if (aer == 0) break; // no events left + if (get_wr_events(&_aio_cmpl_timeout) == jerrno::AIO_TIMEOUT) + throw jexception(jerrno::JERR_JCNTL_AIOCMPLWAIT, "jcntl", "aio_cmpl_wait"); + } +} + + +bool +jcntl::handle_aio_wait(const iores res, iores& resout, const data_tok* dtp) +{ + resout = res; + if (res == RHM_IORES_PAGE_AIOWAIT) + { + while (_wmgr.curr_pg_blocked()) + { + if (_wmgr.get_aio_evt_rem() == 0) { +std::cout << "&&&&&& jcntl::handle_aio_wait() " << _wmgr.status_str() << std::endl; // DEBUG + throw jexception("_wmgr.curr_pg_blocked() with no events remaining"); // TODO - complete exception + } + if (_wmgr.get_events(&_aio_cmpl_timeout, false) == jerrno::AIO_TIMEOUT) + { + std::ostringstream oss; + oss << "get_events() returned JERR_JCNTL_AIOCMPLWAIT; wmgr_status: " << _wmgr.status_str(); + _jrnl_log.log(JournalLog::LOG_CRITICAL, _jid, oss.str()); + throw jexception(jerrno::JERR_JCNTL_AIOCMPLWAIT, "jcntl", "handle_aio_wait"); + } + } + return true; + } + else if (res == RHM_IORES_FILE_AIOWAIT) + { +// while (_wmgr.curr_file_blocked()) +// { +// if (_wmgr.get_events(pmgr::UNUSED, &_aio_cmpl_timeout) == jerrno::AIO_TIMEOUT) +// { +// std::ostringstream oss; +// oss << "get_events() returned JERR_JCNTL_AIOCMPLWAIT; wmgr_status: " << _wmgr.status_str(); +// this->log(LOG_CRITICAL, oss.str()); +// throw jexception(jerrno::JERR_JCNTL_AIOCMPLWAIT, "jcntl", "handle_aio_wait"); +// } +// } +// _wrfc.wr_reset(); + resout = RHM_IORES_SUCCESS; + data_tok::write_state ws = dtp->wstate(); + return ws == data_tok::ENQ_PART || ws == data_tok::DEQ_PART || ws == data_tok::ABORT_PART || + ws == data_tok::COMMIT_PART; + } + return false; +} + +}} diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/jcntl.h b/qpid/cpp/src/qpid/linearstore/jrnl/jcntl.h new file mode 100644 index 0000000000..bb91eac569 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/jcntl.h @@ -0,0 +1,575 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LINEARSTORE_JRNL_JCNTL_H +#define QPID_LINEARSTORE_JRNL_JCNTL_H + +namespace qpid +{ +namespace qls_jrnl +{ + class jcntl; +}} + +#include <cstddef> +#include <deque> +#include <qpid/linearstore/jrnl/LinearFileController.h> +#include "qpid/linearstore/jrnl/jdir.h" +#include "qpid/linearstore/jrnl/RecoveryManager.h" +#include "qpid/linearstore/jrnl/slock.h" +#include "qpid/linearstore/jrnl/smutex.h" +#include "qpid/linearstore/jrnl/wmgr.h" + +namespace qpid +{ +namespace qls_jrnl +{ + class EmptyFilePool; + class EmptyFilePoolManager; + + /** + * \brief Access and control interface for the journal. This is the top-level class for the + * journal. + * + * This is the top-level journal class; one instance of this class controls one instance of the + * journal and all its files and associated control structures. Besides this class, the only + * other class that needs to be used at a higher level is the data_tok class, one instance of + * which is used per data block written to the journal, and is used to track its status through + * the AIO enqueue, read and dequeue process. + */ + class jcntl + { + protected: + /** + * \brief Journal ID + * + * This string uniquely identifies this journal instance. It will most likely be associated + * with the identity of the message queue with which it is associated. + */ + // TODO: This is not included in any files at present, add to file_hdr? + std::string _jid; + + /** + * \brief Journal directory + * + * This string stores the path to the journal directory. It may be absolute or relative, and + * should not end in a file separator character. (e.g. "/fastdisk/jdata" is correct, + * "/fastdisk/jdata/" is not.) + */ + jdir _jdir; + + /** + * \brief Initialized flag + * + * This flag starts out set to false, is set to true once this object has been initialized, + * either by calling initialize() or recover(). + */ + bool _init_flag; + + /** + * \brief Stopped flag + * + * This flag starts out false, and is set to true when stop() is called. At this point, the + * journal will no longer accept messages until either initialize() or recover() is called. + * There is no way other than through initialization to reset this flag. + */ + // TODO: It would be helpful to distinguish between states stopping and stopped. If stop(true) is called, + // then we are stopping, but must wait for all outstanding aios to return before being finally stopped. During + // this period, however, no new enqueue/dequeue/read requests may be accepted. + bool _stop_flag; + + /** + * \brief Read-only state flag used during recover. + * + * When true, this flag prevents journal write operations (enqueue and dequeue), but + * allows read to occur. It is used during recovery, and is reset when recovered() is + * called. + */ + bool _readonly_flag; + + // Journal control structures + JournalLog& _jrnl_log; ///< Ref to Journal Log instance + LinearFileController _linearFileController; ///< Linear File Controller + EmptyFilePool* _emptyFilePoolPtr; ///< Pointer to Empty File Pool for this queue + enq_map _emap; ///< Enqueue map for low water mark management + txn_map _tmap; ///< Transaction map open transactions + wmgr _wmgr; ///< Write page manager which manages AIO + RecoveryManager _recoveryManager; ///< Recovery data used for recovery + smutex _wr_mutex; ///< Mutex for journal writes + + public: + static timespec _aio_cmpl_timeout; ///< Timeout for blocking libaio returns + static timespec _final_aio_cmpl_timeout; ///< Timeout for blocking libaio returns when stopping or finalizing + + /** + * \brief Journal constructor. + * + * Constructor which sets the physical file location and base name. + * + * \param jid A unique identifier for this journal instance. + * \param jdir The directory which will contain the journal files. + * \param base_filename The string which will be used to start all journal filenames. + */ + jcntl(const std::string& jid, + const std::string& jdir, + JournalLog& jrnl_log); + + /** + * \brief Destructor. + */ + virtual ~jcntl(); + + inline const std::string& id() const { return _jid; } + + inline const std::string& jrnl_dir() const { return _jdir.dirname(); } + + /** + * \brief Initialize the journal for storing data. + * + * Initialize the journal by creating new journal data files and initializing internal + * control structures. When complete, the journal will be empty, and ready to store data. + * + * <b>NOTE: Any existing journal will be ignored by this operation.</b> To use recover + * the data from an existing journal, use recover(). + * + * <b>NOTE: If <i>NULL</i> is passed to the deque pointers, they will be internally created + * and deleted.</b> + * + * <b>NOTE: If <i>NULL</i> is passed to the callbacks, internal default callbacks will be + * used.</b> + * + * \param num_jfiles The number of journal files to be created. + * \param auto_expand If true, allows journal file auto-expansion. In this mode, the journal will automatically + * add files to the journal if it runs out of space. No more than ae_max_jfiles may be added. If false, then + * no files are added and an exception will be thrown if the journal runs out of file space. + * \param ae_max_jfiles Upper limit of journal files for auto-expand mode. When auto_expand is true, this is the + * maximum total number of files allowed in the journal (original plus those added by auto-expand mode). If + * this number of files exist and the journal runs out of space, an exception will be thrown. This number + * must be greater than the num_jfiles parameter value but cannot exceed the maximum number of files for a + * single journal; if num_jfiles is already at its maximum value, then auto-expand will be disabled. + * \param jfsize_sblks The size of each journal file expressed in softblocks. + * \param wcache_num_pages The number of write cache pages to create. + * \param wcache_pgsize_sblks The size in sblks of each write cache page. + * \param cbp Pointer to object containing callback functions for read and write operations. May be 0 (NULL). + * + * \exception TODO + */ + void initialize(EmptyFilePool* efpp, + const uint16_t wcache_num_pages, + const uint32_t wcache_pgsize_sblks, + aio_callback* const cbp); + + /** + * /brief Initialize journal by recovering state from previously written journal. + * + * Initialize journal by recovering state from previously written journal. The journal files + * are analyzed, and all records that have not been dequeued and that remain in the journal + * will be available for reading. The journal is placed in a read-only state until + * recovered() is called; any calls to enqueue or dequeue will fail with an exception + * in this state. + * + * <b>NOTE: If <i>NULL</i> is passed to the deque pointers, they will be internally created + * and deleted.</b> + * + * <b>NOTE: If <i>NULL</i> is passed to the callbacks, internal default callbacks will be + * used.</b> + * + * \param num_jfiles The number of journal files to be created. + * \param auto_expand If true, allows journal file auto-expansion. In this mode, the journal will automatically + * add files to the journal if it runs out of space. No more than ae_max_jfiles may be added. If false, then + * no files are added and an exception will be thrown if the journal runs out of file space. + * \param ae_max_jfiles Upper limit of journal files for auto-expand mode. When auto_expand is true, this is the + * maximum total number of files allowed in the journal (original plus those added by auto-expand mode). If + * this number of files exist and the journal runs out of space, an exception will be thrown. This number + * must be greater than the num_jfiles parameter value but cannot exceed the maximum number of files for a + * single journal; if num_jfiles is already at its maximum value, then auto-expand will be disabled. + * \param jfsize_sblks The size of each journal file expressed in softblocks. + * \param wcache_num_pages The number of write cache pages to create. + * \param wcache_pgsize_sblks The size in sblks of each write cache page. + * \param cbp Pointer to object containing callback functions for read and write operations. May be 0 (NULL). + * \param prep_txn_list_ptr + * \param highest_rid Returns the highest rid found in the journal during recover + * + * \exception TODO + */ + void recover(EmptyFilePoolManager* efpm, + const uint16_t wcache_num_pages, + const uint32_t wcache_pgsize_sblks, + aio_callback* const cbp, + const std::vector<std::string>* prep_txn_list_ptr, + uint64_t& highest_rid); + + /** + * \brief Notification to the journal that recovery is complete and that normal operation + * may resume. + * + * This call notifies the journal that recovery is complete and that normal operation + * may resume. The read pointers are reset so that all records read as a part of recover + * may be re-read during normal operation. The read-only flag is then reset, allowing + * enqueue and dequeue operations to resume. + * + * \exception TODO + */ + void recover_complete(); + + /** + * \brief Stops journal and deletes all journal files. + * + * Clear the journal directory of all journal files matching the base filename. + * + * \exception TODO + */ + void delete_jrnl_files(); + + /** + * \brief Enqueue data. + * + * Enqueue data or part thereof. If a large data block is being written, then it may be + * enqueued in parts by setting this_data_len to the size of the data being written in this + * call. The total data size must be known in advance, however, as this is written into the + * record header on the first record write. The state of the write (i.e. how much has been + * written so far) is maintained in the data token dtokp. Partial writes will return in state + * ENQ_PART. + * + * Note that a return value of anything other than RHM_IORES_SUCCESS implies that this write + * operation did not complete successfully or was partially completed. The action taken under + * these conditions depends on the value of the return. For example, RHM_IORES_AIO_WAIT + * implies that all pages in the write page cache are waiting for AIO operations to return, + * and that the call should be remade after waiting a bit. + * + * Example: If a write of 99 kB is divided into three equal parts, then the following states + * and returns would characterize a successful operation: + * <pre> + * dtok. dtok. dtok. + * Pperation Return wstate() dsize() written() Comment + * -----------------+--------+--------+-------+---------+------------------------------------ + * NONE 0 0 Value of dtok before op + * edr(99000, 33000) SUCCESS ENQ_PART 99000 33000 Enqueue part 1 + * edr(99000, 33000) AIO_WAIT ENQ_PART 99000 50000 Enqueue part 2, not completed + * edr(99000, 33000) SUCCESS ENQ_PART 99000 66000 Enqueue part 2 again + * edr(99000, 33000) SUCCESS ENQ 99000 99000 Enqueue part 3 + * </pre> + * + * \param data_buff Pointer to data to be enqueued for this enqueue operation. + * \param tot_data_len Total data length. + * \param this_data_len Amount to be written in this enqueue operation. + * \param dtokp Pointer to data token which contains the details of the enqueue operation. + * \param transient Flag indicating transient persistence (ie, ignored on recover). + * + * \exception TODO + */ + iores enqueue_data_record(const void* const data_buff, + const std::size_t tot_data_len, + const std::size_t this_data_len, + data_tok* dtokp, + const bool transient = false); + + iores enqueue_extern_data_record(const std::size_t tot_data_len, + data_tok* dtokp, + const bool transient = false); + + /** + * \brief Enqueue data. + * + * \param data_buff Pointer to data to be enqueued for this enqueue operation. + * \param tot_data_len Total data length. + * \param this_data_len Amount to be written in this enqueue operation. + * \param dtokp Pointer to data token which contains the details of the enqueue operation. + * \param xid String containing xid. An empty string (i.e. length=0) will be considered + * non-transactional. + * \param transient Flag indicating transient persistence (ie, ignored on recover). + * + * \exception TODO + */ + iores enqueue_txn_data_record(const void* const data_buff, + const std::size_t tot_data_len, + const std::size_t this_data_len, + data_tok* dtokp, + const std::string& xid, + const bool transient = false); + + iores enqueue_extern_txn_data_record(const std::size_t tot_data_len, + data_tok* dtokp, + const std::string& xid, + const bool transient = false); + + /** + * \brief Reads data from the journal. It is the responsibility of the reader to free + * the memory that is allocated through this call - see below for details. + * + * Reads the next non-dequeued data record from the journal. + * + * <b>Note</b> that this call allocates memory into which the data and XID are copied. It + * is the responsibility of the caller to free this memory. The memory for the data and + * XID are allocated in a single call, and the XID precedes the data in the memory space. + * Thus, where an XID exists, freeing the XID pointer will free both the XID and data memory. + * However, if an XID does not exist for the message, the XID pointer xidpp is set to NULL, + * and it is the data pointer datapp that must be freed. Should neither an XID nor data be + * present (ie an empty record), then no memory is allocated, and both pointers will be NULL. + * In this case, there is no need to free memory. + * + * TODO: Fix this lousy interface. The caller should NOT be required to clean up these + * pointers! Rather use a struct, or better still, let the data token carry the data and + * xid pointers and lengths, and have the data token both allocate and delete. + * + * \param datapp Pointer to pointer that will be set to point to memory allocated and + * containing the data. Will be set to NULL if the call fails or there is no data + * in the record. + * \param dsize Ref that will be set to the size of the data. Will be set to 0 if the call + * fails or if there is no data in the record. + * \param xidpp Pointer to pointer that will be set to point to memory allocated and + * containing the XID. Will be set to NULL if the call fails or there is no XID attached + * to this record. + * \param xidsize Ref that will be set to the size of the XID. + * \param transient Ref that will be set true if record is transient. + * \param external Ref that will be set true if record is external. In this case, the data + * pointer datapp will be set to NULL, but dsize will contain the size of the data. + * NOTE: If there is an xid, then xidpp must be freed. + * \param dtokp Pointer to data_tok instance for this data, used to track state of data + * through journal. + * \param ignore_pending_txns When false (default), if the next record to be read is locked + * by a pending transaction, the read fails with RHM_IORES_TXPENDING. However, if set + * to true, then locks are ignored. This is required for reading of the Transaction + * Prepared List (TPL) which may have its entries locked, but may be read from + * time-to-time, and needs all its records (locked and unlocked) to be available. + * + * \exception TODO + */ + iores read_data_record(void** const datapp, + std::size_t& dsize, + void** const xidpp, + std::size_t& xidsize, + bool& transient, + bool& external, + data_tok* const dtokp, + bool ignore_pending_txns = false); + + /** + * \brief Dequeues (marks as no longer needed) data record in journal. + * + * Dequeues (marks as no longer needed) data record in journal. Note that it is possible + * to use the same data token instance used to enqueue this data; it contains the record ID + * needed to correctly mark this data as dequeued in the journal. Otherwise the RID of the + * record to be dequeued and the write state of ENQ must be manually set in a new or reset + * instance of data_tok. + * + * \param dtokp Pointer to data_tok instance for this data, used to track state of data + * through journal. + * \param txn_coml_commit Only used for preparedXID journal. When used for dequeueing + * prepared XID list items, sets whether the complete() was called in commit or abort + * mode. + * + * \exception TODO + */ + iores dequeue_data_record(data_tok* const dtokp, + const bool txn_coml_commit = false); + + /** + * \brief Dequeues (marks as no longer needed) data record in journal. + * + * Dequeues (marks as no longer needed) data record in journal as part of a transaction. + * Note that it is possible to use the same data token instance used to enqueue this data; + * it contains the RID needed to correctly mark this data as dequeued in the journal. + * Otherwise the RID of the record to be dequeued and the write state of ENQ must be + * manually set in a new or reset instance of data_tok. + * + * \param dtokp Pointer to data_tok instance for this data, used to track state of data + * through journal. + * \param xid String containing xid. An empty string (i.e. length=0) will be considered + * non-transactional. + * \param txn_coml_commit Only used for preparedXID journal. When used for dequeueing + * prepared XID list items, sets whether the complete() was called in commit or abort + * mode. + * + * \exception TODO + */ + iores dequeue_txn_data_record(data_tok* const dtokp, + const std::string& xid, + const bool txn_coml_commit = false); + + /** + * \brief Abort the transaction for all records enqueued or dequeued with the matching xid. + * + * Abort the transaction for all records enqueued with the matching xid. All enqueued records + * are effectively deleted from the journal, and can not be read. All dequeued records remain + * as though they had never been dequeued. + * + * \param dtokp Pointer to data_tok instance for this data, used to track state of data + * through journal. + * \param xid String containing xid. + * + * \exception TODO + */ + iores txn_abort(data_tok* const dtokp, + const std::string& xid); + + /** + * \brief Commit the transaction for all records enqueued or dequeued with the matching xid. + * + * Commit the transaction for all records enqueued with the matching xid. All enqueued + * records are effectively released for reading and dequeueing. All dequeued records are + * removed and can no longer be accessed. + * + * \param dtokp Pointer to data_tok instance for this data, used to track state of data + * through journal. + * \param xid String containing xid. + * + * \exception TODO + */ + iores txn_commit(data_tok* const dtokp, + const std::string& xid); + + /** + * \brief Check whether all the enqueue records for the given xid have reached disk. + * + * \param xid String containing xid. + * + * \exception TODO + */ + bool is_txn_synced(const std::string& xid); + + /** + * \brief Forces a check for returned AIO write events. + * + * Forces a check for returned AIO write events. This is normally performed by enqueue() and + * dequeue() operations, but if these operations cease, then this call needs to be made to + * force the processing of any outstanding AIO operations. + */ + int32_t get_wr_events(timespec* const timeout); + + /** + * \brief Stop the journal from accepting any further requests to read or write data. + * + * This operation is used to stop the journal. This is the normal mechanism for bringing the + * journal to an orderly stop. Any outstanding AIO operations or partially written pages in + * the write page cache will by flushed and will complete. + * + * <b>Note:</b> The journal cannot be restarted without either initializing it or restoring + * it. + * + * \param block_till_aio_cmpl If true, will block the thread while waiting for all + * outstanding AIO operations to complete. + */ + void stop(const bool block_till_aio_cmpl = false); + + /** + * \brief Force a flush of the write page cache, creating a single AIO write operation. + */ + iores flush(const bool block_till_aio_cmpl = false); + + inline uint32_t get_enq_cnt() const { return _emap.size(); } // TODO: _emap: Thread safe? + + inline uint32_t get_wr_aio_evt_rem() const { slock l(_wr_mutex); return _wmgr.get_aio_evt_rem(); } + + uint32_t get_wr_outstanding_aio_dblks() const; + + uint32_t get_rd_outstanding_aio_dblks() const; + + LinearFileController& getLinearFileControllerRef(); + + /** + * \brief Check if a particular rid is enqueued. Note that this function will return + * false if the rid is transactionally enqueued and is not committed, or if it is + * locked (i.e. transactionally dequeued, but the dequeue has not been committed). + */ + inline bool is_enqueued(const uint64_t rid, bool ignore_lock = false) { return _emap.is_enqueued(rid, ignore_lock); } + + inline bool is_locked(const uint64_t rid) { + if (_emap.is_enqueued(rid, true) < enq_map::EMAP_OK) + return false; + return _emap.is_locked(rid) == enq_map::EMAP_TRUE; + } + + inline void enq_rid_list(std::vector<uint64_t>& rids) { _emap.rid_list(rids); } + + inline void enq_xid_list(std::vector<std::string>& xids) { _tmap.xid_list(xids); } + + inline uint32_t get_open_txn_cnt() const { return _tmap.size(); } + + // TODO Make this a const, but txn_map must support const first. + inline txn_map& get_txn_map() { return _tmap; } + + /** + * \brief Check if the journal is stopped. + * + * \return <b><i>true</i></b> if the jouranl is stopped; + * <b><i>false</i></b> otherwise. + */ + inline bool is_stopped() { return _stop_flag; } + + /** + * \brief Check if the journal is ready to read and write data. + * + * Checks if the journal is ready to read and write data. This function will return + * <b><i>true</i></b> if the journal has been either initialized or restored, and the stop() + * function has not been called since the initialization. + * + * Note that the journal may also be stopped if an internal error occurs (such as running out + * of data journal file space). + * + * \return <b><i>true</i></b> if the journal is ready to read and write data; + * <b><i>false</i></b> otherwise. + */ + inline bool is_ready() const { return _init_flag && !_stop_flag; } + + inline bool is_read_only() const { return _readonly_flag; } + + /** + * \brief Get the journal directory. + * + * This returns the journal directory as set during initialization. This is the directory + * into which the journal files will be written. + */ + inline const std::string& dirname() const { return _jdir.dirname(); } + + // Management instrumentation callbacks + inline virtual void instr_incr_outstanding_aio_cnt() {} + inline virtual void instr_decr_outstanding_aio_cnt() {} + + protected: + static bool _init; + static bool init_statics(); + + /** + * \brief Check status of journal before allowing write operations. + */ + void check_wstatus(const char* fn_name) const; + + /** + * \brief Check status of journal before allowing read operations. + */ + void check_rstatus(const char* fn_name) const; + + /** + * \brief Call that blocks while waiting for all outstanding AIOs to complete + */ + void aio_cmpl_wait(); + + /** + * \brief Call that blocks until at least one message returns; used to wait for + * AIO wait conditions to clear. + */ + bool handle_aio_wait(const iores res, iores& resout, const data_tok* dtp); + }; + +}} + +#endif // ifndef QPID_LINEARSTORE_JRNL_JCNTL_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/jdir.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/jdir.cpp new file mode 100644 index 0000000000..977c75e37a --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/jdir.cpp @@ -0,0 +1,479 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/jrnl/jdir.h" + +#include <cstdlib> +#include <cstring> +#include <cerrno> +#include <iomanip> +#include "qpid/linearstore/jrnl/jcfg.h" +#include "qpid/linearstore/jrnl/jerrno.h" +#include "qpid/linearstore/jrnl/jexception.h" +#include <sstream> +#include <sys/stat.h> +#include <unistd.h> + +namespace qpid +{ +namespace qls_jrnl +{ + +jdir::jdir(const std::string& dirname/*, const std::string& _base_filename*/): + _dirname(dirname)/*, + _base_filename(_base_filename)*/ +{} + +jdir::~jdir() +{} + +// === create_dir === + +void +jdir::create_dir() +{ + create_dir(_dirname); +} + + +void +jdir::create_dir(const char* dirname) +{ + create_dir(std::string(dirname)); +} + + +void +jdir::create_dir(const std::string& dirname) +{ + std::size_t fdp = dirname.find_last_of('/'); + if (fdp != std::string::npos) + { + std::string parent_dir = dirname.substr(0, fdp); + if (!exists(parent_dir)) + create_dir(parent_dir); + } + if (::mkdir(dirname.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)) + { + if (errno != EEXIST) // Dir exists, ignore + { + std::ostringstream oss; + oss << "dir=\"" << dirname << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_MKDIR, oss.str(), "jdir", "create_dir"); + } + } +} + + +// === clear_dir === + +void +jdir::clear_dir(const bool create_flag) +{ + clear_dir(_dirname/*, _base_filename*/, create_flag); +} + +void +jdir::clear_dir(const char* dirname/*, const char* base_filename*/, const bool create_flag) +{ + clear_dir(std::string(dirname)/*, std::string(base_filename)*/, create_flag); +} + + +void +jdir::clear_dir(const std::string& dirname/*, const std::string& +#ifndef RHM_JOWRITE + base_filename +#endif +*/ + , const bool create_flag) +{ + DIR* dir = ::opendir(dirname.c_str()); + if (!dir) + { + if (errno == 2 && create_flag) // ENOENT (No such file or dir) + { + create_dir(dirname); + return; + } + std::ostringstream oss; + oss << "dir=\"" << dirname << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_OPENDIR, oss.str(), "jdir", "clear_dir"); + } +//#ifndef RHM_JOWRITE + struct dirent* entry; + bool found = false; + std::string bak_dir; + while ((entry = ::readdir(dir)) != 0) + { + // Ignore . and .. + if (std::strcmp(entry->d_name, ".") != 0 && std::strcmp(entry->d_name, "..") != 0) + { + if (std::strlen(entry->d_name) >= 3) // 'bak' + { + if (std::strncmp(entry->d_name, "bak", 3) == 0) + { + if (!found) + { + bak_dir = create_bak_dir(dirname/*, base_filename*/); + found = true; + } + std::ostringstream oldname; + oldname << dirname << "/" << entry->d_name; + std::ostringstream newname; + newname << bak_dir << "/" << entry->d_name; + if (::rename(oldname.str().c_str(), newname.str().c_str())) + { + ::closedir(dir); + std::ostringstream oss; + oss << "file=\"" << oldname.str() << "\" dest=\"" << + newname.str() << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_FMOVE, oss.str(), "jdir", "clear_dir"); + } + } + } + } + } +// FIXME: Find out why this fails with false alarms/errors from time to time... +// While commented out, there is no error capture from reading dir entries. +// check_err(errno, dir, dirname, "clear_dir"); +//#endif + close_dir(dir, dirname, "clear_dir"); +} + +// === push_down === + +std::string +jdir::push_down(const std::string& dirname, const std::string& target_dir/*, const std::string& bak_dir_base*/) +{ + std::string bak_dir_name = create_bak_dir(dirname/*, bak_dir_base*/); + + DIR* dir = ::opendir(dirname.c_str()); + if (!dir) + { + std::ostringstream oss; + oss << "dir=\"" << dirname << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_OPENDIR, oss.str(), "jdir", "push_down"); + } + // Copy contents of targetDirName into bak dir + struct dirent* entry; + while ((entry = ::readdir(dir)) != 0) + { + // Search for targetDirName in storeDirName + if (std::strcmp(entry->d_name, target_dir.c_str()) == 0) + { + std::ostringstream oldname; + oldname << dirname << "/" << target_dir; + std::ostringstream newname; + newname << bak_dir_name << "/" << target_dir; + if (::rename(oldname.str().c_str(), newname.str().c_str())) + { + ::closedir(dir); + std::ostringstream oss; + oss << "file=\"" << oldname.str() << "\" dest=\"" << newname.str() << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_FMOVE, oss.str(), "jdir", "push_down"); + } + break; + } + } + close_dir(dir, dirname, "push_down"); + return bak_dir_name; +} + +// === verify_dir === + +void +jdir::verify_dir() +{ + verify_dir(_dirname/*, _base_filename*/); +} + +void +jdir::verify_dir(const char* dirname/*, const char* base_filename*/) +{ + verify_dir(std::string(dirname)/*, std::string(base_filename)*/); +} + + +void +jdir::verify_dir(const std::string& dirname/*, const std::string& base_filename*/) +{ + if (!is_dir(dirname)) + { + std::ostringstream oss; + oss << "dir=\"" << dirname << "\""; + throw jexception(jerrno::JERR_JDIR_NOTDIR, oss.str(), "jdir", "verify_dir"); + } + + // Read jinf file, then verify all journal files are present +// jinf ji(dirname + "/" + base_filename + "." + QLS_JRNL_FILE_EXTENSION, true); +// for (uint16_t fnum=0; fnum < ji.num_jfiles(); fnum++) +// { +// std::ostringstream oss; +// oss << dirname << "/" << base_filename << "."; +// oss << std::setw(4) << std::setfill('0') << std::hex << fnum; +// oss << "." << QLS_JRNL_FILE_EXTENSION; +// if (!exists(oss.str())) +// throw jexception(jerrno::JERR_JDIR_NOSUCHFILE, oss.str(), "jdir", "verify_dir"); +// } +} + + +// === delete_dir === + +void +jdir::delete_dir(bool children_only) +{ + delete_dir(_dirname, children_only); +} + +void +jdir::delete_dir(const char* dirname, bool children_only) +{ + delete_dir(std::string(dirname), children_only); +} + +void +jdir::delete_dir(const std::string& dirname, bool children_only) +{ + struct dirent* entry; + struct stat s; + DIR* dir = ::opendir(dirname.c_str()); + if (!dir) + { + if (errno == ENOENT) // dir does not exist. + return; + + std::ostringstream oss; + oss << "dir=\"" << dirname << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_OPENDIR, oss.str(), "jdir", "delete_dir"); + } + else + { + while ((entry = ::readdir(dir)) != 0) + { + // Ignore . and .. + if (std::strcmp(entry->d_name, ".") != 0 && std::strcmp(entry->d_name, "..") != 0) + { + std::string full_name(dirname + "/" + entry->d_name); + if (::lstat(full_name.c_str(), &s)) + { + ::closedir(dir); + std::ostringstream oss; + oss << "stat: file=\"" << full_name << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_STAT, oss.str(), "jdir", "delete_dir"); + } + if (S_ISREG(s.st_mode) || S_ISLNK(s.st_mode)) // This is a file or slink + { + if(::unlink(full_name.c_str())) + { + ::closedir(dir); + std::ostringstream oss; + oss << "unlink: file=\"" << entry->d_name << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_UNLINK, oss.str(), "jdir", "delete_dir"); + } + } + else if (S_ISDIR(s.st_mode)) // This is a dir + { + delete_dir(full_name); + } + else // all other types, throw up! + { + ::closedir(dir); + std::ostringstream oss; + oss << "file=\"" << entry->d_name << "\" is not a dir, file or slink."; + oss << " (mode=0x" << std::hex << s.st_mode << std::dec << ")"; + throw jexception(jerrno::JERR_JDIR_BADFTYPE, oss.str(), "jdir", "delete_dir"); + } + } + } + +// FIXME: Find out why this fails with false alarms/errors from time to time... +// While commented out, there is no error capture from reading dir entries. +// check_err(errno, dir, dirname, "delete_dir"); + } + // Now dir is empty, close and delete it + close_dir(dir, dirname, "delete_dir"); + + if (!children_only) + if (::rmdir(dirname.c_str())) + { + std::ostringstream oss; + oss << "dir=\"" << dirname << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_RMDIR, oss.str(), "jdir", "delete_dir"); + } +} + + +std::string +jdir::create_bak_dir(const std::string& dirname) +{ + DIR* dir = ::opendir(dirname.c_str()); + long dir_num = 0L; + if (!dir) + { + std::ostringstream oss; + oss << "dir=\"" << dirname << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_OPENDIR, oss.str(), "jdir", "create_bak_dir"); + } + struct dirent* entry; + while ((entry = ::readdir(dir)) != 0) + { + // Ignore . and .. + if (std::strcmp(entry->d_name, ".") != 0 && std::strcmp(entry->d_name, "..") != 0) + { + if (std::strlen(entry->d_name) == 9) // Format: _bak.XXXX + { + if (std::strncmp(entry->d_name, "_bak.", 5) == 0) + { + long this_dir_num = std::strtol(entry->d_name + 5, 0, 16); + if (this_dir_num > dir_num) + dir_num = this_dir_num; + } + } + } + } +// FIXME: Find out why this fails with false alarms/errors from time to time... +// While commented out, there is no error capture from reading dir entries. +// check_err(errno, dir, dirname, "create_bak_dir"); + close_dir(dir, dirname, "create_bak_dir"); + + std::ostringstream dn; + dn << dirname << "/_bak." << std::hex << std::setw(4) << std::setfill('0') << ++dir_num; + if (::mkdir(dn.str().c_str(), S_IRWXU | S_IRWXG | S_IROTH)) + { + std::ostringstream oss; + oss << "dir=\"" << dn.str() << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_MKDIR, oss.str(), "jdir", "create_bak_dir"); + } + return std::string(dn.str()); +} + +bool +jdir::is_dir(const char* name) +{ + struct stat s; + if (::stat(name, &s)) + { + std::ostringstream oss; + oss << "file=\"" << name << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_STAT, oss.str(), "jdir", "is_dir"); + } + return S_ISDIR(s.st_mode); +} + +bool +jdir::is_dir(const std::string& name) +{ + return is_dir(name.c_str()); +} + +bool +jdir::exists(const char* name) +{ + struct stat s; + if (::stat(name, &s)) + { + if (errno == ENOENT) // No such dir or file + return false; + // Throw for any other condition + std::ostringstream oss; + oss << "file=\"" << name << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_STAT, oss.str(), "jdir", "exists"); + } + return true; +} + +bool +jdir::exists(const std::string& name) +{ + return exists(name.c_str()); +} + +void +jdir::read_dir(const std::string& name, std::vector<std::string>& dir_list, const bool incl_dirs, const bool incl_files, const bool incl_links, const bool return_fqfn) { + struct stat s; + if (is_dir(name)) { + DIR* dir = ::opendir(name.c_str()); + if (dir != 0) { + struct dirent* entry; + while ((entry = ::readdir(dir)) != 0) { + if (std::strcmp(entry->d_name, ".") != 0 && std::strcmp(entry->d_name, "..") != 0) { // Ignore . and .. + std::string full_name(name + "/" + entry->d_name); + if (::stat(full_name.c_str(), &s)) + { + ::closedir(dir); + std::ostringstream oss; + oss << "stat: file=\"" << full_name << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_STAT, oss.str(), "jdir", "delete_dir"); + } + if ((S_ISREG(s.st_mode) && incl_files) || (S_ISDIR(s.st_mode) && incl_dirs) || (S_ISLNK(s.st_mode) && incl_links)) { + if (return_fqfn) { + dir_list.push_back(name + "/" + entry->d_name); + } else { + dir_list.push_back(entry->d_name); + } + } + } + } + } + close_dir(dir, name, "read_dir"); + } +} + +void +jdir::check_err(const int err_num, DIR* dir, const std::string& dir_name, const std::string& fn_name) +{ + if (err_num) + { + std::ostringstream oss; + oss << "dir=\"" << dir_name << "\"" << FORMAT_SYSERR(err_num); + ::closedir(dir); // Try to close, it makes no sense to trap errors here... + throw jexception(jerrno::JERR_JDIR_READDIR, oss.str(), "jdir", fn_name); + } +} + +void +jdir::close_dir(DIR* dir, const std::string& dir_name, const std::string& fn_name) +{ + if (::closedir(dir)) + { + std::ostringstream oss; + oss << "dir=\"" << dir_name << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_CLOSEDIR, oss.str(), "jdir", fn_name); + } +} + +std::ostream& +operator<<(std::ostream& os, const jdir& jdir) +{ + os << jdir._dirname; + return os; +} + +std::ostream& +operator<<(std::ostream& os, const jdir* jdirPtr) +{ + os << jdirPtr->_dirname; + return os; +} + +}} diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/jdir.h b/qpid/cpp/src/qpid/linearstore/jrnl/jdir.h new file mode 100644 index 0000000000..01a08c57fc --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/jdir.h @@ -0,0 +1,369 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LINEARSTORE_JRNL_JDIR_H +#define QPID_LINEARSTORE_JRNL_JDIR_H + +namespace qpid +{ +namespace qls_jrnl +{ +class jdir; +}} + +//#include "qpid/linearstore/jrnl/jinf.h" +#include <dirent.h> +#include <string> +#include <vector> + +namespace qpid +{ +namespace qls_jrnl +{ + + /** + * \class jdir + * \brief Class to manage the %journal directory + */ + class jdir + { + private: + std::string _dirname; + //std::string _base_filename; + + public: + + /** + * \brief Sole constructor + * + * \param dirname Name of directory to be managed. + * \param base_filename Filename root used in the creation of %journal files + * and sub-directories. + */ + jdir(const std::string& dirname/*, const std::string& base_filename*/); + + virtual ~jdir(); + + + /** + * \brief Create %journal directory as set in the dirname parameter of the constructor. + * Recursive creation is supported. + * + * \exception jerrno::JERR_JDIR_MKDIR The creation of dirname failed. + */ + void create_dir(); + + /** + * \brief Static function to create a directory. Recursive creation is supported. + * + * \param dirname C-string containing name of directory. + * + * \exception jerrno::JERR_JDIR_MKDIR The creation of dirname failed. + */ + static void create_dir(const char* dirname); + + /** + * \brief Static function to create a directory. Recursive creation is supported. + * + * \param dirname String containing name of directory. + * + * \exception jerrno::JERR_JDIR_MKDIR The creation of dirname failed. + */ + static void create_dir(const std::string& dirname); + + + /** + * \brief Clear the %journal directory of files matching the base filename + * by moving them into a subdirectory. This fn uses the dirname and base_filename + * that were set on construction. + * + * \param create_flag If set, create dirname if it is non-existent, otherwise throw + * exception. + * + * \exception jerrno::JERR_JDIR_OPENDIR The %journal directory could not be opened. + * \exception jerrno::JERR_JDIR_FMOVE Moving the files from the %journal directory to the created backup + * directory failed. + * \exception jerrno::JERR_JDIR_CLOSEDIR The directory handle could not be closed. + */ + void clear_dir(const bool create_flag = true); + + /** + * \brief Clear the directory dirname of %journal files matching base_filename + * by moving them into a subdirectory. + * + * \param dirname C-string containing name of %journal directory. + * \param base_filename C-string containing base filename of %journal files to be matched + * for moving into subdirectory. + * \param create_flag If set, create dirname if it is non-existent, otherwise throw + * exception + * + * \exception jerrno::JERR_JDIR_OPENDIR The %journal directory could not be opened. + * \exception jerrno::JERR_JDIR_FMOVE Moving the files from the %journal directory to the created backup + * directory failed. + * \exception jerrno::JERR_JDIR_CLOSEDIR The directory handle could not be closed. + */ + static void clear_dir(const char* dirname/*, const char* base_filename*/, + const bool create_flag = true); + + /** + * \brief Clear the directory dirname of %journal files matching base_filename + * by moving them into a subdirectory. + * + * \param dirname String containing name of %journal directory. + * \param base_filename String containing base filename of %journal files to be matched + * for moving into subdirectory. + * \param create_flag If set, create dirname if it is non-existent, otherwise throw + * exception + * + * \exception jerrno::JERR_JDIR_OPENDIR The %journal directory could not be opened. + * \exception jerrno::JERR_JDIR_FMOVE Moving the files from the %journal directory to the created backup + * directory failed. + * \exception jerrno::JERR_JDIR_CLOSEDIR The directory handle could not be closed. + */ + static void clear_dir(const std::string& dirname/*, const std::string& base_filename*/, + const bool create_flag = true); + + + + /** + * \brief Move (push down) the directory target_dir located in directory dirname into a backup directory + * named _bak_dir_base.XXXX (note prepended underscore), where XXXX is an increasing hex serial number + * starting at 0000. + * + * \param dirname Full path to directory containing directory to be pushed down. + * \param target_dir Name of directory in dirname to be pushed down. + * \param bak_dir_base Base name for backup directory to be created in dirname, into which target_dir will be moved. + * \return Name of backup dir into which target_dir was pushed. + */ + static std::string push_down(const std::string& dirname, const std::string& target_dir/*, const std::string& bak_dir_base*/); + + + /** + * \brief Verify that dirname is a valid %journal directory. + * + * The validation reads the .%jinf file, and using this information verifies that all the expected %journal + * (.jdat) files are present. + * + * \exception jerrno::JERR_JDIR_NOTDIR dirname is not a directory + * \exception jerrno::JERR_JDIR_STAT Could not stat dirname + * \exception jerrno::JERR__FILEIO Error reading %jinf file + * \exception jerrno::JERR_JINF_CVALIDFAIL Error validating %jinf file + * \exception jerrno::JERR_JDIR_NOSUCHFILE Expected jdat file is missing + */ + void verify_dir(); + + /** + * \brief Verify that dirname is a valid %journal directory. + * + * The validation reads the .%jinf file, and using this information verifies that all the expected %journal + * (.jdat) files are present. + * + * \param dirname C-string containing name of %journal directory. + * \param base_filename C-string containing base filename of %journal files to be matched for moving into sub-directory. + * + * \exception jerrno::JERR_JDIR_NOTDIR dirname is not a directory + * \exception jerrno::JERR_JDIR_STAT Could not stat dirname + * \exception jerrno::JERR__FILEIO Error reading %jinf file + * \exception jerrno::JERR_JINF_CVALIDFAIL Error validating %jinf file + * \exception jerrno::JERR_JDIR_NOSUCHFILE Expected jdat file is missing + */ + static void verify_dir(const char* dirname/*, const char* base_filename*/); + + /** + * \brief Verify that dirname is a valid %journal directory. + * + * The validation reads the .%jinf file, and using this information verifies that all the expected %journal + * (.jdat) files are present. + * + * \param dirname String containing name of %journal directory. + * \param base_filename String containing base filename of %journal files to be matched for moving into sub-directory. + * + * \exception jerrno::JERR_JDIR_NOTDIR dirname is not a directory + * \exception jerrno::JERR_JDIR_STAT Could not stat dirname + * \exception jerrno::JERR__FILEIO Error reading %jinf file + * \exception jerrno::JERR_JINF_CVALIDFAIL Error validating %jinf file + * \exception jerrno::JERR_JDIR_NOSUCHFILE Expected jdat file is missing + */ + static void verify_dir(const std::string& dirname/*, const std::string& base_filename*/); + + /** + * \brief Delete the %journal directory and all files and sub--directories that it may + * contain. This is equivilent of rm -rf. + * + * FIXME: links are not handled correctly. + * + * \param children_only If true, delete only children of dirname, but leave dirname itself. + * + * \exception jerrno::JERR_JDIR_OPENDIR The %journal directory could not be opened. + * \exception jerrno::JERR_JDIR_STAT Could not stat dirname. + * \exception jerrno::JERR_JDIR_UNLINK A file could not be deleted. + * \exception jerrno::JERR_JDIR_BADFTYPE A dir entry is neiter a file nor a dir. + * \exception jerrno::JERR_JDIR_CLOSEDIR The directory handle could not be closed. + * \exception jerrno::JERR_JDIR_RMDIR A directory could not be deleted. + */ + void delete_dir(bool children_only = false ); + + /** + * \brief Delete the %journal directory and all files and sub--directories that it may + * contain. This is equivilent of rm -rf. + * + * FIXME: links are not handled correctly. + * + * \param dirname C-string containing name of directory to be deleted. + * \param children_only If true, delete only children of dirname, but leave dirname itself. + * + * \exception jerrno::JERR_JDIR_OPENDIR The %journal directory could not be opened. + * \exception jerrno::JERR_JDIR_STAT Could not stat dirname. + * \exception jerrno::JERR_JDIR_UNLINK A file could not be deleted. + * \exception jerrno::JERR_JDIR_BADFTYPE A dir entry is neiter a file nor a dir. + * \exception jerrno::JERR_JDIR_CLOSEDIR The directory handle could not be closed. + * \exception jerrno::JERR_JDIR_RMDIR A directory could not be deleted. + */ + static void delete_dir(const char* dirname, bool children_only = false); + + /** + * \brief Delete the %journal directory and all files and sub--directories that it may + * contain. This is equivilent of rm -rf. + * + * FIXME: links are not handled correctly. + * + * \param dirname String containing name of directory to be deleted. + * \param children_only If true, delete only children of dirname, but leave dirname itself. + * + * \exception jerrno::JERR_JDIR_OPENDIR The %journal directory could not be opened. + * \exception jerrno::JERR_JDIR_STAT Could not stat dirname. + * \exception jerrno::JERR_JDIR_UNLINK A file could not be deleted. + * \exception jerrno::JERR_JDIR_BADFTYPE A dir entry is neiter a file nor a dir. + * \exception jerrno::JERR_JDIR_CLOSEDIR The directory handle could not be closed. + * \exception jerrno::JERR_JDIR_RMDIR A directory could not be deleted. + */ + static void delete_dir(const std::string& dirname, bool children_only = false); + + /** + * \brief Create bakup directory that is next in sequence and move all %journal files + * matching base_filename into it. + * + * In directory dirname, search for existing backup directory using pattern + * "_basename.bak.XXXX" where XXXX is a hexadecimal sequence, and create next directory + * based on highest number found. Move all %journal files which match the base_fileaname + * parameter into this new backup directory. + * + * \param dirname String containing name of %journal directory. + * \param base_filename String containing base filename of %journal files to be matched + * for moving into subdirectory. + * + * \exception jerrno::JERR_JDIR_OPENDIR The %journal directory could not be opened. + * \exception jerrno::JERR_JDIR_CLOSEDIR The directory handle could not be closed. + * \exception jerrno::JERR_JDIR_MKDIR The backup directory could not be deleted. + */ + static std::string create_bak_dir(const std::string& dirname/*, + const std::string& base_filename*/); + + /** + * \brief Return the directory name as a string. + */ + inline const std::string& dirname() const { return _dirname; } + + /** + * \brief Return the %journal base filename name as a string. + */ +// inline const std::string& base_filename() const { return _base_filename; } + + /** + * \brief Test whether the named file is a directory. + * + * \param name Name of file to be tested. + * \return <b><i>true</i></b> if the named file is a directory; <b><i>false</i></b> + * otherwise. + * \exception jerrno::JERR_JDIR_STAT Could not stat name. + */ + static bool is_dir(const char* name); + + /** + * \brief Test whether the named file is a directory. + * + * \param name Name of file to be tested. + * \return <b><i>true</i></b> if the named file is a directory; <b><i>false</i></b> + * otherwise. + * \exception jerrno::JERR_JDIR_STAT Could not stat name. + */ + static bool is_dir(const std::string& name); + + + /** + * \brief Test whether the named entity exists on the filesystem. + * + * If stat() fails with error ENOENT, then this will return <b><i>false</i></b>. If + * stat() succeeds, then <b><i>true</i></b> is returned, irrespective of the file type. + * If stat() fails with any other error, an exception is thrown. + * + * \param name Name of entity to be tested. + * \return <b><i>true</i></b> if the named entity exists; <b><i>false</i></b> + * otherwise. + * \exception jerrno::JERR_JDIR_STAT Could not stat name. + */ + static bool exists(const char* name); + + /** + * \brief Test whether the named entity exists on the filesystem. + * + * If stat() fails with error ENOENT, then this will return <b><i>false</i></b>. If + * stat() succeeds, then <b><i>true</i></b> is returned, irrespective of the file type. + * If stat() fails with any other error, an exception is thrown. + * + * \param name Name of entity to be tested. + * \return <b><i>true</i></b> if the named entity exists; <b><i>false</i></b> + * otherwise. + * \exception jerrno::JERR_JDIR_STAT Could not stat name. + */ + static bool exists(const std::string& name); + + static void read_dir(const std::string& name, std::vector<std::string>& dir_list, const bool incl_dirs, const bool incl_files, const bool incl_links, const bool return_fqfn); + + /** + * \brief Stream operator + */ + friend std::ostream& operator<<(std::ostream& os, const jdir& jdir); + + /** + * \brief Stream operator + */ + friend std::ostream& operator<<(std::ostream& os, const jdir* jdirPtr); + + private: + /** + * \brief Check for error, if non-zero close DIR handle and throw JERR_JDIR_READDIR + * + * \exception jerrno::JERR_JDIR_READDIR Error while reading contents of dir. + */ + static void check_err(const int err_num, DIR* dir, const std::string& dir_name, const std::string& fn_name); + + /** + * \brief Close a DIR handle, throw JERR_JDIR_CLOSEDIR if error occurs during close + * + * \exception jerrno::JERR_JDIR_CLOSEDIR The directory handle could not be closed. + */ + static void close_dir(DIR* dir, const std::string& dir_name, const std::string& fn_name); + }; + +}} + +#endif // ifndef QPID_LINEARSTORE_JRNL_JDIR_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/jerrno.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/jerrno.cpp new file mode 100644 index 0000000000..7c70be1ed7 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/jerrno.cpp @@ -0,0 +1,227 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/jrnl/jerrno.h" + +namespace qpid +{ +namespace qls_jrnl +{ + +std::map<uint32_t, const char*> jerrno::_err_map; +std::map<uint32_t, const char*>::iterator jerrno::_err_map_itr; +bool jerrno::_initialized = jerrno::__init(); + +// generic errors +const uint32_t jerrno::JERR__MALLOC = 0x0100; +const uint32_t jerrno::JERR__UNDERFLOW = 0x0101; +const uint32_t jerrno::JERR__NINIT = 0x0102; +const uint32_t jerrno::JERR__AIO = 0x0103; +const uint32_t jerrno::JERR__FILEIO = 0x0104; +const uint32_t jerrno::JERR__RTCLOCK = 0x0105; +const uint32_t jerrno::JERR__PTHREAD = 0x0106; +const uint32_t jerrno::JERR__TIMEOUT = 0x0107; +const uint32_t jerrno::JERR__UNEXPRESPONSE = 0x0108; +const uint32_t jerrno::JERR__RECNFOUND = 0x0109; +const uint32_t jerrno::JERR__NOTIMPL = 0x010a; +const uint32_t jerrno::JERR__NULL = 0x010b; + +// class jcntl +const uint32_t jerrno::JERR_JCNTL_STOPPED = 0x0200; +const uint32_t jerrno::JERR_JCNTL_READONLY = 0x0201; +const uint32_t jerrno::JERR_JCNTL_AIOCMPLWAIT = 0x0202; +const uint32_t jerrno::JERR_JCNTL_UNKNOWNMAGIC = 0x0203; +const uint32_t jerrno::JERR_JCNTL_NOTRECOVERED = 0x0204; +const uint32_t jerrno::JERR_JCNTL_ENQSTATE = 0x0207; +const uint32_t jerrno::JERR_JCNTL_INVALIDENQHDR = 0x0208; + + +// class jdir +const uint32_t jerrno::JERR_JDIR_NOTDIR = 0x0300; +const uint32_t jerrno::JERR_JDIR_MKDIR = 0x0301; +const uint32_t jerrno::JERR_JDIR_OPENDIR = 0x0302; +const uint32_t jerrno::JERR_JDIR_READDIR = 0x0303; +const uint32_t jerrno::JERR_JDIR_CLOSEDIR = 0x0304; +const uint32_t jerrno::JERR_JDIR_RMDIR = 0x0305; +const uint32_t jerrno::JERR_JDIR_NOSUCHFILE = 0x0306; +const uint32_t jerrno::JERR_JDIR_FMOVE = 0x0307; +const uint32_t jerrno::JERR_JDIR_STAT = 0x0308; +const uint32_t jerrno::JERR_JDIR_UNLINK = 0x0309; +const uint32_t jerrno::JERR_JDIR_BADFTYPE = 0x030a; + +// class JournalFile +const uint32_t jerrno::JERR_JNLF_OPEN = 0x0400; +const uint32_t jerrno::JERR_JNLF_CLOSE = 0x0401; +const uint32_t jerrno::JERR_JNLF_FILEOFFSOVFL = 0x0402; +const uint32_t jerrno::JERR_JNLF_CMPLOFFSOVFL = 0x0403; + +// class LinearFileController +const uint32_t jerrno::JERR_LFCR_SEQNUMNOTFOUND = 0x0500; + +// class jrec, enq_rec, deq_rec, txn_rec +const uint32_t jerrno::JERR_JREC_BADRECHDR = 0x0700; +const uint32_t jerrno::JERR_JREC_BADRECTAIL = 0x0701; + +// class wmgr +const uint32_t jerrno::JERR_WMGR_BADPGSTATE = 0x0801; +const uint32_t jerrno::JERR_WMGR_BADDTOKSTATE = 0x0802; +const uint32_t jerrno::JERR_WMGR_ENQDISCONT = 0x0803; +const uint32_t jerrno::JERR_WMGR_DEQDISCONT = 0x0804; +const uint32_t jerrno::JERR_WMGR_DEQRIDNOTENQ = 0x0805; +const uint32_t jerrno::JERR_WMGR_BADFH = 0x0806; + +// class RecoveryManager +const uint32_t jerrno::JERR_RCVM_OPENRD = 0x0900; +const uint32_t jerrno::JERR_RCVM_READ = 0x0901; +const uint32_t jerrno::JERR_RCVM_WRITE = 0x0902; + +//// class rmgr +//const uint32_t jerrno::JERR_RMGR_UNKNOWNMAGIC = 0x0900; +//const uint32_t jerrno::JERR_RMGR_RIDMISMATCH = 0x0901; +////const uint32_t jerrno::JERR_RMGR_FIDMISMATCH = 0x0902; +//const uint32_t jerrno::JERR_RMGR_ENQSTATE = 0x0903; +//const uint32_t jerrno::JERR_RMGR_BADRECTYPE = 0x0904; + +// class data_tok +const uint32_t jerrno::JERR_DTOK_ILLEGALSTATE = 0x0a00; +// const uint32_t jerrno::JERR_DTOK_RIDNOTSET = 0x0a01; + +// class enq_map, txn_map +const uint32_t jerrno::JERR_MAP_DUPLICATE = 0x0b00; +const uint32_t jerrno::JERR_MAP_NOTFOUND = 0x0b01; +const uint32_t jerrno::JERR_MAP_LOCKED = 0x0b02; + +// EFP errors +const uint32_t jerrno::JERR_EFP_BADPARTITIONNAME = 0x0d01; +const uint32_t jerrno::JERR_EFP_BADPARTITIONDIR = 0x0d02; +const uint32_t jerrno::JERR_EFP_BADEFPDIRNAME = 0x0d03; +const uint32_t jerrno::JERR_EFP_NOEFP = 0x0d04; +const uint32_t jerrno::JERR_EFP_EMPTY = 0x0d05; + +// Negative returns for some functions +const int32_t jerrno::AIO_TIMEOUT = -1; +const int32_t jerrno::LOCK_TAKEN = -2; + + +// static initialization fn + +bool +jerrno::__init() +{ + // generic errors + _err_map[JERR__MALLOC] = "JERR__MALLOC: Buffer memory allocation failed."; + _err_map[JERR__UNDERFLOW] = "JERR__UNDERFLOW: Underflow error"; + _err_map[JERR__NINIT] = "JERR__NINIT: Operation on uninitialized class."; + _err_map[JERR__AIO] = "JERR__AIO: AIO error."; + _err_map[JERR__FILEIO] = "JERR__FILEIO: File read or write failure."; + _err_map[JERR__RTCLOCK] = "JERR__RTCLOCK: Reading real-time clock failed."; + _err_map[JERR__PTHREAD] = "JERR__PTHREAD: pthread failure."; + _err_map[JERR__TIMEOUT] = "JERR__TIMEOUT: Timeout waiting for event."; + _err_map[JERR__UNEXPRESPONSE] = "JERR__UNEXPRESPONSE: Unexpected response to call or event."; + _err_map[JERR__RECNFOUND] = "JERR__RECNFOUND: Record not found."; + _err_map[JERR__NOTIMPL] = "JERR__NOTIMPL: Not implemented"; + _err_map[JERR__NULL] = "JERR__NULL: Operation on null pointer"; + + // class jcntl + _err_map[JERR_JCNTL_STOPPED] = "JERR_JCNTL_STOPPED: Operation on stopped journal."; + _err_map[JERR_JCNTL_READONLY] = "JERR_JCNTL_READONLY: Write operation on read-only journal (during recovery)."; + _err_map[JERR_JCNTL_AIOCMPLWAIT] = "JERR_JCNTL_AIOCMPLWAIT: Timeout waiting for AIOs to complete."; + _err_map[JERR_JCNTL_UNKNOWNMAGIC] = "JERR_JCNTL_UNKNOWNMAGIC: Found record with unknown magic."; + _err_map[JERR_JCNTL_NOTRECOVERED] = "JERR_JCNTL_NOTRECOVERED: Operation requires recover() to be run first."; + _err_map[JERR_JCNTL_ENQSTATE] = "JERR_JCNTL_ENQSTATE: Read error: Record not in ENQ state"; + _err_map[JERR_JCNTL_INVALIDENQHDR] = "JERR_JCNTL_INVALIDENQHDR: Invalid ENQ header"; + + // class jdir + _err_map[JERR_JDIR_NOTDIR] = "JERR_JDIR_NOTDIR: Directory name exists but is not a directory."; + _err_map[JERR_JDIR_MKDIR] = "JERR_JDIR_MKDIR: Directory creation failed."; + _err_map[JERR_JDIR_OPENDIR] = "JERR_JDIR_OPENDIR: Directory open failed."; + _err_map[JERR_JDIR_READDIR] = "JERR_JDIR_READDIR: Directory read failed."; + _err_map[JERR_JDIR_CLOSEDIR] = "JERR_JDIR_CLOSEDIR: Directory close failed."; + _err_map[JERR_JDIR_RMDIR] = "JERR_JDIR_RMDIR: Directory delete failed."; + _err_map[JERR_JDIR_NOSUCHFILE] = "JERR_JDIR_NOSUCHFILE: File does not exist."; + _err_map[JERR_JDIR_FMOVE] = "JERR_JDIR_FMOVE: File move failed."; + _err_map[JERR_JDIR_STAT] = "JERR_JDIR_STAT: File stat failed."; + _err_map[JERR_JDIR_UNLINK] = "JERR_JDIR_UNLINK: File delete failed."; + _err_map[JERR_JDIR_BADFTYPE] = "JERR_JDIR_BADFTYPE: Bad or unknown file type (stat mode)."; + + // class JournalFile + _err_map[JERR_JNLF_OPEN] = "JERR_JNLF_OPEN: Unable to open file for write"; + _err_map[JERR_JNLF_CLOSE] = "JERR_JNLF_CLOSE: Unable to close file"; + _err_map[JERR_JNLF_FILEOFFSOVFL] = "JERR_JNLF_FILEOFFSOVFL: Attempted to increase submitted offset past file size."; + _err_map[JERR_JNLF_CMPLOFFSOVFL] = "JERR_JNLF_CMPLOFFSOVFL: Attempted to increase completed file offset past submitted offset."; + + // class LinearFileController + _err_map[JERR_LFCR_SEQNUMNOTFOUND] = "JERR_LFCR_SEQNUMNOTFOUND: File sequence number not found"; + + // class jrec, enq_rec, deq_rec, txn_rec + _err_map[JERR_JREC_BADRECHDR] = "JERR_JREC_BADRECHDR: Invalid data record header."; + _err_map[JERR_JREC_BADRECTAIL] = "JERR_JREC_BADRECTAIL: Invalid data record tail."; + + // class wmgr + _err_map[JERR_WMGR_BADPGSTATE] = "JERR_WMGR_BADPGSTATE: Page buffer in illegal state for operation."; + _err_map[JERR_WMGR_BADDTOKSTATE] = "JERR_WMGR_BADDTOKSTATE: Data token in illegal state for operation."; + _err_map[JERR_WMGR_ENQDISCONT] = "JERR_WMGR_ENQDISCONT: Enqueued new dtok when previous enqueue returned partly completed (state ENQ_PART)."; + _err_map[JERR_WMGR_DEQDISCONT] = "JERR_WMGR_DEQDISCONT: Dequeued new dtok when previous dequeue returned partly completed (state DEQ_PART)."; + _err_map[JERR_WMGR_DEQRIDNOTENQ] = "JERR_WMGR_DEQRIDNOTENQ: Dequeue rid is not enqueued."; + _err_map[JERR_WMGR_BADFH] = "JERR_WMGR_BADFH: Bad file handle."; + + // class RecoveryManager + _err_map[JERR_RCVM_OPENRD] = "JERR_JCNTL_OPENRD: Unable to open file for write"; + _err_map[JERR_RCVM_READ] = "JERR_JCNTL_READ: Read error: no or insufficient data to read"; + _err_map[JERR_RCVM_WRITE] = "JERR_RCVM_WRITE: Write error"; +// // class rmgr +// _err_map[JERR_RMGR_UNKNOWNMAGIC] = "JERR_RMGR_UNKNOWNMAGIC: Found record with unknown magic."; +// _err_map[JERR_RMGR_RIDMISMATCH] = "JERR_RMGR_RIDMISMATCH: RID mismatch between current record and dtok RID"; +// //_err_map[JERR_RMGR_FIDMISMATCH] = "JERR_RMGR_FIDMISMATCH: FID mismatch between emap and rrfc"; +// _err_map[JERR_RMGR_ENQSTATE] = "JERR_RMGR_ENQSTATE: Attempted read when data token wstate was not ENQ"; +// _err_map[JERR_RMGR_BADRECTYPE] = "JERR_RMGR_BADRECTYPE: Attempted operation on inappropriate record type"; + + // class data_tok + _err_map[JERR_DTOK_ILLEGALSTATE] = "JERR_MTOK_ILLEGALSTATE: Attempted to change to illegal state."; + //_err_map[JERR_DTOK_RIDNOTSET] = "JERR_DTOK_RIDNOTSET: Record ID not set."; + + // class enq_map, txn_map + _err_map[JERR_MAP_DUPLICATE] = "JERR_MAP_DUPLICATE: Attempted to insert record into map using duplicate key."; + _err_map[JERR_MAP_NOTFOUND] = "JERR_MAP_NOTFOUND: Key not found in map."; + _err_map[JERR_MAP_LOCKED] = "JERR_MAP_LOCKED: Record ID locked by a pending transaction."; + + // EFP errors + _err_map[JERR_EFP_BADPARTITIONNAME] = "JERR_EFP_BADPARTITIONNAME: Invalid partition name (must be \'pNNN\' where NNN is a non-zero number)"; + _err_map[JERR_EFP_BADEFPDIRNAME] = "JERR_EFP_BADEFPDIRNAME: Bad Empty File Pool directory name (must be \'NNNk\', where NNN is a number which is a multiple of 4)"; + _err_map[JERR_EFP_BADPARTITIONDIR] = "JERR_EFP_BADPARTITIONDIR: Invalid partition directory"; + _err_map[JERR_EFP_NOEFP] = "JERR_EFP_NOEFP: No Empty File Pool found for given partition and empty file size"; + _err_map[JERR_EFP_EMPTY] = "JERR_EFP_EMPTY: Empty File Pool is empty"; + + //_err_map[] = ""; + + return true; +} + +const char* +jerrno::err_msg(const uint32_t err_no) throw () +{ + _err_map_itr = _err_map.find(err_no); + if (_err_map_itr == _err_map.end()) + return "<Unknown error code>"; + return _err_map_itr->second; +} + +}} diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/jerrno.h b/qpid/cpp/src/qpid/linearstore/jrnl/jerrno.h new file mode 100644 index 0000000000..7bd521ecbf --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/jerrno.h @@ -0,0 +1,154 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_JRNL_JERRNO_H +#define QPID_LEGACYSTORE_JRNL_JERRNO_H + +namespace qpid +{ +namespace qls_jrnl +{ +class jerrno; +}} + +#include <map> +#include <stdint.h> +#include <string> + +namespace qpid +{ +namespace qls_jrnl +{ + + /** + * \class jerrno + * \brief Class containing static error definitions and static map for error messages. + */ + class jerrno + { + static std::map<uint32_t, const char*> _err_map; ///< Map of error messages + static std::map<uint32_t, const char*>::iterator _err_map_itr; ///< Iterator + static bool _initialized; ///< Dummy flag, used to initialise map. + + public: + // generic errors + static const uint32_t JERR__MALLOC; ///< Buffer memory allocation failed + static const uint32_t JERR__UNDERFLOW; ///< Underflow error + static const uint32_t JERR__NINIT; ///< Operation on uninitialized class + static const uint32_t JERR__AIO; ///< AIO failure + static const uint32_t JERR__FILEIO; ///< File read or write failure + static const uint32_t JERR__RTCLOCK; ///< Reading real-time clock failed + static const uint32_t JERR__PTHREAD; ///< pthread failure + static const uint32_t JERR__TIMEOUT; ///< Timeout waiting for an event + static const uint32_t JERR__UNEXPRESPONSE; ///< Unexpected response to call or event + static const uint32_t JERR__RECNFOUND; ///< Record not found + static const uint32_t JERR__NOTIMPL; ///< Not implemented + static const uint32_t JERR__NULL; ///< Operation on null pointer + + // class jcntl + static const uint32_t JERR_JCNTL_STOPPED; ///< Operation on stopped journal + static const uint32_t JERR_JCNTL_READONLY; ///< Write operation on read-only journal + static const uint32_t JERR_JCNTL_AIOCMPLWAIT; ///< Timeout waiting for AIOs to complete + static const uint32_t JERR_JCNTL_UNKNOWNMAGIC; ///< Found record with unknown magic + static const uint32_t JERR_JCNTL_NOTRECOVERED; ///< Req' recover() to be called first + static const uint32_t JERR_JCNTL_ENQSTATE; ///< Read error: Record not in ENQ state + static const uint32_t JERR_JCNTL_INVALIDENQHDR;///< Invalid ENQ header + + // class jdir + static const uint32_t JERR_JDIR_NOTDIR; ///< Exists but is not a directory + static const uint32_t JERR_JDIR_MKDIR; ///< Directory creation failed + static const uint32_t JERR_JDIR_OPENDIR; ///< Directory open failed + static const uint32_t JERR_JDIR_READDIR; ///< Directory read failed + static const uint32_t JERR_JDIR_CLOSEDIR; ///< Directory close failed + static const uint32_t JERR_JDIR_RMDIR; ///< Directory delete failed + static const uint32_t JERR_JDIR_NOSUCHFILE; ///< File does not exist + static const uint32_t JERR_JDIR_FMOVE; ///< File move failed + static const uint32_t JERR_JDIR_STAT; ///< File stat failed + static const uint32_t JERR_JDIR_UNLINK; ///< File delete failed + static const uint32_t JERR_JDIR_BADFTYPE; ///< Bad or unknown file type (stat mode) + + // class JournalFile + static const uint32_t JERR_JNLF_OPEN; ///< Unable to open file for write + static const uint32_t JERR_JNLF_CLOSE; ///< Unable to close file + static const uint32_t JERR_JNLF_FILEOFFSOVFL; ///< Increased offset past file size + static const uint32_t JERR_JNLF_CMPLOFFSOVFL; ///< Increased cmpl offs past subm offs + + // class LinearFileController + static const uint32_t JERR_LFCR_SEQNUMNOTFOUND;///< File sequence number not found + + // class jrec, enq_rec, deq_rec, txn_rec + static const uint32_t JERR_JREC_BADRECHDR; ///< Invalid data record header + static const uint32_t JERR_JREC_BADRECTAIL; ///< Invalid data record tail + + // class wmgr + static const uint32_t JERR_WMGR_BADPGSTATE; ///< Page buffer in illegal state. + static const uint32_t JERR_WMGR_BADDTOKSTATE; ///< Data token in illegal state. + static const uint32_t JERR_WMGR_ENQDISCONT; ///< Enq. new dtok when previous part compl. + static const uint32_t JERR_WMGR_DEQDISCONT; ///< Deq. new dtok when previous part compl. + static const uint32_t JERR_WMGR_DEQRIDNOTENQ; ///< Deq. rid not enqueued + static const uint32_t JERR_WMGR_BADFH; ///< Bad file handle + + // class RecoveryManager + static const uint32_t JERR_RCVM_OPENRD; ///< Unable to open file for read + static const uint32_t JERR_RCVM_READ; ///< Read error: no or insufficient data to read + static const uint32_t JERR_RCVM_WRITE; ///< Write error + +// // class rmgr +// static const uint32_t JERR_RMGR_UNKNOWNMAGIC; ///< Found record with unknown magic +// static const uint32_t JERR_RMGR_RIDMISMATCH; ///< RID mismatch between rec and dtok +// //static const uint32_t JERR_RMGR_FIDMISMATCH; ///< FID mismatch between emap and rrfc +// static const uint32_t JERR_RMGR_ENQSTATE; ///< Attempted read when wstate not ENQ +// static const uint32_t JERR_RMGR_BADRECTYPE; ///< Attempted op on incorrect rec type + + // class data_tok + static const uint32_t JERR_DTOK_ILLEGALSTATE; ///< Attempted to change to illegal state +// static const uint32_t JERR_DTOK_RIDNOTSET; ///< Record ID not set + + // class enq_map, txn_map + static const uint32_t JERR_MAP_DUPLICATE; ///< Attempted to insert using duplicate key + static const uint32_t JERR_MAP_NOTFOUND; ///< Key not found in map + static const uint32_t JERR_MAP_LOCKED; ///< rid locked by pending txn + + // EFP errors + static const uint32_t JERR_EFP_BADPARTITIONNAME; ///< Partition name invalid or of value 0 + static const uint32_t JERR_EFP_BADEFPDIRNAME; ///< Empty File Pool directory name invalid + static const uint32_t JERR_EFP_BADPARTITIONDIR; ///< Invalid partition directory + static const uint32_t JERR_EFP_NOEFP; ///< No EFP found for given partition and file size + static const uint32_t JERR_EFP_EMPTY; ///< EFP empty + + // Negative returns for some functions + static const int32_t AIO_TIMEOUT; ///< Timeout waiting for AIO return + static const int32_t LOCK_TAKEN; ///< Attempted to take lock, but it was taken by another thread + /** + * \brief Method to access error message from known error number. + */ + static const char* err_msg(const uint32_t err_no) throw (); + + private: + /** + * \brief Static function to initialize map. + */ + static bool __init(); + }; + +}} + +#endif // ifndef QPID_LEGACYSTORE_JRNL_JERRNO_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/jexception.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/jexception.cpp new file mode 100644 index 0000000000..38ef85ddb8 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/jexception.cpp @@ -0,0 +1,171 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/jrnl/jexception.h" + +#include <iomanip> +#include <sstream> +#include "qpid/linearstore/jrnl/jerrno.h" + +#define CATLEN(p) MAX_MSG_SIZE - std::strlen(p) - 1 + +namespace qpid +{ +namespace qls_jrnl +{ + +jexception::jexception() throw (): + std::exception(), + _err_code(0) +{ + format(); +} + +jexception::jexception(const uint32_t err_code) throw (): + std::exception(), + _err_code(err_code) +{ + format(); +} + +jexception::jexception(const char* additional_info) throw (): + std::exception(), + _err_code(0), + _additional_info(additional_info) +{ + format(); +} + +jexception::jexception(const std::string& additional_info) throw (): + std::exception(), + _err_code(0), + _additional_info(additional_info) +{ + format(); +} + +jexception::jexception(const uint32_t err_code, const char* additional_info) throw (): + std::exception(), + _err_code(err_code), + _additional_info(additional_info) +{ + format(); +} + +jexception::jexception(const uint32_t err_code, const std::string& additional_info) throw (): + std::exception(), + _err_code(err_code), + _additional_info(additional_info) +{ + format(); +} + +jexception::jexception(const uint32_t err_code, const char* throwing_class, + const char* throwing_fn) throw (): + std::exception(), + _err_code(err_code), + _throwing_class(throwing_class), + _throwing_fn(throwing_fn) +{ + format(); +} + +jexception::jexception(const uint32_t err_code, const std::string& throwing_class, + const std::string& throwing_fn) throw (): + std::exception(), + _err_code(err_code), + _throwing_class(throwing_class), + _throwing_fn(throwing_fn) +{ + format(); +} + +jexception::jexception(const uint32_t err_code, const char* additional_info, + const char* throwing_class, const char* throwing_fn) throw (): + std::exception(), + _err_code(err_code), + _additional_info(additional_info), + _throwing_class(throwing_class), + _throwing_fn(throwing_fn) +{ + format(); +} + +jexception::jexception(const uint32_t err_code, const std::string& additional_info, + const std::string& throwing_class, const std::string& throwing_fn) throw (): + std::exception(), + _err_code(err_code), + _additional_info(additional_info), + _throwing_class(throwing_class), + _throwing_fn(throwing_fn) +{ + format(); +} + +jexception::~jexception() throw () +{} + +void +jexception::format() +{ + const bool ai = !_additional_info.empty(); + const bool tc = !_throwing_class.empty(); + const bool tf = !_throwing_fn.empty(); + std::ostringstream oss; + oss << "jexception 0x" << std::hex << std::setfill('0') << std::setw(4) << _err_code << " "; + if (tc) + { + oss << _throwing_class; + if (tf) + oss << "::"; + else + oss << " "; + } + if (tf) + oss << _throwing_fn << "() "; + if (tc || tf) + oss << "threw " << jerrno::err_msg(_err_code); + if (ai) + oss << " (" << _additional_info << ")"; + _what.assign(oss.str()); +} + +const char* +jexception::what() const throw () +{ + return _what.c_str(); +} + +std::ostream& +operator<<(std::ostream& os, const jexception& je) +{ + os << je.what(); + return os; +} + +std::ostream& +operator<<(std::ostream& os, const jexception* jePtr) +{ + os << jePtr->what(); + return os; +} + +}} diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/jexception.h b/qpid/cpp/src/qpid/linearstore/jrnl/jexception.h new file mode 100644 index 0000000000..a3a34691df --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/jexception.h @@ -0,0 +1,126 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_JRNL_JEXCEPTION_H +#define QPID_LEGACYSTORE_JRNL_JEXCEPTION_H + +namespace qpid +{ +namespace qls_jrnl +{ +class jexception; +}} + +#include <cerrno> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <exception> +#include "qpid/linearstore/jrnl/jerrno.h" +#include <sstream> +#include <string> + +// Macro for formatting commom system errors +#define FORMAT_SYSERR(errno) " errno=" << errno << " (" << std::strerror(errno) << ")" + +#define MALLOC_CHK(ptr, var, cls, fn) if(ptr == 0) { \ + clean(); \ + std::ostringstream oss; \ + oss << var << ": malloc() failed: " << FORMAT_SYSERR(errno); \ + throw jexception(jerrno::JERR__MALLOC, oss.str(), cls, fn); \ + } + +// TODO: The following is a temporary bug-tracking aid which forces a core. +// Replace with the commented out version below when BZ484048 is resolved. +#define PTHREAD_CHK(err, pfn, cls, fn) if(err != 0) { \ + std::ostringstream oss; \ + oss << cls << "::" << fn << "(): " << pfn; \ + errno = err; \ + ::perror(oss.str().c_str()); \ + ::abort(); \ + } +/* +#define PTHREAD_CHK(err, pfn, cls, fn) if(err != 0) { \ + std::ostringstream oss; \ + oss << pfn << " failed: " << FORMAT_SYSERR(err); \ + throw jexception(jerrno::JERR__PTHREAD, oss.str(), cls, fn); \ + } +*/ + +#define ASSERT(cond, msg) if(cond == 0) { \ + std::cerr << msg << std::endl; \ + ::abort(); \ + } + +namespace qpid +{ +namespace qls_jrnl +{ + /** + * \class jexception + * \brief Generic journal exception class + */ + class jexception : public std::exception + { + private: + uint32_t _err_code; + std::string _additional_info; + std::string _throwing_class; + std::string _throwing_fn; + std::string _what; + void format(); + + public: + jexception() throw (); + + jexception(const uint32_t err_code) throw (); + + jexception(const char* additional_info) throw (); + jexception(const std::string& additional_info) throw (); + + jexception(const uint32_t err_code, const char* additional_info) throw (); + jexception(const uint32_t err_code, const std::string& additional_info) throw (); + + jexception(const uint32_t err_code, const char* throwing_class, const char* throwing_fn) + throw (); + jexception(const uint32_t err_code, const std::string& throwing_class, + const std::string& throwing_fn) throw (); + + jexception(const uint32_t err_code, const char* additional_info, + const char* throwing_class, const char* throwing_fn) throw (); + jexception(const uint32_t err_code, const std::string& additional_info, + const std::string& throwing_class, const std::string& throwing_fn) throw (); + + virtual ~jexception() throw (); + virtual const char* what() const throw (); // override std::exception::what() + + inline uint32_t err_code() const throw () { return _err_code; } + inline const std::string additional_info() const throw () { return _additional_info; } + inline const std::string throwing_class() const throw () { return _throwing_class; } + inline const std::string throwing_fn() const throw () { return _throwing_fn; } + + friend std::ostream& operator<<(std::ostream& os, const jexception& je); + friend std::ostream& operator<<(std::ostream& os, const jexception* jePtr); + }; // class jexception + +}} + +#endif // ifndef QPID_LEGACYSTORE_JRNL_JEXCEPTION_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/jrec.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/jrec.cpp new file mode 100644 index 0000000000..87e0b0c04c --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/jrec.cpp @@ -0,0 +1,107 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/jrnl/jrec.h" + +#include <iomanip> +#include "qpid/linearstore/jrnl/jerrno.h" +#include "qpid/linearstore/jrnl/jexception.h" +#include <sstream> + +namespace qpid +{ +namespace qls_jrnl +{ + +jrec::jrec() {} +jrec::~jrec() {} + +void +jrec::chk_hdr(const rec_hdr_t& hdr) +{ + if (hdr._magic == 0) + { + std::ostringstream oss; + oss << std::hex << std::setfill('0'); + oss << "enq magic NULL: rid=0x" << hdr._rid; + throw jexception(jerrno::JERR_JREC_BADRECHDR, oss.str(), "jrec", "chk_hdr"); + } + if (hdr._version != QLS_JRNL_VERSION) + { + std::ostringstream oss; + oss << std::hex << std::setfill('0'); + oss << "version: rid=0x" << hdr._rid; + oss << ": expected=0x" << std::setw(2) << (int)QLS_JRNL_VERSION; + oss << " read=0x" << std::setw(2) << (int)hdr._version; + throw jexception(jerrno::JERR_JREC_BADRECHDR, oss.str(), "jrec", "chk_hdr"); + } +//#if defined (JRNL_LITTLE_ENDIAN) +// uint8_t endian_flag = RHM_LENDIAN_FLAG; +//#else +// uint8_t endian_flag = RHM_BENDIAN_FLAG; +//#endif +// if (hdr._eflag != endian_flag) +// { +// std::ostringstream oss; +// oss << std::hex << std::setfill('0'); +// oss << "endian_flag: rid=" << hdr._rid; +// oss << ": expected=0x" << std::setw(2) << (int)endian_flag; +// oss << " read=0x" << std::setw(2) << (int)hdr._eflag; +// throw jexception(jerrno::JERR_JREC_BADRECHDR, oss.str(), "jrec", "chk_hdr"); +// } +} + +void +jrec::chk_rid(const rec_hdr_t& hdr, const uint64_t rid) +{ + if (hdr._rid != rid) + { + std::ostringstream oss; + oss << std::hex << std::setfill('0'); + oss << "rid mismatch: expected=0x" << rid; + oss << " read=0x" << hdr._rid; + throw jexception(jerrno::JERR_JREC_BADRECHDR, oss.str(), "jrec", "chk_hdr"); + } +} + +void +jrec::chk_tail(const rec_tail_t& tail, const rec_hdr_t& hdr) +{ + if (tail._xmagic != ~hdr._magic) + { + std::ostringstream oss; + oss << std::hex << std::setfill('0'); + oss << "magic: rid=0x" << hdr._rid; + oss << ": expected=0x" << ~hdr._magic; + oss << " read=0x" << tail._xmagic; + throw jexception(jerrno::JERR_JREC_BADRECTAIL, oss.str(), "jrec", "chk_tail"); + } + if (tail._rid != hdr._rid) + { + std::ostringstream oss; + oss << std::hex << std::setfill('0'); + oss << "rid: rid=0x" << hdr._rid; + oss << ": read=0x" << tail._rid; + throw jexception(jerrno::JERR_JREC_BADRECTAIL, oss.str(), "jrec", "chk_tail"); + } +} + +}} diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/jrec.h b/qpid/cpp/src/qpid/linearstore/jrnl/jrec.h new file mode 100644 index 0000000000..619cd0bdec --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/jrec.h @@ -0,0 +1,170 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_JRNL_JREC_H +#define QPID_LEGACYSTORE_JRNL_JREC_H + +namespace qpid +{ +namespace qls_jrnl +{ +class jrec; +}} + +#include <cstddef> +#include <fstream> +#include "qpid/linearstore/jrnl/jcfg.h" +#include "qpid/linearstore/jrnl/utils/rec_hdr.h" +#include "qpid/linearstore/jrnl/utils/rec_tail.h" +#include <string> + +namespace qpid +{ +namespace qls_jrnl +{ + + /** + * \class jrec + * \brief Abstract class for all file jrecords, both data and log. This class establishes + * the common data format and structure for these jrecords. + */ + class jrec + { + public: + jrec(); + virtual ~jrec(); + + /** + * \brief Encode this instance of jrec into the write buffer at the disk-block-aligned + * pointer wptr starting at position rec_offs_dblks in the encoded record to a + * maximum size of max_size_dblks. + * + * This call encodes the content of the data contianed in this instance of jrec into a + * disk-softblock-aligned (defined by JRNL_SBLK_SIZE) buffer pointed to by parameter + * wptr. No more than paramter max_size_dblks data-blocks may be written to the buffer. + * The parameter rec_offs_dblks is the offset in data-blocks within the fully encoded + * data block this instance represents at which to start encoding. + * + * Encoding entails writing the record header (struct enq_hdr), the data and the record tail + * (struct enq_tail). The record must be data-block-aligned (defined by JRNL_DBLK_SIZE), + * thus any remaining space in the final data-block is ignored; the returned value is the + * number of data-blocks consumed from the page by the encode action. Provided the initial + * alignment requirements are met, records may be of arbitrary size and may span multiple + * data-blocks, disk-blocks and/or pages. + * + * Since the record size in data-blocks is known, the general usage pattern is to call + * encode() as many times as is needed to fully encode the data. Each call to encode() + * will encode as much of the record as it can to what remains of the current page cache, + * and will return the number of data-blocks actually encoded. + * + * <b>Example:</b> Assume that record r1 was previously written to page 0, and that this + * is an instance representing record r2. Being larger than the page size ps, r2 would span + * multiple pages as follows: + * <pre> + * |<---ps--->| + * +----------+----------+----------+----... + * | |r2a| r2b | r2c | | + * |<-r1-><----------r2----------> | + * +----------+----------+----------+----... + * page: p0 p1 p2 + * </pre> + * Encoding record r2 will require multiple calls to encode; one for each page which + * is involved. Record r2 is divided logically into sections r2a, r2b and r2c at the + * points where the page boundaries intersect with the record. Assuming a page size + * of ps, the page boundary pointers are represented by their names p0, p1... and the + * sizes of the record segments are represented by their names r1, r2a, r2b..., the calls + * should be as follows: + * <pre> + * encode(p0+r1, 0, ps-r1); (returns r2a data-blocks) + * encode(p1, r2a, ps); (returns r2b data-blocks which equals ps) + * encode(p2, r2a+r2b, ps); (returns r2c data-blocks) + * </pre> + * + * \param wptr Data-block-aligned pointer to position in page buffer where encoding is to + * take place. + * \param rec_offs_dblks Offset in data-blocks within record from which to start encoding. + * \param max_size_dblks Maximum number of data-blocks to write to pointer wptr. + * \returns Number of data-blocks encoded. + */ + virtual uint32_t encode(void* wptr, uint32_t rec_offs_dblks, + uint32_t max_size_dblks) = 0; + + /** + * \brief Decode into this instance of jrec from the read buffer at the disk-block-aligned + * pointer rptr starting at position jrec_offs_dblks in the encoded record to a + * maximum size of max_size_blks. + * + * This call decodes a record in the page buffer pointed to by the data-block-aligned + * (defined by JRNL_DBLK_SIZE) parameter rptr into this instance of jrec. No more than + * paramter max_size_dblks data-blocks may be read from the buffer. The parameter + * jrec_offs_dblks is the offset in data-blocks within the encoded record at which to start + * decoding. + * + * Decoding entails reading the record header, the data and the tail. The record is + * data-block-aligned (defined by JRNL_DBLK_SIZE); the returned value is the number of + * data-blocks read from the buffer by the decode action. As the record data size is only + * known once the header is read, the number of calls required to complete reading the + * record will depend on the vlaues within this instance which are set when the + * header is decoded. + * + * A non-zero value for jrec_offs_dblks implies that this is not the first call to + * decode and the record data will be appended at this offset. + * + * \param h Reference to instance of struct hdr, already read from page buffer and used + * to determine record type + * \param rptr Data-block-aligned pointer to position in page buffer where decoding is to + * begin. + * \param rec_offs_dblks Offset within record from which to start appending the decoded + * record. + * \param max_size_dblks Maximum number of data-blocks to read from pointer rptr. + * \returns Number of data-blocks read (consumed). + */ + virtual uint32_t decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, + uint32_t max_size_dblks) = 0; + + virtual bool rcv_decode(rec_hdr_t h, std::ifstream* ifsp, std::size_t& rec_offs) = 0; + + virtual std::string& str(std::string& str) const = 0; + virtual std::size_t data_size() const = 0; + virtual std::size_t xid_size() const = 0; + virtual std::size_t rec_size() const = 0; + inline virtual uint32_t rec_size_dblks() const { return size_dblks(rec_size()); } + static inline uint32_t size_dblks(const std::size_t size) + { return size_blks(size, QLS_DBLK_SIZE_BYTES); } + static inline uint32_t size_sblks(const std::size_t size) + { return size_blks(size, QLS_SBLK_SIZE_BYTES); } + static inline uint32_t size_blks(const std::size_t size, const std::size_t blksize) + { return (size + blksize - 1)/blksize; } + virtual uint64_t rid() const = 0; + + protected: + virtual void chk_hdr() const = 0; + virtual void chk_hdr(uint64_t rid) const = 0; + virtual void chk_tail() const = 0; + static void chk_hdr(const rec_hdr_t& hdr); + static void chk_rid(const rec_hdr_t& hdr, uint64_t rid); + static void chk_tail(const rec_tail_t& tail, const rec_hdr_t& hdr); + virtual void clean() = 0; + }; // class jrec + +}} + +#endif // ifndef QPID_LEGACYSTORE_JRNL_JREC_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/pmgr.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/pmgr.cpp new file mode 100644 index 0000000000..32b024a09d --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/pmgr.cpp @@ -0,0 +1,202 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/jrnl/pmgr.h" + +#include <cerrno> +#include <cstdlib> +#include <cstring> +#include "qpid/linearstore/jrnl/jcfg.h" +#include "qpid/linearstore/jrnl/jcntl.h" +#include "qpid/linearstore/jrnl/jerrno.h" +#include <sstream> + +namespace qpid +{ +namespace qls_jrnl +{ + +pmgr::page_cb::page_cb(uint16_t index): + _index(index), + _state(UNUSED), + _frid(0), + _wdblks(0), + _rdblks(0), + _pdtokl(0), + _jfp(0), + _pbuff(0) +{} + +// TODO: almost identical to pmgr::page_state_str() below - resolve +const char* +pmgr::page_cb::state_str() const +{ + switch(_state) + { + case UNUSED: + return "UNUSED"; + case IN_USE: + return "IN_USE"; + case AIO_PENDING: + return "AIO_PENDING"; + } + return "<unknown>"; +} + +// static +const uint32_t pmgr::_sblkSizeBytes = QLS_SBLK_SIZE_BYTES; + +pmgr::pmgr(jcntl* jc, enq_map& emap, txn_map& tmap): + _cache_pgsize_sblks(0), + _cache_num_pages(0), + _jc(jc), + _emap(emap), + _tmap(tmap), + _page_base_ptr(0), + _page_ptr_arr(0), + _page_cb_arr(0), + _aio_cb_arr(0), + _aio_event_arr(0), + _ioctx(0), + _pg_index(0), + _pg_cntr(0), + _pg_offset_dblks(0), + _aio_evt_rem(0), + _cbp(0), + _enq_rec(), + _deq_rec(), + _txn_rec() +{} + +pmgr::~pmgr() +{ + pmgr::clean(); +} + +void +pmgr::initialize(aio_callback* const cbp, const uint32_t cache_pgsize_sblks, const uint16_t cache_num_pages) +{ + // As static use of this class keeps old values around, clean up first... + pmgr::clean(); + _pg_index = 0; + _pg_cntr = 0; + _pg_offset_dblks = 0; + _aio_evt_rem = 0; + _cache_pgsize_sblks = cache_pgsize_sblks; + _cache_num_pages = cache_num_pages; + _cbp = cbp; + + // 1. Allocate page memory (as a single block) + std::size_t cache_pgsize = _cache_num_pages * _cache_pgsize_sblks * _sblkSizeBytes; + if (::posix_memalign(&_page_base_ptr, QLS_AIO_ALIGN_BOUNDARY_BYTES, cache_pgsize)) + { + clean(); + std::ostringstream oss; + oss << "posix_memalign(): alignment=" << QLS_AIO_ALIGN_BOUNDARY_BYTES << " size=" << cache_pgsize; + oss << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR__MALLOC, oss.str(), "pmgr", "initialize"); + } + + // 2. Allocate array of page pointers + _page_ptr_arr = (void**)std::malloc(_cache_num_pages * sizeof(void*)); + MALLOC_CHK(_page_ptr_arr, "_page_ptr_arr", "pmgr", "initialize"); + + // 3. Allocate and initialize page control block (page_cb) array + _page_cb_arr = (page_cb*)std::malloc(_cache_num_pages * sizeof(page_cb)); + MALLOC_CHK(_page_cb_arr, "_page_cb_arr", "pmgr", "initialize"); + std::memset(_page_cb_arr, 0, _cache_num_pages * sizeof(page_cb)); + + // 4. Allocate IO control block (iocb) array + _aio_cb_arr = (aio_cb*)std::malloc(_cache_num_pages * sizeof(aio_cb)); + MALLOC_CHK(_aio_cb_arr, "_aio_cb_arr", "pmgr", "initialize"); + + // 5. Set page pointers in _page_ptr_arr, _page_cb_arr and iocbs to pages within page block + for (uint16_t i=0; i<_cache_num_pages; i++) + { + _page_ptr_arr[i] = (void*)((char*)_page_base_ptr + _cache_pgsize_sblks * _sblkSizeBytes * i); + _page_cb_arr[i]._index = i; + _page_cb_arr[i]._state = UNUSED; + _page_cb_arr[i]._pbuff = _page_ptr_arr[i]; + _page_cb_arr[i]._pdtokl = new std::deque<data_tok*>; + _page_cb_arr[i]._pdtokl->clear(); + _aio_cb_arr[i].data = (void*)&_page_cb_arr[i]; + } + + // 6. Allocate io_event array, max one event per cache page plus one for each file + const uint16_t max_aio_evts = _cache_num_pages + 1; // One additional event for file header writes + _aio_event_arr = (aio_event*)std::malloc(max_aio_evts * sizeof(aio_event)); + MALLOC_CHK(_aio_event_arr, "_aio_event_arr", "pmgr", "initialize"); + + // 7. Initialize AIO context + if (int ret = aio::queue_init(max_aio_evts, &_ioctx)) + { + std::ostringstream oss; + oss << "io_queue_init() failed: " << FORMAT_SYSERR(-ret); + throw jexception(jerrno::JERR__AIO, oss.str(), "pmgr", "initialize"); + } +} + +void +pmgr::clean() +{ + // Clean up allocated memory here + + if (_ioctx) + aio::queue_release(_ioctx); + + std::free(_page_base_ptr); + _page_base_ptr = 0; + + if (_page_cb_arr) + { + for (int i=0; i<_cache_num_pages; i++) + delete _page_cb_arr[i]._pdtokl; + std::free(_page_ptr_arr); + _page_ptr_arr = 0; + } + + std::free(_page_cb_arr); + _page_cb_arr = 0; + + std::free(_aio_cb_arr); + _aio_cb_arr = 0; + + std::free(_aio_event_arr); + _aio_event_arr = 0; +} + +// TODO: almost identical to pmgr::page_cb::state_str() above - resolve +const char* +pmgr::page_state_str(page_state ps) +{ + switch (ps) + { + case UNUSED: + return "UNUSED"; + case IN_USE: + return "IN_USE"; + case AIO_PENDING: + return "AIO_PENDING"; + } + return "<page_state unknown>"; +} + +}} diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/pmgr.h b/qpid/cpp/src/qpid/linearstore/jrnl/pmgr.h new file mode 100644 index 0000000000..64143a9e6e --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/pmgr.h @@ -0,0 +1,127 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_JRNL_PMGR_H +#define QPID_LEGACYSTORE_JRNL_PMGR_H + +namespace qpid +{ +namespace qls_jrnl +{ + class pmgr; + class jcntl; +}} + +#include <deque> +#include "qpid/linearstore/jrnl/aio.h" +#include "qpid/linearstore/jrnl/aio_callback.h" +#include "qpid/linearstore/jrnl/data_tok.h" +#include "qpid/linearstore/jrnl/deq_rec.h" +#include "qpid/linearstore/jrnl/enq_map.h" +#include "qpid/linearstore/jrnl/enq_rec.h" +#include "qpid/linearstore/jrnl/txn_map.h" +#include "qpid/linearstore/jrnl/txn_rec.h" + +namespace qpid +{ +namespace qls_jrnl +{ +class JournalFile; + + /** + * \brief Abstract class for managing either read or write page cache of arbitrary size and + * number of cache_num_pages. + */ + class pmgr + { + public: + /** + * \brief Enumeration of possible stats of a page within a page cache. + */ + enum page_state + { + UNUSED, ///< A page is uninitialized, contains no data. + IN_USE, ///< Page is in use. + AIO_PENDING ///< An AIO request outstanding. + }; + + /** + * \brief Page control block, carries control and state information for each page in the + * cache. + */ + struct page_cb + { + uint16_t _index; ///< Index of this page + page_state _state; ///< Status of page + uint64_t _frid; ///< First rid in page (used for fhdr init) + uint32_t _wdblks; ///< Total number of dblks in page so far + uint32_t _rdblks; ///< Total number of dblks in page + std::deque<data_tok*>* _pdtokl; ///< Page message tokens list + JournalFile* _jfp; ///< Journal file for incrementing compl counts + void* _pbuff; ///< Page buffer + + page_cb(uint16_t index); ///< Convenience constructor + const char* state_str() const; ///< Return state as string for this pcb + }; + + protected: + static const uint32_t _sblkSizeBytes; ///< Disk softblock size + uint32_t _cache_pgsize_sblks; ///< Size of page cache cache_num_pages + uint16_t _cache_num_pages; ///< Number of page cache cache_num_pages + jcntl* _jc; ///< Pointer to journal controller + enq_map& _emap; ///< Ref to enqueue map + txn_map& _tmap; ///< Ref to transaction map + void* _page_base_ptr; ///< Base pointer to page memory + void** _page_ptr_arr; ///< Array of pointers to cache_num_pages in page memory + page_cb* _page_cb_arr; ///< Array of page_cb structs + aio_cb* _aio_cb_arr; ///< Array of iocb structs + aio_event* _aio_event_arr; ///< Array of io_events + io_context_t _ioctx; ///< AIO context for read/write operations + uint16_t _pg_index; ///< Index of current page being used + uint32_t _pg_cntr; ///< Page counter; determines if file rotation req'd + uint32_t _pg_offset_dblks; ///< Page offset (used so far) in data blocks + uint32_t _aio_evt_rem; ///< Remaining AIO events + aio_callback* _cbp; ///< Pointer to callback object + + enq_rec _enq_rec; ///< Enqueue record used for encoding/decoding + deq_rec _deq_rec; ///< Dequeue record used for encoding/decoding + txn_rec _txn_rec; ///< Transaction record used for encoding/decoding + + public: + pmgr(jcntl* jc, enq_map& emap, txn_map& tmap); + virtual ~pmgr(); + + virtual int32_t get_events(timespec* const timeout, bool flush) = 0; + inline uint32_t get_aio_evt_rem() const { return _aio_evt_rem; } + static const char* page_state_str(page_state ps); + inline uint32_t cache_pgsize_sblks() const { return _cache_pgsize_sblks; } + inline uint16_t cache_num_pages() const { return _cache_num_pages; } + + protected: + virtual void initialize(aio_callback* const cbp, const uint32_t cache_pgsize_sblks, + const uint16_t cache_num_pages); + virtual void rotate_page() = 0; + virtual void clean(); + }; + +}} + +#endif // ifndef QPID_LEGACYSTORE_JRNL_PMGR_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/slock.h b/qpid/cpp/src/qpid/linearstore/jrnl/slock.h new file mode 100644 index 0000000000..bbfee37353 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/slock.h @@ -0,0 +1,73 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_JRNL_SLOCK_H +#define QPID_LEGACYSTORE_JRNL_SLOCK_H + +#include "qpid/linearstore/jrnl/jexception.h" +#include "qpid/linearstore/jrnl/smutex.h" +#include <pthread.h> + +namespace qpid +{ +namespace qls_jrnl +{ + + // Ultra-simple scoped lock class, auto-releases mutex when it goes out-of-scope + class slock + { + protected: + const smutex& _sm; + public: + inline slock(const smutex& sm) : _sm(sm) + { + PTHREAD_CHK(::pthread_mutex_lock(_sm.get()), "::pthread_mutex_lock", "slock", "slock"); + } + inline ~slock() + { + PTHREAD_CHK(::pthread_mutex_unlock(_sm.get()), "::pthread_mutex_unlock", "slock", "~slock"); + } + }; + + // Ultra-simple scoped try-lock class, auto-releases mutex when it goes out-of-scope + class stlock + { + protected: + const smutex& _sm; + bool _locked; + public: + inline stlock(const smutex& sm) : _sm(sm), _locked(false) + { + int ret = ::pthread_mutex_trylock(_sm.get()); + _locked = (ret == 0); // check if lock obtained + if (!_locked && ret != EBUSY) PTHREAD_CHK(ret, "::pthread_mutex_trylock", "stlock", "stlock"); + } + inline ~stlock() + { + if (_locked) + PTHREAD_CHK(::pthread_mutex_unlock(_sm.get()), "::pthread_mutex_unlock", "stlock", "~stlock"); + } + inline bool locked() const { return _locked; } + }; + +}} + +#endif // ifndef QPID_LEGACYSTORE_JRNL_SLOCK_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/smutex.h b/qpid/cpp/src/qpid/linearstore/jrnl/smutex.h new file mode 100644 index 0000000000..087e19730b --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/smutex.h @@ -0,0 +1,52 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_JRNL_SMUTEX_H +#define QPID_LEGACYSTORE_JRNL_SMUTEX_H + +#include "qpid/linearstore/jrnl/jexception.h" +#include <pthread.h> + +namespace qpid +{ +namespace qls_jrnl +{ + + // Ultra-simple scoped mutex class that allows a posix mutex to be initialized and destroyed with error checks + class smutex + { + protected: + mutable pthread_mutex_t _m; + public: + inline smutex() + { + PTHREAD_CHK(::pthread_mutex_init(&_m, 0), "::pthread_mutex_init", "smutex", "smutex"); + } + inline virtual ~smutex() + { + PTHREAD_CHK(::pthread_mutex_destroy(&_m), "::pthread_mutex_destroy", "smutex", "~smutex"); + } + inline pthread_mutex_t* get() const { return &_m; } + }; + +}} + +#endif // ifndef QPID_LEGACYSTORE_JRNL_SMUTEX_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/time_ns.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/time_ns.cpp new file mode 100644 index 0000000000..49e9a61704 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/time_ns.cpp @@ -0,0 +1,43 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/jrnl/time_ns.h" + +#include <sstream> + +namespace qpid +{ +namespace qls_jrnl +{ + +const std::string +time_ns::str(int precision) const +{ + const double t = tv_sec + (tv_nsec/1e9); + std::ostringstream oss; + oss.setf(std::ios::fixed, std::ios::floatfield); + oss.precision(precision); + oss << t; + return oss.str(); +} + + +}} diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/time_ns.h b/qpid/cpp/src/qpid/linearstore/jrnl/time_ns.h new file mode 100644 index 0000000000..3319573001 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/time_ns.h @@ -0,0 +1,93 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_JRNL_TIME_NS_H +#define QPID_LEGACYSTORE_JRNL_TIME_NS_H + +#include <cerrno> +#include <ctime> +#include <string> + +namespace qpid +{ +namespace qls_jrnl +{ + +struct time_ns : public timespec +{ + inline time_ns() { tv_sec = 0; tv_nsec = 0; } + inline time_ns(const std::time_t sec, const long nsec = 0) { tv_sec = sec; tv_nsec = nsec; } + inline time_ns(const time_ns& t) { tv_sec = t.tv_sec; tv_nsec = t.tv_nsec; } + + inline void set_zero() { tv_sec = 0; tv_nsec = 0; } + inline bool is_zero() const { return tv_sec == 0 && tv_nsec == 0; } + inline int now() { if(::clock_gettime(CLOCK_REALTIME, this)) return errno; return 0; } + const std::string str(int precision = 6) const; + + inline time_ns& operator=(const time_ns& rhs) + { tv_sec = rhs.tv_sec; tv_nsec = rhs.tv_nsec; return *this; } + inline time_ns& operator+=(const time_ns& rhs) + { + tv_nsec += rhs.tv_nsec; + if (tv_nsec >= 1000000000L) { tv_sec++; tv_nsec -= 1000000000L; } + tv_sec += rhs.tv_sec; + return *this; + } + inline time_ns& operator+=(const long ns) + { + tv_nsec += ns; + if (tv_nsec >= 1000000000L) { tv_sec++; tv_nsec -= 1000000000L; } + return *this; + } + inline time_ns& operator-=(const long ns) + { + tv_nsec -= ns; + if (tv_nsec < 0) { tv_sec--; tv_nsec += 1000000000L; } + return *this; + } + inline time_ns& operator-=(const time_ns& rhs) + { + tv_nsec -= rhs.tv_nsec; + if (tv_nsec < 0) { tv_sec--; tv_nsec += 1000000000L; } + tv_sec -= rhs.tv_sec; + return *this; + } + inline const time_ns operator+(const time_ns& rhs) + { time_ns t(*this); t += rhs; return t; } + inline const time_ns operator-(const time_ns& rhs) + { time_ns t(*this); t -= rhs; return t; } + inline bool operator==(const time_ns& rhs) + { return tv_sec == rhs.tv_sec && tv_nsec == rhs.tv_nsec; } + inline bool operator!=(const time_ns& rhs) + { return tv_sec != rhs.tv_sec || tv_nsec != rhs.tv_nsec; } + inline bool operator>(const time_ns& rhs) + { if(tv_sec == rhs.tv_sec) return tv_nsec > rhs.tv_nsec; return tv_sec > rhs.tv_sec; } + inline bool operator>=(const time_ns& rhs) + { if(tv_sec == rhs.tv_sec) return tv_nsec >= rhs.tv_nsec; return tv_sec >= rhs.tv_sec; } + inline bool operator<(const time_ns& rhs) + { if(tv_sec == rhs.tv_sec) return tv_nsec < rhs.tv_nsec; return tv_sec < rhs.tv_sec; } + inline bool operator<=(const time_ns& rhs) + { if(tv_sec == rhs.tv_sec) return tv_nsec <= rhs.tv_nsec; return tv_sec <= rhs.tv_sec; } +}; + +}} + +#endif // ifndef QPID_LEGACYSTORE_JRNL_TIME_NS_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/txn_map.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/txn_map.cpp new file mode 100644 index 0000000000..d1972ea1f2 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/txn_map.cpp @@ -0,0 +1,248 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/jrnl/txn_map.h" + +#include <iomanip> +#include "qpid/linearstore/jrnl/jerrno.h" +#include "qpid/linearstore/jrnl/jexception.h" +#include "qpid/linearstore/jrnl/slock.h" +#include <sstream> + +namespace qpid +{ +namespace qls_jrnl +{ + +// return/error codes +int16_t txn_map::TMAP_RID_NOT_FOUND = -2; +int16_t txn_map::TMAP_XID_NOT_FOUND = -1; +int16_t txn_map::TMAP_OK = 0; +int16_t txn_map::TMAP_NOT_SYNCED = 0; +int16_t txn_map::TMAP_SYNCED = 1; + +txn_data_struct::txn_data_struct(const uint64_t rid, const uint64_t drid, const uint16_t pfid, + const bool enq_flag, const bool commit_flag): + _rid(rid), + _drid(drid), + _pfid(pfid), + _enq_flag(enq_flag), + _commit_flag(commit_flag), + _aio_compl(false) +{} + +txn_map::txn_map(): + _map()/*, + _pfid_txn_cnt()*/ +{} + +txn_map::~txn_map() {} + +/* +void +txn_map::set_num_jfiles(const uint16_t num_jfiles) +{ + _pfid_txn_cnt.resize(num_jfiles, 0); +} +*/ + +/* +uint32_t +txn_map::get_txn_pfid_cnt(const uint16_t pfid) const +{ + return _pfid_txn_cnt.at(pfid); +} +*/ + +bool +txn_map::insert_txn_data(const std::string& xid, const txn_data& td) +{ + bool ok = true; + slock s(_mutex); + xmap_itr itr = _map.find(xid); + if (itr == _map.end()) // not found in map + { + txn_data_list list; + list.push_back(td); + std::pair<xmap_itr, bool> ret = _map.insert(xmap_param(xid, list)); + if (!ret.second) // duplicate + ok = false; + } + else + itr->second.push_back(td); +// _pfid_txn_cnt.at(td._pfid)++; + return ok; +} + +const txn_data_list +txn_map::get_tdata_list(const std::string& xid) +{ + slock s(_mutex); + return get_tdata_list_nolock(xid); +} + +const txn_data_list +txn_map::get_tdata_list_nolock(const std::string& xid) +{ + xmap_itr itr = _map.find(xid); + if (itr == _map.end()) // not found in map + return _empty_data_list; + return itr->second; +} + +const txn_data_list +txn_map::get_remove_tdata_list(const std::string& xid) +{ + slock s(_mutex); + xmap_itr itr = _map.find(xid); + if (itr == _map.end()) // not found in map + return _empty_data_list; + txn_data_list list = itr->second; + _map.erase(itr); +// for (tdl_itr i=list.begin(); i!=list.end(); i++) +// _pfid_txn_cnt.at(i->_pfid)--; + return list; +} + +bool +txn_map::in_map(const std::string& xid) +{ + slock s(_mutex); + xmap_itr itr= _map.find(xid); + return itr != _map.end(); +} + +uint32_t +txn_map::enq_cnt() +{ + return cnt(true); +} + +uint32_t +txn_map::deq_cnt() +{ + return cnt(true); +} + +uint32_t +txn_map::cnt(const bool enq_flag) +{ + slock s(_mutex); + uint32_t c = 0; + for (xmap_itr i = _map.begin(); i != _map.end(); i++) + { + for (tdl_itr j = i->second.begin(); j < i->second.end(); j++) + { + if (j->_enq_flag == enq_flag) + c++; + } + } + return c; +} + +int16_t +txn_map::is_txn_synced(const std::string& xid) +{ + slock s(_mutex); + xmap_itr itr = _map.find(xid); + if (itr == _map.end()) // not found in map + return TMAP_XID_NOT_FOUND; + bool is_synced = true; + for (tdl_itr litr = itr->second.begin(); litr < itr->second.end(); litr++) + { + if (!litr->_aio_compl) + { + is_synced = false; + break; + } + } + return is_synced ? TMAP_SYNCED : TMAP_NOT_SYNCED; +} + +int16_t +txn_map::set_aio_compl(const std::string& xid, const uint64_t rid) +{ + slock s(_mutex); + xmap_itr itr = _map.find(xid); + if (itr == _map.end()) // xid not found in map + return TMAP_XID_NOT_FOUND; + for (tdl_itr litr = itr->second.begin(); litr < itr->second.end(); litr++) + { + if (litr->_rid == rid) + { + litr->_aio_compl = true; + return TMAP_OK; // rid found + } + } + // xid present, but rid not found + return TMAP_RID_NOT_FOUND; +} + +bool +txn_map::data_exists(const std::string& xid, const uint64_t rid) +{ + bool found = false; + { + slock s(_mutex); + txn_data_list tdl = get_tdata_list_nolock(xid); + tdl_itr itr = tdl.begin(); + while (itr != tdl.end() && !found) + { + found = itr->_rid == rid; + itr++; + } + } + return found; +} + +bool +txn_map::is_enq(const uint64_t rid) +{ + bool found = false; + { + slock s(_mutex); + for (xmap_itr i = _map.begin(); i != _map.end() && !found; i++) + { + txn_data_list list = i->second; + for (tdl_itr j = list.begin(); j < list.end() && !found; j++) + { + if (j->_enq_flag) + found = j->_rid == rid; + else + found = j->_drid == rid; + } + } + } + return found; +} + +void +txn_map::xid_list(std::vector<std::string>& xv) +{ + xv.clear(); + { + slock s(_mutex); + for (xmap_itr itr = _map.begin(); itr != _map.end(); itr++) + xv.push_back(itr->first); + } +} + +}} diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/txn_map.h b/qpid/cpp/src/qpid/linearstore/jrnl/txn_map.h new file mode 100644 index 0000000000..b69d17da1c --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/txn_map.h @@ -0,0 +1,145 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_JRNL_TXN_MAP_H +#define QPID_LEGACYSTORE_JRNL_TXN_MAP_H + +namespace qpid +{ +namespace qls_jrnl +{ + class txn_map; +}} + +#include "qpid/linearstore/jrnl/smutex.h" +#include <map> +#include <pthread.h> +#include <string> +#include <vector> + +namespace qpid +{ +namespace qls_jrnl +{ + + /** + * \struct txn_data_struct + * \brief Struct encapsulating transaction data necessary for processing a transaction + * in the journal once it is closed with either a commit or abort. + */ + struct txn_data_struct + { + uint64_t _rid; ///< Record id for this operation + uint64_t _drid; ///< Dequeue record id for this operation + uint16_t _pfid; ///< Physical file id, to be used when transferring to emap on commit + bool _enq_flag; ///< If true, enq op, otherwise deq op + bool _commit_flag; ///< (2PC transactions) Records 2PC complete c/a mode + bool _aio_compl; ///< Initially false, set to true when record AIO returns + txn_data_struct(const uint64_t rid, const uint64_t drid, const uint16_t pfid, + const bool enq_flag, const bool commit_flag = false); + }; + typedef txn_data_struct txn_data; + typedef std::vector<txn_data> txn_data_list; + typedef txn_data_list::iterator tdl_itr; + + /** + * \class txn_map + * \brief Class for storing transaction data for each open (ie not committed or aborted) + * xid in the store. If aborted, records are discarded; if committed, they are + * transferred to the enqueue map. + * + * The data is encapsulated by struct txn_data_struct. A vector containing the information + * for each operation included as part of the same transaction is mapped against the + * xid. + * + * The aio_compl flag is set true as each AIO write operation for the enqueue or dequeue + * returns. Checking that all of these flags are true for a given xid is the mechanism + * used to determine if the transaction is syncronized (through method is_txn_synced()). + * + * On transaction commit, then each operation is handled as follows: + * + * If an enqueue (_enq_flag is true), then the rid and pfid are transferred to the enq_map. + * If a dequeue (_enq_flag is false), then the rid stored in the drid field is used to + * remove the corresponding record from the enq_map. + * + * On transaction abort, then each operation is handled as follows: + * + * If an enqueue (_enq_flag is true), then the data is simply discarded. + * If a dequeue (_enq_flag is false), then the lock for the corresponding enqueue in enq_map + * (if not a part of the same transaction) is removed, and the data discarded. + * + * <pre> + * key data + * + * xid1 --- vector< [ rid, drid, pfid, enq_flag, commit_flag, aio_compl ] > + * xid2 --- vector< [ rid, drid, pfid, enq_flag, commit_flag, aio_compl ] > + * xid3 --- vector< [ rid, drid, pfid, enq_flag, commit_flag, aio_compl ] > + * ... + * </pre> + */ + class txn_map + { + public: + // return/error codes + static int16_t TMAP_RID_NOT_FOUND; + static int16_t TMAP_XID_NOT_FOUND; + static int16_t TMAP_OK; + static int16_t TMAP_NOT_SYNCED; + static int16_t TMAP_SYNCED; + + private: + typedef std::pair<std::string, txn_data_list> xmap_param; + typedef std::map<std::string, txn_data_list> xmap; + typedef xmap::iterator xmap_itr; + + xmap _map; + smutex _mutex; +// std::vector<uint32_t> _pfid_txn_cnt; + const txn_data_list _empty_data_list; + + public: + txn_map(); + virtual ~txn_map(); + +// void set_num_jfiles(const uint16_t num_jfiles); +// uint32_t get_txn_pfid_cnt(const uint16_t pfid) const; + bool insert_txn_data(const std::string& xid, const txn_data& td); + const txn_data_list get_tdata_list(const std::string& xid); + const txn_data_list get_remove_tdata_list(const std::string& xid); + bool in_map(const std::string& xid); + uint32_t enq_cnt(); + uint32_t deq_cnt(); + int16_t is_txn_synced(const std::string& xid); // -1=xid not found; 0=not synced; 1=synced + int16_t set_aio_compl(const std::string& xid, const uint64_t rid); // -2=rid not found; -1=xid not found; 0=done + bool data_exists(const std::string& xid, const uint64_t rid); + bool is_enq(const uint64_t rid); + inline void clear() { _map.clear(); } + inline bool empty() const { return _map.empty(); } + inline size_t size() const { return _map.size(); } + void xid_list(std::vector<std::string>& xv); + private: + uint32_t cnt(const bool enq_flag); + const txn_data_list get_tdata_list_nolock(const std::string& xid); + }; + +}} + +#endif // ifndef QPID_LEGACYSTORE_JRNL_TXN_MAP_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/txn_rec.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/txn_rec.cpp new file mode 100644 index 0000000000..3c78903520 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/txn_rec.cpp @@ -0,0 +1,440 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/jrnl/txn_rec.h" + +#include <cassert> +#include <cerrno> +#include <cstdlib> +#include <cstring> +#include <iomanip> +#include "qpid/linearstore/jrnl/jerrno.h" +#include "qpid/linearstore/jrnl/jexception.h" +#include <sstream> + +namespace qpid +{ +namespace qls_jrnl +{ + +txn_rec::txn_rec(): +// _txn_hdr(), + _xidp(0), + _buff(0) +// _txn_tail() +{ + ::txn_hdr_init(&_txn_hdr, 0, QLS_JRNL_VERSION, 0, 0, 0); + ::rec_tail_init(&_txn_tail, 0, 0, 0); +} + +txn_rec::txn_rec(const uint32_t magic, const uint64_t rid, const void* const xidp, + const std::size_t xidlen/*, const bool owi*/): +// _txn_hdr(magic, RHM_JDAT_VERSION, rid, xidlen, owi), + _xidp(xidp), + _buff(0) +// _txn_tail(_txn_hdr) +{ + ::txn_hdr_init(&_txn_hdr, magic, QLS_JRNL_VERSION, 0, rid, xidlen); + ::rec_tail_copy(&_txn_tail, &_txn_hdr._rhdr, 0); +} + +txn_rec::~txn_rec() +{ + clean(); +} + +void +txn_rec::reset(const uint32_t magic) +{ + _txn_hdr._rhdr._magic = magic; + _txn_hdr._rhdr._rid = 0; + _txn_hdr._xidsize = 0; + _xidp = 0; + _buff = 0; + _txn_tail._xmagic = ~magic; + _txn_tail._rid = 0; +} + +void +txn_rec::reset(const uint32_t magic, const uint64_t rid, const void* const xidp, + const std::size_t xidlen/*, const bool owi*/) +{ + _txn_hdr._rhdr._magic = magic; + _txn_hdr._rhdr._rid = rid; +// _txn_hdr.set_owi(owi); + _txn_hdr._xidsize = xidlen; + _xidp = xidp; + _buff = 0; + _txn_tail._xmagic = ~magic; + _txn_tail._rid = rid; +} + +uint32_t +txn_rec::encode(void* wptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks) +{ + assert(wptr != 0); + assert(max_size_dblks > 0); + assert(_xidp != 0 && _txn_hdr._xidsize > 0); + + std::size_t rec_offs = rec_offs_dblks * QLS_DBLK_SIZE_BYTES; + std::size_t rem = max_size_dblks * QLS_DBLK_SIZE_BYTES; + std::size_t wr_cnt = 0; + if (rec_offs_dblks) // Continuation of split dequeue record (over 2 or more pages) + { + if (size_dblks(rec_size()) - rec_offs_dblks > max_size_dblks) // Further split required + { + rec_offs -= sizeof(txn_hdr_t); + std::size_t wsize = _txn_hdr._xidsize > rec_offs ? _txn_hdr._xidsize - rec_offs : 0; + std::size_t wsize2 = wsize; + if (wsize) + { + if (wsize > rem) + wsize = rem; + std::memcpy(wptr, (const char*)_xidp + rec_offs, wsize); + wr_cnt += wsize; + rem -= wsize; + } + rec_offs -= _txn_hdr._xidsize - wsize2; + if (rem) + { + wsize = sizeof(_txn_tail) > rec_offs ? sizeof(_txn_tail) - rec_offs : 0; + wsize2 = wsize; + if (wsize) + { + if (wsize > rem) + wsize = rem; + std::memcpy((char*)wptr + wr_cnt, (char*)&_txn_tail + rec_offs, wsize); + wr_cnt += wsize; + rem -= wsize; + } + rec_offs -= sizeof(_txn_tail) - wsize2; + } + assert(rem == 0); + assert(rec_offs == 0); + } + else // No further split required + { + rec_offs -= sizeof(txn_hdr_t); + std::size_t wsize = _txn_hdr._xidsize > rec_offs ? _txn_hdr._xidsize - rec_offs : 0; + if (wsize) + { + std::memcpy(wptr, (const char*)_xidp + rec_offs, wsize); + wr_cnt += wsize; + } + rec_offs -= _txn_hdr._xidsize - wsize; + wsize = sizeof(_txn_tail) > rec_offs ? sizeof(_txn_tail) - rec_offs : 0; + if (wsize) + { + std::memcpy((char*)wptr + wr_cnt, (char*)&_txn_tail + rec_offs, wsize); + wr_cnt += wsize; +#ifdef QLS_CLEAN + std::size_t rec_offs = rec_offs_dblks * QLS_DBLK_SIZE_BYTES; + std::size_t dblk_rec_size = size_dblks(rec_size() - rec_offs) * QLS_DBLK_SIZE_BYTES; + std::memset((char*)wptr + wr_cnt, QLS_CLEAN_CHAR, dblk_rec_size - wr_cnt); +#endif + } + rec_offs -= sizeof(_txn_tail) - wsize; + assert(rec_offs == 0); + } + } + else // Start at beginning of data record + { + // Assumption: the header will always fit into the first dblk + std::memcpy(wptr, (void*)&_txn_hdr, sizeof(txn_hdr_t)); + wr_cnt = sizeof(txn_hdr_t); + if (size_dblks(rec_size()) > max_size_dblks) // Split required + { + std::size_t wsize; + rem -= sizeof(txn_hdr_t); + if (rem) + { + wsize = rem >= _txn_hdr._xidsize ? _txn_hdr._xidsize : rem; + std::memcpy((char*)wptr + wr_cnt, _xidp, wsize); + wr_cnt += wsize; + rem -= wsize; + } + if (rem) + { + wsize = rem >= sizeof(_txn_tail) ? sizeof(_txn_tail) : rem; + std::memcpy((char*)wptr + wr_cnt, (void*)&_txn_tail, wsize); + wr_cnt += wsize; + rem -= wsize; + } + assert(rem == 0); + } + else // No split required + { + std::memcpy((char*)wptr + wr_cnt, _xidp, _txn_hdr._xidsize); + wr_cnt += _txn_hdr._xidsize; + std::memcpy((char*)wptr + wr_cnt, (void*)&_txn_tail, sizeof(_txn_tail)); + wr_cnt += sizeof(_txn_tail); +#ifdef QLS_CLEAN + std::size_t dblk_rec_size = size_dblks(rec_size()) * QLS_DBLK_SIZE_BYTES; + std::memset((char*)wptr + wr_cnt, QLS_CLEAN_CHAR, dblk_rec_size - wr_cnt); +#endif + } + } + return size_dblks(wr_cnt); +} + +uint32_t +txn_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks) +{ + assert(rptr != 0); + assert(max_size_dblks > 0); + + std::size_t rd_cnt = 0; + if (rec_offs_dblks) // Continuation of record on new page + { + const uint32_t hdr_xid_dblks = size_dblks(sizeof(txn_hdr_t) + _txn_hdr._xidsize); + const uint32_t hdr_xid_tail_dblks = size_dblks(sizeof(txn_hdr_t) + _txn_hdr._xidsize + sizeof(rec_tail_t)); + const std::size_t rec_offs = rec_offs_dblks * QLS_DBLK_SIZE_BYTES; + + if (hdr_xid_tail_dblks - rec_offs_dblks <= max_size_dblks) + { + // Remainder of xid fits within this page + if (rec_offs - sizeof(txn_hdr_t) < _txn_hdr._xidsize) + { + // Part of xid still outstanding, copy remainder of xid and tail + const std::size_t xid_offs = rec_offs - sizeof(txn_hdr_t); + const std::size_t xid_rem = _txn_hdr._xidsize - xid_offs; + std::memcpy((char*)_buff + xid_offs, rptr, xid_rem); + rd_cnt = xid_rem; + std::memcpy((void*)&_txn_tail, ((char*)rptr + rd_cnt), sizeof(_txn_tail)); + chk_tail(); + rd_cnt += sizeof(_txn_tail); + } + else + { + // Tail or part of tail only outstanding, complete tail + const std::size_t tail_offs = rec_offs - sizeof(txn_hdr_t) - _txn_hdr._xidsize; + const std::size_t tail_rem = sizeof(rec_tail_t) - tail_offs; + std::memcpy((char*)&_txn_tail + tail_offs, rptr, tail_rem); + chk_tail(); + rd_cnt = tail_rem; + } + } + else if (hdr_xid_dblks - rec_offs_dblks <= max_size_dblks) + { + // Remainder of xid fits within this page, tail split + const std::size_t xid_offs = rec_offs - sizeof(txn_hdr_t); + const std::size_t xid_rem = _txn_hdr._xidsize - xid_offs; + std::memcpy((char*)_buff + xid_offs, rptr, xid_rem); + rd_cnt += xid_rem; + const std::size_t tail_rem = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt; + if (tail_rem) + { + std::memcpy((void*)&_txn_tail, ((char*)rptr + xid_rem), tail_rem); + rd_cnt += tail_rem; + } + } + else + { + // Remainder of xid split + const std::size_t xid_cp_size = (max_size_dblks * QLS_DBLK_SIZE_BYTES); + std::memcpy((char*)_buff + rec_offs - sizeof(txn_hdr_t), rptr, xid_cp_size); + rd_cnt += xid_cp_size; + } + } + else // Start of record + { + // Get and check header + //_txn_hdr.hdr_copy(h); + ::rec_hdr_copy(&_txn_hdr._rhdr, &h); + rd_cnt = sizeof(rec_hdr_t); +#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT) + rd_cnt += sizeof(uint32_t); // Filler 0 +#endif + _txn_hdr._xidsize = *(std::size_t*)((char*)rptr + rd_cnt); + rd_cnt = sizeof(txn_hdr_t); + chk_hdr(); + _buff = std::malloc(_txn_hdr._xidsize); + MALLOC_CHK(_buff, "_buff", "txn_rec", "decode"); + const uint32_t hdr_xid_dblks = size_dblks(sizeof(txn_hdr_t) + _txn_hdr._xidsize); + const uint32_t hdr_xid_tail_dblks = size_dblks(sizeof(txn_hdr_t) + _txn_hdr._xidsize + + sizeof(rec_tail_t)); + + // Check if record (header + xid + tail) fits within this page, we can check the + // tail before the expense of copying data to memory + if (hdr_xid_tail_dblks <= max_size_dblks) + { + // Entire header, xid and tail fits within this page + std::memcpy(_buff, (char*)rptr + rd_cnt, _txn_hdr._xidsize); + rd_cnt += _txn_hdr._xidsize; + std::memcpy((void*)&_txn_tail, (char*)rptr + rd_cnt, sizeof(_txn_tail)); + rd_cnt += sizeof(_txn_tail); + chk_tail(); + } + else if (hdr_xid_dblks <= max_size_dblks) + { + // Entire header and xid fit within this page, tail split + std::memcpy(_buff, (char*)rptr + rd_cnt, _txn_hdr._xidsize); + rd_cnt += _txn_hdr._xidsize; + const std::size_t tail_rem = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt; + if (tail_rem) + { + std::memcpy((void*)&_txn_tail, (char*)rptr + rd_cnt, tail_rem); + rd_cnt += tail_rem; + } + } + else + { + // Header fits within this page, xid split + const std::size_t xid_cp_size = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt; + std::memcpy(_buff, (char*)rptr + rd_cnt, xid_cp_size); + rd_cnt += xid_cp_size; + } + } + return size_dblks(rd_cnt); +} + +bool +txn_rec::rcv_decode(rec_hdr_t h, std::ifstream* ifsp, std::size_t& rec_offs) +{ + if (rec_offs == 0) + { + // Read header, allocate for xid + //_txn_hdr.hdr_copy(h); + ::rec_hdr_copy(&_txn_hdr._rhdr, &h); +#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT) + ifsp->ignore(sizeof(uint32_t)); // _filler0 +#endif + ifsp->read((char*)&_txn_hdr._xidsize, sizeof(std::size_t)); +#if defined(JRNL_LITTLE_ENDIAN) && defined(JRNL_32_BIT) + ifsp->ignore(sizeof(uint32_t)); // _filler0 +#endif + rec_offs = sizeof(txn_hdr_t); + _buff = std::malloc(_txn_hdr._xidsize); + MALLOC_CHK(_buff, "_buff", "txn_rec", "rcv_decode"); + } + if (rec_offs < sizeof(txn_hdr_t) + _txn_hdr._xidsize) + { + // Read xid (or continue reading xid) + std::size_t offs = rec_offs - sizeof(txn_hdr_t); + ifsp->read((char*)_buff + offs, _txn_hdr._xidsize - offs); + std::size_t size_read = ifsp->gcount(); + rec_offs += size_read; + if (size_read < _txn_hdr._xidsize - offs) + { + assert(ifsp->eof()); + // As we may have read past eof, turn off fail bit + ifsp->clear(ifsp->rdstate()&(~std::ifstream::failbit)); + assert(!ifsp->fail() && !ifsp->bad()); + return false; + } + } + if (rec_offs < sizeof(txn_hdr_t) + _txn_hdr._xidsize + sizeof(rec_tail_t)) + { + // Read tail (or continue reading tail) + std::size_t offs = rec_offs - sizeof(txn_hdr_t) - _txn_hdr._xidsize; + ifsp->read((char*)&_txn_tail + offs, sizeof(rec_tail_t) - offs); + std::size_t size_read = ifsp->gcount(); + rec_offs += size_read; + if (size_read < sizeof(rec_tail_t) - offs) + { + assert(ifsp->eof()); + // As we may have read past eof, turn off fail bit + ifsp->clear(ifsp->rdstate()&(~std::ifstream::failbit)); + assert(!ifsp->fail() && !ifsp->bad()); + return false; + } + } + ifsp->ignore(rec_size_dblks() * QLS_DBLK_SIZE_BYTES - rec_size()); + chk_tail(); // Throws if tail invalid or record incomplete + assert(!ifsp->fail() && !ifsp->bad()); + return true; +} + +std::size_t +txn_rec::get_xid(void** const xidpp) +{ + if (!_buff) + { + *xidpp = 0; + return 0; + } + *xidpp = _buff; + return _txn_hdr._xidsize; +} + +std::string& +txn_rec::str(std::string& str) const +{ + std::ostringstream oss; + if (_txn_hdr._rhdr._magic == QLS_TXA_MAGIC) + oss << "dtxa_rec: m=" << _txn_hdr._rhdr._magic; + else + oss << "dtxc_rec: m=" << _txn_hdr._rhdr._magic; + oss << " v=" << (int)_txn_hdr._rhdr._version; + oss << " rid=" << _txn_hdr._rhdr._rid; + oss << " xid=\"" << _xidp << "\""; + str.append(oss.str()); + return str; +} + +std::size_t +txn_rec::xid_size() const +{ + return _txn_hdr._xidsize; +} + +std::size_t +txn_rec::rec_size() const +{ + return sizeof(txn_hdr_t) + _txn_hdr._xidsize + sizeof(rec_tail_t); +} + +void +txn_rec::chk_hdr() const +{ + jrec::chk_hdr(_txn_hdr._rhdr); + if (_txn_hdr._rhdr._magic != QLS_TXA_MAGIC && _txn_hdr._rhdr._magic != QLS_TXC_MAGIC) + { + std::ostringstream oss; + oss << std::hex << std::setfill('0'); + oss << "dtx magic: rid=0x" << std::setw(16) << _txn_hdr._rhdr._rid; + oss << ": expected=(0x" << std::setw(8) << QLS_TXA_MAGIC; + oss << " or 0x" << QLS_TXC_MAGIC; + oss << ") read=0x" << std::setw(2) << (int)_txn_hdr._rhdr._magic; + throw jexception(jerrno::JERR_JREC_BADRECHDR, oss.str(), "txn_rec", "chk_hdr"); + } +} + +void +txn_rec::chk_hdr(uint64_t rid) const +{ + chk_hdr(); + jrec::chk_rid(_txn_hdr._rhdr, rid); +} + +void +txn_rec::chk_tail() const +{ + jrec::chk_tail(_txn_tail, _txn_hdr._rhdr); +} + +void +txn_rec::clean() +{ + // clean up allocated memory here +} + +}} diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/txn_rec.h b/qpid/cpp/src/qpid/linearstore/jrnl/txn_rec.h new file mode 100644 index 0000000000..4e9700e392 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/txn_rec.h @@ -0,0 +1,88 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_JRNL_TXN_REC_H +#define QPID_LEGACYSTORE_JRNL_TXN_REC_H + +namespace qpid +{ +namespace qls_jrnl +{ +class txn_rec; +}} + +#include <cstddef> +#include "qpid/linearstore/jrnl/jrec.h" +#include "qpid/linearstore/jrnl/utils/txn_hdr.h" + +namespace qpid +{ +namespace qls_jrnl +{ + + /** + * \class txn_rec + * \brief Class to handle a single journal DTX commit or abort record. + */ + class txn_rec : public jrec + { + private: + txn_hdr_t _txn_hdr; ///< transaction header + const void* _xidp; ///< xid pointer for encoding (writing to disk) + void* _buff; ///< Pointer to buffer to receive data read from disk + rec_tail_t _txn_tail; ///< Record tail + + public: + // constructor used for read operations and xid must have memory allocated + txn_rec(); + // constructor used for write operations, where xid already exists + txn_rec(const uint32_t magic, const uint64_t rid, const void* const xidp, + const std::size_t xidlen/*, const bool owi*/); + virtual ~txn_rec(); + + // Prepare instance for use in reading data from journal + void reset(const uint32_t magic); + // Prepare instance for use in writing data to journal + void reset(const uint32_t magic, const uint64_t rid, const void* const xidp, + const std::size_t xidlen/*, const bool owi*/); + uint32_t encode(void* wptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks); + uint32_t decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, + uint32_t max_size_dblks); + // Decode used for recover + bool rcv_decode(rec_hdr_t h, std::ifstream* ifsp, std::size_t& rec_offs); + + std::size_t get_xid(void** const xidpp); + std::string& str(std::string& str) const; + inline std::size_t data_size() const { return 0; } // This record never carries data + std::size_t xid_size() const; + std::size_t rec_size() const; + inline uint64_t rid() const { return _txn_hdr._rhdr._rid; } + + private: + void chk_hdr() const; + void chk_hdr(uint64_t rid) const; + void chk_tail() const; + virtual void clean(); + }; // class txn_rec + +}} + +#endif // ifndef QPID_LEGACYSTORE_JRNL_TXN_REC_H diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/utils/deq_hdr.c b/qpid/cpp/src/qpid/linearstore/jrnl/utils/deq_hdr.c new file mode 100644 index 0000000000..0cdab3580c --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/utils/deq_hdr.c @@ -0,0 +1,46 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "deq_hdr.h" + +/*static const uint16_t DEQ_HDR_TXNCMPLCOMMIT_MASK = 0x10;*/ + +void deq_hdr_init(deq_hdr_t* dest, const uint32_t magic, const uint16_t version, const uint16_t uflag, const uint64_t + rid, const uint64_t deq_rid, const uint64_t xidsize) { + rec_hdr_init(&dest->_rhdr, magic, version, uflag, rid); + dest->_deq_rid = deq_rid; + dest->_xidsize = xidsize; +} + +void deq_hdr_copy(deq_hdr_t* dest, const deq_hdr_t* src) { + rec_hdr_copy(&dest->_rhdr, &src->_rhdr); + dest->_deq_rid = src->_deq_rid; + dest->_xidsize = src->_xidsize; +} + +bool is_txn_coml_commit(const deq_hdr_t *dh) { + return dh->_rhdr._uflag & DEQ_HDR_TXNCMPLCOMMIT_MASK; +} + +void set_txn_coml_commit(deq_hdr_t *dh, const bool commit) { + dh->_rhdr._uflag = commit ? dh->_rhdr._uflag | DEQ_HDR_TXNCMPLCOMMIT_MASK : + dh->_rhdr._uflag & (~DEQ_HDR_TXNCMPLCOMMIT_MASK); +} diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/utils/deq_hdr.h b/qpid/cpp/src/qpid/linearstore/jrnl/utils/deq_hdr.h new file mode 100644 index 0000000000..25cfc71efb --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/utils/deq_hdr.h @@ -0,0 +1,81 @@ +#ifndef QPID_LINEARSTORE_JRNL_UTILS_DEQ_HDR_H +#define QPID_LINEARSTORE_JRNL_UTILS_DEQ_HDR_H +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include <stdbool.h> +#include "rec_hdr.h" + +#ifdef __cplusplus +extern "C"{ +#endif + +#pragma pack(1) + +/** + * \brief Struct for dequeue record. + * + * Struct for dequeue record. If this record has a non-zero xidsize field (i.e., there is a + * valid XID), then this header is followed by the XID of xidsize bytes and a rec_tail. If, + * on the other hand, this record has a zero xidsize (i.e., there is no XID), then the rec_tail + * is absent. + * + * Note that this record had its own rid distinct from the rid of the record it is dequeueing. + * The rid field below is the rid of the dequeue record itself; the deq-rid field is the rid of a + * previous enqueue record being dequeued by this record. + * + * Record header info in binary format (32 bytes): + * <pre> + * 0 7 + * +---+---+---+---+---+---+---+---+ -+ + * | magic | ver | flags | | + * +---+---+---+---+---+---+---+---+ | struct rec_hdr_t + * | rid | | + * +---+---+---+---+---+---+---+---+ -+ + * | deq-rid | + * +---+---+---+---+---+---+---+---+ + * | xidsize | + * +---+---+---+---+---+---+---+---+ + * + * deq-rid = dequeue record ID + * </pre> + */ +typedef struct deq_hdr_t { + rec_hdr_t _rhdr; /**< Common record header struct */ + uint64_t _deq_rid; /**< Record ID of record being dequeued */ + uint64_t _xidsize; /**< XID size */ +} deq_hdr_t; + +static const uint16_t DEQ_HDR_TXNCMPLCOMMIT_MASK = 0x10; + +void deq_hdr_init(deq_hdr_t* dest, const uint32_t magic, const uint16_t version, const uint16_t uflag, + const uint64_t rid, const uint64_t deq_rid, const uint64_t xidsize); +void deq_hdr_copy(deq_hdr_t* dest, const deq_hdr_t* src); +bool is_txn_coml_commit(const deq_hdr_t *dh); +void set_txn_coml_commit(deq_hdr_t *dh, const bool commit); + +#pragma pack() + +#ifdef __cplusplus +} +#endif + +#endif /* ifndef QPID_LINEARSTORE_JRNL_UTILS_DEQ_HDR_H */ diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/utils/enq_hdr.c b/qpid/cpp/src/qpid/linearstore/jrnl/utils/enq_hdr.c new file mode 100644 index 0000000000..b8d2da7722 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/utils/enq_hdr.c @@ -0,0 +1,63 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "enq_hdr.h" + +//static const uint16_t ENQ_HDR_TRANSIENT_MASK = 0x10; +//static const uint16_t ENQ_HDR_EXTERNAL_MASK = 0x20; + +void enq_hdr_init(enq_hdr_t* dest, const uint32_t magic, const uint16_t version, const uint16_t uflag, + const uint64_t rid, const uint64_t xidsize, const uint64_t dsize) { + rec_hdr_init(&dest->_rhdr, magic, version, uflag, rid); + dest->_xidsize = xidsize; + dest->_dsize = dsize; +} + +void enq_hdr_copy(enq_hdr_t* dest, const enq_hdr_t* src) { + rec_hdr_copy(&dest->_rhdr, &src->_rhdr); + dest->_xidsize = src->_xidsize; + dest->_dsize = src->_dsize; +} + +bool is_enq_transient(const enq_hdr_t *eh) { + return eh->_rhdr._uflag & ENQ_HDR_TRANSIENT_MASK; +} + +void set_enq_transient(enq_hdr_t *eh, const bool transient) { + eh->_rhdr._uflag = transient ? eh->_rhdr._uflag | ENQ_HDR_TRANSIENT_MASK : + eh->_rhdr._uflag & (~ENQ_HDR_TRANSIENT_MASK); +} + +bool is_enq_external(const enq_hdr_t *eh) { + return eh->_rhdr._uflag & ENQ_HDR_EXTERNAL_MASK; +} + +void set_enq_external(enq_hdr_t *eh, const bool external) { + eh->_rhdr._uflag = external ? eh->_rhdr._uflag | ENQ_HDR_EXTERNAL_MASK : + eh->_rhdr._uflag & (~ENQ_HDR_EXTERNAL_MASK); +} + +bool validate_enq_hdr(enq_hdr_t *eh, const uint32_t magic, const uint16_t version, const uint64_t rid) { + return eh->_rhdr._magic == magic && + eh->_rhdr._version == version && + rid > 0 ? eh->_rhdr._rid == rid /* If rid == 0, don't compare rids */ + : true; +} diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/utils/enq_hdr.h b/qpid/cpp/src/qpid/linearstore/jrnl/utils/enq_hdr.h new file mode 100644 index 0000000000..1beaef1db8 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/utils/enq_hdr.h @@ -0,0 +1,81 @@ +#ifndef QPID_LINEARSTORE_JRNL_UTILS_ENQ_HDR_H +#define QPID_LINEARSTORE_JRNL_UTILS_ENQ_HDR_H +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include <stdbool.h> +#include "rec_hdr.h" + +#ifdef __cplusplus +extern "C"{ +#endif + +#pragma pack(1) + +/** + * \brief Struct for enqueue record. + * + * Struct for enqueue record. In addition to the common data, this header includes both the + * xid and data blob sizes. + * + * This header precedes all enqueue data in journal files. + * + * Record header info in binary format (32 bytes): + * <pre> + * 0 7 + * +---+---+---+---+---+---+---+---+ -+ + * | magic | ver | flags | | + * +---+---+---+---+---+---+---+---+ | struct rec_hdr_t + * | rid | | + * +---+---+---+---+---+---+---+---+ -+ + * | xidsize | + * +---+---+---+---+---+---+---+---+ + * | dsize | + * +---+---+---+---+---+---+---+---+ + * v = file version (If the format or encoding of this file changes, then this + * number should be incremented) + * </pre> + */ +typedef struct enq_hdr_t { + rec_hdr_t _rhdr; /**< Common record header struct */ + uint64_t _xidsize; /**< XID size in octets */ + uint64_t _dsize; /**< Record data size in octets */ +} enq_hdr_t; + +static const uint16_t ENQ_HDR_TRANSIENT_MASK = 0x10; +static const uint16_t ENQ_HDR_EXTERNAL_MASK = 0x20; + +void enq_hdr_init(enq_hdr_t* dest, const uint32_t magic, const uint16_t version, const uint16_t uflag, + const uint64_t rid, const uint64_t xidsize, const uint64_t dsize); +void enq_hdr_copy(enq_hdr_t* dest, const enq_hdr_t* src); +bool is_enq_transient(const enq_hdr_t *eh); +void set_enq_transient(enq_hdr_t *eh, const bool transient); +bool is_enq_external(const enq_hdr_t *eh); +void set_enq_external(enq_hdr_t *eh, const bool external); +bool validate_enq_hdr(enq_hdr_t *eh, const uint32_t magic, const uint16_t version, const uint64_t rid); + +#pragma pack() + +#ifdef __cplusplus +} +#endif + +#endif /* ifndef QPID_LINEARSTORE_JRNL_UTILS_ENQ_HDR_H */ diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/utils/file_hdr.c b/qpid/cpp/src/qpid/linearstore/jrnl/utils/file_hdr.c new file mode 100644 index 0000000000..7e53be5e18 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/utils/file_hdr.c @@ -0,0 +1,105 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "file_hdr.h" +#include <string.h> + +void file_hdr_create(file_hdr_t* dest, const uint32_t magic, const uint16_t version, const uint16_t fhdr_size_sblks, + const uint16_t efp_partition, const uint64_t file_size) { + rec_hdr_init(&dest->_rhdr, magic, version, 0, 0); + dest->_fhdr_size_sblks = fhdr_size_sblks; + dest->_efp_partition = efp_partition; + dest->_reserved = 0; + dest->_file_size_kib = file_size; + dest->_fro = 0; + dest->_ts_nsec = 0; + dest->_ts_sec = 0; + dest->_file_number = 0; + dest->_queue_name_len = 0; +} + +int file_hdr_init(void* dest, const uint64_t dest_len, const uint16_t uflag, const uint64_t rid, const uint64_t fro, + const uint64_t file_number, const uint16_t queue_name_len, const char* queue_name) { + file_hdr_t* fhp = (file_hdr_t*)dest; + fhp->_rhdr._uflag = uflag; + fhp->_rhdr._rid = rid; + fhp->_fro = fro; + fhp->_file_number = file_number; + if (sizeof(file_hdr_t) + queue_name_len < MAX_FILE_HDR_LEN) { + fhp->_queue_name_len = queue_name_len; + } else { + fhp->_queue_name_len = MAX_FILE_HDR_LEN - sizeof(file_hdr_t); + } + fhp->_queue_name_len = queue_name_len; + memcpy((char*)dest + sizeof(file_hdr_t), queue_name, queue_name_len); + memset((char*)dest + sizeof(file_hdr_t) + queue_name_len, 0, dest_len - sizeof(file_hdr_t) - queue_name_len); + return set_time_now(dest); +} + +void file_hdr_copy(file_hdr_t* dest, const file_hdr_t* src) { + rec_hdr_copy(&dest->_rhdr, &src->_rhdr); + dest->_fhdr_size_sblks = src->_fhdr_size_sblks; // Should this be copied? + dest->_efp_partition = src->_efp_partition; // Should this be copied? + dest->_file_size_kib = src->_file_size_kib; + dest->_fro = src->_fro; + dest->_ts_sec = src->_ts_sec; + dest->_ts_nsec = src->_ts_nsec; + dest->_file_number = src->_file_number; +} + +void file_hdr_reset(file_hdr_t* target) { + target->_rhdr._uflag = 0; + target->_rhdr._rid = 0; + target->_fro = 0; + target->_ts_sec = 0; + target->_ts_nsec = 0; + target->_file_number = 0; + target->_queue_name_len = 0; +} + +int is_file_hdr_reset(file_hdr_t* target) { + return target->_rhdr._uflag == 0 && + target->_rhdr._rid == 0 && + target->_ts_sec == 0 && + target->_ts_nsec == 0 && + target->_file_number == 0 && + target->_queue_name_len == 0; +} + +int set_time_now(file_hdr_t *fh) +{ + struct timespec ts; + int err = clock_gettime(CLOCK_REALTIME, &ts); + if (err) + return err; + fh->_ts_sec = ts.tv_sec; + fh->_ts_nsec = ts.tv_nsec; + return 0; +} + + +void set_time(file_hdr_t *fh, struct timespec *ts) +{ + fh->_ts_sec = ts->tv_sec; + fh->_ts_nsec = ts->tv_nsec; +} + + diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/utils/file_hdr.h b/qpid/cpp/src/qpid/linearstore/jrnl/utils/file_hdr.h new file mode 100644 index 0000000000..efdd97b624 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/utils/file_hdr.h @@ -0,0 +1,106 @@ +#ifndef QPID_LINEARSTORE_JRNL_UTILS_FILE_HDR_H +#define QPID_LINEARSTORE_JRNL_UTILS_FILE_HDR_H +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include <time.h> +#include "rec_hdr.h" + +#ifdef __cplusplus +extern "C"{ +#endif + +#define MAX_FILE_HDR_LEN 4096 // Set to 1 sblk + +#pragma pack(1) + +/** + * \brief Struct for data common to the head of all journal files. In addition to + * the common data, this includes the record ID and offset of the first record in + * the file. + * + * This header precedes all data in journal files and occupies the first complete + * block in the file. The record ID and offset are updated on each overwrite of the + * file. + * + * File header info in binary format (66 bytes + size of file name in octets): + * <pre> + * 0 7 + * +---+---+---+---+---+---+---+---+ -+ + * | magic | ver | flags | | + * +---+---+---+---+---+---+---+---+ | struct rec_hdr_t + * | first rid in file | | + * +---+---+---+---+---+---+---+---+ -+ + * | fs | partn | reserved | + * +---+---+---+---+---+---+---+---+ + * | file-size | + * +---+---+---+---+---+---+---+---+ + * | fro | + * +---+---+---+---+---+---+---+---+ + * | timestamp (sec) | + * +---+---+---+---+---+---+---+---+ + * | timestamp (ns) | + * +---+---+---+---+---+---+---+---+ + * | file-number | + * +---+---+---+---+---+---+---+---+ + * | qnl | Queue Name... | + * +-------+ | + * | | + * +---+---+---+---+---+---+---+---+ + * + * ver = Journal version + * rid = Record ID + * fs = File header size in sblks (defined by JRNL_SBLK_SIZE) + * partn = EFP partition from which this file came + * fro = First Record Offset + * qnl = Length of the queue name in octets. + * </pre> + */ +typedef struct file_hdr_t { + rec_hdr_t _rhdr; /**< Common record header struct, but rid field is used for rid of first compete record in file */ + uint16_t _fhdr_size_sblks; /**< File header size in sblks (defined by JRNL_SBLK_SIZE) */ + uint16_t _efp_partition; /**< EFP Partition number from which this file was obtained */ + uint32_t _reserved; + uint64_t _file_size_kib; /**< Size of this file in KiB, excluding header sblk */ + uint64_t _fro; /**< First Record Offset (FRO) */ + uint64_t _ts_sec; /**< Time stamp (seconds part) */ + uint64_t _ts_nsec; /**< Time stamp (nanoseconds part) */ + uint64_t _file_number; /**< The logical number of this file in a monotonically increasing sequence */ + uint16_t _queue_name_len; /**< Length of the queue name in octets, which follows this struct in the header */ +} file_hdr_t; + +void file_hdr_create(file_hdr_t* dest, const uint32_t magic, const uint16_t version, const uint16_t fhdr_size_sblks, + const uint16_t efp_partition, const uint64_t file_size); +int file_hdr_init(void* dest, const uint64_t dest_len, const uint16_t uflag, const uint64_t rid, const uint64_t fro, + const uint64_t file_number, const uint16_t queue_name_len, const char* queue_name); +void file_hdr_reset(file_hdr_t* target); +int is_file_hdr_reset(file_hdr_t* target); +void file_hdr_copy(file_hdr_t* dest, const file_hdr_t* src); +int set_time_now(file_hdr_t *fh); +void set_time(file_hdr_t *fh, struct timespec *ts); + +#pragma pack() + +#ifdef __cplusplus +} +#endif + +#endif /* ifndef QPID_LINEARSTORE_JRNL_UTILS_FILE_HDR_H */ diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/utils/rec_hdr.c b/qpid/cpp/src/qpid/linearstore/jrnl/utils/rec_hdr.c new file mode 100644 index 0000000000..be56068b9e --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/utils/rec_hdr.c @@ -0,0 +1,15 @@ +#include "rec_hdr.h" + +void rec_hdr_init(rec_hdr_t* dest, const uint32_t magic, const uint16_t version, const uint16_t uflag, const uint64_t rid) { + dest->_magic = magic; + dest->_version = version; + dest->_uflag = uflag; + dest->_rid = rid; +} + +void rec_hdr_copy(rec_hdr_t* dest, const rec_hdr_t* src) { + dest->_magic = src->_magic; + dest->_version = src->_version; + dest->_uflag = src->_uflag; + dest->_rid = src->_rid; +} diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/utils/rec_hdr.h b/qpid/cpp/src/qpid/linearstore/jrnl/utils/rec_hdr.h new file mode 100644 index 0000000000..c843e21316 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/utils/rec_hdr.h @@ -0,0 +1,67 @@ +#ifndef QPID_LINEARSTORE_JRNL_UTILS_REC_HDR_H +#define QPID_LINEARSTORE_JRNL_UTILS_REC_HDR_H +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include <stdint.h> + +#ifdef __cplusplus +extern "C"{ +#endif + +#pragma pack(1) + +/** + * \brief Struct for data common to the head of all journal files and records. + * This includes identification for the file type, the encoding version, endian + * indicator and a record ID. + * + * File header info in binary format (16 bytes): + * <pre> + * 0 7 + * +---+---+---+---+---+---+---+---+ + * | magic | ver | flags | + * +---+---+---+---+---+---+---+---+ + * | rid | + * +---+---+---+---+---+---+---+---+ + * + * ver = file version (If the format or encoding of this file changes, then this + * number should be incremented) + * rid = Record ID + * </pre> + */ +typedef struct rec_hdr_t { + uint32_t _magic; /**< File type identifier (magic number) */ + uint16_t _version; /**< File encoding version */ + uint16_t _uflag; /**< User-defined flags */ + uint64_t _rid; /**< Record ID (rotating 64-bit counter) */ +} rec_hdr_t; + +void rec_hdr_init(rec_hdr_t* dest, const uint32_t magic, const uint16_t version, const uint16_t uflag, const uint64_t rid); +void rec_hdr_copy(rec_hdr_t* dest, const rec_hdr_t* src); + +#pragma pack() + +#ifdef __cplusplus +} +#endif + +#endif /* ifndef QPID_LINEARSTORE_JRNL_UTILS_REC_HDR_H */ diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/utils/rec_tail.c b/qpid/cpp/src/qpid/linearstore/jrnl/utils/rec_tail.c new file mode 100644 index 0000000000..2e604eb6c6 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/utils/rec_tail.c @@ -0,0 +1,34 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "rec_tail.h" + +void rec_tail_init(rec_tail_t* dest, const uint32_t xmagic, const uint32_t checksum, const uint64_t rid) { + dest->_xmagic = xmagic; + dest->_checksum = checksum; + dest->_rid = rid; +} + +void rec_tail_copy(rec_tail_t* dest, const rec_hdr_t* src, const uint32_t checksum) { + dest->_xmagic = ~(src->_magic); + dest->_checksum = checksum; + dest->_rid = src->_rid; +} diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/utils/rec_tail.h b/qpid/cpp/src/qpid/linearstore/jrnl/utils/rec_tail.h new file mode 100644 index 0000000000..9fbd186ea0 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/utils/rec_tail.h @@ -0,0 +1,71 @@ +#ifndef QPID_LEGACYSTORE_JRNL_UTILS_REC_TAIL_H +#define QPID_LEGACYSTORE_JRNL_UTILS_REC_TAIL_H +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include <stdint.h> +#include "rec_hdr.h" + +#ifdef __cplusplus +extern "C"{ +#endif + +#pragma pack(1) + +/** + * \brief Struct for data common to the tail of all records. The magic number + * used here is the binary inverse (1's complement) of the magic used in the + * record header; this minimizes possible confusion with other headers that may + * be present during recovery. The tail is used with all records that have either + * XIDs or data - ie any size-variable content. Currently the only records that + * do NOT use the tail are non-transactional dequeues and filler records. + * + * The checksum is used to verify the xid and/or data portion of the record + * on recovery, and excludes the header and tail. + * + * Record header info in binary format (16 bytes): + * <pre> + * 0 7 + * +---+---+---+---+---+---+---+---+ + * | ~(magic) | checksum | + * +---+---+---+---+---+---+---+---+ + * | rid | + * +---+---+---+---+---+---+---+---+ + * + * rid = Record ID + * </pre> + */ +typedef struct rec_tail_t { + uint32_t _xmagic; /**< Binary inverse (1's complement) of hdr magic number */ + uint32_t _checksum; /**< Checksum of xid and data */ + uint64_t _rid; /**< ID (rotating 64-bit counter) */ +} rec_tail_t; + +void rec_tail_init(rec_tail_t* dest, const uint32_t xmagic, const uint32_t checksum, const uint64_t rid); +void rec_tail_copy(rec_tail_t* dest, const rec_hdr_t* src, const uint32_t checksum); + +#pragma pack() + +#ifdef __cplusplus +} +#endif + +#endif /* ifnedf QPID_LEGACYSTORE_JRNL_UTILS_REC_TAIL_H */ diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/utils/txn_hdr.c b/qpid/cpp/src/qpid/linearstore/jrnl/utils/txn_hdr.c new file mode 100644 index 0000000000..3b94bf3bb8 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/utils/txn_hdr.c @@ -0,0 +1,33 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "txn_hdr.h" + +void txn_hdr_init(txn_hdr_t* dest, const uint32_t magic, const uint16_t version, const uint16_t uflag, + const uint64_t rid, const uint64_t xidsize) { + rec_hdr_init(&dest->_rhdr, magic, version, uflag, rid); + dest->_xidsize = xidsize; +} + +void txn_hdr_copy(txn_hdr_t* dest, const txn_hdr_t* src) { + rec_hdr_copy(&dest->_rhdr, &src->_rhdr); + dest->_xidsize = src->_xidsize; +} diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/utils/txn_hdr.h b/qpid/cpp/src/qpid/linearstore/jrnl/utils/txn_hdr.h new file mode 100644 index 0000000000..02c00e09b3 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/utils/txn_hdr.h @@ -0,0 +1,70 @@ +#ifndef QPID_LINEARSTORE_JRNL_UTILS_TXN_HDR_H +#define QPID_LINEARSTORE_JRNL_UTILS_TXN_HDR_H +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "rec_hdr.h" + +#ifdef __cplusplus +extern "C"{ +#endif + +#pragma pack(1) + +/** + * \brief Struct for transaction commit and abort records. + * + * Struct for local and DTX commit and abort records. Only the magic distinguishes between them. + * Since this record must be used in the context of a valid XID, the xidsize field must not be + * zero. Immediately following this record is the XID itself which is xidsize bytes long, + * followed by a rec_tail. + * + * Note that this record had its own rid distinct from the rids of the record(s) making up the + * transaction it is committing or aborting. + * + * Record header info in binary format (24 bytes): + * <pre> + * 0 7 + * +---+---+---+---+---+---+---+---+ -+ + * | magic | v | e | flags | | + * +---+---+---+---+---+---+---+---+ | struct rec_hdr_t + * | rid | | + * +---+---+---+---+---+---+---+---+ -+ + * | xidsize | + * +---+---+---+---+---+---+---+---+ + * </pre> + */ +typedef struct txn_hdr_t { + rec_hdr_t _rhdr; /**< Common record header struct */ + uint64_t _xidsize; /**< XID size */ +} txn_hdr_t; + +void txn_hdr_init(txn_hdr_t* dest, const uint32_t magic, const uint16_t version, const uint16_t uflag, + const uint64_t rid, const uint64_t xidsize); +void txn_hdr_copy(txn_hdr_t* dest, const txn_hdr_t* src); + +#pragma pack() + +#ifdef __cplusplus +} +#endif + +#endif /* ifndef QPID_LINEARSTORE_JRNL_UTILS_TXN_HDR_H */ diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/wmgr.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/wmgr.cpp new file mode 100644 index 0000000000..a4ed6cbf19 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/wmgr.cpp @@ -0,0 +1,1031 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/linearstore/jrnl/wmgr.h" + +#include <cassert> +#include <cerrno> +#include <cstdlib> +#include <cstring> +#include "qpid/linearstore/jrnl/utils/file_hdr.h" +#include "qpid/linearstore/jrnl/jcfg.h" +#include "qpid/linearstore/jrnl/jcntl.h" +#include "qpid/linearstore/jrnl/jerrno.h" +#include "qpid/linearstore/jrnl/JournalFile.h" +#include <sstream> +#include <stdint.h> + +#include <iostream> // DEBUG + +namespace qpid +{ +namespace qls_jrnl +{ + +wmgr::wmgr(jcntl* jc, + enq_map& emap, + txn_map& tmap, + LinearFileController& lfc): + pmgr(jc, emap, tmap), + _lfc(lfc), + _max_dtokpp(0), + _max_io_wait_us(0), + _cached_offset_dblks(0), + _enq_busy(false), + _deq_busy(false), + _abort_busy(false), + _commit_busy(false), + _txn_pending_set() +{} + +wmgr::wmgr(jcntl* jc, + enq_map& emap, + txn_map& tmap, + LinearFileController& lfc, + const uint32_t max_dtokpp, + const uint32_t max_iowait_us): + pmgr(jc, emap, tmap), + _lfc(lfc), + _max_dtokpp(max_dtokpp), + _max_io_wait_us(max_iowait_us), + _cached_offset_dblks(0), + _enq_busy(false), + _deq_busy(false), + _abort_busy(false), + _commit_busy(false), + _txn_pending_set() +{} + +wmgr::~wmgr() +{ + wmgr::clean(); +} + +void +wmgr::initialize(aio_callback* const cbp, + const uint32_t wcache_pgsize_sblks, + const uint16_t wcache_num_pages, + const uint32_t max_dtokpp, + const uint32_t max_iowait_us, + std::size_t eo) +{ + _enq_busy = false; + _deq_busy = false; + _abort_busy = false; + _commit_busy = false; + _max_dtokpp = max_dtokpp; + _max_io_wait_us = max_iowait_us; + + initialize(cbp, wcache_pgsize_sblks, wcache_num_pages); + + if (eo) + { + const uint32_t wr_pg_size_dblks = _cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS; + uint32_t data_dblks = (eo / QLS_DBLK_SIZE_BYTES) - 4; // 4 dblks for file hdr + _pg_cntr = data_dblks / wr_pg_size_dblks; + _pg_offset_dblks = data_dblks - (_pg_cntr * wr_pg_size_dblks); + } +} + +iores +wmgr::enqueue(const void* const data_buff, + const std::size_t tot_data_len, + const std::size_t this_data_len, + data_tok* dtokp, + const void* const xid_ptr, + const std::size_t xid_len, + const bool transient, + const bool external) +{ + if (xid_len) + assert(xid_ptr != 0); + + if (_deq_busy || _abort_busy || _commit_busy) { + std::ostringstream oss; + oss << "RHM_IORES_BUSY: enqueue while part way through another op:"; + oss << " _deq_busy=" << (_deq_busy?"T":"F"); + oss << " _abort_busy=" << (_abort_busy?"T":"F"); + oss << " _commit_busy=" << (_commit_busy?"T":"F"); + throw jexception(oss.str()); // TODO: complete exception + } + + if (this_data_len != tot_data_len && !external) { + throw jexception("RHM_IORES_NOTIMPL: partial enqueues not implemented"); // TODO: complete exception; + } + + iores res = pre_write_check(WMGR_ENQUEUE, dtokp, xid_len, tot_data_len, external); + if (res != RHM_IORES_SUCCESS) + return res; + + bool cont = false; + if (_enq_busy) // If enqueue() exited last time with RHM_IORES_FULL or RHM_IORES_PAGE_AIOWAIT + { + if (dtokp->wstate() == data_tok::ENQ_PART) + cont = true; + else + { + std::ostringstream oss; + oss << "This data_tok: id=" << dtokp->id() << " state=" << dtokp->wstate_str(); + throw jexception(jerrno::JERR_WMGR_ENQDISCONT, oss.str(), "wmgr", "enqueue"); + } + } + + uint64_t rid = (dtokp->external_rid() | cont) ? dtokp->rid() : _lfc.getNextRecordId(); + _enq_rec.reset(rid, data_buff, tot_data_len, xid_ptr, xid_len, transient, external); + if (!cont) + { + dtokp->set_rid(rid); + dtokp->set_dequeue_rid(0); + if (xid_len) + dtokp->set_xid(xid_ptr, xid_len); + else + dtokp->clear_xid(); + _enq_busy = true; + } +//std::cout << "---+++ wmgr::enqueue() ENQ rid=0x" << std::hex << rid << " " << std::dec << std::flush; // DEBUG + bool done = false; + while (!done) + { +//std::cout << "*" << std::flush; // DEBUG + assert(_pg_offset_dblks < _cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS); + void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * QLS_DBLK_SIZE_BYTES); + uint32_t data_offs_dblks = dtokp->dblocks_written(); + uint32_t ret = _enq_rec.encode(wptr, data_offs_dblks, + (_cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS) - _pg_offset_dblks); + + // Remember fid which contains the record header in case record is split over several files + if (data_offs_dblks == 0) { + dtokp->set_fid(_lfc.getCurrentFileSeqNum()); + } + _pg_offset_dblks += ret; + _cached_offset_dblks += ret; + dtokp->incr_dblocks_written(ret); + dtokp->incr_pg_cnt(); + _page_cb_arr[_pg_index]._pdtokl->push_back(dtokp); + + // Is the encoding of this record complete? + if (dtokp->dblocks_written() >= _enq_rec.rec_size_dblks()) + { +//std::cout << "!" << std::flush; // DEBUG + // TODO: Incorrect - must set state to ENQ_CACHED; ENQ_SUBM is set when AIO returns. + dtokp->set_wstate(data_tok::ENQ_SUBM); + dtokp->set_dsize(tot_data_len); + // Only add this data token to page token list when submit is complete, this way + // long multi-page messages have their token on the page containing the END of the + // message. AIO callbacks will then only process this token when entire message is + // enqueued. + _lfc.incrEnqueuedRecordCount(dtokp->fid()); +//std::cout << "[0x" << std::hex << _lfc.getEnqueuedRecordCount() << std::dec << std::flush; // DEBUG + + if (xid_len) // If part of transaction, add to transaction map + { + std::string xid((const char*)xid_ptr, xid_len); + _tmap.insert_txn_data(xid, txn_data(rid, 0, dtokp->fid(), true)); + } + else + { + if (_emap.insert_pfid(rid, dtokp->fid(), 0) < enq_map::EMAP_OK) // fail + { + // The only error code emap::insert_pfid() returns is enq_map::EMAP_DUP_RID. + std::ostringstream oss; + oss << std::hex << "rid=0x" << rid << " _pfid=0x" << dtokp->fid(); + throw jexception(jerrno::JERR_MAP_DUPLICATE, oss.str(), "wmgr", "enqueue"); + } + } + + done = true; + } else { +//std::cout << "$" << std::endl << std::flush; // DEBUG + dtokp->set_wstate(data_tok::ENQ_PART); + } + + file_header_check(rid, cont, _enq_rec.rec_size_dblks() - data_offs_dblks); + flush_check(res, cont, done, rid); + } + if (dtokp->wstate() >= data_tok::ENQ_SUBM) + _enq_busy = false; +//std::cout << " res=" << iores_str(res) << " _enq_busy=" << (_enq_busy?"T":"F") << std::endl << std::flush; // DEBUG + return res; +} + +iores +wmgr::dequeue(data_tok* dtokp, + const void* const xid_ptr, + const std::size_t xid_len, + const bool txn_coml_commit) +{ + if (xid_len) + assert(xid_ptr != 0); + + if (_enq_busy || _abort_busy || _commit_busy) { + std::ostringstream oss; + oss << "RHM_IORES_BUSY: dequeue while part way through another op:"; + oss << " _enq_busy=" << (_enq_busy?"T":"F"); + oss << " _abort_busy=" << (_abort_busy?"T":"F"); + oss << " _commit_busy=" << (_commit_busy?"T":"F"); + throw jexception(oss.str()); // TODO: complete exception + } + + iores res = pre_write_check(WMGR_DEQUEUE, dtokp); + if (res != RHM_IORES_SUCCESS) + return res; + + bool cont = false; + if (_deq_busy) // If dequeue() exited last time with RHM_IORES_FULL or RHM_IORES_PAGE_AIOWAIT + { + if (dtokp->wstate() == data_tok::DEQ_PART) + cont = true; + else + { + std::ostringstream oss; + oss << "This data_tok: id=" << dtokp->id() << " state=" << dtokp->wstate_str(); + throw jexception(jerrno::JERR_WMGR_DEQDISCONT, oss.str(), "wmgr", "dequeue"); + } + } + + const bool ext_rid = dtokp->external_rid(); + uint64_t rid = (ext_rid | cont) ? dtokp->rid() : _lfc.getNextRecordId(); + uint64_t dequeue_rid = (ext_rid | cont) ? dtokp->dequeue_rid() : dtokp->rid(); + _deq_rec.reset(rid, dequeue_rid, xid_ptr, xid_len/*, _wrfc.owi()*/, txn_coml_commit); + if (!cont) + { + if (!ext_rid) + { + dtokp->set_rid(rid); + dtokp->set_dequeue_rid(dequeue_rid); + } + if (xid_len) + dtokp->set_xid(xid_ptr, xid_len); + else + dtokp->clear_xid(); + dequeue_check(dtokp->xid(), dequeue_rid); + dtokp->set_dblocks_written(0); // Reset dblks_written from previous op + _deq_busy = true; + } +//std::cout << "---+++ wmgr::dequeue() DEQ rid=0x" << std::hex << rid << " drid=0x" << dequeue_rid << " " << std::dec << std::flush; // DEBUG + bool done = false; + while (!done) + { +//std::cout << "*" << std::flush; // DEBUG + assert(_pg_offset_dblks < _cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS); + void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * QLS_DBLK_SIZE_BYTES); + uint32_t data_offs_dblks = dtokp->dblocks_written(); + uint32_t ret = _deq_rec.encode(wptr, data_offs_dblks, + (_cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS) - _pg_offset_dblks); + + // Remember fid which contains the record header in case record is split over several files + if (data_offs_dblks == 0) { + dtokp->set_fid(_lfc.getCurrentFileSeqNum()); + } + _pg_offset_dblks += ret; + _cached_offset_dblks += ret; + dtokp->incr_dblocks_written(ret); + dtokp->incr_pg_cnt(); + _page_cb_arr[_pg_index]._pdtokl->push_back(dtokp); + + // Is the encoding of this record complete? + if (dtokp->dblocks_written() >= _deq_rec.rec_size_dblks()) + { +//std::cout << "!" << std::flush; // DEBUG + // TODO: Incorrect - must set state to ENQ_CACHED; ENQ_SUBM is set when AIO returns. + dtokp->set_wstate(data_tok::DEQ_SUBM); + + if (xid_len) // If part of transaction, add to transaction map + { + // If the enqueue is part of a pending txn, it will not yet be in emap + _emap.lock(dequeue_rid); // ignore rid not found error + std::string xid((const char*)xid_ptr, xid_len); + _tmap.insert_txn_data(xid, txn_data(rid, dequeue_rid, dtokp->fid(), false)); + } + else + { + uint64_t fid; + short eres = _emap.get_remove_pfid(dtokp->dequeue_rid(), fid); + if (eres < enq_map::EMAP_OK) // fail + { + if (eres == enq_map::EMAP_RID_NOT_FOUND) + { + std::ostringstream oss; + oss << std::hex << "rid=0x" << rid; + throw jexception(jerrno::JERR_MAP_NOTFOUND, oss.str(), "wmgr", "dequeue"); + } + if (eres == enq_map::EMAP_LOCKED) + { + std::ostringstream oss; + oss << std::hex << "rid=0x" << rid; + throw jexception(jerrno::JERR_MAP_LOCKED, oss.str(), "wmgr", "dequeue"); + } + } +//std::cout << "[0x" << std::hex << _lfc.getEnqueuedRecordCount(fid) << std::dec << std::flush; // DEBUG +//try { + _lfc.decrEnqueuedRecordCount(fid); +//} catch (std::exception& e) { std::cout << "***OOPS*** " << e.what() << " cfid=" << _lfc.getCurrentFileSeqNum() << " fid=" << fid << std::flush; throw; } + } + + done = true; + } else { +//std::cout << "$" << std::flush; // DEBUG + dtokp->set_wstate(data_tok::DEQ_PART); + } + + file_header_check(rid, cont, _deq_rec.rec_size_dblks() - data_offs_dblks); + flush_check(res, cont, done, rid); + } + if (dtokp->wstate() >= data_tok::DEQ_SUBM) + _deq_busy = false; +//std::cout << " res=" << iores_str(res) << " _deq_busy=" << (_deq_busy?"T":"F") << std::endl << std::flush; // DEBUG + return res; +} + +iores +wmgr::abort(data_tok* dtokp, + const void* const xid_ptr, + const std::size_t xid_len) +{ + // commit and abort MUST have a valid xid + assert(xid_ptr != 0 && xid_len > 0); + + if (_enq_busy || _deq_busy || _commit_busy) { + std::ostringstream oss; + oss << "RHM_IORES_BUSY: abort while part way through another op:"; + oss << " _enq_busy=" << (_enq_busy?"T":"F"); + oss << " _deq_busy=" << (_deq_busy?"T":"F"); + oss << " _commit_busy=" << (_commit_busy?"T":"F"); + throw jexception(oss.str()); // TODO: complete exception + } + + iores res = pre_write_check(WMGR_ABORT, dtokp); + if (res != RHM_IORES_SUCCESS) + return res; + + bool cont = false; + if (_abort_busy) // If abort() exited last time with RHM_IORES_FULL or RHM_IORES_PAGE_AIOWAIT + { + if (dtokp->wstate() == data_tok::ABORT_PART) + cont = true; + else + { + std::ostringstream oss; + oss << "This data_tok: id=" << dtokp->id() << " state=" << dtokp->wstate_str(); + throw jexception(jerrno::JERR_WMGR_DEQDISCONT, oss.str(), "wmgr", "abort"); + } + } + + uint64_t rid = (dtokp->external_rid() | cont) ? dtokp->rid() : _lfc.getNextRecordId(); + _txn_rec.reset(QLS_TXA_MAGIC, rid, xid_ptr, xid_len/*, _wrfc.owi()*/); + if (!cont) + { + dtokp->set_rid(rid); + dtokp->set_dequeue_rid(0); + dtokp->set_xid(xid_ptr, xid_len); + dtokp->set_dblocks_written(0); // Reset dblks_written from previous op + _abort_busy = true; + } + bool done = false; + while (!done) + { + assert(_pg_offset_dblks < _cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS); + void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * QLS_DBLK_SIZE_BYTES); + uint32_t data_offs_dblks = dtokp->dblocks_written(); + uint32_t ret = _txn_rec.encode(wptr, data_offs_dblks, + (_cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS) - _pg_offset_dblks); + + // Remember fid which contains the record header in case record is split over several files + if (data_offs_dblks == 0) + dtokp->set_fid(_lfc.getCurrentFileSeqNum()); + _pg_offset_dblks += ret; + _cached_offset_dblks += ret; + dtokp->incr_dblocks_written(ret); + dtokp->incr_pg_cnt(); + _page_cb_arr[_pg_index]._pdtokl->push_back(dtokp); + + // Is the encoding of this record complete? + if (dtokp->dblocks_written() >= _txn_rec.rec_size_dblks()) + { + dtokp->set_wstate(data_tok::ABORT_SUBM); + + // Delete this txn from tmap, unlock any locked records in emap + std::string xid((const char*)xid_ptr, xid_len); + txn_data_list tdl = _tmap.get_remove_tdata_list(xid); // tdl will be empty if xid not found + for (tdl_itr itr = tdl.begin(); itr != tdl.end(); itr++) + { + if (!itr->_enq_flag) + _emap.unlock(itr->_drid); // ignore rid not found error + if (itr->_enq_flag) + _lfc.decrEnqueuedRecordCount(itr->_pfid); + } + std::pair<std::set<std::string>::iterator, bool> res = _txn_pending_set.insert(xid); + if (!res.second) + { + std::ostringstream oss; + oss << std::hex << "_txn_pending_set: xid=\"" << xid << "\""; + throw jexception(jerrno::JERR_MAP_DUPLICATE, oss.str(), "wmgr", "abort"); + } + + done = true; + } + else + dtokp->set_wstate(data_tok::ABORT_PART); + + file_header_check(rid, cont, _txn_rec.rec_size_dblks() - data_offs_dblks); + flush_check(res, cont, done, rid); + } + if (dtokp->wstate() >= data_tok::ABORT_SUBM) + _abort_busy = false; + return res; +} + +iores +wmgr::commit(data_tok* dtokp, + const void* const xid_ptr, + const std::size_t xid_len) +{ + // commit and abort MUST have a valid xid + assert(xid_ptr != 0 && xid_len > 0); + + if (_enq_busy || _deq_busy || _abort_busy) { + std::ostringstream oss; + oss << "RHM_IORES_BUSY: commit while part way through another op:"; + oss << " _enq_busy=" << (_enq_busy?"T":"F"); + oss << " _deq_busy=" << (_deq_busy?"T":"F"); + oss << " _abort_busy=" << (_abort_busy?"T":"F"); + throw jexception(oss.str()); // TODO: complete exception + } + + iores res = pre_write_check(WMGR_COMMIT, dtokp); + if (res != RHM_IORES_SUCCESS) + return res; + + bool cont = false; + if (_commit_busy) // If commit() exited last time with RHM_IORES_FULL or RHM_IORES_PAGE_AIOWAIT + { + if (dtokp->wstate() == data_tok::COMMIT_PART) + cont = true; + else + { + std::ostringstream oss; + oss << "This data_tok: id=" << dtokp->id() << " state=" << dtokp->wstate_str(); + throw jexception(jerrno::JERR_WMGR_DEQDISCONT, oss.str(), "wmgr", "commit"); + } + } + + uint64_t rid = (dtokp->external_rid() | cont) ? dtokp->rid() : _lfc.getNextRecordId(); + _txn_rec.reset(QLS_TXC_MAGIC, rid, xid_ptr, xid_len/*, _wrfc.owi()*/); + if (!cont) + { + dtokp->set_rid(rid); + dtokp->set_dequeue_rid(0); + dtokp->set_xid(xid_ptr, xid_len); + dtokp->set_dblocks_written(0); // Reset dblks_written from previous op + _commit_busy = true; + } + bool done = false; + while (!done) + { + assert(_pg_offset_dblks < _cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS); + void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * QLS_DBLK_SIZE_BYTES); + uint32_t data_offs_dblks = dtokp->dblocks_written(); + uint32_t ret = _txn_rec.encode(wptr, data_offs_dblks, + (_cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS) - _pg_offset_dblks); + + // Remember fid which contains the record header in case record is split over several files + if (data_offs_dblks == 0) + dtokp->set_fid(_lfc.getCurrentFileSeqNum()); + _pg_offset_dblks += ret; + _cached_offset_dblks += ret; + dtokp->incr_dblocks_written(ret); + dtokp->incr_pg_cnt(); + _page_cb_arr[_pg_index]._pdtokl->push_back(dtokp); + + // Is the encoding of this record complete? + if (dtokp->dblocks_written() >= _txn_rec.rec_size_dblks()) + { + dtokp->set_wstate(data_tok::COMMIT_SUBM); + + // Delete this txn from tmap, process records into emap + std::string xid((const char*)xid_ptr, xid_len); + txn_data_list tdl = _tmap.get_remove_tdata_list(xid); // tdl will be empty if xid not found + for (tdl_itr itr = tdl.begin(); itr != tdl.end(); itr++) + { + if (itr->_enq_flag) // txn enqueue + { + if (_emap.insert_pfid(itr->_rid, itr->_pfid, 0) < enq_map::EMAP_OK) // fail + { + // The only error code emap::insert_pfid() returns is enq_map::EMAP_DUP_RID. + std::ostringstream oss; + oss << std::hex << "rid=0x" << itr->_rid << " _pfid=0x" << itr->_pfid; + throw jexception(jerrno::JERR_MAP_DUPLICATE, oss.str(), "wmgr", "commit"); + } + } + else // txn dequeue + { + uint64_t fid; + short eres = _emap.get_remove_pfid(itr->_drid, fid, true); + if (eres < enq_map::EMAP_OK) // fail + { + if (eres == enq_map::EMAP_RID_NOT_FOUND) + { + std::ostringstream oss; + oss << std::hex << "rid=0x" << rid; + throw jexception(jerrno::JERR_MAP_NOTFOUND, oss.str(), "wmgr", "dequeue"); + } + if (eres == enq_map::EMAP_LOCKED) + { + std::ostringstream oss; + oss << std::hex << "rid=0x" << rid; + throw jexception(jerrno::JERR_MAP_LOCKED, oss.str(), "wmgr", "dequeue"); + } + } + _lfc.decrEnqueuedRecordCount(fid); + } + } + std::pair<std::set<std::string>::iterator, bool> res = _txn_pending_set.insert(xid); + if (!res.second) + { + std::ostringstream oss; + oss << std::hex << "_txn_pending_set: xid=\"" << xid << "\""; + throw jexception(jerrno::JERR_MAP_DUPLICATE, oss.str(), "wmgr", "commit"); + } + + done = true; + } + else + dtokp->set_wstate(data_tok::COMMIT_PART); + + file_header_check(rid, cont, _txn_rec.rec_size_dblks() - data_offs_dblks); + flush_check(res, cont, done, rid); + } + if (dtokp->wstate() >= data_tok::COMMIT_SUBM) + _commit_busy = false; + return res; +} + +void +wmgr::file_header_check(const uint64_t rid, + const bool cont, + const uint32_t rec_dblks_rem) +{ + if (_lfc.isEmpty()) // File never written (i.e. no header or data) + { + std::size_t fro = 0; + if (cont) { + bool file_fit = rec_dblks_rem <= _lfc.dataSize_sblks() * QLS_SBLK_SIZE_DBLKS; // Will fit within this journal file + bool file_full = rec_dblks_rem == _lfc.dataSize_sblks() * QLS_SBLK_SIZE_DBLKS; // Will exactly fill this journal file + if (file_fit && !file_full) { + fro = (rec_dblks_rem + (QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_DBLKS)) * QLS_DBLK_SIZE_BYTES; + } + } else { + fro = QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_BYTES; + } + _lfc.asyncFileHeaderWrite(_ioctx, 0, rid, fro); + _aio_evt_rem++; + } +} + +void +wmgr::flush_check(iores& res, + bool& cont, + bool& done, const uint64_t /*rid*/) // DEBUG +{ + // Is page is full, flush + if (_pg_offset_dblks >= _cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS) + { + res = write_flush(); + assert(res == RHM_IORES_SUCCESS); + + if (_page_cb_arr[_pg_index]._state == AIO_PENDING && !done) + { + res = RHM_IORES_PAGE_AIOWAIT; + done = true; + } + + // If file is full, rotate to next file + uint32_t fileSize_pgs = _lfc.fileSize_sblks() / _cache_pgsize_sblks; + if (_pg_cntr >= fileSize_pgs) + { + get_next_file(); + if (!done) { + cont = true; + } +//std::cout << "***** wmgr::flush_check(): GET NEXT FILE: rid=0x" << std::hex << rid << std::dec << " res=" << iores_str(res) << " cont=" << (cont?"T":"F") << " done=" << (done?"T":"F") << std::endl; // DEBUG + } + } +} + +iores +wmgr::flush() +{ + iores res = write_flush(); + uint32_t fileSize_pgs = _lfc.fileSize_sblks() / _cache_pgsize_sblks; + if (res == RHM_IORES_SUCCESS && _pg_cntr >= fileSize_pgs) { + get_next_file(); + } + return res; +} + +iores +wmgr::write_flush() +{ + iores res = RHM_IORES_SUCCESS; + // Don't bother flushing an empty page or one that is still in state AIO_PENDING + if (_cached_offset_dblks) + { + if (_page_cb_arr[_pg_index]._state == AIO_PENDING) { +//std::cout << "#"; // DEBUG + res = RHM_IORES_PAGE_AIOWAIT; + } else { + if (_page_cb_arr[_pg_index]._state != IN_USE) + { + std::ostringstream oss; + oss << "pg_index=" << _pg_index << " state=" << _page_cb_arr[_pg_index].state_str(); + throw jexception(jerrno::JERR_WMGR_BADPGSTATE, oss.str(), "wmgr", "write_flush"); + } + + // Send current page using AIO + + // In manual flushes, dblks may not coincide with sblks, add filler records ("RHMx") if necessary. + dblk_roundup(); + + std::size_t pg_offs = (_pg_offset_dblks - _cached_offset_dblks) * QLS_DBLK_SIZE_BYTES; + aio_cb* aiocbp = &_aio_cb_arr[_pg_index]; + _lfc.asyncPageWrite(_ioctx, aiocbp, (char*)_page_ptr_arr[_pg_index] + pg_offs, _cached_offset_dblks); + _page_cb_arr[_pg_index]._state = AIO_PENDING; + _aio_evt_rem++; +//std::cout << "." << _aio_evt_rem << std::flush; // DEBUG + _cached_offset_dblks = 0; + _jc->instr_incr_outstanding_aio_cnt(); + + rotate_page(); // increments _pg_index, resets _pg_offset_dblks if req'd + if (_page_cb_arr[_pg_index]._state == UNUSED) + _page_cb_arr[_pg_index]._state = IN_USE; + } + } + get_events(0, false); + if (_page_cb_arr[_pg_index]._state == UNUSED) + _page_cb_arr[_pg_index]._state = IN_USE; + return res; +} + +void +wmgr::get_next_file() +{ + _pg_cntr = 0; +//std::cout << "&&&&& wmgr::get_next_file(): " << status_str() << std::endl; // DEBUG + _lfc.pullEmptyFileFromEfp(); +} + +int32_t +wmgr::get_events(timespec* const timeout, + bool flush) +{ + if (_aio_evt_rem == 0) // no events to get + return 0; + + int ret = 0; + if ((ret = aio::getevents(_ioctx, flush ? _aio_evt_rem : 1, _aio_evt_rem, _aio_event_arr, timeout)) < 0) + { + if (ret == -EINTR) // Interrupted by signal + return 0; + std::ostringstream oss; + oss << "io_getevents() failed: " << std::strerror(-ret) << " (" << ret << ")"; + throw jexception(jerrno::JERR__AIO, oss.str(), "wmgr", "get_events"); + } + + if (ret == 0 && timeout) + return jerrno::AIO_TIMEOUT; + + int32_t tot_data_toks = 0; + for (int i=0; i<ret; i++) // Index of returned AIOs + { + if (_aio_evt_rem == 0) + { + std::ostringstream oss; + oss << "_aio_evt_rem; evt " << (i + 1) << " of " << ret; + throw jexception(jerrno::JERR__UNDERFLOW, oss.str(), "wmgr", "get_events"); + } + _aio_evt_rem--; +//std::cout << "'" << _aio_evt_rem; // DEBUG + aio_cb* aiocbp = _aio_event_arr[i].obj; // This I/O control block (iocb) + page_cb* pcbp = (page_cb*)(aiocbp->data); // This page control block (pcb) + long aioret = (long)_aio_event_arr[i].res; + if (aioret < 0) { + std::ostringstream oss; + oss << "AIO write operation failed: " << std::strerror(-aioret) << " (" << aioret << ") ["; + if (pcbp) { + oss << "pg=" << pcbp->_index; + } else { + file_hdr_t* fhp = (file_hdr_t*)aiocbp->u.c.buf; + oss << "fnum=" << fhp->_file_number; + oss << " qname=" << std::string((char*)fhp + sizeof(file_hdr_t), fhp->_queue_name_len); + } + oss << " size=" << aiocbp->u.c.nbytes; + oss << " offset=" << aiocbp->u.c.offset << " fh=" << aiocbp->aio_fildes << "]"; + throw jexception(jerrno::JERR__AIO, oss.str(), "wmgr", "get_events"); + } + if (pcbp) // Page writes have pcb + { +//std::cout << "p"; // DEBUG + uint32_t s = pcbp->_pdtokl->size(); + std::vector<data_tok*> dtokl; + dtokl.reserve(s); + for (uint32_t k=0; k<s; k++) + { + data_tok* dtokp = pcbp->_pdtokl->at(k); + if (dtokp->decr_pg_cnt() == 0) + { + std::set<std::string>::iterator it; + switch (dtokp->wstate()) + { + case data_tok::ENQ_SUBM: + dtokl.push_back(dtokp); + tot_data_toks++; + dtokp->set_wstate(data_tok::ENQ); + if (dtokp->has_xid()) + // Ignoring return value here. A non-zero return can signify that the transaction + // has committed or aborted, and which was completed prior to the aio returning. + _tmap.set_aio_compl(dtokp->xid(), dtokp->rid()); + break; + case data_tok::DEQ_SUBM: + dtokl.push_back(dtokp); + tot_data_toks++; + dtokp->set_wstate(data_tok::DEQ); + if (dtokp->has_xid()) + // Ignoring return value - see note above. + _tmap.set_aio_compl(dtokp->xid(), dtokp->rid()); + break; + case data_tok::ABORT_SUBM: + dtokl.push_back(dtokp); + tot_data_toks++; + dtokp->set_wstate(data_tok::ABORTED); + it = _txn_pending_set.find(dtokp->xid()); + if (it == _txn_pending_set.end()) + { + std::ostringstream oss; + oss << std::hex << "_txn_pending_set: abort xid=\""; + oss << dtokp->xid() << "\""; + throw jexception(jerrno::JERR_MAP_NOTFOUND, oss.str(), "wmgr", + "get_events"); + } + _txn_pending_set.erase(it); + break; + case data_tok::COMMIT_SUBM: + dtokl.push_back(dtokp); + tot_data_toks++; + dtokp->set_wstate(data_tok::COMMITTED); + it = _txn_pending_set.find(dtokp->xid()); + if (it == _txn_pending_set.end()) + { + std::ostringstream oss; + oss << std::hex << "_txn_pending_set: commit xid=\""; + oss << dtokp->xid() << "\""; + throw jexception(jerrno::JERR_MAP_NOTFOUND, oss.str(), "wmgr", + "get_events"); + } + _txn_pending_set.erase(it); + break; + case data_tok::ENQ_PART: + case data_tok::DEQ_PART: + case data_tok::ABORT_PART: + case data_tok::COMMIT_PART: + // ignore these + break; + default: + // throw for anything else + std::ostringstream oss; + oss << "dtok_id=" << dtokp->id() << " dtok_state=" << dtokp->wstate_str(); + throw jexception(jerrno::JERR_WMGR_BADDTOKSTATE, oss.str(), "wmgr", + "get_events"); + } + } + } + + // Increment the completed write offset + // NOTE: We cannot use _wrfc here, as it may have rotated since submitting count. + // Use stored pointer to fcntl in the pcb instead. + pcbp->_jfp->addCompletedDblkCount(pcbp->_wdblks); + pcbp->_jfp->decrOutstandingAioOperationCount(); + _jc->instr_decr_outstanding_aio_cnt(); + + // Clean up this pcb's data_tok list + pcbp->_pdtokl->clear(); + pcbp->_state = UNUSED; +//std::cout << "c" << pcbp->_index << pcbp->state_str(); // DEBUG + + // Perform AIO return callback + if (_cbp && tot_data_toks) + _cbp->wr_aio_cb(dtokl); + } + else // File header writes have no pcb + { +//std::cout << "f"; // DEBUG + file_hdr_t* fhp = (file_hdr_t*)aiocbp->u.c.buf; + _lfc.addWriteCompletedDblkCount(fhp->_file_number, QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_DBLKS); + _lfc.decrOutstandingAioOperationCount(fhp->_file_number); + } + } + + return tot_data_toks; +} + +bool +wmgr::is_txn_synced(const std::string& xid) +{ + // Ignore xid not found error here + if (_tmap.is_txn_synced(xid) == txn_map::TMAP_NOT_SYNCED) + return false; + // Check for outstanding commit/aborts + std::set<std::string>::iterator it = _txn_pending_set.find(xid); + return it == _txn_pending_set.end(); +} + +void +wmgr::initialize(aio_callback* const cbp, + const uint32_t wcache_pgsize_sblks, + const uint16_t wcache_num_pages) +{ + + pmgr::initialize(cbp, wcache_pgsize_sblks, wcache_num_pages); + wmgr::clean(); + _page_cb_arr[0]._state = IN_USE; + _ddtokl.clear(); + _cached_offset_dblks = 0; + _enq_busy = false; +} + +iores +wmgr::pre_write_check(const _op_type op, + const data_tok* const dtokp, + const std::size_t /*xidsize*/, + const std::size_t /*dsize*/, + const bool /*external*/) const +{ + // Check status of current file + // TODO: Replace for LFC +/* + if (!_wrfc.is_wr_reset()) + { + if (!_wrfc.wr_reset()) + return RHM_IORES_FULL; + } +*/ + + // Check status of current page is ok for writing + if (_page_cb_arr[_pg_index]._state != IN_USE) + { + if (_page_cb_arr[_pg_index]._state == UNUSED) + _page_cb_arr[_pg_index]._state = IN_USE; + else if (_page_cb_arr[_pg_index]._state == AIO_PENDING) + return RHM_IORES_PAGE_AIOWAIT; + else + { + std::ostringstream oss; + oss << "jrnl=" << _jc->id() << " op=" << _op_str[op]; + oss << " index=" << _pg_index << " pg_state=" << _page_cb_arr[_pg_index].state_str(); + throw jexception(jerrno::JERR_WMGR_BADPGSTATE, oss.str(), "wmgr", "pre_write_check"); + } + } + + // operation-specific checks + switch (op) + { + case WMGR_ENQUEUE: + { + if (!dtokp->is_writable()) + { + std::ostringstream oss; + oss << "jrnl=" << _jc->id() << " op=" << _op_str[op]; + oss << " dtok_id=" << dtokp->id() << " dtok_state=" << dtokp->wstate_str(); + throw jexception(jerrno::JERR_WMGR_BADDTOKSTATE, oss.str(), "wmgr", + "pre_write_check"); + } + } + break; + case WMGR_DEQUEUE: + if (!dtokp->is_dequeueable()) + { + std::ostringstream oss; + oss << "jrnl=" << _jc->id() << " op=" << _op_str[op]; + oss << " dtok_id=" << dtokp->id() << " dtok_state=" << dtokp->wstate_str(); + throw jexception(jerrno::JERR_WMGR_BADDTOKSTATE, oss.str(), "wmgr", + "pre_write_check"); + } + break; + case WMGR_ABORT: + break; + case WMGR_COMMIT: + break; + } + + return RHM_IORES_SUCCESS; +} + +void +wmgr::dequeue_check(const std::string& xid, + const uint64_t drid) +{ + // First check emap + bool found = false; + uint64_t fid; + short eres = _emap.get_pfid(drid, fid); + if (eres < enq_map::EMAP_OK) { // fail + if (eres == enq_map::EMAP_RID_NOT_FOUND) { + if (xid.size()) { + found = _tmap.data_exists(xid, drid); + } + } else if (eres == enq_map::EMAP_LOCKED) { + std::ostringstream oss; + oss << std::hex << "drid=0x" << drid; + throw jexception(jerrno::JERR_MAP_LOCKED, oss.str(), "wmgr", "dequeue_check"); + } + } else { + found = true; + } + if (!found) { + std::ostringstream oss; + oss << "jrnl=" << _jc->id() << " drid=0x" << std::hex << drid; + throw jexception(jerrno::JERR_WMGR_DEQRIDNOTENQ, oss.str(), "wmgr", "dequeue_check"); + } +} + +void +wmgr::dblk_roundup() +{ + const uint32_t xmagic = QLS_EMPTY_MAGIC; + uint32_t wdblks = jrec::size_blks(_cached_offset_dblks, QLS_SBLK_SIZE_DBLKS) * QLS_SBLK_SIZE_DBLKS; + while (_cached_offset_dblks < wdblks) + { + void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * QLS_DBLK_SIZE_BYTES); + std::memcpy(wptr, (const void*)&xmagic, sizeof(xmagic)); +#ifdef QLS_CLEAN + std::memset((char*)wptr + sizeof(xmagic), QLS_CLEAN_CHAR, QLS_DBLK_SIZE_BYTES - sizeof(xmagic)); +#endif + _pg_offset_dblks++; + _cached_offset_dblks++; + } +} + +void +wmgr::rotate_page() +{ +//std::cout << "^^^^^ wmgr::rotate_page() " << status_str() << " pi=" << _pg_index; // DEBUG + if (_pg_offset_dblks >= _cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS) + { + _pg_offset_dblks = 0; + _pg_cntr++; + } + if (++_pg_index >= _cache_num_pages) + _pg_index = 0; +//std::cout << "->" << _pg_index << std::endl; // DEBUG +} + +void +wmgr::clean() { + // Clean up allocated memory here +} + +const std::string +wmgr::status_str() const +{ + std::ostringstream oss; + oss << "wmgr: pi=" << _pg_index << " pc=" << _pg_cntr; + oss << " po=" << _pg_offset_dblks << " aer=" << _aio_evt_rem; + oss << " edac=" << (_enq_busy?"T":"F") << (_deq_busy?"T":"F"); + oss << (_abort_busy?"T":"F") << (_commit_busy?"T":"F"); + oss << " ps=["; + for (int i=0; i<_cache_num_pages; i++) + { + switch (_page_cb_arr[i]._state) + { + case UNUSED: oss << "-"; break; + case IN_USE: oss << "U"; break; + case AIO_PENDING: oss << "A"; break; + default: oss << _page_cb_arr[i]._state; + } + } + oss << "] "; + return oss.str(); +} + +// static + +const char* wmgr::_op_str[] = {"enqueue", "dequeue", "abort", "commit"}; + +}} diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/wmgr.h b/qpid/cpp/src/qpid/linearstore/jrnl/wmgr.h new file mode 100644 index 0000000000..6b455febd9 --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/jrnl/wmgr.h @@ -0,0 +1,159 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifndef QPID_LEGACYSTORE_JRNL_WMGR_H +#define QPID_LEGACYSTORE_JRNL_WMGR_H + +namespace qpid +{ +namespace qls_jrnl +{ +class wmgr; +}} + +#include <cstring> +#include "qpid/linearstore/jrnl/EmptyFilePoolTypes.h" +#include "qpid/linearstore/jrnl/enums.h" +#include "qpid/linearstore/jrnl/pmgr.h" +#include <set> + +class file_hdr_t; + +namespace qpid +{ +namespace qls_jrnl +{ + class LinearFileController; + + /** + * \brief Class for managing a write page cache of arbitrary size and number of pages. + * + * The write page cache works on the principle of caching the write data within a page until + * that page is either full or flushed; this initiates a single AIO write operation to store + * the data on disk. + * + * The maximum disk throughput is achieved by keeping the write operations of uniform size. + * Waiting for a page cache to fill achieves this; and in high data volume/throughput situations + * achieves the optimal disk throughput. Calling flush() forces a write of the current page cache + * no matter how full it is, and disrupts the uniformity of the write operations. This should + * normally only be done if throughput drops and there is a danger of a page of unwritten data + * waiting around for excessive time. + * + * The usual tradeoff between data storage latency and throughput performance applies. + */ + class wmgr : public pmgr + { + private: + LinearFileController& _lfc; ///< Linear File Controller ref + uint32_t _max_dtokpp; ///< Max data writes per page + uint32_t _max_io_wait_us; ///< Max wait in microseconds till submit + uint32_t _cached_offset_dblks; ///< Amount of unwritten data in page (dblocks) + std::deque<data_tok*> _ddtokl; ///< Deferred dequeue data_tok list + + // TODO: Convert _enq_busy etc into a proper threadsafe lock + // TODO: Convert to enum? Are these encodes mutually exclusive? + bool _enq_busy; ///< Flag true if enqueue is in progress + bool _deq_busy; ///< Flag true if dequeue is in progress + bool _abort_busy; ///< Flag true if abort is in progress + bool _commit_busy; ///< Flag true if commit is in progress + + enum _op_type { WMGR_ENQUEUE = 0, WMGR_DEQUEUE, WMGR_ABORT, WMGR_COMMIT }; + static const char* _op_str[]; + + enq_rec _enq_rec; ///< Enqueue record used for encoding/decoding + deq_rec _deq_rec; ///< Dequeue record used for encoding/decoding + txn_rec _txn_rec; ///< Transaction record used for encoding/decoding + std::set<std::string> _txn_pending_set; ///< Set containing xids of pending commits/aborts + + public: + wmgr(jcntl* jc, + enq_map& emap, + txn_map& tmap, + LinearFileController& lfc); + wmgr(jcntl* jc, + enq_map& emap, + txn_map& tmap, + LinearFileController& lfc, + const uint32_t max_dtokpp, + const uint32_t max_iowait_us); + virtual ~wmgr(); + + void initialize(aio_callback* const cbp, + const uint32_t wcache_pgsize_sblks, + const uint16_t wcache_num_pages, + const uint32_t max_dtokpp, + const uint32_t max_iowait_us, + std::size_t eo = 0); + iores enqueue(const void* const data_buff, + const std::size_t tot_data_len, + const std::size_t this_data_len, + data_tok* dtokp, + const void* const xid_ptr, + const std::size_t xid_len, + const bool transient, + const bool external); + iores dequeue(data_tok* dtokp, + const void* const xid_ptr, + const std::size_t xid_len, + const bool txn_coml_commit); + iores abort(data_tok* dtokp, + const void* const xid_ptr, + const std::size_t xid_len); + iores commit(data_tok* dtokp, + const void* const xid_ptr, + const std::size_t xid_len); + iores flush(); + int32_t get_events(timespec* const timeout, + bool flush); + bool is_txn_synced(const std::string& xid); + inline bool curr_pg_blocked() const { return _page_cb_arr[_pg_index]._state != UNUSED; } + inline uint32_t unflushed_dblks() { return _cached_offset_dblks; } + + // Debug aid + const std::string status_str() const; + + private: + void initialize(aio_callback* const cbp, + const uint32_t wcache_pgsize_sblks, + const uint16_t wcache_num_pages); + iores pre_write_check(const _op_type op, + const data_tok* const dtokp, + const std::size_t xidsize = 0, + const std::size_t dsize = 0, + const bool external = false) const; + void dequeue_check(const std::string& xid, + const uint64_t drid); + void file_header_check(const uint64_t rid, + const bool cont, + const uint32_t rec_dblks_rem); + void flush_check(iores& res, + bool& cont, + bool& done, const uint64_t rid); + iores write_flush(); + void get_next_file(); + void dblk_roundup(); + void rotate_page(); + void clean(); + }; + +}} + +#endif // ifndef QPID_LEGACYSTORE_JRNL_WMGR_H diff --git a/qpid/cpp/src/qpid/linearstore/management-schema.xml b/qpid/cpp/src/qpid/linearstore/management-schema.xml new file mode 100644 index 0000000000..dce8f7988a --- /dev/null +++ b/qpid/cpp/src/qpid/linearstore/management-schema.xml @@ -0,0 +1,99 @@ +<schema package="org.apache.qpid.linearstore"> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + + <class name="Store"> + <property name="brokerRef" type="objId" access="RO" references="qpid.Broker" index="y" parentRef="y"/> + <property name="location" type="sstr" access="RO" desc="Logical directory on disk"/> + <!--property name="defaultInitialFileCount" type="uint16" access="RO" unit="file" desc="Default number of files initially allocated to each journal"/--> + <!--property name="defaultDataFileSize" type="uint32" access="RO" unit="RdPg" desc="Default size of each journal data file"/--> + <property name="tplIsInitialized" type="bool" access="RO" desc="Transaction prepared list has been initialized by a transactional prepare"/> + <property name="tplDirectory" type="sstr" access="RO" desc="Transaction prepared list directory"/> + <property name="tplWritePageSize" type="uint32" access="RO" unit="byte" desc="Page size in transaction prepared list write-page-cache"/> + <property name="tplWritePages" type="uint32" access="RO" unit="wpage" desc="Number of pages in transaction prepared list write-page-cache"/> + <!--property name="tplInitialFileCount" type="uint16" access="RO" unit="file" desc="Number of files initially allocated to transaction prepared list journal"/--> + <!--property name="tplDataFileSize" type="uint32" access="RO" unit="byte" desc="Size of each journal data file in transaction prepared list journal"/--> + <!--property name="tplCurrentFileCount" type="uint32" access="RO" unit="file" desc="Number of files currently allocated to transaction prepared list journal"/--> + + <statistic name="tplTransactionDepth" type="hilo32" unit="txn" desc="Number of currently enqueued prepared transactions"/> + <statistic name="tplTxnPrepares" type="count64" unit="record" desc="Total transaction prepares on transaction prepared list"/> + <statistic name="tplTxnCommits" type="count64" unit="record" desc="Total transaction commits on transaction prepared list"/> + <statistic name="tplTxnAborts" type="count64" unit="record" desc="Total transaction aborts on transaction prepared list"/> + <statistic name="tplOutstandingAIOs" type="hilo32" unit="aio_op" desc="Number of currently outstanding AIO requests in Async IO system"/> + </class> + + <class name="Journal"> + <property name="queueRef" type="objId" access="RO" references="qpid.Queue" isGeneralReference="y"/> + <property name="name" type="sstr" access="RO" index="y"/> + <property name="directory" type="sstr" access="RO" desc="Directory containing journal files"/> + <property name="baseFileName" type="sstr" access="RO" desc="Base filename prefix for journal"/> + <property name="writePageSize" type="uint32" access="RO" unit="byte" desc="Page size in write-page-cache"/> + <property name="writePages" type="uint32" access="RO" unit="wpage" desc="Number of pages in write-page-cache"/> + <property name="readPageSize" type="uint32" access="RO" unit="byte" desc="Page size in read-page-cache"/> + <property name="readPages" type="uint32" access="RO" unit="rpage" desc="Number of pages in read-page-cache"/> + <!--property name="initialFileCount" type="uint16" access="RO" unit="file" desc="Number of files initially allocated to this journal"/--> + <!--property name="autoExpand" type="bool" access="RO" desc="Auto-expand enabled"/--> + <!--property name="currentFileCount" type="uint16" access="RO" unit="file" desc="Number of files currently allocated to this journal"/--> + <!--property name="maxFileCount" type="uint16" access="RO" unit="file" desc="Max number of files allowed for this journal"/--> + <!--property name="dataFileSize" type="uint32" access="RO" unit="byte" desc="Size of each journal data file"/--> + + <statistic name="recordDepth" type="hilo32" unit="record" desc="Number of currently enqueued records (durable messages)"/> + <statistic name="enqueues" type="count64" unit="record" desc="Total enqueued records on journal"/> + <statistic name="dequeues" type="count64" unit="record" desc="Total dequeued records on journal"/> + <statistic name="txn" type="count32" unit="record" desc="Total open transactions (xids) on journal"/> + <statistic name="txnEnqueues" type="count64" unit="record" desc="Total transactional enqueued records on journal"/> + <statistic name="txnDequeues" type="count64" unit="record" desc="Total transactional dequeued records on journal"/> + <statistic name="txnCommits" type="count64" unit="record" desc="Total transactional commit records on journal"/> + <statistic name="txnAborts" type="count64" unit="record" desc="Total transactional abort records on journal"/> + <statistic name="outstandingAIOs" type="hilo32" unit="aio_op" desc="Number of currently outstanding AIO requests in Async IO system"/> + +<!-- + The following are not yet "wired up" in JournalImpl.cpp +--> + <statistic name="freeFileCount" type="hilo32" unit="file" desc="Number of files free on this journal. Includes free files trapped in holes."/> + <statistic name="availableFileCount" type="hilo32" unit="file" desc="Number of files available to be written. Excluding holes"/> + <statistic name="writeWaitFailures" type="count64" unit="record" desc="AIO Wait failures on write"/> + <statistic name="writeBusyFailures" type="count64" unit="record" desc="AIO Busy failures on write"/> + <statistic name="readRecordCount" type="count64" unit="record" desc="Records read from the journal"/> + <statistic name="readBusyFailures" type="count64" unit="record" desc="AIO Busy failures on read"/> + <statistic name="writePageCacheDepth" type="hilo32" unit="wpage" desc="Current depth of write-page-cache"/> + <statistic name="readPageCacheDepth" type="hilo32" unit="rpage" desc="Current depth of read-page-cache"/> + + <!--method name="expand" desc="Increase number of files allocated for this journal"> + <arg name="by" type="uint32" dir="I" desc="Number of files to increase journal size by"/> + </method--> + </class> + + <eventArguments> + <!--arg name="autoExpand" type="bool" desc="Journal auto-expand enabled"/--> + <arg name="fileSize" type="uint32" desc="Journal file size in bytes"/> + <arg name="jrnlId" type="sstr" desc="Journal Id"/> + <arg name="numEnq" type="uint32" desc="Number of recovered enqueues"/> + <arg name="numFiles" type="uint16" desc="Number of journal files"/> + <arg name="numTxn" type="uint32" desc="Number of recovered transactions"/> + <arg name="numTxnDeq" type="uint32" desc="Number of recovered transactional dequeues"/> + <arg name="numTxnEnq" type="uint32" desc="Number of recovered transactional enqueues"/> + <arg name="what" type="sstr" desc="Description of event"/> + </eventArguments> + <event name="enqThresholdExceeded" sev="warn" args="jrnlId, what"/> + <event name="created" sev="notice" args="jrnlId, fileSize, numFiles"/> + <event name="full" sev="error" args="jrnlId, what"/> + <event name="recovered" sev="notice" args="jrnlId, fileSize, numFiles, numEnq, numTxn, numTxnEnq, numTxnDeq"/> +</schema> diff --git a/qpid/cpp/src/tests/linearstore/linearstoredirsetup.sh b/qpid/cpp/src/tests/linearstore/linearstoredirsetup.sh new file mode 100755 index 0000000000..b4b8721edd --- /dev/null +++ b/qpid/cpp/src/tests/linearstore/linearstoredirsetup.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +STORE_DIR=/tmp +LINEARSTOREDIR=~/RedHat/linearstore + +rm -rf $STORE_DIR/qls +rm -rf $STORE_DIR/p002 +rm $STORE_DIR/p004 + +mkdir $STORE_DIR/qls +mkdir $STORE_DIR/p002 +touch $STORE_DIR/p004 +mkdir $STORE_DIR/qls/p001 +touch $STORE_DIR/qls/p003 +ln -s $STORE_DIR/p002 $STORE_DIR/qls/p002 +ln -s $STORE_DIR/p004 $STORE_DIR/qls/p004 + +${LINEARSTOREDIR}/tools/src/py/linearstore/efptool.py $STORE_DIR/qls/ -a -p 1 -s 2048 -n 25 +${LINEARSTOREDIR}/tools/src/py/linearstore/efptool.py $STORE_DIR/qls/ -a -p 1 -s 512 -n 25 +${LINEARSTOREDIR}/tools/src/py/linearstore/efptool.py $STORE_DIR/qls/ -a -p 2 -s 2048 -n 25 + +${LINEARSTOREDIR}/tools/src/py/linearstore/efptool.py $STORE_DIR/qls/ -l +tree -la $STORE_DIR/qls + |
