diff options
245 files changed, 13767 insertions, 9997 deletions
@@ -6,10 +6,6 @@ Files: doc/* Copyright: (c) 2010-2012 New Dream Network and contributors License: Creative Commons Attribution-ShareAlike (CC BY-SA) -Files: src/client/hadoop/ceph -Copyright: Copyright (C) New Dream Network and contributors -License: Apache License v2 - Files: src/mount/canonicalize.c Copyright: Copyright (C) 1993 Rick Sladkey <jrs@world.std.com> License: LGPL2 or later @@ -96,3 +92,35 @@ License: Files: src/test/common/Throttle.cc src/test/filestore/chain_xattr.cc Copyright: Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com> License: LGPL2 or later + +Files: src/osd/ErasureCodePluginJerasure/*.{c,h} +Copyright (c) 2011, James S. Plank <plank@cs.utk.edu> + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + - Neither the name of the University of Tennessee nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/PendingReleaseNotes b/PendingReleaseNotes index 6609acbd5a8..e7fcd7201bb 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -1,28 +1,2 @@ -v0.68 +v0.69 ~~~~~ - -* 'ceph osd crush set <id> <weight> <loc..>' no longer adds the osd to the - specified location, as that's a job for 'ceph osd crush add'. It will - however continue to work just the same as long as the osd already exists - in the crush map. - -* The OSD now enforces that class write methods cannot both mutate an - object and return data. The rbd.assign_bid method, the lone - offender, has been removed. This breaks compatibility with - pre-bobtail librbd clients by preventing them from creating new - images. - -* librados now returns on commit instead of ack for synchronous calls. - This is a bit safer in the case where both OSDs and the client crash, and - is probably how it should have been acting from the beginning. Users are - unlikely to notice but it could result in lower performance in some - circumstances. Those who care should switch to using the async interfaces, - which let you specify safety semantics precisely. - -* The C++ librados AioComplete::get_version() method was incorrectly - returning an int (usually 32-bits). To avoid breaking library - compatibility, a get_version64() method is added that returns the - full-width value. The old method is deprecated and will be removed - in a future release. Users of the C++ librados API that make use of - the get_version() method should modify their code to avoid getting a - value that is truncated from 64 to to 32 bits.
\ No newline at end of file diff --git a/admin/doc-requirements.txt b/admin/doc-requirements.txt index b088df083db..ce4654c6600 100644 --- a/admin/doc-requirements.txt +++ b/admin/doc-requirements.txt @@ -1,3 +1,3 @@ -Sphinx >=1.1.2 +Sphinx == 1.1.3 -e git+https://github.com/ceph/sphinx-ditaa.git#egg=sphinx-ditaa -e git+https://github.com/ceph/asphyxiate.git#egg=asphyxiate diff --git a/ceph.spec.in b/ceph.spec.in index 8091018c1dc..851ee7acfd5 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -278,7 +278,6 @@ export RPM_OPT_FLAGS=`echo $RPM_OPT_FLAGS | sed -e 's/i386/i486/'` --localstatedir=/var \ --sysconfdir=/etc \ --docdir=%{_docdir}/ceph \ - --without-hadoop \ --with-nss \ --without-cryptopp \ --with-rest-bench \ diff --git a/configure.ac b/configure.ac index b45828f1602..bdea8b18ceb 100644 --- a/configure.ac +++ b/configure.ac @@ -106,6 +106,13 @@ AC_DEFUN([AC_CHECK_CC_FLAG], AC_CHECK_CC_FLAG([-Wtype-limits], [WARN_TYPE_LIMITS]) AC_CHECK_CC_FLAG([-Wignored-qualifiers], [WARN_IGNORED_QUALIFIERS]) +# Checks for architecture stuff +AM_CONDITIONAL([ENABLE_FPU_NEON], [case $target_cpu in arm*) true;; *) false;; esac]) + +# Check for compiler VTA support +AX_CHECK_COMPILE_FLAG([-fvar-tracking-assignments], [HAS_VTA_SUPPORT=1], [HAS_VTA_SUPPORT=0]) +AM_CONDITIONAL(COMPILER_HAS_VTA, [test "$HAS_VTA_SUPPORT" = 1]) + # Checks for libraries. ACX_PTHREAD AC_CHECK_LIB([uuid], [uuid_parse], [true], AC_MSG_FAILURE([libuuid not found])) @@ -326,14 +333,18 @@ if test "x$enable_cephfs_java" = "xyes"; then # setup defaults for Debian default-jdk package (without --with-jdk-dir) AS_IF([test -z "$with_jdk_dir"], [ - # This works with Debian's default-jdk package - dir='/usr/lib/jvm/default-java/' - javac_prog=`find $dir -name javac | head -n 1` - AS_IF([test -x "$javac_prog"], [ - EXTRA_JDK_BIN_DIR=`dirname $javac_prog`]) - jnih=`find $dir -name jni.h | head -n 1` - AS_IF([test -r "$jnih"], [ - EXTRA_JDK_INC_DIR=`dirname $jnih`])]) + # This works with Debian's and CentOS' default-jdk package + for dir in '/usr/lib/jvm/default-java/' '/usr/lib/jvm/java/' ; do + # only test if a suitable path has not yet been found + AS_IF([test "$EXTRA_JDK_BIN_DIR" == ""], [ + AS_IF([test -x "$javac_prog"], [ + EXTRA_JDK_BIN_DIR=`dirname $javac_prog`]) + jnih=`find $dir -name jni.h | head -n 1` + AS_IF([test -r "$jnih"], [ + EXTRA_JDK_INC_DIR=`dirname $jnih`]) + ]) + done + ]) # cephfs_java_test only makes sense if java is already turned on # setup CLASSPATH for Debian default junit4.jar package @@ -361,10 +372,6 @@ if test "x$enable_cephfs_java" = "xyes"; then CLASSPATH=$CLASSPATH:$EXTRA_CLASSPATH_JAR export CLASSPATH AC_MSG_NOTICE([classpath - $CLASSPATH]) - AS_IF([test "$have_junit4" = "1"], [ - AC_CHECK_CLASS([org.junit.rules.ExternalResource], [], [ - AC_MSG_NOTICE(Could not find org.junit.rules.ExternalResource) - have_junit4=0])]) # Check for jni.h CPPFLAGS_save=$CPPFLAGS @@ -384,25 +391,6 @@ if test "x$enable_cephfs_java" = "xyes"; then fi AM_CONDITIONAL(HAVE_JUNIT4, [test "$have_junit4" = "1"]) -# jni? -# clear cache (from java above) -- this whole thing will get -# folded into the bigger java package later -- for now maintain -# backward compat -AS_UNSET(ac_cv_header_jni_h) -AC_ARG_WITH([hadoop], - [AS_HELP_STRING([--with-hadoop], [build hadoop client])], - [], - [with_hadoop=check]) -AS_IF([test "x$with_hadoop" != xno], - [AC_CHECK_HEADER([jni.h], - [HAVE_JNI=1], - [if test "x$with_hadoop" != xcheck; then - AC_MSG_FAILURE( - [--with-hadoop was given but jni.h not found]) - fi - ])]) -AM_CONDITIONAL(WITH_HADOOPCLIENT, [test "$HAVE_JNI" = "1"]) - # # FreeBSD has it in base. # diff --git a/debian/copyright b/debian/copyright index aa91a149853..d11a0f7f5da 100644 --- a/debian/copyright +++ b/debian/copyright @@ -7,10 +7,6 @@ Files: * Copyright: (c) 2004-2010 by Sage Weil <sage@newdream.net> License: LGPL2.1 (see /usr/share/common-licenses/LGPL-2.1) -Files: src/client/hadoop/ceph -Copyright: Copyright (C) New Dream Network and contributors -License: Apache License v2 - Files: src/mount/canonicalize.c Copyright: Copyright (C) 1993 Rick Sladkey <jrs@world.std.com> License: LGPL2 or later diff --git a/do_autogen.sh b/do_autogen.sh index 32e9df4623b..bc6749e9e5d 100755 --- a/do_autogen.sh +++ b/do_autogen.sh @@ -10,7 +10,6 @@ do_autogen.sh: make a ceph build by running autogen, etc. level 1: -g level 3: -Wextra level 4: even more... --H --with-hadoop -T --without-tcmalloc -e <path> dump encoded objects to <path> -P profiling build @@ -46,8 +45,6 @@ do h) usage exit 0;; - H) CONFIGURE_FLAGS="$CONFIGURE_FLAGS --with-hadoop";; - T) CONFIGURE_FLAGS="$CONFIGURE_FLAGS --without-tcmalloc";; j) CONFIGURE_FLAGS="$CONFIGURE_FLAGS --enable-cephfs-java";; diff --git a/doc/changelog/v0.67.3.txt b/doc/changelog/v0.67.3.txt new file mode 100644 index 00000000000..d6b1f2b2742 --- /dev/null +++ b/doc/changelog/v0.67.3.txt @@ -0,0 +1,700 @@ +commit 408cd61584c72c0d97b774b3d8f95c6b1b06341a +Author: Gary Lowell <gary.lowell@inktank.com> +Date: Mon Sep 9 12:50:11 2013 -0700 + + v0.67.3 + +commit 17a7342b3b935c06610c58ab92a9a1d086923d32 +Merge: b4252bf 10433bb +Author: Sage Weil <sage@inktank.com> +Date: Sat Sep 7 13:34:45 2013 -0700 + + Merge pull request #574 from dalgaaf/fix/da-dumpling-cherry-picks + + init-radosgw*: fix status return value if radosgw isn't running + + Reviewed-by: Sage Weil <sage@inktank.com> + +commit 10433bbe72dbf8eae8fae836e557a043610eb54e +Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de> +Date: Sat Sep 7 11:30:15 2013 +0200 + + init-radosgw*: fix status return value if radosgw isn't running + + Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de> + (cherry picked from commit b5137baf651eaaa9f67e3864509e437f9d5c3d5a) + +commit b4252bff79150a95e9d075dd0b5e146ba9bf2ee5 +Author: Samuel Just <sam.just@inktank.com> +Date: Thu Aug 22 11:19:37 2013 -0700 + + FileStore: add config option to disable the wbthrottle + + Backport: dumpling + Signed-off-by: Samuel Just <sam.just@inktank.com> + Reviewed-by: Sage Weil <sage@inktank.com> + (cherry picked from commit 3528100a53724e7ae20766344e467bf762a34163) + +commit 699324e0910e5e07a1ac68df8cf1108e5671ec15 +Author: Samuel Just <sam.just@inktank.com> +Date: Thu Aug 22 11:19:52 2013 -0700 + + WBThrottle: use fdatasync instead of fsync + + Backport: dumpling + Signed-off-by: Samuel Just <sam.just@inktank.com> + Reviewed-by: Sage Weil <sage@inktank.com> + (cherry picked from commit d571825080f0bff1ed3666e95e19b78a738ecfe8) + +commit 074717b4b49ae1a55bc867e5c34d43c51edc84a5 +Author: Samuel Just <sam.just@inktank.com> +Date: Thu Aug 29 15:08:58 2013 -0700 + + PGLog: initialize writeout_from in PGLog constructor + + Fixes: 6151 + Backport: dumpling + Signed-off-by: Samuel Just <sam.just@inktank.com> + Introduced: f808c205c503f7d32518c91619f249466f84c4cf + Reviewed-by: Sage Weil <sage@inktank.com> + (cherry picked from commit 42d65b0a7057696f4b8094f7c686d467c075a64d) + +commit c22d980cf42e580818dc9f526327518c0ddf8ff5 +Author: Samuel Just <sam.just@inktank.com> +Date: Tue Aug 27 08:49:14 2013 -0700 + + PGLog: maintain writeout_from and trimmed + + This way, we can avoid omap_rmkeyrange in the common append + and trim cases. + + Fixes: #6040 + Backport: Dumpling + Signed-off-by: Samuel Just <sam.just@inktank.com> + (cherry picked from commit f808c205c503f7d32518c91619f249466f84c4cf) + +commit 53c7ab4db00ec7034f5aa555231f9ee167f43201 +Author: Samuel Just <sam.just@inktank.com> +Date: Tue Aug 27 07:27:26 2013 -0700 + + PGLog: don't maintain log_keys_debug if the config is disabled + + Fixes: #6040 + Backport: Dumpling + Signed-off-by: Samuel Just <sam.just@inktank.com> + (cherry picked from commit 1c0d75db1075a58d893d30494a5d7280cb308899) + +commit 40dc489351383c2e35b91c3d4e76b633309716df +Author: Samuel Just <sam.just@inktank.com> +Date: Mon Aug 26 23:19:45 2013 -0700 + + PGLog: move the log size check after the early return + + There really are stl implementations (like the one on my ubuntu 12.04 + machine) which have a list::size() which is linear in the size of the + list. That assert, therefore, is quite expensive! + + Fixes: #6040 + Backport: Dumpling + Signed-off-by: Samuel Just <sam.just@inktank.com> + (cherry picked from commit fe68b15a3d82349f8941f5b9f70fcbb5d4bc7f97) + +commit 4261eb5ec105b9c27605360910602dc367fd79f5 +Author: Sage Weil <sage@inktank.com> +Date: Tue Aug 13 17:16:08 2013 -0700 + + rbd.cc: relicense as LGPL2 + + All past authors for rbd.cc have consented to relicensing from GPL to + LGPL2 via email: + + --- + + Date: Sat, 27 Jul 2013 01:59:36 +0200 + From: Sylvain Munaut <s.munaut@whatever-company.com> + Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change + + I hereby consent to the relicensing of any contribution I made to the + aforementioned rbd.cc file from GPL to LGPL2.1. + + (I hope that'll be impressive enough, I did my best :p) + + btw, tnt@246tNt.com and s.munaut@whatever-company.com are both me. + + Cheers, + + Sylvain + + --- + + Date: Fri, 26 Jul 2013 17:00:48 -0700 + From: Yehuda Sadeh <yehuda@inktank.com> + Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change + + I consent. + + --- + + Date: Fri, 26 Jul 2013 17:02:24 -0700 + From: Josh Durgin <josh.durgin@inktank.com> + Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change + + I consent. + + --- + + Date: Fri, 26 Jul 2013 18:17:46 -0700 + From: Stanislav Sedov <stas@freebsd.org> + Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change + + I consent. + + Thanks for taking care of it! + + --- + + Date: Fri, 26 Jul 2013 18:24:15 -0700 + From: Colin McCabe <cmccabe@alumni.cmu.edu> + + I consent. + + cheers, + Colin + + --- + + Date: Sat, 27 Jul 2013 07:08:12 +0200 + From: Christian Brunner <christian@brunner-muc.de> + Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change + + I consent + + Christian + + --- + + Date: Sat, 27 Jul 2013 12:17:34 +0300 + From: Stratos Psomadakis <psomas@grnet.gr> + Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change + + Hi, + + I consent with the GPL -> LGL2.1 re-licensing. + + Thanks + Stratos + + --- + + Date: Sat, 27 Jul 2013 16:13:13 +0200 + From: Wido den Hollander <wido@42on.com> + Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change + + I consent! + + You have my permission to re-license the code I wrote for rbd.cc to LGPL2.1 + + --- + + Date: Sun, 11 Aug 2013 10:40:32 +0200 + From: Danny Al-Gaaf <danny.al-gaaf@bisect.de> + Subject: Re: btw + + Hi Sage, + + I agree to switch the license of ceph_argparse.py and rbd.cc from GPL2 + to LGPL2. + + Regards + + Danny Al-Gaaf + + --- + + Date: Tue, 13 Aug 2013 17:15:24 -0700 + From: Dan Mick <dan.mick@inktank.com> + Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change + + I consent to relicense any contributed code that I wrote under LGPL2.1 license. + + --- + + ...and I consent too. Drop the exception from COPYING and debian/copyright + files. + + Signed-off-by: Sage Weil <sage@inktank.com> + (cherry picked from commit 2206f55761c675b31078dea4e7dd66f2666d7d03) + +commit 211c5f13131e28b095a1f3b72426128f1db22218 +Author: Yehuda Sadeh <yehuda@inktank.com> +Date: Fri Aug 23 15:39:20 2013 -0700 + + rgw: flush pending data when completing multipart part upload + + Fixes: #6111 + Backport: dumpling + When completing the part upload we need to flush any data that we + aggregated and didn't flush yet. With earlier code didn't have to deal + with it as for multipart upload we didn't have any pending data. + What we do now is we call the regular atomic data completion + function that takes care of it. + + Reviewed-by: Josh Durgin <josh.durgin@inktank.com> + Signed-off-by: Yehuda Sadeh <yehuda@inktank.com> + (cherry picked from commit 9a551296e0811f2b65972377b25bb28dbb42f575) + +commit 1a9651010aab51c9be2edeccd80e9bd11f5177ce +Author: Yehuda Sadeh <yehuda@inktank.com> +Date: Mon Aug 26 19:46:43 2013 -0700 + + rgw: check object name after rebuilding it in S3 POST + + Fixes: #6088 + Backport: bobtail, cuttlefish, dumpling + + When posting an object it is possible to provide a key + name that refers to the original filename, however we + need to verify that in the end we don't end up with an + empty object name. + + Reviewed-by: Josh Durgin <josh.durgin@inktank.com> + Signed-off-by: Yehuda Sadeh <yehuda@inktank.com> + (cherry picked from commit c8ec532fadc0df36e4b265fe20a2ff3e35319744) + +commit 1bd74a020b93f154b2d4129d512f6334387de7c7 +Author: Sage Weil <sage@inktank.com> +Date: Thu Aug 22 17:46:45 2013 -0700 + + mon/MonClient: release pending outgoing messages on shutdown + + This fixes a small memory leak when we have messages queued for the mon + when we shut down. It is harmless except for the valgrind leak check + noise that obscures real leaks. + + Backport: dumpling + Signed-off-by: Sage Weil <sage@inktank.com> + (cherry picked from commit 309569a6d0b7df263654b7f3f15b910a72f2918d) + +commit 24f2669783e2eb9d9af5ecbe106efed93366ba63 +Author: Yehuda Sadeh <yehuda@inktank.com> +Date: Thu Aug 29 13:06:33 2013 -0700 + + rgw: change watch init ordering, don't distribute if can't + + Backport: dumpling + + Moving back the watch initialization after the zone init, + as the zone info holds the control pool name. Since zone + init might need to create a new system object (that needs + to distribute cache), don't try to distribute cache if + watch is not yet initialized. + + Reviewed-by: Sage Weil <sage@inktank.com> + Signed-off-by: Yehuda Sadeh <yehuda@inktank.com> + (cherry picked from commit 1d1f7f18dfbdc46fdb09a96ef973475cd29feef5) + +commit a708c8ab52e5b1476405a1f817c23b8845fbaab3 +Author: Sage Weil <sage@inktank.com> +Date: Fri Aug 30 09:41:29 2013 -0700 + + ceph-post-file: use mktemp instead of tempfile + + tempfile is a debian thing, apparently; mktemp is present everywhere. + + Signed-off-by: Sage Weil <sage@inktank.com> + (cherry picked from commit e60d4e09e9f11e3c34a05cd122341e06c7c889bb) + +commit 625f13ee0d6cca48d61dfd65e00517d092552d1c +Author: Sage Weil <sage@inktank.com> +Date: Wed Aug 28 09:50:11 2013 -0700 + + mon: discover mon addrs, names during election state too + + Currently we only detect new mon addrs and names during the probing phase. + For non-trivial clusters, this means we can get into a sticky spot when + we discover enough peers to form an quorum, but not all of them, and the + undiscovered ones are enough to break the mon ranks and prevent an + election. + + One way to work around this is to continue addr and name discovery during + the election. We should also consider making the ranks less sensitive to + the undefined addrs; that is a separate change. + + Fixes: #4924 + Backport: dumpling + Signed-off-by: Sage Weil <sage@inktank.com> + Tested-by: Bernhard Glomm <bernhard.glomm@ecologic.eu> + (cherry picked from commit c24028570015cacf1d9e154ffad80bec06a61e7c) + +commit 83cfd4386c1fd0fa41aea345704e27f82b524ece +Author: Dan Mick <dan.mick@inktank.com> +Date: Thu Aug 22 17:30:24 2013 -0700 + + ceph_rest_api.py: create own default for log_file + + common/config thinks the default log_file for non-daemons should be "". + Override that so that the default is + /var/log/ceph/{cluster}-{name}.{pid}.log + since ceph-rest-api is more of a daemon than a client. + + Fixes: #6099 + Backport: dumpling + Signed-off-by: Dan Mick <dan.mick@inktank.com> + (cherry picked from commit 2031f391c3df68e0d9e381a1ef3fe58d8939f0a8) + +commit 8a1da62d9564a32f7b8963fe298e1ac3ad0ea3d9 +Author: Sage Weil <sage@inktank.com> +Date: Fri Aug 16 17:59:11 2013 -0700 + + ceph-post-file: single command to upload a file to cephdrop + + Use sftp to upload to a directory that only this user and ceph devs can + access. + + Distribute an ssh key to connect to the account. This will let us revoke + the key in the future if we feel the need. Also distribute a known_hosts + file so that users have some confidence that they are connecting to the + real ceph drop account and not some third party. + + Signed-off-by: Sage Weil <sage@inktank.com> + Reviewed-by: Dan Mick <dan.mick@inktank.com> + (cherry picked from commit d08e05e463f1f7106a1f719d81b849435790a3b9) + +commit 3f8663477b585dcb528fdd7047c50d9a52d24b95 +Author: Gary Lowell <glowell@inktank.com> +Date: Thu Aug 22 13:29:32 2013 -0700 + + ceph.spec.in: remove trailing paren in previous commit + + Signed-off-by: Gary Lowell <gary.lowell@inktank.com> + +commit 23fb908cb3ac969c874ac12755d20ed2f636e1b9 +Author: Gary Lowell <glowell@inktank.com> +Date: Thu Aug 22 11:07:16 2013 -0700 + + ceph.spec.in: Don't invoke debug_package macro on centos. + + If the redhat-rpm-config package is installed, the debuginfo rpms will + be built by default. The build will fail when the package installed + and the specfile also invokes the macro. + + Signed-off-by: Gary Lowell <gary.lowell@inktank.com> + +commit 11f5853d8178ab60ab948d373c1a1f67324ce3bd +Author: Sage Weil <sage@inktank.com> +Date: Sat Aug 24 14:04:09 2013 -0700 + + osd: install admin socket commands after signals + + This lets us tell by the presence of the admin socket commands whether + a signal will make us shut down cleanly. See #5924. + + Signed-off-by: Sage Weil <sage@inktank.com> + Reviewed-by: Samuel Just <sam.just@inktank.com> + (cherry picked from commit c5b5ce120a8ce9116be52874dbbcc39adec48b5c) + +commit 39adc0195e6016ce36828885515be1bffbc10ae1 +Author: Sage Weil <sage@inktank.com> +Date: Tue Aug 20 22:39:09 2013 -0700 + + ceph-disk: partprobe after creating journal partition + + At least one user reports that a partprobe is needed after creating the + journal partition. It is not clear why sgdisk is not doing it, but this + fixes ceph-disk for them, and should be harmless for other users. + + Fixes: #5599 + Tested-by: lurbs in #ceph + Signed-off-by: Sage Weil <sage@inktank.com> + (cherry picked from commit 2af59d5e81c5e3e3d7cfc50d9330d7364659c5eb) + (cherry picked from commit 3e42df221315679605d68b2875aab6c7eb6b3cc4) + +commit 6a4fe7b9b068ae990d6404921a46631fe9ebcd31 +Author: Sage Weil <sage@inktank.com> +Date: Tue Aug 20 11:27:23 2013 -0700 + + mon/Paxos: always refresh after any store_state + + If we store any new state, we need to refresh the services, even if we + are still in the midst of Paxos recovery. This is because the + subscription path will share any committed state even when paxos is + still recovering. This prevents a race like: + + - we have maps 10..20 + - we drop out of quorum + - we are elected leader, paxos recovery starts + - we get one LAST with committed states that trim maps 10..15 + - we get a subscribe for map 10..20 + - we crash because 10 is no longer on disk because the PaxosService + is out of sync with the on-disk state. + + Fixes: #6045 + Backport: dumpling + Signed-off-by: Sage Weil <sage@inktank.com> + Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com> + (cherry picked from commit 981eda9f7787c83dc457f061452685f499e7dd27) + +commit 13d396e46ed9200e4b9f21db2f0a8efbc5998d82 +Author: Sage Weil <sage@inktank.com> +Date: Tue Aug 20 11:27:09 2013 -0700 + + mon/Paxos: return whether store_state stored anything + + Signed-off-by: Sage Weil <sage@inktank.com> + Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com> + (cherry picked from commit 7e0848d8f88f156a05eef47a9f730b772b64fbf2) + +commit f248383bacff76203fa94716cfdf6cf766da24a7 +Author: Sage Weil <sage@inktank.com> +Date: Tue Aug 20 11:26:57 2013 -0700 + + mon/Paxos: cleanup: use do_refresh from handle_commit + + This avoid duplicated code by using the helper created exactly for this + purpose. + + Signed-off-by: Sage Weil <sage@inktank.com> + Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com> + (cherry picked from commit b9dee2285d9fe8533fa98c940d5af7b0b81f3d33) + +commit 02608a12d4e7592784148a62a47d568efc24079d +Author: Sage Weil <sage@inktank.com> +Date: Thu Aug 15 21:48:06 2013 -0700 + + osdc/ObjectCacher: do not merge rx buffers + + We do not try to merge rx buffers currently. Make that explicit and + documented in the code that it is not supported. (Otherwise the + last_read_tid values will get lost and read results won't get applied + to the cache properly.) + + Signed-off-by: Sage Weil <sage@inktank.com> + (cherry picked from commit 1c50c446152ab0e571ae5508edb4ad7c7614c310) + +commit 0e2bfe71965eeef29b47e8032637ea820a7ce49c +Author: Sage Weil <sage@inktank.com> +Date: Thu Aug 15 21:47:18 2013 -0700 + + osdc/ObjectCacher: match reads with their original rx buffers + + Consider a sequence like: + + 1- start read on 100~200 + 100~200 state rx + 2- truncate to 200 + 100~100 state rx + 3- start read on 200~200 + 100~100 state rx + 200~200 state rx + 4- get 100~200 read result + + Currently this makes us crash on + + osdc/ObjectCacher.cc: 738: FAILED assert(bh->length() <= start+(loff_t)length-opos) + + when processing the second 200~200 bufferhead (it is too big). The + larger issue, though, is that we should not be looking at this data at + all; it has been truncated away. + + Fix this by marking each rx buffer with the read request that is sent to + fill it, and only fill it from that read request. Then the first reply + will fill the first 100~100 extend but not touch the other extent; the + second read will do that. + + Signed-off-by: Sage Weil <sage@inktank.com> + (cherry picked from commit b59f930ae147767eb4c9ff18c3821f6936a83227) + +commit 6b51c960715971a0351e8203d4896cb0c4138a3f +Author: Sage Weil <sage@inktank.com> +Date: Thu Aug 22 15:54:48 2013 -0700 + + mon/Paxos: fix another uncommitted value corner case + + It is possible that we begin the paxos recovery with an uncommitted + value for, say, commit 100. During last/collect we discover 100 has been + committed already. But also, another node provides an uncommitted value + for 101 with the same pn. Currently, we refuse to learn it, because the + pn is not strictly > than our current uncommitted pn... even though it is + the next last_committed+1 value that we need. + + There are two possible fixes here: + + - make this a >= as we can accept newer values from the same pn. + - discard our uncommitted value metadata when we commit the value. + + Let's do both! + + Fixes: #6090 + Signed-off-by: Sage Weil <sage@inktank.com> + (cherry picked from commit fe5010380a3a18ca85f39403e8032de1dddbe905) + +commit b3a280d5af9d06783d2698bd434940de94ab0fda +Author: Sage Weil <sage@inktank.com> +Date: Fri Aug 23 11:45:35 2013 -0700 + + os: make readdir_r buffers larger + + PATH_MAX isn't quite big enough. + + Backport: dumpling, cuttlefish, bobtail + Signed-off-by: Sage Weil <sage@inktank.com> + (cherry picked from commit 99a2ff7da99f8cf70976f05d4fe7aa28dd7afae5) + +commit 989a664ef0d1c716cab967f249112f595cf98c43 +Author: Sage Weil <sage@inktank.com> +Date: Fri Aug 23 11:45:08 2013 -0700 + + os: fix readdir_r buffer size + + The buffer needs to be big or else we're walk all over the stack. + + Backport: dumpling, cuttlefish, bobtail + Signed-off-by: Sage Weil <sage@inktank.com> + (cherry picked from commit 2df66d9fa214e90eb5141df4d5755b57e8ba9413) + + Conflicts: + + src/os/BtrfsFileStoreBackend.cc + +commit a4cca31c82bf0e84272e01eb1b3188dfdb5b5615 +Author: Yehuda Sadeh <yehuda@inktank.com> +Date: Thu Aug 22 10:53:12 2013 -0700 + + rgw: fix crash when creating new zone on init + + Moving the watch/notify init before the zone init, + as we might need to send a notification. + + Reviewed-by: Sage Weil <sage@inktank.com> + Signed-off-by: Yehuda Sadeh <yehuda@inktank.com> + (cherry picked from commit 3d55534268de7124d29bd365ea65da8d2f63e501) + +commit 4cf6996803ef66f2b6083f73593259d45e2740a3 +Author: Yehuda Sadeh <yehuda@inktank.com> +Date: Mon Aug 19 08:40:16 2013 -0700 + + rgw: change cache / watch-notify init sequence + + Fixes: #6046 + We were initializing the watch-notify (through the cache + init) before reading the zone info which was much too + early, as we didn't have the control pool name yet. Now + simplifying init/cleanup a bit, cache doesn't call watch/notify + init and cleanup directly, but rather states its need + through a virtual callback. + + Signed-off-by: Yehuda Sadeh <yehuda@inktank.com> + Reviewed-by: Sage Weil <sage@inktank.com> + (cherry picked from commit d26ba3ab0374e77847c742dd00cb3bc9301214c2) + +commit aea6de532b0b843c3a8bb76d10bab8476f0d7c09 +Author: Alexandre Oliva <oliva@gnu.org> +Date: Thu Aug 22 03:40:22 2013 -0300 + + enable mds rejoin with active inodes' old parent xattrs + + When the parent xattrs of active inodes that the mds attempts to open + during rejoin lack pool info (struct_v < 5), this field will be filled + in with -1, causing the mds to retry fetching a backtrace with a pool + number that matches the expected value, which fails and causes the + err==-ENOENT branch to be taken and retry pool 1, which succeeds, but + with pool -1, and so keeps on bouncing between the two retry cases + forever. + + This patch arranges for the mds to go along with pool -1 instead of + insisting that it be refetched, enabling it to complete recovery + instead of eating cpu, network bandwidth and metadata osd's resources + like there's no tomorrow, in what AFAICT is an infinite and very busy + loop. + + This is not a new problem: I've had it even before upgrading from + Cuttlefish to Dumpling, I'd just never managed to track it down, and + force-unmounting the filesystem and then restarting the mds was an + easier (if inconvenient) work-around, particularly because it always + hit when the filesystem was under active, heavy-ish use (or there + wouldn't be much reason for caps recovery ;-) + + There are two issues not addressed in this patch, however. One is + that nothing seems to proactively update the parent xattr when it is + found to be outdated, so it remains out of date forever. Not even + renaming top-level directories causes the xattrs to be recursively + rewritten. AFAICT that's a bug. + + The other is that inodes that don't have a parent xattr (created by + even older versions of ceph) are reported as non-existing in the mds + rejoin message, because the absence of the parent xattr is signaled as + a missing inode (?failed to reconnect caps for missing inodes?). I + suppose this may cause more serious recovery problems. + + I suppose a global pass over the filesystem tree updating parent + xattrs that are out-of-date would be desirable, if we find any parent + xattrs still lacking current information; it might make sense to + activate it as a background thread from the backtrace decoding + function, when it finds a parent xattr that's too out-of-date, or as a + separate client (ceph-fsck?). + + Backport: dumpling, cuttlefish + Signed-off-by: Alexandre Oliva <oliva@gnu.org> + Reviewed-by: Zheng, Yan <zheng.z.yan@intel.com> + (cherry picked from commit 617dc36d477fd83b2d45034fe6311413aa1866df) + +commit 0738bdf92f5e5eb93add152a4135310ac7ea1c91 +Author: David Disseldorp <ddiss@suse.de> +Date: Mon Jul 29 17:05:44 2013 +0200 + + mds: remove waiting lock before merging with neighbours + + CephFS currently deadlocks under CTDB's ping_pong POSIX locking test + when run concurrently on multiple nodes. + The deadlock is caused by failed removal of a waiting_locks entry when + the waiting lock is merged with an existing lock, e.g: + + Initial MDS state (two clients, same file): + held_locks -- start: 0, length: 1, client: 4116, pid: 7899, type: 2 + start: 2, length: 1, client: 4110, pid: 40767, type: 2 + waiting_locks -- start: 1, length: 1, client: 4116, pid: 7899, type: 2 + + Waiting lock entry 4116@1:1 fires: + handle_client_file_setlock: start: 1, length: 1, + client: 4116, pid: 7899, type: 2 + + MDS state after lock is obtained: + held_locks -- start: 0, length: 2, client: 4116, pid: 7899, type: 2 + start: 2, length: 1, client: 4110, pid: 40767, type: 2 + waiting_locks -- start: 1, length: 1, client: 4116, pid: 7899, type: 2 + + Note that the waiting 4116@1:1 lock entry is merged with the existing + 4116@0:1 held lock to become a 4116@0:2 held lock. However, the now + handled 4116@1:1 waiting_locks entry remains. + + When handling a lock request, the MDS calls adjust_locks() to merge + the new lock with available neighbours. If the new lock is merged, + then the waiting_locks entry is not located in the subsequent + remove_waiting() call because adjust_locks changed the new lock to + include the old locks. + This fix ensures that the waiting_locks entry is removed prior to + modification during merge. + + Signed-off-by: David Disseldorp <ddiss@suse.de> + Reviewed-by: Greg Farnum <greg@inktank.com> + (cherry picked from commit 476e4902907dfadb3709ba820453299ececf990b) + +commit a0ac88272511d670b5c3756dda2d02c93c2e9776 +Author: Dan Mick <dan.mick@inktank.com> +Date: Tue Aug 20 11:10:42 2013 -0700 + + mon/PGMap: OSD byte counts 4x too large (conversion to bytes overzealous) + + Fixes: #6049 + Signed-off-by: Dan Mick <dan.mick@inktank.com> + (cherry picked from commit eca53bbf583027397f0d5e050a76498585ecb059) + +commit 87b19c33ce29e2ca4fc49a2adeb12d3f14ca90a9 +Author: Alfredo Deza <alfredo.deza@inktank.com> +Date: Fri Aug 23 08:56:07 2013 -0400 + + ceph-disk: specify the filetype when mounting + + Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com> + Reviewed-by: Sage Weil <sage@inktank.com> + (cherry picked from commit f040020fb2a7801ebbed23439159755ff8a3edbd) diff --git a/doc/dev/corpus.rst b/doc/dev/corpus.rst index 92f480a01a3..64f71c12fdf 100644 --- a/doc/dev/corpus.rst +++ b/doc/dev/corpus.rst @@ -22,71 +22,71 @@ We can generate an object corpus for a particular version of ceph like so. #. Checkout a clean repo (best not to do this where you normally work):: - git clone ceph.git - cd ceph - git submodule update --init + git clone ceph.git + cd ceph + git submodule update --init #. Build with flag to dump objects to /tmp/foo:: - rm -rf /tmp/foo ; mkdir /tmp/foo - ./do_autogen.sh -e /tmp/foo - make + rm -rf /tmp/foo ; mkdir /tmp/foo + ./do_autogen.sh -e /tmp/foo + make #. Start via vstart:: - cd src - MON=3 OSD=3 MDS=3 RGW=1 ./vstart.sh -n -x + cd src + MON=3 OSD=3 MDS=3 RGW=1 ./vstart.sh -n -x #. Use a much functionality of the cluster as you can, to exercise as many object encoder methods as possible:: - ./rados -p rbd bench 10 write -b 123 - ./ceph osd out 0 - ./init-ceph stop osd.1 - for f in ../qa/workunits/cls/*.sh ; do PATH=".:$PATH" $f ; done - ../qa/workunits/rados/test.sh - ./ceph_test_librbd - ./ceph_test_libcephfs - ./init-ceph restart mds.a + ./rados -p rbd bench 10 write -b 123 + ./ceph osd out 0 + ./init-ceph stop osd.1 + for f in ../qa/workunits/cls/*.sh ; do PATH=".:$PATH" $f ; done + ../qa/workunits/rados/test.sh + ./ceph_test_librbd + ./ceph_test_libcephfs + ./init-ceph restart mds.a Do some more stuff with rgw if you know how. #. Stop:: - ./stop.sh + ./stop.sh #. Import the corpus (this will take a few minutes):: - test/encoding/import.sh /tmp/foo `./ceph-dencoder version` ../ceph-object-corpus/archive - test/encoding/import-generated.sh ../ceph-object-corpus/archive + test/encoding/import.sh /tmp/foo `./ceph-dencoder version` ../ceph-object-corpus/archive + test/encoding/import-generated.sh ../ceph-object-corpus/archive #. Prune it! There will be a bazillion copies of various objects, and we only want a representative sample.:: - pushd ../ceph-object-corpus - bin/prune-archive.sh - popd + pushd ../ceph-object-corpus + bin/prune-archive.sh + popd #. Verify the tests pass:: - make check-local + make check-local #. Commit it to the corpus repo and push:: - pushd ../ceph-object-corpus - git checkout -b wip-new - git add archive/`../src/ceph-dencoder version` - git commit -m `../src/ceph-dencoder version` - git remote add cc ceph.com:/git/ceph-object-corpus.git - git push cc wip-new - popd + pushd ../ceph-object-corpus + git checkout -b wip-new + git add archive/`../src/ceph-dencoder version` + git commit -m `../src/ceph-dencoder version` + git remote add cc ceph.com:/git/ceph-object-corpus.git + git push cc wip-new + popd #. Go test it out:: - cd my/regular/tree - cd ceph-object-corpus - git fetch origin - git checkout wip-new - cd ../src - make check-local + cd my/regular/tree + cd ceph-object-corpus + git fetch origin + git checkout wip-new + cd ../src + make check-local #. If everything looks good, update the submodule master branch, and commit the submodule in ceph.git. diff --git a/doc/dev/mon-bootstrap.rst b/doc/dev/mon-bootstrap.rst index 9ce0070b791..0a4a9a2981e 100644 --- a/doc/dev/mon-bootstrap.rst +++ b/doc/dev/mon-bootstrap.rst @@ -42,7 +42,7 @@ with a command like:: When creating a new monitor cluster, the keyring should also contain a ``client.admin`` key that can be used to administer the system:: - ceph-authtool /path/to/keyring --gen-key -n client.admin + ceph-authtool /path/to/keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow' The resulting keyring is fed to ``ceph-mon --mkfs`` with the ``--keyring <keyring>`` command-line argument. diff --git a/doc/dev/osd_internals/erasure_coding/pgbackend.rst b/doc/dev/osd_internals/erasure_coding/pgbackend.rst index 4519ee34f9c..c16354f5116 100644 --- a/doc/dev/osd_internals/erasure_coding/pgbackend.rst +++ b/doc/dev/osd_internals/erasure_coding/pgbackend.rst @@ -79,10 +79,13 @@ Core Changes: APPEND, DELETE, (SET|RM)ATTR log entries. - The filestore needs to be able to deal with multiply versioned hobjects. This probably means adapting the filestore internally to - use a vhobject which is basically a pair<version_t, hobject_t>. The - version needs to be included in the on-disk filename. An interface - needs to be added to get all versions of a particular hobject_t or - the most recently versioned instance of a particular hobject_t. + use a ghobject which is basically a tuple<hobject_t, gen_t, + shard_t>. The gen_t + shard_t need to be included in the on-disk + filename. gen_t is a unique object identifier to make sure there + are no name collisions when object N is created + + deleted + created again. An interface needs to be added to get all + versions of a particular hobject_t or the most recently versioned + instance of a particular hobject_t. PGBackend Interfaces: @@ -178,7 +181,7 @@ acting set have different pieces of the erasure coding scheme and are not interchangeable. Worse, crush might cause chunk 2 to be written to an osd which happens already to contain an (old) copy of chunk 4. This means that the OSD and PG messages need to work in terms of a -type like pair<chunk_id_t, pg_t> in order to distinguish different pg +type like pair<shard_t, pg_t> in order to distinguish different pg chunks on a single OSD. Because the mapping of object name to object in the filestore must @@ -188,14 +191,14 @@ include the chunk id in the object key. Core changes: -- The filestore `vhobject_t needs to also include a chunk id +- The filestore `ghobject_t needs to also include a chunk id <http://tracker.ceph.com/issues/5862>`_ making it more like - tuple<hobject_t, version_t, chunk_id_t>. -- coll_t needs to include a chunk_id_t. + tuple<hobject_t, gen_t, shard_t>. +- coll_t needs to include a shard_t. - The `OSD pg_map and similar pg mappings need to work in terms of a cpg_t <http://tracker.ceph.com/issues/5863>`_ (essentially - pair<pg_t, chunk_id_t>). Similarly, pg->pg messages need to include - a chunk_id_t + pair<pg_t, shard_t>). Similarly, pg->pg messages need to include + a shard_t - For client->PG messages, the OSD will need a way to know which PG chunk should get the message since the OSD may contain both a primary and non-primary chunk for the same pg @@ -254,10 +257,10 @@ Core changes: - Ensure that crush behaves as above for INDEP. -`Recovery <http://tracker.ceph.com/issues/5857>`_ +Recovery -------- -The logic for recovering an object depends on the backend. With +See `Issue #5857`_. The logic for recovering an object depends on the backend. With the current replicated strategy, we first pull the object replica to the primary and then concurrently push it out to the replicas. With the erasure coded strategy, we probably want to read the @@ -271,6 +274,7 @@ PGBackend impementation will have to be able to direct the search for pg replicas with unrecoverable object chunks and to be able to determine whether a particular object is recoverable. + Core changes: - s/unfound/unrecoverable @@ -281,10 +285,10 @@ PGBackend interfaces: - recoverable() - recover_object() -`Backfill <http://tracker.ceph.com/issues/5856>`_ +Backfill -------- -For the most part, backfill itself should behave similarly between +See `Issue #5856`_. For the most part, backfill itself should behave similarly between replicated and erasure coded pools with a few exceptions: 1. We probably want to be able to backfill multiple osds concurrently @@ -311,3 +315,7 @@ PGBackend interfaces: - choose_backfill(): allows the implementation to determine which osds should be backfilled in a particular interval. + + +.. _Issue #5857: http://tracker.ceph.com/issues/5857 +.. _Issue #5856: http://tracker.ceph.com/issues/5856
\ No newline at end of file diff --git a/doc/dev/osd_internals/snaps.rst b/doc/dev/osd_internals/snaps.rst index a1e9020ba02..63b1bc0503c 100644 --- a/doc/dev/osd_internals/snaps.rst +++ b/doc/dev/osd_internals/snaps.rst @@ -70,18 +70,19 @@ See ReplicatedPG::SnapTrimmer, SnapMapper This trimming is performed asynchronously by the snap_trim_wq while the pg is clean and not scrubbing. - 1. The next snap in PG::snaptrimq is selected for trimming - 2. We determine the next object for trimming out of PG::snap_mapper. + #. The next snap in PG::snaptrimq is selected for trimming + #. We determine the next object for trimming out of PG::snap_mapper. For each object, we create a log entry and repop updating the object info and the snap set (including adjusting the overlaps). - 3. We also locally update our *SnapMapper* instance with the object's + #. We also locally update our *SnapMapper* instance with the object's new snaps. - 4. The log entry containing the modification of the object also + #. The log entry containing the modification of the object also contains the new set of snaps, which the replica uses to update its own *SnapMapper* instance. - 6. The primary shares the info with the replica, which persists + #. The primary shares the info with the replica, which persists the new set of purged_snaps along with the rest of the info. + Recovery -------- Because the trim operations are implemented using repops and log entries, diff --git a/doc/dev/release-process.rst b/doc/dev/release-process.rst index 656d5ca76c5..b48934831bb 100644 --- a/doc/dev/release-process.rst +++ b/doc/dev/release-process.rst @@ -26,7 +26,7 @@ Prior to building, it's necessary to update the pbuilder seed tarballs:: The release key should be present:: pub 4096R/17ED316D 2012-05-20 - uid Ceph Release Key <sage@newdream.net> + uid Ceph Release Key <sage@newdream.net> 3. Set up build area @@ -48,24 +48,25 @@ Checkout the submodules:: 4. Update Build version numbers ================================ -Substitute the ceph release number where indicated below by the string 0.xx:: +Substitute the ceph release number where indicated below by the string ``0.xx``. Edit configure.ac and update the version number. Example diff:: - -AC_INIT([ceph], [0.54], [ceph-devel@vger.kernel.org]) - +AC_INIT([ceph], [0.55], [ceph-devel@vger.kernel.org]) + -AC_INIT([ceph], [0.54], [ceph-devel@vger.kernel.org]) + +AC_INIT([ceph], [0.55], [ceph-devel@vger.kernel.org]) Update the version number in the debian change log:: - DEBEMAIL user@host dch -v 0.xx-1 + DEBEMAIL user@host dch -v 0.xx-1 Commit the changes:: - git commit -a + git commit -a Tag the release:: - ../ceph-build/tag-release v0.xx + ../ceph-build/tag-release v0.xx + 5. Create Makefiles =================== @@ -74,7 +75,7 @@ The actual configure options used to build packages are in the ``ceph.spec.in`` and ``debian/rules`` files. At this point we just need to create a Makefile.:: - ./do_autogen.sh + ./do_autogen.sh 6. Run the release scripts @@ -84,7 +85,8 @@ This creates tarballs and copies them, with other needed files to the build hosts listed in deb_hosts and rpm_hosts, runs a local build script, then rsyncs the results back tot the specified release directory.:: - ../ceph-build/do_release.sh /tmp/release + ../ceph-build/do_release.sh /tmp/release + 7. Create RPM Repo ================== @@ -92,35 +94,38 @@ script, then rsyncs the results back tot the specified release directory.:: Copy the rpms to the destination repo, creates the yum repository rpm and indexes.:: - ../ceph-build/push_to_rpm_repo.sh /tmp/release /tmp/rpm-repo 0.xx + ../ceph-build/push_to_rpm_repo.sh /tmp/release /tmp/rpm-repo 0.xx + 8. Create debian repo ===================== The key-id used below is the id of the ceph release key from step 2:: - mkdir /tmp/debian-repo - ../ceph-build/gen_reprepro_conf.sh /tmp/debian-repo key-id - ../ceph-build/push_to_deb_repo.sh /tmp/release /tmp/debian-repo 0.xx main + mkdir /tmp/debian-repo + ../ceph-build/gen_reprepro_conf.sh /tmp/debian-repo key-id + ../ceph-build/push_to_deb_repo.sh /tmp/release /tmp/debian-repo 0.xx main + 9. Push repos to ceph.org ========================== For a development release:: - rcp ceph-0.xx.tar.bz2 ceph-0.xx.tar.gz \ - ceph_site@ceph.com:ceph.com/downloads/. - rsync -av /tmp/rpm-repo/0.xx/ ceph_site@ceph.com:ceph.com/rpm-testing - rsync -av /tmp/debian-repo/ ceph_site@ceph.com:ceph.com/debian-testing + rcp ceph-0.xx.tar.bz2 ceph-0.xx.tar.gz \ + ceph_site@ceph.com:ceph.com/downloads/. + rsync -av /tmp/rpm-repo/0.xx/ ceph_site@ceph.com:ceph.com/rpm-testing + rsync -av /tmp/debian-repo/ ceph_site@ceph.com:ceph.com/debian-testing For a stable release, replace {CODENAME} with the release codename (e.g., ``argonaut`` or ``bobtail``):: - rcp ceph-0.xx.tar.bz2 \ - ceph_site@ceph.com:ceph.com/downloads/ceph-0.xx{CODENAME}.tar.bz2 - rcp ceph-0.xx.tar.gz \ - ceph_site@ceph.com:ceph.com/downloads/ceph-0.xx{CODENAME}.tar.gz - rsync -av /tmp/rpm-repo/0.xx/ ceph_site@ceph.com:ceph.com/rpm-{CODENAME} - rsync -auv /tmp/debian-repo/ ceph_site@ceph.com:ceph.com/debian-{CODENAME} + rcp ceph-0.xx.tar.bz2 \ + ceph_site@ceph.com:ceph.com/downloads/ceph-0.xx{CODENAME}.tar.bz2 + rcp ceph-0.xx.tar.gz \ + ceph_site@ceph.com:ceph.com/downloads/ceph-0.xx{CODENAME}.tar.gz + rsync -av /tmp/rpm-repo/0.xx/ ceph_site@ceph.com:ceph.com/rpm-{CODENAME} + rsync -auv /tmp/debian-repo/ ceph_site@ceph.com:ceph.com/debian-{CODENAME} + 10. Update Git ============== @@ -148,15 +153,16 @@ Similarly, for a development release, for both ``teuthology.git`` and ``ceph-qa- Stable release -------------- -For ``ceph.git``: +For ``ceph.git``:: + + git push origin stable - git push origin stable Point release ------------- -Just push the new tag: +Just push the new tag:: - git push origin v0.xx + git push origin v0.xx diff --git a/doc/dev/repo-lab-access.rst b/doc/dev/repo-lab-access.rst index 706f02e395c..2d1b328325a 100644 --- a/doc/dev/repo-lab-access.rst +++ b/doc/dev/repo-lab-access.rst @@ -4,11 +4,11 @@ Notes on Ceph repositories and test lab Special branches ---------------- -* ``master'': current tip (integration branch) -* ``next'': pending release (feature frozen, bugfixes only) -* ``last'': last/previous release -* ``dumpling'', ``cuttlefish'', ``bobtail'', ``argonaut'', etc.: stable release branches -* ``dumpling-next'': backports for stable release, pending testing +* ``master``: current tip (integration branch) +* ``next``: pending release (feature frozen, bugfixes only) +* ``last``: last/previous release +* ``dumpling``, ``cuttlefish``, ``bobtail``, ``argonaut``, etc.: stable release branches +* ``dumpling-next``: backports for stable release, pending testing Rules ----- @@ -18,7 +18,7 @@ The source repos are all on github. * Any branch pushed to ceph.git will kick off builds that will either run unit tests or generate packages for gitbuilder.ceph.com. Try not to generate unnecessary load. For private, unreviewed work, - only push to branches named ``wip-*''. This avoids colliding with + only push to branches named ``wip-*``. This avoids colliding with any special branches. * Nothing should every reach a special branch unless it has been @@ -34,7 +34,7 @@ The source repos are all on github. Reviewed-by: directly to the commit so that it is also visible when the patch is cherry-picked for backports. -* All backports should use ``git cherry-pick -x'' to capture which +* All backports should use ``git cherry-pick -x`` to capture which commit they are cherry-picking from. @@ -63,21 +63,21 @@ Locking machines * All tests pull their builds from gitbuilder.ceph.com. * Anybody can lock machines with ``teuthology-lock --lock-many NUM - --machine-type TYPE''. + --machine-type TYPE``. -* Machines are locked as ``whoami''@``hostname -s''. --owner to +* Machines are locked as ``whoami''@``hostname -s``. --owner to choose otherwise. -* Automated tests current run on the ``plana''; please avoid locking +* Automated tests current run on the ``plana``; please avoid locking these for personal use. -* To unlock, please use ``teuthology-nuke -t list.yaml -r -u'', which +* To unlock, please use ``teuthology-nuke -t list.yaml -r -u``, which will reboot and clean up any leftover test state before unlocking - (or fail to unlock). It looks for a ``targets::'' section in the + (or fail to unlock). It looks for a ``targets::`` section in the yaml, so the regular job yaml will work. You can get a list of all - locked machines with ``teuthology-lock --list-targets''. + locked machines with ``teuthology-lock --list-targets``. -* ``teuthology-lock -a --brief'' or ``teuthology-lock --summary'' to +* ``teuthology-lock -a --brief`` or ``teuthology-lock --summary`` to see what is locked and by whom. * Be conscientious about scheduling entire qa runs. Coordinate @@ -85,4 +85,4 @@ Locking machines ceph-qa-suite.git and teuthology.git. * Results for scheduled runs appear in /a/$jobname on the teuthology - machine. ``ls -alt | head'' to find them. + machine. ``ls -alt | head`` to find them. diff --git a/doc/dev/versions.rst b/doc/dev/versions.rst index 4fc4d8a12b2..bf5ee252cd5 100644 --- a/doc/dev/versions.rst +++ b/doc/dev/versions.rst @@ -1,6 +1,7 @@ -============== +================== Public OSD Version -============== +================== + We maintain two versions on disk: an eversion_t pg_log.head and a version_t info.user_version. Each object is tagged with both the pg version and user_version it was last modified with. The PG version is @@ -13,11 +14,11 @@ user_at_version is modified only in ReplicatedPG::prepare_transaction when the op was a "user modify" (a non-watch write), and the durable user_version is updated according to the following rules: 1) set user_at_version to the maximum of ctx->new_obs.oi.user_version+1 - and info.last_user_version+1. +and info.last_user_version+1. 2) set user_at_version to the maximum of itself and - ctx->at_version.version. +ctx->at_version.version. 3) ctx->new_obs.oi.user_version = ctx->user_at_version (to change the - object's user_version) +object's user_version) This set of update semantics mean that for traditional pools the user_version will be equal to the past reassert_version, while for diff --git a/doc/install/rpm.rst b/doc/install/rpm.rst index 72934cc8d5e..ea96d394c7a 100644 --- a/doc/install/rpm.rst +++ b/doc/install/rpm.rst @@ -202,9 +202,7 @@ Installing Ceph Object Storage ServerName {FQDN} -#. Create an Apache httpd virtual host in ``/etc/httpd/conf.d/rgw.conf``. - -.. code-block:: ini +#. Create an Apache httpd virtual host in ``/etc/httpd/conf.d/rgw.conf``. :: FastCgiExternalServer /var/www/s3gw.fcgi -socket /tmp/radosgw.sock <VirtualHost *:80> @@ -235,7 +233,7 @@ Installing Ceph Object Storage #FastCgiWrapper On -#. Add a ``fastcgi`` script. :: +#. Add a ``fastcgi`` script with the following path ``/var/www/s3gw.fcgi``. :: #!/bin/sh exec /usr/bin/radosgw -c /etc/ceph/ceph.conf -n client.radosgw.gateway @@ -243,7 +241,7 @@ Installing Ceph Object Storage #. Make ``s3gw.fcgi`` executable:: - chmod +x /var/www/rgw/s3gw.fcgi + chmod +x /var/www/s3gw.fcgi #. Create a user key. :: diff --git a/doc/man/8/ceph-conf.rst b/doc/man/8/ceph-conf.rst index 7b4f83a9282..48d63d8fb07 100644 --- a/doc/man/8/ceph-conf.rst +++ b/doc/man/8/ceph-conf.rst @@ -84,4 +84,3 @@ See also ======== :doc:`ceph <ceph>`\(8), -:doc:`mkcephfs <mkcephfs>`\(8) diff --git a/doc/man/8/ceph-osd.rst b/doc/man/8/ceph-osd.rst index 451ca6d956b..9a08bef7e04 100644 --- a/doc/man/8/ceph-osd.rst +++ b/doc/man/8/ceph-osd.rst @@ -46,8 +46,7 @@ Options .. option:: --mkfs - Create an empty object repository. Normally invoked by - :doc:`mkcephfs <mkcephfs>`\(8). This also initializes the journal + Create an empty object repository. This also initializes the journal (if one is defined). .. option:: --mkkey diff --git a/doc/man/8/ceph-post-file.rst b/doc/man/8/ceph-post-file.rst index 5625843eaa6..f16fbd8a63c 100644 --- a/doc/man/8/ceph-post-file.rst +++ b/doc/man/8/ceph-post-file.rst @@ -1,6 +1,6 @@ -======================================== +================================================== ceph-post-file -- post files for ceph developers -======================================== +================================================== .. program:: ceph-post-file diff --git a/doc/man/8/ceph.rst b/doc/man/8/ceph.rst index 634c82c433e..661d93ad5b7 100644 --- a/doc/man/8/ceph.rst +++ b/doc/man/8/ceph.rst @@ -95,4 +95,3 @@ See also ======== :doc:`ceph <ceph>`\(8), -:doc:`mkcephfs <mkcephfs>`\(8) diff --git a/doc/man/8/crushtool.rst b/doc/man/8/crushtool.rst index 187352a5c04..97303cc4bfa 100644 --- a/doc/man/8/crushtool.rst +++ b/doc/man/8/crushtool.rst @@ -7,8 +7,8 @@ Synopsis ======== -| **crushtool** ( -d *map* | -c *map.txt* | --build *numosds* - *layer1* *...* ) [ -o *outfile* [ --clobber ]] +| **crushtool** ( -d *map* | -c *map.txt* | --build --num_osds *numosds* + *layer1* *...* ) [ -o *outfile* ] Description @@ -41,9 +41,11 @@ The tool has four modes of operation. will create a relatively generic map with the given layer structure. See below for examples. -.. option:: --test ... +.. option:: --test + will perform a dry run of a CRUSH mapping for a range of input object - names, see crushtool --help for more information. + names, see crushtool --help for more information. + Options ======= @@ -52,10 +54,7 @@ Options will specify the output file. -.. option:: --clobber - - will allow the tool to overwrite an existing outfile (it will normally refuse). - + Building a map ============== @@ -83,22 +82,26 @@ preceding layer. Example ======= -Suppose we have 128 devices, each grouped into shelves with 4 devices -each, and 8 shelves per rack. We could create a three level hierarchy -with:: +Suppose we have two rows with two racks each and 20 nodes per rack. Suppose +each node contains 4 storage devices for Ceph OSD Daemons. This configuration +allows us to deploy 320 Ceph OSD Daemons. Lets assume a 42U rack with 2U nodes, +leaving an extra 2U for a rack switch. + +To reflect our hierarchy of devices, nodes, racks and rows, we would execute +the following:: - crushtool --build 128 shelf uniform 4 rack straw 8 root straw 0 -o map + crushtool -o crushmap --build --num_osds 320 node straw 4 rack straw 20 row straw 2 To adjust the default (generic) mapping rules, we can run:: # decompile - crushtool -d map -o map.txt + crushtool -d crushmap -o map.txt # edit vi map.txt # recompile - crushtool -c map.txt -o map + crushtool -c map.txt -o crushmap Availability @@ -114,4 +117,3 @@ See also :doc:`ceph <ceph>`\(8), :doc:`osdmaptool <osdmaptool>`\(8), -:doc:`mkcephfs <mkcephfs>`\(8) diff --git a/doc/man/8/mkcephfs.rst b/doc/man/8/mkcephfs.rst deleted file mode 100644 index 054a8deae1d..00000000000 --- a/doc/man/8/mkcephfs.rst +++ /dev/null @@ -1,123 +0,0 @@ -======================================= - mkcephfs -- create a ceph file system -======================================= - -.. program:: mkcephfs - -Synopsis -======== - -| **mkcephfs** -c *ceph.conf* [ --mkfs ] [ -a, --all-hosts [ -k - */path/to/admin.keyring* ] ] - - -Description -=========== - -**mkcephfs** is used to create an empty Ceph file system, possibly -spanning multiple hosts. The ceph.conf file describes the composition -of the entire Ceph cluster, including which hosts are participating, -which daemons run where, and which paths are used to store file system -data or metadata. - -The mkcephfs tool can be used in two ways. If -a is used, it will use -ssh and scp to connect to remote hosts on your behalf and do the setup -of the entire cluster. This is the easiest solution, but can also be -inconvenient (if you don't have ssh to connect without prompting for -passwords) or slow (if you have a large cluster). - -Alternatively, you can run each setup phase manually. First, you need -to prepare a monmap that will be shared by each node:: - - # prepare - master# mkdir /tmp/foo - master# mkcephfs -c /etc/ceph/ceph.conf \ - --prepare-monmap -d /tmp/foo - -Share the ``/tmp/foo`` directory with other nodes in whatever way is -convenient for you. On each OSD and MDS node:: - - osdnode# mkcephfs --init-local-daemons osd -d /tmp/foo - mdsnode# mkcephfs --init-local-daemons mds -d /tmp/foo - -Collect the contents of the /tmp/foo directories back onto a single -node, and then:: - - master# mkcephfs --prepare-mon -d /tmp/foo - -Finally, distribute ``/tmp/foo`` to all monitor nodes and, on each of -those nodes:: - - monnode# mkcephfs --init-local-daemons mon -d /tmp/foo - - -Options -======= - -.. option:: -a, --allhosts - - Performs the necessary initialization steps on all hosts in the - cluster, executing commands via SSH. - -.. option:: -c ceph.conf, --conf=ceph.conf - - Use the given conf file instead of the default ``/etc/ceph/ceph.conf``. - -.. option:: -k /path/to/keyring - - When ``-a`` is used, we can specify a location to copy the - client.admin keyring, which is used to administer the cluster. The - default is ``/etc/ceph/keyring`` (or whatever is specified in the - config file). - -.. option:: --mkfs - - Create and mount the file systems specified in the ceph.conf for - OSD data storage using mkfs.$type. The ``devs`` option in ceph.conf - must specify the device(s) and the ``osd mkfs type`` option must - specify the file system type (normally one of btrfs, xfs, or ext4). - -.. option:: --no-copy-conf - - By default, mkcephfs with -a will copy the new configuration to - /etc/ceph/ceph.conf on each node in the cluster. This option - disables that behavior. - -Subcommands -=========== - -The sub-commands performed during cluster setup can be run individually with - -.. option:: --prepare-monmap -d dir -c ceph.conf - - Create an initial monmap with a random fsid/uuid and store it and - the ceph.conf in dir. - -.. option:: --init-local-daemons type -d dir - - Initialize any daemons of type type on the local host using the - monmap in dir. For types osd and mds, the resulting authentication - keys will be placed in dir. For type mon, the initial data files - generated by --prepare-mon (below) are expected in dir. - -.. option:: --prepare-mon -d dir - - Prepare the initial monitor data based on the monmap, OSD, and MDS - authentication keys collected in dir, and put the result in dir. - - -Availability -============ - -**mkcephfs** is part of the Ceph distributed file system. Please refer -to the Ceph documentation at http://ceph.com/docs for more -information. - - -See also -======== - -:doc:`ceph <ceph>`\(8), -:doc:`monmaptool <monmaptool>`\(8), -:doc:`osdmaptool <osdmaptool>`\(8), -:doc:`crushtool <crushtool>`\(8) diff --git a/doc/man/8/monmaptool.rst b/doc/man/8/monmaptool.rst index 8415ba4136a..42af2faea8b 100644 --- a/doc/man/8/monmaptool.rst +++ b/doc/man/8/monmaptool.rst @@ -103,4 +103,3 @@ See also :doc:`ceph <ceph>`\(8), :doc:`crushtool <crushtool>`\(8), -:doc:`mkcephfs <mkcephfs>`\(8) diff --git a/doc/man/8/osdmaptool.rst b/doc/man/8/osdmaptool.rst index 27ffb6eba72..c815e485685 100644 --- a/doc/man/8/osdmaptool.rst +++ b/doc/man/8/osdmaptool.rst @@ -73,4 +73,3 @@ See also :doc:`ceph <ceph>`\(8), :doc:`crushtool <crushtool>`\(8), -:doc:`mkcephfs <mkcephfs>`\(8) diff --git a/doc/rados/configuration/ceph-conf.rst b/doc/rados/configuration/ceph-conf.rst index fc316478034..e0633483120 100644 --- a/doc/rados/configuration/ceph-conf.rst +++ b/doc/rados/configuration/ceph-conf.rst @@ -504,12 +504,8 @@ See `General Settings`_, `OSD Settings`_, `Monitor Settings`_, `MDS Settings`_, .. _RGW Settings: ../../../radosgw/config-ref/ .. _Log Settings: ../log-and-debug-ref -When deploying the Ceph configuration file, ensure that you use the cluster name -in your command line syntax. For example:: - ssh myserver01 sudo tee /etc/ceph/openstack.conf < /etc/ceph/openstack.conf - -When creating default directories or files, you should also use the cluster +When creating default directories or files, you should use the cluster name at the appropriate places in the path. For example:: sudo mkdir /var/lib/ceph/osd/openstack-0 @@ -520,10 +516,10 @@ name at the appropriate places in the path. For example:: have monitors using port 6789, use a different port for your other cluster(s). To invoke a cluster other than the default ``ceph`` cluster, use the -``--cluster=clustername`` option with the ``ceph`` command. For example:: - - ceph --cluster=openstack health +``-c {filename}.conf`` option with the ``ceph`` command. For example:: + ceph -c {cluster-name}.conf health + ceph -c openstack.conf health .. _Hardware Recommendations: ../../../install/hardware-recommendations diff --git a/doc/rados/deployment/ceph-deploy-osd.rst b/doc/rados/deployment/ceph-deploy-osd.rst index 9b27ac41094..d504b7eebbf 100644 --- a/doc/rados/deployment/ceph-deploy-osd.rst +++ b/doc/rados/deployment/ceph-deploy-osd.rst @@ -8,10 +8,6 @@ write data to the disk and to journals. So you need to provide a disk for the OSD and a path to the journal partition (i.e., this is the most common configuration, but you may configure your system to your own needs). -By default, ``ceph-deploy`` will create an OSD with the XFS filesystem. You may -override the filesystem type by providing a ``--fs-type FS_TYPE`` argument, -where ``FS_TYPE`` is an alternate filesystem such as ``ext4`` or ``btrfs``. - In Ceph v0.60 and later releases, Ceph supports ``dm-crypt`` on disk encryption. You may specify the ``--dm-crypt`` argument when preparing an OSD to tell ``ceph-deploy`` that you want to use encryption. You may also specify the diff --git a/doc/rados/man/index.rst b/doc/rados/man/index.rst index d7bbe9d3acb..e4f0f23bfa2 100644 --- a/doc/rados/man/index.rst +++ b/doc/rados/man/index.rst @@ -21,3 +21,9 @@ ../../man/8/monmaptool.rst ../../man/8/osdmaptool.rst ../../man/8/rados.rst + + +.. toctree:: + :hidden: + + ../../man/8/ceph-post-file.rst
\ No newline at end of file diff --git a/doc/rados/operations/add-or-rm-mons.rst b/doc/rados/operations/add-or-rm-mons.rst index 0a15781c6ea..17ae9d86b85 100644 --- a/doc/rados/operations/add-or-rm-mons.rst +++ b/doc/rados/operations/add-or-rm-mons.rst @@ -164,7 +164,7 @@ a cluster that has placement groups that are persistently not ``active + clean`` ceph mon dump ssh {mon-host} -#. Stop the ``ceph-mon'' daemon and extract a copy of the monap file. :: +#. Stop the ``ceph-mon`` daemon and extract a copy of the monap file. :: service ceph stop mon || stop ceph-mon-all ceph-mon -i {mon-id} --extract-monmap {map-path} diff --git a/doc/rados/troubleshooting/log-and-debug.rst b/doc/rados/troubleshooting/log-and-debug.rst index 2f2e5e4abc0..7d1ea43d1db 100644 --- a/doc/rados/troubleshooting/log-and-debug.rst +++ b/doc/rados/troubleshooting/log-and-debug.rst @@ -243,7 +243,7 @@ to their default level or to a level suitable for normal operations. +--------------------+-----------+--------------+ | ``rgw`` | 1 | 5 | +--------------------+-----------+--------------+ -| ``hadoop`` | 1 | 5 | +| ``javaclient`` | 1 | 5 | +--------------------+-----------+--------------+ | ``asok`` | 1 | 5 | +--------------------+-----------+--------------+ diff --git a/doc/radosgw/config-ref.rst b/doc/radosgw/config-ref.rst index 790fafdc854..ea650f0683a 100644 --- a/doc/radosgw/config-ref.rst +++ b/doc/radosgw/config-ref.rst @@ -3,9 +3,9 @@ ====================================== The following settings may added to the Ceph configuration file (i.e., usually -``ceph.conf``) under the ``[client.radosgw.gateway]`` section. The settings may -contain default values. If you do not specify each setting in the Ceph -configuration file, the default value will be set automatically. +``ceph.conf``) under the ``[client.radosgw.{instance-name}]`` section. The +settings may contain default values. If you do not specify each setting in the +Ceph configuration file, the default value will be set automatically. ``rgw data`` @@ -91,70 +91,222 @@ configuration file, the default value will be set automatically. :Default: None +``rgw print continue`` -``rgw swift url`` +:Description: Enable ``100-continue`` if it is operational. +:Type: Boolean +:Default: ``true`` + + +``rgw remote addr param`` + +:Description: The remote address parameter. For example, the HTTP field + containing the remote address, or the ``X-Forwarded-For`` + address if a reverse proxy is operational. -:Description: The URL for the Ceph Object Gateway Swift API. :Type: String -:Default: None - +:Default: ``REMOTE_ADDR`` -``rgw swift url prefix`` -:Description: The URL prefix for the Swift API. -:Default: ``swift`` -:Example: http://fqdn.com/swift +``rgw op thread timeout`` + +:Description: The timeout in seconds for open threads. +:Type: Integer +:Default: 600 -``rgw swift auth url`` +``rgw op thread suicide timeout`` + +:Description: The time ``timeout`` in seconds before a Ceph Object Gateway + process dies. Disbled if set to ``0``. -:Description: Default URL for verifying v1 auth tokens (if not using internal - Swift auth). +:Type: Integer +:Default: ``0`` -:Type: String -:Default: None +``rgw thread pool size`` + +:Description: The size of the thread pool. +:Type: Integer +:Default: 100 threads. -``rgw swifth auth entry`` -:Description: The entry point for a Swift auth URL. -:Type: String -:Default: ``auth`` +``rgw num control oids`` +:Description: The number of notification objects used for cache synchronization + between different ``rgw`` instances. -``rgw keystone url`` +:Type: Integer +:Default: ``8`` + + +``rgw init timeout`` + +:Description: The number of seconds before Ceph Object Gateway gives up on + initialization. + +:Type: Integer +:Default: ``30`` + + +``rgw mime types file`` + +:Description: The path and location of the MIME types. Used for Swift + auto-detection of object types. -:Description: The URL for the Keystone server. :Type: String -:Default: None +:Default: ``/etc/mime.types`` -``rgw keystone admin token`` +``rgw gc max objs`` + +:Description: The maximum number of objects that may be handled by + garbage collection in one garbage collection processing cycle. + +:Type: Integer +:Default: ``32`` + + +``rgw gc obj min wait`` + +:Description: The minimum wait time before the object may be removed + and handled by garbage collection processing. + +:Type: Integer +:Default: ``2 * 3600`` + + +``rgw gc processor max time`` + +:Description: The maximum time between the beginning of two consecutive garbage + collection processing cycles. + +:Type: Integer +:Default: ``3600`` + + +``rgw gc processor period`` + +:Description: The cycle time for garbage collection processing. +:Type: Integer +:Default: ``3600`` + + +``rgw s3 success create obj status`` + +:Description: The alternate success status response for ``create-obj``. +:Type: Integer +:Default: ``0`` + + +``rgw resolve cname`` + +:Description: Whether ``rgw`` should use DNS CNAME record of the request + hostname field (if hostname is not equal to ``rgw dns name``). + +:Type: Boolean +:Default: ``false`` + + +``rgw object stripe size`` + +:Description: The size of an object stripe for Ceph Object Gateway objects. + See `Architecture`_ for details on striping. + +:Type: Integer +:Default: ``4 << 20`` + + +``rgw extended http attrs`` + +:Description: Add new set of attributes that could be set on an object. These + extra attributes can be set through HTTP header fields when + putting the objects. If set, these attributes will return as HTTP + fields when doing GET/HEAD on the object. -:Description: The Keystone admin token (shared secret). :Type: String :Default: None +:Example: "content_foo, content_bar" -``rgw keystone accepted roles`` +``rgw exit timeout secs`` -:Description: The roles requires to serve requests. -:Type: String -:Default: ``Member, admin`` +:Description: Number of seconds to wait for a process before exiting + unconditionally. +:Type: Integer +:Default: ``120`` -``rgw keystone token cache size`` -:Description: The maximum number of entries in each Keystone token cache. +``rgw get obj window size`` + +:Description: The window size in bytes for a single object request. :Type: Integer -:Default: ``10000`` +:Default: ``16 << 20`` -``rgw keystone revocation interval`` +``rgw get obj max req size`` + +:Description: The maximum request size of a single get operation sent to the + Ceph Storage Cluster. -:Description: The number of seconds between token revocation checks. :Type: Integer -:Default: ``15 * 60`` +:Default: ``4 << 20`` + + +``rgw relaxed s3 bucket names`` + +:Description: Enables relaxed S3 bucket names rules for US region buckets. +:Type: Boolean +:Default: ``false`` + + +``rgw list buckets max chunk`` + +:Description: The maximum number of buckets to retrieve in a single operation + when listing user buckets. + +:Type: Integer +:Default: ``1000`` + + +``rgw num zone opstate shards`` + +:Description: The maximum number of shards for keeping inter-region copy + progress information. + +:Type: Integer +:Default: ``128`` + + +``rgw opstate ratelimit sec`` + +:Description: The minimum time between opstate updates on a single upload. + ``0`` disables the ratelimit. + +:Type: Integer +:Default: ``30`` + + +``rgw curl wait timeout ms`` + +:Description: The timeout in milliseconds for certain ``curl`` calls. +:Type: Integer +:Default: ``1000`` + + +``rgw copy obj progress`` + +:Description: Enables output of object progress during long copy operations. +:Type: Boolean +:Default: ``true`` + + +``rgw copy obj progress every bytes`` + +:Description: The minimum bytes between copy progress output. +:Type: Integer +:Default: ``1024 * 1024`` ``rgw admin entry`` @@ -162,69 +314,337 @@ configuration file, the default value will be set automatically. :Description: The entry point for an admin request URL. :Type: String :Default: ``admin`` - -``rgw enforce swift acls`` -:Description: Enforces the Swift Access Control List (ACL) settings. -:Type: Boolean -:Default: ``true`` +Regions +======= + +In Ceph v0.67 and beyond, Ceph Object Gateway supports federated deployments and +a global namespace via the notion of regions. A region defines the geographic +location of one or more Ceph Object Gateway instances within one or more zones. + + +Configuring regions differs from typical configuration procedures, because not +all of the settings end up in a Ceph configuration file. In Ceph v0.67 and +beyond, you can list regions, get a region configuration and set a region +configuration. + + +List Regions +------------ + +A Ceph cluster contains a list of regions. To list the regions, execute:: + + sudo radosgw-admin regions list + +The ``radosgw-admin`` returns a JSON formatted list of regions. + +.. code-block:: javascript + + { "default_info": { "default_region": "default"}, + "regions": [ + "default"]} + + +Get a Region Map +---------------- + +To list the details of each region, execute:: + + sudo radosgw-admin region-map get -``rgw swift token expiration`` +.. note:: If you receive a ``failed to read region map`` error, run + ``sudo radosgw-admin region-map update`` first. -:Description: The time in seconds for expiring a Swift token. -:Type: Integer -:Default: ``24 * 3600`` +Get a Region +------------ -``rgw print continue`` +To view the configuration of a region, execute:: -:Description: Enable ``100-continue`` if it is operational. -:Type: Boolean -:Default: ``true`` + radosgw-admin region get [--rgw-region=<region>] +The ``default`` region looks like this: -``rgw remote addr param`` +.. code-block:: javascript -:Description: The remote address parameter. For example, the HTTP field - containing the remote address, or the ``X-Forwarded-For`` - address if a reverse proxy is operational. + {"name": "default", + "api_name": "", + "is_master": "true", + "endpoints": [], + "master_zone": "", + "zones": [ + {"name": "default", + "endpoints": [], + "log_meta": "false", + "log_data": "false"} + ], + "placement_targets": [ + {"name": "default-placement", + "tags": [] }], + "default_placement": "default-placement"} + +Set a Region +------------ + +Defining a region consists of creating a JSON object, specifying at least the +required settings: + +#. ``name``: The name of the region. Required. + +#. ``api_name``: The API name for the region. Optional. + +#. ``is_master``: Determines if the region is the master region. Required. + **note:** You can only have one master region. + +#. ``endpoints``: A list of all the endpoints in the region. For example, + you may use multiple domain names to refer to the same region. Remember to + escape the forward slashes (``\/``). You may also specify a + port (``fgdn:port``) for each endpoint. Optional. + +#. ``master_zone``: The master zone for the region. Optional. Uses the default + zone if not specified. **note:** You can only have one master zone per + region. + +#. ``zones``: A list of all zones within the region. Each zone has a + name (required), a list of endpoints (optional), and whether or not the + gateway will log metadata and data operations (false by default). + +#. ``placement_targets``: A list of placement targets (optional). Each + placement target contains a name (required) and a list of tags (optional). + +#. ``default_placement``: The set of default placement pools for the object + index and object data. Set to ``default-placement`` by default. + +To set a region, create a JSON object consisting of the required fields, save +the object to a file (e.g., ``region.json``); then, execute the following +command:: + + + sudo radosgw-admin region set --infile region.json + +Where ``region.json`` is the JSON file you created. + +.. important:: The ``default`` region ``is_master`` setting is ``true`` by + default. If you create a new region and want to make it the master region, + you must either set the ``default`` region ``is_master`` setting to + ``false``, or delete the ``default`` region. + +Finally, update the map. :: + + sudo radosgw-admin region-map update + + +Set a Region Map +---------------- + +Setting a region map consists of creating a JSON object consisting of one or more +regions, and setting the ``master_region`` for the cluster. Each region in the +region map consists of a key/value pair, where the ``key`` setting is equivalent to +the ``name`` setting for an individual region configuration, and the ``val`` is +a JSON object consisting of an individual region configuration. + +You may only have one region with ``is_master`` equal to ``true``, and it must be +specified as the ``master_region`` at the end of the region map. The following +JSON object is an example of a default region map. + + +.. code-block:: javascript + + { "regions": [ + { "key": "default", + "val": { "name": "default", + "api_name": "", + "is_master": "true", + "endpoints": [], + "master_zone": "", + "zones": [ + { "name": "default", + "endpoints": [], + "log_meta": "false", + "log_data": "false"}], + "placement_targets": [ + { "name": "default-placement", + "tags": []}], + "default_placement": "default-placement" + } + } + ], + "master_region": "default" + } + +To set a region map, execute the following:: + + sudo radosgw-admin region-map set --infile regionmap.json + +Where ``regionmap.json`` is the JSON file you created. Ensure that you have +zones created for the ones specified in the region map. Finally, update the map. +:: + + sudo radosgw-admin regionmap update + + +Zones +===== + +In Ceph v0.67 and beyond, Ceph Object Gateway supports the notion of zones. A +zone defines a logical group consisting of one or more Ceph Object Gateway +instances. + +Configuring zones differs from typical configuration procedures, because not +all of the settings end up in a Ceph configuration file. In Ceph v0.67 and +beyond, you can list zones, get a zone configuration and set a zone +configuration. + + +List Zones +---------- + +To list the zones in a cluster, execute:: + + sudo radosgw-admin zone list + + +Get a Zone +---------- + +To get the configuration of a zone, execute:: + + sudo radosgw-admin zone [--rgw-zone=<zone>] + +The ``default`` zone looks like this: + +.. code-block:: javascript + + { "domain_root": ".rgw", + "control_pool": ".rgw.control", + "gc_pool": ".rgw.gc", + "log_pool": ".log", + "intent_log_pool": ".intent-log", + "usage_log_pool": ".usage", + "user_keys_pool": ".users", + "user_email_pool": ".users.email", + "user_swift_pool": ".users.swift", + "user_uid_pool": ".users.uid", + "system_key": { "access_key": "", "secret_key": ""}, + "placement_pools": [ + { "key": "default-placement", + "val": { "index_pool": ".rgw.buckets.index", + "data_pool": ".rgw.buckets"} + } + ] + } + + +Set a Zone +---------- + +Configuring a zone involves specifying a series of Ceph Object Gateway pools. +For consistency, we recommend using a pool prefix that is +the same as the zone name. See `Pools`_ for details of configuring pools. + +To set a zone, create a JSON object consisting of the pools, save +the object to a file (e.g., ``zone.json``); then, execute the following +command, replacing ``{zone-name}`` with the name of the zone:: + + sudo radosgw-admin zone set --rgw-zone={zone-name} --infile zone.json +Where ``zone.json`` is the JSON file you created. + + +Region/Zone Settings +==================== + +You may include the following settings in your Ceph configuration +file under each ``[client.radosgw.{instance-name}]`` instance. + + +.. versionadded:: v.67 + +``rgw zone`` + +:Description: The name of the zone for the gateway instance. :Type: String -:Default: ``REMOTE_ADDR`` +:Default: None -``rgw op thread timeout`` - -:Description: The timeout in seconds for open threads. -:Type: Integer -:Default: 600 - +.. versionadded:: v.67 -``rgw op thread suicide timeout`` - -:Description: The time ``timeout`` in seconds before a Ceph Object Gateway - process dies. Disbled if set to ``0``. +``rgw region`` -:Type: Integer -:Default: ``0`` +:Description: The name of the region for the gateway instance. +:Type: String +:Default: None -``rgw thread pool size`` +.. versionadded:: v.67 -:Description: The size of the thread pool. -:Type: Integer -:Default: 100 threads. +``rgw default region info oid`` +:Description: The OID for storing the default region. We do not recommend + changing this setting. + +:Type: String +:Default: ``default.region`` -``rgw num control oids`` -:Description: The number of notification objects used for cache synchronization - between different ``rgw`` instances. -:Type: Integer -:Default: ``8`` +Pools +===== + +Ceph zones map to a series of Ceph Storage Cluster pools. + +.. topic:: Manually Created Pools vs. Generated Pools + + If you provide write capabilities to the user key for your Ceph Object + Gateway, the gateway has the ability to create pools automatically. This + is convenient, but the Ceph Object Storage Cluster uses the default + values for the number of placement groups (which may not be ideal) or the + values you specified in your Ceph configuration file. If you allow the + Ceph Object Gateway to create pools automatically, ensure that you have + reasonable defaults for the number of placement groups. See + `Pool Configuration`_ for details. See `Cluster Pools`_ for details on + creating pools. + +The default pools for the Ceph Object Gateway's default zone include: + +- ``.rgw`` +- ``.rgw.control`` +- ``.rgw.gc`` +- ``.log`` +- ``.intent-log`` +- ``.usage`` +- ``.users`` +- ``.users.email`` +- ``.users.swift`` +- ``.users.uid`` + +You have significant discretion in determining how you want a zone to access +pools. You can create pools on a per zone basis, or use the same pools for +multiple zones. As a best practice, we recommend having a separate set of pools +for your master zone and your secondary zones in each region. When creating +pools for a specific zone, consider prepending the region name and zone name to +the default pool names. For example: + +- ``.region1-zone1.rgw`` +- ``.region1-zone1.rgw.control`` +- ``.region1-zone1.rgw.gc`` +- ``.region1-zone1.log`` +- ``.region1-zone1.intent-log`` +- ``.region1-zone1.usage`` +- ``.region1-zone1.users`` +- ``.region1-zone1.users.email`` +- ``.region1-zone1.users.swift`` +- ``.region1-zone1.users.uid`` + + +Ceph Object Gateways store data for the bucket index (``index_pool``) and bucket +data (``data_pool``) in placement pools. These may overlap--i.e., you may use +the same pool for the the index and the data. The index pool for default +placement is ``.rgw.buckets.index`` and for the data pool for default placement +is ``.rgw.buckets``. See `Zones`_ for details on specifying pools in a zone +configuration. .. deprecated:: v.67 @@ -243,11 +663,12 @@ configuration file, the default value will be set automatically. .. versionadded:: v.67 -``rgw zone`` +``rgw region root pool`` -:Description: The name of the zone for the gateway instance. +:Description: The pool for storing all region-specific information. :Type: String -:Default: None +:Default: ``.rgw.root`` + .. versionadded:: v.67 @@ -259,30 +680,56 @@ configuration file, the default value will be set automatically. :Default: ``.rgw.root`` -.. versionadded:: v.67 +Swift Settings +============== -``rgw region`` +``rgw enforce swift acls`` -:Description: The name of the region for the gateway instance. +:Description: Enforces the Swift Access Control List (ACL) settings. +:Type: Boolean +:Default: ``true`` + + +``rgw swift token expiration`` + +:Description: The time in seconds for expiring a Swift token. +:Type: Integer +:Default: ``24 * 3600`` + + +``rgw swift url`` + +:Description: The URL for the Ceph Object Gateway Swift API. :Type: String :Default: None + -.. versionadded:: v.67 +``rgw swift url prefix`` -``rgw region root pool`` +:Description: The URL prefix for the Swift API. +:Default: ``swift`` +:Example: http://fqdn.com/swift + -:Description: The pool for storing all region-specific information. -:Type: String -:Default: ``.rgw.root`` +``rgw swift auth url`` +:Description: Default URL for verifying v1 auth tokens (if not using internal + Swift auth). -.. versionadded:: v.67 +:Type: String +:Default: None -``rgw default region info oid`` -:Description: The OID for storing the default region. +``rgw swift auth entry`` + +:Description: The entry point for a Swift auth URL. :Type: String -:Default: ``default.region`` +:Default: ``auth`` + + + +Logging Settings +================ ``rgw log nonexistent bucket`` @@ -401,134 +848,41 @@ configuration file, the default value will be set automatically. :Default: ``false`` -``rgw init timeout`` - -:Description: The number of seconds before Ceph Object Gateway gives up on - initialization. +``rgw data log window`` +:Description: The data log entries window in seconds. :Type: Integer :Default: ``30`` -``rgw mime types file`` - -:Description: The path and location of the MIME types. Used for Swift - auto-detection of object types. - -:Type: String -:Default: ``/etc/mime.types`` - - -``rgw gc max objs`` - -:Description: The maximum number of objects that may be handled by - garbage collection in one garbage collection processing cycle. - -:Type: Integer -:Default: ``32`` - - -``rgw gc obj min wait`` - -:Description: The minimum wait time before the object may be removed - and handled by garbage collection processing. - -:Type: Integer -:Default: ``2 * 3600`` - - -``rgw gc processor max time`` - -:Description: The maximum time between the beginning of two consecutive garbage - collection processing cycles. - -:Type: Integer -:Default: ``3600`` - - -``rgw gc processor period`` - -:Description: The cycle time for garbage collection processing. -:Type: Integer -:Default: ``3600`` - - -``rgw s3 success create obj status`` +``rgw data log changes size`` -:Description: The alternate success status response for ``create-obj``. +:Description: The number of in-memory entries to hold for the data changes log. :Type: Integer -:Default: ``0`` - - -``rgw resolve cname`` - -:Description: Whether ``rgw`` should use DNS CNAME record of the request - hostname field (if hostname is not equal to ``rgw dns name``). - -:Type: Boolean -:Default: ``false`` +:Default: ``1000`` -``rgw object stripe size`` +``rgw data log num shards`` -:Description: The size of an object stripe for Ceph Object Gateway objects. - See `Architecture`_ for details on striping. +:Description: The number of shards (objects) on which to keep the + data changes log. :Type: Integer -:Default: ``4 << 20`` - +:Default: ``128`` -``rgw extended http attrs`` -:Description: Add new set of attributes that could be set on an object. These - extra attributes can be set through HTTP header fields when - putting the objects. If set, these attributes will return as HTTP - fields when doing GET/HEAD on the object. +``rgw data log obj prefix`` +:Description: The object name prefix for the data log. :Type: String -:Default: None -:Example: "content_foo, content_bar" - - -``rgw exit timeout secs`` - -:Description: Number of seconds to wait for a process before exiting - unconditionally. - -:Type: Integer -:Default: ``120`` - - -``rgw get obj window size`` - -:Description: The window size in bytes for a single object request. -:Type: Integer -:Default: ``16 << 20`` - - -``rgw get obj max req size`` - -:Description: The maximum request size of a single get operation sent to the - Ceph Storage Cluster. - -:Type: Integer -:Default: ``4 << 20`` - - -``rgw relaxed s3 bucket names`` - -:Description: Enables relaxed S3 bucket names rules for US region buckets. -:Type: Boolean -:Default: ``false`` - +:Default: ``data_log`` -``rgw list buckets max chunk`` -:Description: The maximum number of buckets to retrieve in a single operation - when listing user buckets. +``rgw replica log obj prefix`` -:Type: Integer -:Default: ``1000`` +:Description: The object name prefix for the replica log. +:Type: String +:Default: ``replica log`` ``rgw md log max shards`` @@ -538,80 +892,47 @@ configuration file, the default value will be set automatically. :Default: ``64`` -``rgw num zone opstate shards`` - -:Description: The maximum number of shards for keeping inter-region copy - progress information. - -:Type: Integer -:Default: ``128`` +Keystone Settings +================= -``rgw opstate ratelimit sec`` -:Description: The minimum time between opstate updates on a single upload. - ``0`` disables the ratelimit. - -:Type: Integer -:Default: ``30`` - - -``rgw curl wait timeout ms`` - -:Description: The timeout in milliseconds for certain ``curl`` calls. -:Type: Integer -:Default: ``1000`` - - -``rgw copy obj progress`` +``rgw keystone url`` -:Description: Enables output of object progress during long copy operations. -:Type: Boolean -:Default: ``true`` +:Description: The URL for the Keystone server. +:Type: String +:Default: None -``rgw copy obj progress every bytes`` +``rgw keystone admin token`` -:Description: The minimum bytes between copy progress output. -:Type: Integer -:Default: ``1024 * 1024`` +:Description: The Keystone admin token (shared secret). +:Type: String +:Default: None -``rgw data log window`` +``rgw keystone accepted roles`` -:Description: The data log entries window in seconds. -:Type: Integer -:Default: ``30`` +:Description: The roles requires to serve requests. +:Type: String +:Default: ``Member, admin`` -``rgw data log changes size`` +``rgw keystone token cache size`` -:Description: The number of in-memory entries to hold for the data changes log. +:Description: The maximum number of entries in each Keystone token cache. :Type: Integer -:Default: ``1000`` - +:Default: ``10000`` -``rgw data log num shards`` -:Description: The number of shards (objects) on which to keep the - data changes log. +``rgw keystone revocation interval`` +:Description: The number of seconds between token revocation checks. :Type: Integer -:Default: ``128`` - - -``rgw data log obj prefix`` - -:Description: The object name prefix for the data log. -:Type: String -:Default: ``data_log`` - - -``rgw replica log obj prefix`` +:Default: ``15 * 60`` -:Description: The object name prefix for the replica log. -:Type: String -:Default: ``replica log`` -.. _Architecture: ../../architecture#data-striping
\ No newline at end of file +.. _Architecture: ../../architecture#data-striping +.. _Pool Configuration: ../../rados/configuration/pool-pg-config-ref/ +.. _Cluster Pools: ../../rados/operations/pools
\ No newline at end of file diff --git a/doc/radosgw/config.rst b/doc/radosgw/config.rst index 7dd9f9a93dd..684a50649ec 100644 --- a/doc/radosgw/config.rst +++ b/doc/radosgw/config.rst @@ -333,7 +333,7 @@ subuser and a Swift access key. :: - sudo radosgw-admin key create --subuser=johndoe:swift --key-type=swift + sudo radosgw-admin key create --subuser=johndoe:swift --key-type=swift --gen-secret .. code-block:: javascript @@ -449,4 +449,4 @@ on client the machine(s). .. _Pool Configuration: ../../rados/configuration/pool-pg-config-ref/ .. _Pools: ../../rados/operations/pools .. _Cephx Guide: ../../rados/operations/authentication/#cephx-guide -.. _Operating a Cluster: ../../rados/rados/operations/operating
\ No newline at end of file +.. _Operating a Cluster: ../../rados/rados/operations/operating diff --git a/doc/release-notes.rst b/doc/release-notes.rst index 92254cf95e6..604b4fa296b 100644 --- a/doc/release-notes.rst +++ b/doc/release-notes.rst @@ -2,6 +2,159 @@ Release Notes =============== +v0.69 +----- + +Upgrading +~~~~~~~~~ + +* Users of the librados C++ API should replace users of get_version() + with get_version64() as the old method only returns a 32-bit value + for a 64-bit field. The existing 32-bit get_version() method is now + deprecated. + +* The OSDs are now more picky that request payload match their + declared size. A write operation across N bytes that includes M + bytes of data will now be rejected. No known clients do this, but + the because the server-side behavior has changed it is possible that + an application misusing the interface may now get errors. + +Notable Changes +~~~~~~~~~~~~~~~ + +* build cleanly under clang (Christophe Courtaut) +* common: migrate SharedPtrRegistry to use boost::shared_ptr<> (Loic Dachary) +* doc: erasure coding design notes (Loic Dachary) +* improved intel-optimized crc32c support (~8x faster on my laptop!) +* librados: get_version64() method for C++ API +* mds: fix locking deadlock (David Disseldorp) +* mon, osd: initial CLI for configuring tiering +* mon: allow cap strings with . to be unquoted +* mon: continue to discover peer addr info during election phase +* mon: fix 'osd crush move ...' command for buckets (Joao Luis) +* mon: warn when mon data stores grow very large (Joao Luis) +* objecter, librados: redirect requests based on cache tier config +* osd, librados: add new COPY_FROM rados operation +* osd, librados: add new COPY_GET rados operations (used by COPY_FROM) +* osd: add 'osd heartbeat min healthy ratio' configurable (was hard-coded at 33%) +* osd: add option to disable pg log debug code (which burns CPU) +* osd: allow cap strings with . to be unquoted +* osd: fix version value returned by various operations (Greg Farnum) +* osd: infrastructure to copy objects from other OSDs +* osd: use fdatasync(2) instead of fsync(2) to improve performance (Sam Just) +* rgw: fix major CPU utilization bug with internal caching (Yehuda Sadeh, Mark Nelson) +* rgw: fix ordering of write operations (preventing data loss on crash) (Yehuda Sadeh) +* rgw: fix ordering of writes for mulitpart upload (Yehuda Sadeh) +* rgw: fix various CORS bugs (Yehuda Sadeh) +* rgw: improve help output (Christophe Courtaut) +* rgw: validate S3 tokens against keystone (Roald J. van Loon) +* rgw: wildcard support for keystone roles (Christophe Courtaut) +* sysvinit radosgw: fix status return code (Danny Al-Gaaf) +* sysvinit rbdmap: fix error 'service rbdmap stop' (Laurent Barbe) + +v0.68 +----- + +Upgrading +~~~~~~~~~ + +* 'ceph osd crush set <id> <weight> <loc..>' no longer adds the osd to the + specified location, as that's a job for 'ceph osd crush add'. It will + however continue to work just the same as long as the osd already exists + in the crush map. + +* The OSD now enforces that class write methods cannot both mutate an + object and return data. The rbd.assign_bid method, the lone + offender, has been removed. This breaks compatibility with + pre-bobtail librbd clients by preventing them from creating new + images. + +* librados now returns on commit instead of ack for synchronous calls. + This is a bit safer in the case where both OSDs and the client crash, and + is probably how it should have been acting from the beginning. Users are + unlikely to notice but it could result in lower performance in some + circumstances. Those who care should switch to using the async interfaces, + which let you specify safety semantics precisely. + +* The C++ librados AioComplete::get_version() method was incorrectly + returning an int (usually 32-bits). To avoid breaking library + compatibility, a get_version64() method is added that returns the + full-width value. The old method is deprecated and will be removed + in a future release. Users of the C++ librados API that make use of + the get_version() method should modify their code to avoid getting a + value that is truncated from 64 to to 32 bits. + + + +Notable Changes +~~~~~~~~~~~~~~~ + +* ceph-fuse: fix problem with readahead vs truncate race (Yan, Zheng) +* ceph-post-file: new command to easily share logs or other files with ceph devs +* ceph: parse CEPH_ARGS env variable +* librados: fix async aio completion wakeup +* librados: hello_world example (Greg Farnum) +* librados: sync calls now return on commit (instead of ack) (Greg Farnum) +* mds: fix mds rejoin with legacy parent backpointer xattrs (Alexandre Oliva) +* mds: fix rare restart/failure race during fs creation +* mds: notify clients about deleted files (so they can release from their cache) (Yan, Zheng) +* mds: several bug fixes with clustered mds (Yan, Zheng) +* mon: allow logging level of cluster log (/var/log/ceph/ceph.log) to be adjusted +* mon: do not expose uncommitted state from 'osd crush {add,set} ...' (Joao Luis) +* mon: fix byte counts (off by factor of 4) (Dan Mick, Joao Luis) +* mon: fix paxos corner case +* mon: modify 'auth add' semantics to make a bit more sense (Joao Luis) +* mon: new 'osd perf' command to dump recent performance information (Samuel Just) +* mon: new and improved 'ceph -s' or 'ceph status' command (more info, easier to read) +* monc: fix small memory leak +* new wireshark patches pulled into the tree (Kevin Jones) +* objecter: fix possible hang when cluster is unpaused (Josh Durgin) +* osd: 'osd recover clone overlap limit' option to limit cloning during recovery (Samuel Just) +* osd: cls_hello OSD class example +* osd: experiemental support for ZFS (zfsonlinux.org) (Yan, Zheng) +* osd: instrument peering states (David Zafman) +* osd: properly enforce RD/WR flags for rados classes +* osd: remove old pg log on upgrade (Samuel Just) +* rgw: complete in-progress requests before shutting down +* rgw: fix S3 auth with response-* query string params (Sylvain Munaut, Yehuda Sadeh) +* sysvinit: add condrestart command (Dan van der Ster) + + +v0.67.3 "Dumpling" +------------------ + +This point release fixes a few important performance regressions with +the OSD (both with CPU and disk utilization), as well as several other +important but less common problems. We recommend that all production users +upgrade. + +Notable Changes +~~~~~~~~~~~~~~~ + +* ceph-disk: partprobe after creation journal partition +* ceph-disk: specify fs type when mounting +* ceph-post-file: new utility to help share logs and other files with ceph developers +* libcephfs: fix truncate vs readahead race (crash) +* mds: fix flock/fcntl lock deadlock +* mds: fix rejoin loop when encountering pre-dumpling backpointers +* mon: allow name and addr discovery during election stage +* mon: always refresh after Paxos store_state (fixes recovery corner case) +* mon: fix off-by-4x bug with osd byte counts +* osd: add and disable 'pg log keys debug' by default +* osd: add option to disable throttling +* osd: avoid leveldb iterators for pg log append and trim +* osd: fix readdir_r invocations +* osd: use fdatasync instead of sync +* radosgw: fix sysvinit script return status +* rbd: relicense as LGPL2 +* rgw: flush pending data on multipart upload +* rgw: recheck object name during S3 POST +* rgw: reorder init/startup +* rpm: fix debuginfo package build + +For more detailed information, see :download:`the complete changelog <changelog/v0.67.3.txt>`. + + v0.67.2 "Dumpling" ------------------ @@ -165,11 +318,11 @@ In addition to the above notes about upgrading from v0.66: * The 'ceph osd tell ...' and 'ceph mon tell ...' commands are no longer supported. Any callers should use:: - ceph tell osd.<id or *> ... - ceph tell mon.<id or name or *> ... + ceph tell osd.<id or *> ... + ceph tell mon.<id or name or *> ... The 'ceph mds tell ...' command is still there, but will soon also - transition to 'ceph tell mds.<id or name or *> ...' + transition to 'ceph tell mds.<id or name or \*> ...' * The 'ceph osd crush add ...' command used to take one of two forms:: @@ -194,7 +347,7 @@ In addition to the above notes about upgrading from v0.66: renamed to 'mon osd min down {reporters|reports}', and the documentation has been updated to reflect that these options apply to the monitors (who process failure reports) and not OSDs. If you - have adjusted these settings, please update your ``ceph.conf'' + have adjusted these settings, please update your ``ceph.conf`` accordingly. @@ -398,11 +551,11 @@ Upgrading * The 'ceph osd tell ...' and 'ceph mon tell ...' commands are no longer supported. Any callers should use:: - ceph tell osd.<id or *> ... - ceph tell mon.<id or name or *> ... + ceph tell osd.<id or *> ... + ceph tell mon.<id or name or *> ... The 'ceph mds tell ...' command is still there, but will soon also - transition to 'ceph tell mds.<id or name or *> ...' + transition to 'ceph tell mds.<id or name or \*> ...' * The 'ceph osd crush add ...' command used to take one of two forms:: @@ -496,7 +649,7 @@ Upgrading renamed to 'mon osd min down {reporters|reports}', and the documentation has been updated to reflect that these options apply to the monitors (who process failure reports) and not OSDs. If you - have adjusted these settings, please update your ``ceph.conf'' + have adjusted these settings, please update your ``ceph.conf`` accordingly. Notable Changes diff --git a/doc/start/quick-rgw.rst b/doc/start/quick-rgw.rst index 76d4e2630a3..af48a3154c1 100644 --- a/doc/start/quick-rgw.rst +++ b/doc/start/quick-rgw.rst @@ -306,7 +306,7 @@ Next, create a subuser for the Swift-compatible interface. :: :: - sudo radosgw-admin key create --subuser=johndoe:swift --key-type=swift + sudo radosgw-admin key create --subuser=johndoe:swift --key-type=swift --gen-secret .. code-block:: javascript @@ -348,4 +348,4 @@ tutorials. See the `S3-compatible`_ and `Swift-compatible`_ APIs for details. .. _Ceph Object Storage Manual Install: ../../radosgw/manual-install .. _RGW Configuration: ../../radosgw/config .. _S3-compatible: ../../radosgw/s3 -.. _Swift-compatible: ../../radosgw/swift
\ No newline at end of file +.. _Swift-compatible: ../../radosgw/swift diff --git a/doc/start/quick-start-preflight.rst b/doc/start/quick-start-preflight.rst index 58068f1df22..74dc403c211 100644 --- a/doc/start/quick-start-preflight.rst +++ b/doc/start/quick-start-preflight.rst @@ -10,17 +10,17 @@ demo cluster to explore some of the functionality. This **Preflight Checklist** will help you prepare an admin node and a server node for use with ``ceph-deploy``. -.. ditaa:: +.. ditaa:: /----------------\ /----------------\ | Admin Node |<------->| Server Node | | cCCC | | cCCC | \----------------/ \----------------/ - + Before you can deploy Ceph using ``ceph-deploy``, you need to ensure that you have a few things set up first on your admin node and on nodes running Ceph daemons. - + Install an Operating System =========================== @@ -42,7 +42,7 @@ SSH server. :: Create a User ============= -Create a user on nodes running Ceph daemons. +Create a user on nodes running Ceph daemons. .. tip:: We recommend a username that brute force attackers won't guess easily (e.g., something other than ``root``, ``ceph``, etc). @@ -55,12 +55,12 @@ Create a user on nodes running Ceph daemons. ``ceph-deploy`` installs packages onto your nodes. This means that -the user you create requires passwordless ``sudo`` privileges. +the user you create requires passwordless ``sudo`` privileges. -.. note:: We **DO NOT** recommend enabling the ``root`` password - for security reasons. +.. note:: We **DO NOT** recommend enabling the ``root`` password + for security reasons. -To provide full privileges to the user, add the following to +To provide full privileges to the user, add the following to ``/etc/sudoers.d/ceph``. :: echo "ceph ALL = (root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/ceph @@ -81,22 +81,25 @@ running Ceph daemons (leave the passphrase empty). :: Your identification has been saved in /ceph-client/.ssh/id_rsa. Your public key has been saved in /ceph-client/.ssh/id_rsa.pub. -Copy the key to each node running Ceph daemons:: +Copy the key to each node running Ceph daemons:: ssh-copy-id ceph@ceph-server -Modify your ~/.ssh/config file of your admin node so that it defaults +Modify your ~/.ssh/config file of your admin node so that it defaults to logging in as the user you created when no username is specified. :: Host ceph-server Hostname ceph-server.fqdn-or-ip-address.com User ceph +.. note:: Do not call ceph-deploy with ``sudo`` or run as ``root`` if you are + login in as a different user (as in the ssh config above) because it + will not issue ``sudo`` commands needed on the remote host. Install ceph-deploy =================== -To install ``ceph-deploy``, execute the following:: +To install ``ceph-deploy``, execute the following:: wget -q -O- 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc' | sudo apt-key add - echo deb http://ceph.com/debian-dumpling/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list @@ -112,7 +115,7 @@ node (e.g., ensure ``iptables``, ``ufw`` or other tools that may prevent connections, traffic forwarding, etc. to allow what you need). .. tip:: The ``ceph-deploy`` tool is new and you may encounter some issues - without effective error messages. + without effective error messages. Once you have completed this pre-flight checklist, you are ready to begin using ``ceph-deploy``. @@ -147,7 +150,7 @@ Once you have passwordless ``ssh`` connectivity, passwordless ``sudo``, installed ``ceph-deploy``, and you have ensured appropriate connectivity, proceed to the `Storage Cluster Quick Start`_. -.. tip:: The ``ceph-deploy`` utility can install Ceph packages on remote +.. tip:: The ``ceph-deploy`` utility can install Ceph packages on remote machines from the admin node! .. _Storage Cluster Quick Start: ../quick-ceph-deploy diff --git a/m4/ax_check_compile_flag.m4 b/m4/ax_check_compile_flag.m4 new file mode 100644 index 00000000000..c3a8d695a1b --- /dev/null +++ b/m4/ax_check_compile_flag.m4 @@ -0,0 +1,72 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_COMPILE_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the current language's compiler +# or gives an error. (Warnings, however, are ignored) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the current language's default +# flags (e.g. CFLAGS) when the check is done. The check is thus made with +# the flags: "CFLAGS EXTRA-FLAGS FLAG". This can for example be used to +# force the compiler to issue an error when a bad flag is given. +# +# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this +# macro in sync with AX_CHECK_{PREPROC,LINK}_FLAG. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim <guidod@gmx.de> +# Copyright (c) 2011 Maarten Bosmans <mkbosmans@gmail.com> +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 2 + +AC_DEFUN([AX_CHECK_COMPILE_FLAG], +[AC_PREREQ(2.59)dnl for _AC_LANG_PREFIX +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]flags_$4_$1])dnl +AC_CACHE_CHECK([whether _AC_LANG compiler accepts $1], CACHEVAR, [ + ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS + _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $4 $1" + AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], + [AS_VAR_SET(CACHEVAR,[yes])], + [AS_VAR_SET(CACHEVAR,[no])]) + _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags]) +AS_IF([test x"AS_VAR_GET(CACHEVAR)" = xyes], + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_COMPILE_FLAGS diff --git a/qa/workunits/mon/pool_ops.sh b/qa/workunits/mon/pool_ops.sh index e98e1e4121e..2436cc4837e 100755 --- a/qa/workunits/mon/pool_ops.sh +++ b/qa/workunits/mon/pool_ops.sh @@ -2,7 +2,8 @@ set -e -ceph osd pool create foo 123 123 +ceph osd pool create foo 123 123 key1=+++ && exit 1 || true +ceph osd pool create foo 123 123 key1=value1 key2 key3=value3 ceph osd pool create fooo 123 ceph osd pool create foo 123 # idempotent diff --git a/qa/workunits/mon/rbd_snaps_ops.sh b/qa/workunits/mon/rbd_snaps_ops.sh new file mode 100755 index 00000000000..29e94df7cad --- /dev/null +++ b/qa/workunits/mon/rbd_snaps_ops.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# attempt to trigger #6047 + + +cmd_no=0 +expect() +{ + cmd_no=$(($cmd_no+1)) + cmd="$1" + expected=$2 + echo "[$cmd_no] $cmd" + eval $cmd + ret=$? + if [[ $ret -ne $expected ]]; then + echo "[$cmd_no] unexpected return '$ret', expected '$expected'" + exit 1 + fi +} + +expect 'ceph osd pool create test 256 256' 0 +expect 'ceph osd pool mksnap test snapshot' 0 +expect 'ceph osd pool rmsnap test snapshot' 0 + +expect 'rbd --pool=test create --size=102400 image' 0 +expect 'rbd --pool=test snap create image@snapshot' 22 + +expect 'ceph osd pool delete test test --yes-i-really-really-mean-it' 0 +expect 'ceph osd pool create test 256 256' 0 +expect 'rbd --pool=test create --size=102400 image' 0 +expect 'rbd --pool=test snap create image@snapshot' 0 +expect 'rbd --pool=test snap ls image' 0 +expect 'rbd --pool=test snap rm image@snapshot' 0 + +expect 'ceph osd pool mksnap test snapshot' 22 + +expect 'ceph osd pool delete test test --yes-i-really-really-mean-it' 0 + +echo OK diff --git a/qa/workunits/rados/caching_redirects.sh b/qa/workunits/rados/caching_redirects.sh index a8eda487246..19b940b5b4c 100755 --- a/qa/workunits/rados/caching_redirects.sh +++ b/qa/workunits/rados/caching_redirects.sh @@ -11,9 +11,9 @@ expect_false() #create pools, set up tier relationship ceph osd pool create base_pool 2 -ceph osd pool create empty_cache 2 +ceph osd pool create partial_cache 2 ceph osd pool create data_cache 2 -ceph osd tier add base_pool empty_cache +ceph osd tier add base_pool partial_cache ceph osd tier add base_pool data_cache # populate base_pool and data_cache with some data @@ -25,6 +25,8 @@ rados -p base_pool put barobj bar.txt # data_cache is backwards so we can tell we read from it rados -p data_cache put fooobj bar.txt rados -p data_cache put barobj foo.txt +# partial_cache gets barobj backwards +rados -p partial_cache put barobj foo.txt # get the objects back before setting a caching pool rados -p base_pool get fooobj tmp.txt @@ -32,20 +34,18 @@ diff -q tmp.txt foo.txt rados -p base_pool get barobj tmp.txt diff -q tmp.txt bar.txt -# set up redirect and make sure we get nothing -ceph osd tier set-overlay base_pool empty_cache -expect_false rados -p base_pool get fooobj tmp.txt -expect_false rados -p base_pool get barobj tmp.txt -#let's write as well -rados -p base_pool put fooobj baz.txt -rados -p base_pool put barobj baz.txt -#and make sure we can look at the cache pool directly -rados -p empty_cache get fooobj tmp.txt -diff -q tmp.txt baz.txt +# set up redirect and make sure we get redirect-based results +ceph osd tier set-overlay base_pool partial_cache +ceph osd tier cache-mode partial_cache writeback +rados -p base_pool get fooobj tmp.txt +diff -q tmp.txt foo.txt +rados -p base_pool get barobj tmp.txt +diff -q tmp.txt foo.txt # switch cache pools and make sure contents differ ceph osd tier remove-overlay base_pool ceph osd tier set-overlay base_pool data_cache +ceph osd tier cache-mode data_cache writeback rados -p base_pool get fooobj tmp.txt diff -q tmp.txt bar.txt rados -p base_pool get barobj tmp.txt diff --git a/src/Makefile-env.am b/src/Makefile-env.am new file mode 100644 index 00000000000..cc9ffc62f12 --- /dev/null +++ b/src/Makefile-env.am @@ -0,0 +1,172 @@ +AUTOMAKE_OPTIONS = gnu subdir-objects + +SUBDIRS = +DIST_SUBDIRS = +BUILT_SOURCES = +EXTRA_DIST = +CLEANFILES = + +noinst_HEADERS = +bin_PROGRAMS = +bin_SCRIPTS = +sbin_PROGRAMS = +sbin_SCRIPTS = +dist_bin_SCRIPTS = +lib_LTLIBRARIES = +noinst_LTLIBRARIES = +noinst_LIBRARIES = +radoslib_LTLIBRARIES = + +# like bin_PROGRAMS, but these targets are only built for debug builds +bin_DEBUGPROGRAMS = + +# like sbin_SCRIPTS but can be used to install to e.g. /usr/sbin +ceph_sbindir = $(exec_prefix)$(sbindir) + +# C/C++ tests to build will be appended to this +check_PROGRAMS = + +# when doing a debug build, make sure to make the targets +if WITH_DEBUG +bin_PROGRAMS += $(bin_DEBUGPROGRAMS) +endif + + +################################## +## automake environment + +AM_COMMON_FLAGS = \ + -D__CEPH__ \ + -D_FILE_OFFSET_BITS=64 \ + -D_REENTRANT \ + -D_THREAD_SAFE \ + -D__STDC_FORMAT_MACROS \ + -D_GNU_SOURCE \ + -rdynamic \ + -Wall \ + ${WARN_TYPE_LIMITS} \ + ${WARN_IGNORED_QUALIFIERS} \ + -Winit-self \ + -Wpointer-arith \ + -Werror=format-security \ + -fno-strict-aliasing \ + -fsigned-char + +AM_CFLAGS = $(AM_COMMON_FLAGS) +AM_CPPFLAGS = \ + $(AM_COMMON_FLAGS) \ + -DCEPH_LIBDIR=\"${libdir}\" +AM_CXXFLAGS = \ + @AM_CXXFLAGS@ \ + $(AM_COMMON_FLAGS) \ + -DCEPH_LIBDIR=\"${libdir}\" \ + -Wnon-virtual-dtor \ + -Wno-invalid-offsetof \ + -Wstrict-null-sentinel + +# note: this is position dependant, it affects the -l options that +# come after it on the command line. when you use ${AM_LDFLAGS} in +# later rules, take care where you place it. for more information, see +# http://blog.flameeyes.eu/2008/11/19/relationship-between-as-needed-and-no-undefined-part-1-what-do-they-do +# http://blog.flameeyes.eu/2008/11/20/misguided-link-and-as-needed +# http://www.gentoo.org/proj/en/qa/asneeded.xml +# http://gcc.gnu.org/ml/gcc-help/2010-12/msg00338.html +# http://sigquit.wordpress.com/2011/02/16/why-asneeded-doesnt-work-as-expected-for-your-libraries-on-your-autotools-project/ +AM_LDFLAGS = -Wl,--as-needed + +if USE_BOOST_SPIRIT_OLD_HDR +AM_CXXFLAGS += -DUSE_BOOST_SPIRIT_OLD_HDR +endif + +if WITH_LIBATOMIC +AM_LDFLAGS += -latomic_ops +endif + +if ENABLE_COVERAGE +AM_CFLAGS += -fprofile-arcs -ftest-coverage +AM_CXXFLAGS += -fprofile-arcs -ftest-coverage -O0 +endif + +CCAS = ${srcdir}/yasm-wrapper +AM_CCASFLAGS = -f elf64 + + +##################### +## library definitions and dependencies + +EXTRALIBS = -luuid -lm -lkeyutils +if FREEBSD +EXTRALIBS += -lexecinfo +endif # FREEBSD + +if LINUX +EXTRALIBS += -lrt +endif # LINUX + +if WITH_PROFILER +EXTRALIBS += -lprofiler +endif # PROFILER + +LIBGLOBAL = libglobal.la +LIBCOMMON = libcommon.la +LIBARCH = libarch.la +LIBPERFGLUE = libperfglue.la +LIBAUTH = libauth.la +LIBMSG = libmsg.la +LIBCRUSH = libcrush.la +LIBJSON_SPIRIT = libjson_spirit.la +LIBLOG = liblog.la +LIBOS = libos.la +LIBOSD = libosd.la +LIBOSDC = libosdc.la +LIBMON = libmon.la +LIBMDS = libmds.la +LIBCLIENT = libclient.la +LIBCLIENT_FUSE = libclient_fuse.la +LIBRADOS = librados.la +LIBRGW = librgw.la +LIBRBD = librbd.la +LIBCEPHFS = libcephfs.la + +if WITH_LIBAIO +LIBOS += -laio +endif # WITH_LIBAIO + +if WITH_LIBZFS +LIBOS += libos_zfs.a -lzfs +endif # WITH_LIBZFS + +if WITH_TCMALLOC +LIBPERFGLUE += -ltcmalloc +endif # WITH_TCMALLOC + +if ENABLE_COVERAGE +EXTRALIBS += -lgcov +endif # ENABLE_COVERAGE + +# Libosd always needs osdc and os +LIBOSD += $(LIBOSDC) $(LIBOS) + +# These have references to syms like ceph_using_tcmalloc(), glue libperfglue to them +LIBMON += $(LIBPERFGLUE) +LIBOSD += $(LIBPERFGLUE) +LIBMDS += $(LIBPERFGLUE) + +# Always use system leveldb +LIBOS += -lleveldb -lsnappy + +# Use this for binaries requiring libglobal +CEPH_GLOBAL = $(LIBGLOBAL) $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS) + +# This is set by [lib]/Makefile.am and used for build tests +LIBCOMMON_DEPS = +LIBRADOS_DEPS = +LIBRGW_DEPS = + +# This is used by the dencoder test +DENCODER_SOURCES = +DENCODER_DEPS = + + +radoslibdir = $(libdir)/rados-classes + diff --git a/src/Makefile.am b/src/Makefile.am index 3bdec278c6f..c0ed016006d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,1287 +1,125 @@ -AUTOMAKE_OPTIONS = gnu -SUBDIRS = ocf java -DIST_SUBDIRS = gtest ocf libs3 java - -EXTRA_DIST = \ - libs3/COPYING \ - libs3/ChangeLog \ - libs3/GNUmakefile \ - libs3/GNUmakefile.mingw \ - libs3/GNUmakefile.osx \ - libs3/INSTALL \ - libs3/LICENSE \ - libs3/README \ - libs3/TODO \ - libs3/archlinux \ - libs3/debian \ - libs3/doxyfile \ - libs3/inc \ - libs3/libs3.spec \ - libs3/mswin \ - libs3/src \ - libs3/test \ - unittest_bufferlist.sh \ - yasm-wrapper - -CLEANFILES = -bin_PROGRAMS = -# like bin_PROGRAMS, but these targets are only built for debug builds -bin_DEBUGPROGRAMS = -sbin_PROGRAMS = -# like sbin_SCRIPTS but can be used to install to e.g. /usr/sbin -ceph_sbindir = $(exec_prefix)$(sbindir) -ceph_sbin_SCRIPTS = \ - ceph-disk \ - ceph-disk-prepare \ - ceph-disk-activate \ - ceph-disk-udev \ - ceph-create-keys - -sbin_SCRIPTS = \ - mount.fuse.ceph - -bin_SCRIPTS = ceph ceph-run ceph-rest-api ceph-clsinfo ceph-debugpack ceph-rbdnamer ceph-post-file -dist_bin_SCRIPTS = -# C/C++ tests to build will be appended to this -check_PROGRAMS = -# tests to actually run on "make check"; if you need extra, non-test, -# executables built, you need to replace this with manual assignments -# target by target -TESTS = $(check_PROGRAMS) unittest_bufferlist.sh - -check-local: - $(srcdir)/test/encoding/check-generated.sh - $(srcdir)/test/encoding/readable.sh ../ceph-object-corpus - -EXTRALIBS = -luuid -if FREEBSD -EXTRALIBS += -lexecinfo -endif -if LINUX -EXTRALIBS += -lrt -endif -if WITH_PROFILER -EXTRALIBS += -lprofiler -endif - -LIBGLOBAL_LDA = libglobal.la $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS) - -LIBOS_LDA = libos.a - -if WITH_LIBAIO -LIBOS_LDA += -laio -endif - -if WITH_LIBZFS -LIBOS_LDA += libos_zfs.a -lzfs -endif - -# use system leveldb -LIBOS_LDA += -lleveldb -lsnappy - -# monitor -ceph_mon_SOURCES = ceph_mon.cc common/TextTable.cc -ceph_mon_LDFLAGS = $(AM_LDFLAGS) -ceph_mon_LDADD = libmon.a $(LIBOS_LDA) $(LIBGLOBAL_LDA) -ceph_mon_CXXFLAGS = ${CRYPTO_CXXFLAGS} ${AM_CXXFLAGS} +include Makefile-env.am + +SUBDIRS += ocf java +DIST_SUBDIRS += gtest ocf libs3 java + + +# subdirs + +include arch/Makefile.am +include auth/Makefile.am +include crush/Makefile.am +include mon/Makefile.am +include mds/Makefile.am +include os/Makefile.am +include osd/Makefile.am +include osdc/Makefile.am +include client/Makefile.am +include global/Makefile.am +include json_spirit/Makefile.am +include log/Makefile.am +include perfglue/Makefile.am +include common/Makefile.am +include msg/Makefile.am +include messages/Makefile.am +include include/Makefile.am +include librados/Makefile.am +include librbd/Makefile.am +include rgw/Makefile.am +include cls/Makefile.am +include key_value_store/Makefile.am +include test/Makefile.am +include tools/Makefile.am + + +# core daemons + +ceph_mon_SOURCES = ceph_mon.cc +ceph_mon_LDADD = $(LIBMON) $(LIBOS) $(CEPH_GLOBAL) bin_PROGRAMS += ceph-mon -ceph_mon_store_converter_SOURCES = mon_store_converter.cc \ - mon/MonitorStore.cc -ceph_mon_store_converter_LDFLAGS = ${AM_LDFLAGS} -ceph_mon_store_converter_LDADD = $(LIBOS_LDA) $(LIBGLOBAL_LDA) -ceph_mon_store_converter_CXXFLAGS = ${AM_CXXFLAGS} -bin_PROGRAMS += ceph_mon_store_converter - - -# osd ceph_osd_SOURCES = ceph_osd.cc -ceph_osd_LDADD = libosd.a libosdc.la $(LIBOS_LDA) $(LIBGLOBAL_LDA) -ceph_osd_CXXFLAGS = ${AM_CXXFLAGS} -bin_PROGRAMS += ceph-osd - +ceph_osd_LDADD = $(LIBOSD) $(CEPH_GLOBAL) if LINUX ceph_osd_LDADD += -ldl -endif +endif # LINUX +bin_PROGRAMS += ceph-osd -# mds ceph_mds_SOURCES = ceph_mds.cc -ceph_mds_LDADD = libmds.a libosdc.la $(LIBGLOBAL_LDA) +ceph_mds_LDADD = $(LIBMDS) $(LIBOSDC) $(CEPH_GLOBAL) bin_PROGRAMS += ceph-mds -ceph_mds_CXXFLAGS = ${AM_CXXFLAGS} + # admin tools -ceph_conf_SOURCES = ceph_conf.cc -ceph_conf_LDADD = $(LIBGLOBAL_LDA) -ceph_conf_CXXFLAGS = ${AM_CXXFLAGS} -ceph_authtool_SOURCES = ceph_authtool.cc -ceph_authtool_LDADD = $(LIBGLOBAL_LDA) -ceph_filestore_dump_SOURCES = tools/ceph-filestore-dump.cc -ceph_filestore_dump_SOURCES += perfglue/disabled_heap_profiler.cc -ceph_filestore_dump_LDADD = libosd.a libosdc.la $(LIBOS_LDA) $(LIBGLOBAL_LDA) -lboost_program_options -if LINUX -ceph_filestore_dump_LDADD += -ldl -endif -bin_PROGRAMS += ceph-conf ceph-authtool ceph_filestore_dump - -ceph_osdomap_tool_SOURCES = tools/ceph-osdomap-tool.cc \ - os/LevelDBStore.cc -ceph_osdomap_tool_LDFLAGS = ${AM_LDFLAGS} -ceph_osdomap_tool_LDADD = $(LIBOS_LDA) $(LIBGLOBAL_LDA) -lboost_program_options -ceph_osdomap_tool_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph-osdomap-tool - -ceph_monstore_tool_SOURCES = tools/ceph-monstore-tool.cc \ - os/LevelDBStore.cc -ceph_monstore_tool_LDFLAGS = ${AM_LDFLAGS} -ceph_monstore_tool_LDADD = $(LIBOS_LDA) $(LIBGLOBAL_LDA) -lboost_program_options -ceph_monstore_tool_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph-monstore-tool - -monmaptool_SOURCES = monmaptool.cc -monmaptool_LDADD = $(LIBGLOBAL_LDA) -crushtool_SOURCES = crushtool.cc -crushtool_LDADD = $(LIBGLOBAL_LDA) -osdmaptool_SOURCES = osdmaptool.cc -osdmaptool_LDADD = $(LIBGLOBAL_LDA) -bin_PROGRAMS += monmaptool crushtool osdmaptool - -rgw_dencoder_src = rgw/rgw_dencoder.cc \ - rgw/rgw_acl.cc \ - rgw/rgw_common.cc \ - rgw/rgw_env.cc \ - rgw/rgw_json_enc.cc - -ceph_dencoder_SOURCES = test/encoding/ceph_dencoder.cc ${rgw_dencoder_src} perfglue/disabled_heap_profiler.cc -ceph_dencoder_CXXFLAGS = ${AM_CXXFLAGS} -ceph_dencoder_LDADD = $(LIBGLOBAL_LDA) libcls_lock_client.a \ - libcls_rgw_client.a \ - libcls_replica_log_client.a \ - libcls_refcount_client.a \ - libosd.a libmds.a libosdc.la $(LIBOS_LDA) libmon.a -bin_PROGRAMS += ceph-dencoder - -mount_ceph_SOURCES = mount/mount.ceph.c common/armor.c common/safe_io.c common/secret.c include/addr_parsing.c -mount_ceph_LDADD = -lkeyutils + +# user tools + +mount_ceph_SOURCES = mount/mount.ceph.c +mount_ceph_LDADD = $(LIBCOMMON) if LINUX sbin_PROGRAMS += mount.ceph -endif +endif # LINUX +sbin_SCRIPTS += mount.fuse.ceph -# user tools cephfs_SOURCES = cephfs.cc -cephfs_LDADD = libcommon.la +cephfs_LDADD = $(LIBCOMMON) bin_PROGRAMS += cephfs librados_config_SOURCES = librados-config.cc -librados_config_LDADD = librados.la $(LIBGLOBAL_LDA) +librados_config_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) bin_PROGRAMS += librados-config -# synthetic client -ceph_syn_SOURCES = ceph_syn.cc client/SyntheticClient.cc -ceph_syn_LDADD = libclient.la $(LIBGLOBAL_LDA) +ceph_syn_SOURCES = ceph_syn.cc +ceph_syn_SOURCES += client/SyntheticClient.cc # uses g_conf.. needs cleanup +ceph_syn_LDADD = $(LIBCLIENT) $(CEPH_GLOBAL) bin_PROGRAMS += ceph-syn -base: ceph-mon ceph-osd ceph-mds \ - cephfs \ - ceph-syn \ - rados radosgw librados-config \ - ceph-conf monmaptool osdmaptool crushtool ceph-authtool \ - init-ceph mkcephfs mon_store_converter ceph-post-file +rbd_SOURCES = rbd.cc +rbd_LDADD = $(LIBRBD) $(LIBRADOS) $(CEPH_GLOBAL) +if LINUX +bin_PROGRAMS += rbd +endif #LINUX + -# fuse targets? +# Fuse targets + if WITH_FUSE -ceph_fuse_SOURCES = ceph_fuse.cc client/fuse_ll.cc -ceph_fuse_LDADD = -lfuse libclient.la $(LIBGLOBAL_LDA) -ceph_fuse_CXXFLAGS = ${AM_CXXFLAGS} +ceph_fuse_SOURCES = ceph_fuse.cc +ceph_fuse_LDADD = $(LIBCLIENT_FUSE) $(CEPH_GLOBAL) bin_PROGRAMS += ceph-fuse rbd_fuse_SOURCES = rbd_fuse/rbd-fuse.c -rbd_fuse_LDADD = -lfuse librados.la librbd.la $(LIBGLOBAL_LDA) -rbd_fuse_CXXFLAGS = ${AM_CXXFLAGS} +rbd_fuse_LDADD = -lfuse $(LIBRBD) $(LIBRADOS) $(CEPH_GLOBAL) bin_PROGRAMS += rbd-fuse +endif # WITH_FUSE -endif -# tcmalloc? -if WITH_TCMALLOC -tcmalloc_safety_flags = -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free -ceph_osd_LDADD += -ltcmalloc -ceph_osd_CXXFLAGS += ${tcmalloc_safety_flags} -ceph_osd_SOURCES += perfglue/heap_profiler.cc -ceph_mds_LDADD += -ltcmalloc -ceph_mds_CXXFLAGS += ${tcmalloc_safety_flags} -ceph_mds_SOURCES += perfglue/heap_profiler.cc -ceph_mon_LDADD += -ltcmalloc -ceph_mon_CXXFLAGS += ${tcmalloc_safety_flags} -ceph_mon_SOURCES += perfglue/heap_profiler.cc -if WITH_FUSE -ceph_fuse_LDADD += -ltcmalloc -ceph_fuse_CXXFLAGS += ${tcmalloc_safety_flags} -endif #WITH_FUSE -else -ceph_osd_SOURCES += perfglue/disabled_heap_profiler.cc -ceph_mds_SOURCES += perfglue/disabled_heap_profiler.cc -ceph_mon_SOURCES += perfglue/disabled_heap_profiler.cc -endif # WITH_TCMALLOC - -# debug targets -ceph_psim_SOURCES = psim.cc -ceph_psim_LDADD = $(LIBGLOBAL_LDA) -bin_DEBUGPROGRAMS += ceph_psim - -ceph_test_mutate_SOURCES = test/test_mutate.cc -ceph_test_mutate_LDADD = librados.la $(LIBGLOBAL_LDA) -bin_DEBUGPROGRAMS += ceph_test_mutate - -ceph_test_rewrite_latency_SOURCES = test/test_rewrite_latency.cc -ceph_test_rewrite_latency_LDADD = libcommon.la $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS) -bin_DEBUGPROGRAMS += ceph_test_rewrite_latency - -ceph_test_msgr_SOURCES = testmsgr.cc -ceph_test_msgr_LDADD = $(LIBGLOBAL_LDA) -bin_DEBUGPROGRAMS += ceph_test_msgr - -ceph_test_ioctls_SOURCES = client/test_ioctls.c -bin_DEBUGPROGRAMS += ceph_test_ioctls - -ceph_dupstore_SOURCES = dupstore.cc -ceph_dupstore_CXXFLAGS= ${CRYPTO_CXXFLAGS} ${AM_CXXFLAGS} -ceph_dupstore_LDADD = $(LIBOS_LDA) $(LIBGLOBAL_LDA) -ceph_streamtest_SOURCES = streamtest.cc -ceph_streamtest_CXXFLAGS= ${CRYPTO_CXXFLAGS} ${AM_CXXFLAGS} -ceph_streamtest_LDADD = $(LIBOS_LDA) $(LIBGLOBAL_LDA) -bin_DEBUGPROGRAMS += ceph_dupstore ceph_streamtest - -ceph_test_trans_SOURCES = test_trans.cc -ceph_test_trans_CXXFLAGS= ${CRYPTO_CXXFLAGS} ${AM_CXXFLAGS} -ceph_test_trans_LDADD = $(LIBOS_LDA) $(LIBGLOBAL_LDA) -bin_DEBUGPROGRAMS += ceph_test_trans - -ceph_test_rados_SOURCES = test/osd/TestRados.cc test/osd/TestOpStat.cc test/osd/Object.cc test/osd/RadosModel.cc -ceph_test_rados_LDADD = librados.la $(LIBGLOBAL_LDA) -bin_DEBUGPROGRAMS += ceph_test_rados - -ceph_smalliobench_SOURCES = test/bench/small_io_bench.cc test/bench/rados_backend.cc test/bench/detailed_stat_collector.cc test/bench/bencher.cc -ceph_smalliobench_LDADD = librados.la -lboost_program_options $(LIBGLOBAL_LDA) -bin_DEBUGPROGRAMS += ceph_smalliobench - -ceph_smalliobenchfs_SOURCES = test/bench/small_io_bench_fs.cc test/bench/testfilestore_backend.cc test/bench/detailed_stat_collector.cc test/bench/bencher.cc -ceph_smalliobenchfs_LDADD = librados.la -lboost_program_options $(LIBOS_LDA) $(LIBGLOBAL_LDA) -ceph_smalliobenchfs_CXXFLAGS = ${CRYPTO_CXXFLAGS} ${AM_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_smalliobenchfs - -ceph_smalliobenchdumb_SOURCES = test/bench/small_io_bench_dumb.cc test/bench/dumb_backend.cc test/bench/detailed_stat_collector.cc test/bench/bencher.cc -ceph_smalliobenchdumb_LDADD = librados.la -lboost_program_options $(LIBOS_LDA) $(LIBGLOBAL_LDA) -ceph_smalliobenchdumb_CXXFLAGS = ${CRYPTO_CXXFLAGS} ${AM_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_smalliobenchdumb - -ceph_smalliobenchrbd_SOURCES = test/bench/small_io_bench_rbd.cc test/bench/rbd_backend.cc test/bench/detailed_stat_collector.cc test/bench/bencher.cc -ceph_smalliobenchrbd_LDADD = librados.la librbd.la -lboost_program_options $(LIBGLOBAL_LDA) -bin_DEBUGPROGRAMS += ceph_smalliobenchrbd - -ceph_tpbench_SOURCES = test/bench/tp_bench.cc test/bench/detailed_stat_collector.cc -ceph_tpbench_LDADD = librados.la -lboost_program_options $(LIBOS_LDA) $(LIBGLOBAL_LDA) -bin_DEBUGPROGRAMS += ceph_tpbench - -ceph_omapbench_SOURCES = test/omap_bench.cc -ceph_omapbench_LDADD = librados.la $(LIBGLOBAL_LDA) -bin_DEBUGPROGRAMS += ceph_omapbench - -ceph_kvstorebench_SOURCES = test/kv_store_bench.cc key_value_store/kv_flat_btree_async.cc -ceph_kvstorebench_LDADD = librados.la $(LIBGLOBAL_LDA) -bin_DEBUGPROGRAMS += ceph_kvstorebench - -ceph_multi_stress_watch_SOURCES = test/multi_stress_watch.cc test/librados/test.cc -ceph_multi_stress_watch_LDADD = librados.la $(LIBGLOBAL_LDA) -bin_DEBUGPROGRAMS += ceph_multi_stress_watch - -if WITH_BUILD_TESTS -ceph_test_libcommon_build_SOURCES = test/test_libcommon_build.cc $(libcommon_files) -ceph_test_libcommon_build_LDADD = $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS) -bin_DEBUGPROGRAMS += ceph_test_libcommon_build - -ceph_test_librados_build_SOURCES = test/test_libcommon_build.cc $(libcommon_files) $(librados_SOURCES) -ceph_test_librados_build_LDADD = $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS) -ceph_test_librados_build_CXXFLAGS = $(AM_CXXFLAGS) -bin_DEBUGPROGRAMS += ceph_test_librados_build - -ceph_test_librgw_build_SOURCES = test/test_libcommon_build.cc $(libcommon_files) \ - $(librados_SOURCES) $(librgw_la_SOURCES) -ceph_test_librgw_build_LDADD = -lexpat $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS) -ceph_test_librgw_build_CXXFLAGS = $(AM_CXXFLAGS) -bin_DEBUGPROGRAMS += ceph_test_librgw_build - -ceph_test_libcephfs_build_SOURCES = test/test_libcommon_build.cc $(libcommon_files) \ - $(libosdc_la_SOURCES) -ceph_test_libcephfs_build_LDADD = libcephfs.la -lexpat $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS) -ceph_test_libcephfs_build_CXXFLAGS = $(AM_CXXFLAGS) -bin_DEBUGPROGRAMS += ceph_test_libcephfs_build -endif +# libcephfs (this should go somewhere else in the future) -if WITH_HADOOPCLIENT -ceph_test_libhadoopcephfs_build_SOURCES = test/test_libcommon_build.cc \ - $(libhadoopcephfs_la_SOURCES) \ - $(libosdc_la_SOURCES) $(libcommon_files) -ceph_test_libhadoopcephfs_build_LDADD = libcephfs.la -lexpat $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS) -ceph_test_libhadoopcephfs_build_CXXFLAGS = $(AM_CXXFLAGS) -bin_DEBUGPROGRAMS += ceph_test_libhadoopcephfs_build -endif - -########## -BUILT_SOURCES = -lib_LTLIBRARIES = -noinst_LTLIBRARIES = -noinst_LIBRARIES = - -# libcephfs -libcephfs_la_SOURCES = \ - libcephfs.cc -libcephfs_la_CFLAGS= ${CRYPTO_CFLAGS} ${AM_CFLAGS} -libcephfs_la_CXXFLAGS= ${AM_CXXFLAGS} -libcephfs_la_LIBADD = libclient.la -libcephfs_la_LDFLAGS = $(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS) \ - ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '^ceph_.*' +libcephfs_la_SOURCES = libcephfs.cc +libcephfs_la_LIBADD = $(LIBCLIENT) $(LIBCOMMON) $(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS) +libcephfs_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '^ceph_.*' lib_LTLIBRARIES += libcephfs.la -ceph_test_timers_SOURCES = test/TestTimers.cc -ceph_test_timers_LDADD = $(LIBGLOBAL_LDA) -bin_DEBUGPROGRAMS += ceph_test_timers - -ceph_test_signal_handlers_SOURCES = test/TestSignalHandlers.cc -ceph_test_signal_handlers_LDADD = $(LIBGLOBAL_LDA) -bin_DEBUGPROGRAMS += ceph_test_signal_handlers - -# librados -librados_SOURCES = \ - librados/librados.cc \ - librados/RadosClient.cc \ - librados/IoCtxImpl.cc \ - osdc/Objecter.cc \ - osdc/Striper.cc \ - cls/lock/cls_lock_client.cc \ - cls/lock/cls_lock_types.cc \ - cls/lock/cls_lock_ops.cc -librados_la_SOURCES = ${librados_SOURCES} -librados_la_CFLAGS = ${CRYPTO_CFLAGS} ${AM_CFLAGS} -librados_la_CXXFLAGS = ${AM_CXXFLAGS} -librados_la_LIBADD = libcommon.la $(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS) -librados_la_LDFLAGS = ${AM_LDFLAGS} -version-info 2:0:0 -export-symbols-regex '^rados_.*' -lib_LTLIBRARIES += librados.la - -if WITH_RADOSGW - -# rgw -librgw_a_SOURCES = \ - rgw/librgw.cc \ - rgw/rgw_acl.cc \ - rgw/rgw_acl_s3.cc \ - rgw/rgw_acl_swift.cc \ - rgw/rgw_client_io.cc \ - rgw/rgw_fcgi.cc \ - rgw/rgw_xml.cc \ - rgw/rgw_usage.cc \ - rgw/rgw_json_enc.cc \ - rgw/rgw_user.cc \ - rgw/rgw_bucket.cc\ - rgw/rgw_tools.cc \ - rgw/rgw_rados.cc \ - rgw/rgw_http_client.cc \ - rgw/rgw_rest_client.cc \ - rgw/rgw_rest_conn.cc \ - rgw/rgw_op.cc \ - rgw/rgw_common.cc \ - rgw/rgw_cache.cc \ - rgw/rgw_formats.cc \ - rgw/rgw_log.cc \ - rgw/rgw_multi.cc \ - rgw/rgw_policy_s3.cc \ - rgw/rgw_gc.cc \ - rgw/rgw_multi_del.cc \ - rgw/rgw_env.cc \ - rgw/rgw_cors.cc \ - rgw/rgw_cors_s3.cc \ - rgw/rgw_auth_s3.cc \ - rgw/rgw_metadata.cc \ - rgw/rgw_replica_log.cc \ - rgw/rgw_keystone.cc -librgw_a_CFLAGS = ${CRYPTO_CFLAGS} ${AM_CFLAGS} -librgw_a_CXXFLAGS = -Woverloaded-virtual ${AM_CXXFLAGS} -noinst_LIBRARIES += librgw.a - -my_radosgw_ldadd = \ - librgw.a librados.la libcls_rgw_client.a libcls_log_client.a \ - libcls_statelog_client.a libcls_replica_log_client.a libcls_lock_client.a \ - libcls_refcount_client.a libcls_version_client.a -lcurl -lexpat \ - $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS) $(LIBGLOBAL_LDA) - -radosgw_SOURCES = \ - rgw/rgw_resolve.cc \ - rgw/rgw_rest.cc \ - rgw/rgw_rest_swift.cc \ - rgw/rgw_rest_s3.cc \ - rgw/rgw_rest_usage.cc \ - rgw/rgw_rest_user.cc \ - rgw/rgw_rest_bucket.cc \ - rgw/rgw_rest_metadata.cc \ - rgw/rgw_replica_log.cc \ - rgw/rgw_rest_log.cc \ - rgw/rgw_rest_opstate.cc \ - rgw/rgw_rest_replica_log.cc \ - rgw/rgw_rest_config.cc \ - rgw/rgw_http_client.cc \ - rgw/rgw_swift.cc \ - rgw/rgw_swift_auth.cc \ - rgw/rgw_main.cc -radosgw_LDADD = $(my_radosgw_ldadd) -lfcgi -lresolv -radosgw_CXXFLAGS = ${AM_CXXFLAGS} -bin_PROGRAMS += radosgw - -radosgw_admin_SOURCES = rgw/rgw_admin.cc -radosgw_admin_CXXFLAGS = ${AM_CXXFLAGS} -radosgw_admin_LDADD = $(my_radosgw_ldadd) -bin_PROGRAMS += radosgw-admin - -ceph_rgw_multiparser_SOURCES = rgw/rgw_multiparser.cc -ceph_rgw_multiparser_CXXFLAGS = ${AM_CXXFLAGS} -ceph_rgw_multiparser_LDADD = $(my_radosgw_ldadd) -bin_DEBUGPROGRAMS += ceph_rgw_multiparser - -ceph_rgw_jsonparser_SOURCES = rgw/rgw_jsonparser.cc rgw/rgw_common.cc rgw/rgw_env.cc rgw/rgw_json_enc.cc -ceph_rgw_jsonparser_CXXFLAGS = ${CRYPTO_CXXFLAGS} ${AM_CXXFLAGS} -ceph_rgw_jsonparser_LDADD = $(my_radosgw_ldadd) -bin_DEBUGPROGRAMS += ceph_rgw_jsonparser +# jni library (java source is in src/java) -endif - -# librbd -librbd_la_SOURCES = \ - librbd/librbd.cc \ - librbd/AioCompletion.cc \ - librbd/AioRequest.cc \ - cls/rbd/cls_rbd_client.cc \ - librbd/ImageCtx.cc \ - librbd/internal.cc \ - librbd/LibrbdWriteback.cc \ - librbd/WatchCtx.cc \ - osdc/ObjectCacher.cc \ - osdc/Striper.cc \ - librados/snap_set_diff.cc \ - cls/lock/cls_lock_client.cc \ - cls/lock/cls_lock_types.cc \ - cls/lock/cls_lock_ops.cc \ - common/util.cc -librbd_la_CFLAGS = ${AM_CFLAGS} ${CRYPTO_CFLAGS} -librbd_la_CXXFLAGS = ${AM_CXXFLAGS} -librbd_la_LIBADD = librados.la -librbd_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 \ - -export-symbols-regex '^rbd_.*' $(PTHREAD_LIBS) $(EXTRALIBS) -lib_LTLIBRARIES += librbd.la - -rados_SOURCES = rados.cc rados_import.cc rados_export.cc rados_sync.cc common/obj_bencher.cc -rados_LDADD = libcls_lock_client.a librados.la $(LIBGLOBAL_LDA) -bin_PROGRAMS += rados - -if WITH_REST_BENCH - -rest_bench_SOURCES = tools/rest_bench.cc common/obj_bencher.cc -rest_bench_LDADD = $(LIBGLOBAL_LDA) -rest_bench_CXXFLAGS = ${AM_CXXFLAGS} -bin_PROGRAMS += rest-bench - -if WITH_SYSTEM_LIBS3 -rest_bench_LDADD += -ls3 -else -rest_bench_LDADD += libs3/build/lib/libs3.a -lcurl -lxml2 -rest_bench_CXXFLAGS += -I$(top_srcdir)/src/libs3/inc -SUBDIRS += libs3 -endif - -endif - -ceph_scratchtool_SOURCES = scratchtool.c -ceph_scratchtool_LDADD = librados.la $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS) -ceph_scratchtoolpp_SOURCES = scratchtoolpp.cc -ceph_scratchtoolpp_LDADD = librados.la $(PTHREAD_LIBS) -lm -ceph_radosacl_SOURCES = radosacl.cc -ceph_radosacl_LDADD = librados.la $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS) -bin_DEBUGPROGRAMS += ceph_scratchtool ceph_scratchtoolpp ceph_radosacl - -rbd_SOURCES = rbd.cc common/secret.c common/TextTable.cc common/util.cc -rbd_CXXFLAGS = ${AM_CXXFLAGS} -rbd_LDADD = librbd.la librados.la $(LIBGLOBAL_LDA) -lkeyutils -if LINUX -bin_PROGRAMS += rbd -endif - - -ceph_test_crypto_SOURCES = testcrypto.cc -ceph_test_crypto_LDADD = $(LIBGLOBAL_LDA) -ceph_test_crypto_CXXFLAGS = ${AM_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_crypto - -ceph_test_keys_SOURCES = testkeys.cc -ceph_test_keys_LDADD = libmon.a $(LIBGLOBAL_LDA) -ceph_test_keys_CXXFLAGS = ${AM_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_keys - -if WITH_TCMALLOC -ceph_test_keys_LDADD += -ltcmalloc -ceph_test_keys_CXXFLAGS += ${tcmalloc_safety_flags} -ceph_test_keys_SOURCES += perfglue/heap_profiler.cc -endif - - -## rados object classes - -radoslibdir = $(libdir)/rados-classes -radoslib_LTLIBRARIES = - -# hello world class -libcls_hello_la_SOURCES = cls/hello/cls_hello.cc -libcls_hello_la_CFLAGS = ${AM_CFLAGS} -libcls_hello_la_CXXFLAGS= ${AM_CXXFLAGS} -libcls_hello_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) -libcls_hello_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' - -radoslib_LTLIBRARIES += libcls_hello.la - -# rbd: rados block device class -libcls_rbd_la_SOURCES = cls/rbd/cls_rbd.cc -libcls_rbd_la_CFLAGS = ${AM_CFLAGS} -libcls_rbd_la_CXXFLAGS= ${AM_CXXFLAGS} -libcls_rbd_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) -libcls_rbd_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' - -radoslib_LTLIBRARIES += libcls_rbd.la - -# lock class -libcls_lock_la_SOURCES = cls/lock/cls_lock.cc -libcls_lock_la_CFLAGS = ${AM_CFLAGS} -libcls_lock_la_CXXFLAGS= ${AM_CXXFLAGS} -libcls_lock_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) -libcls_lock_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' -radoslib_LTLIBRARIES += libcls_lock.la - -# refcount class -libcls_refcount_la_SOURCES = \ - cls/refcount/cls_refcount.cc \ - cls/refcount/cls_refcount_ops.cc \ - common/ceph_json.cc \ - json_spirit/json_spirit_reader.cpp \ - json_spirit/json_spirit_writer.cpp \ - json_spirit/json_spirit_value.cpp -libcls_refcount_la_CFLAGS = ${AM_CFLAGS} -libcls_refcount_la_CXXFLAGS= ${AM_CXXFLAGS} -libcls_refcount_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) -libcls_refcount_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' - -radoslib_LTLIBRARIES += libcls_refcount.la - -# version class -libcls_version_la_SOURCES = cls/version/cls_version.cc -libcls_version_la_CFLAGS = ${AM_CFLAGS} -libcls_version_la_CXXFLAGS= ${AM_CXXFLAGS} -libcls_version_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) -libcls_version_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' - -radoslib_LTLIBRARIES += libcls_version.la - -# log class -libcls_log_la_SOURCES = cls/log/cls_log.cc -libcls_log_la_CFLAGS = ${AM_CFLAGS} -libcls_log_la_CXXFLAGS= ${AM_CXXFLAGS} -libcls_log_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) -libcls_log_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' - -radoslib_LTLIBRARIES += libcls_log.la - -libcls_statelog_la_SOURCES = cls/statelog/cls_statelog.cc -libcls_statelog_la_CFLAGS = ${AM_CFLAGS} -libcls_statelog_la_CXXFLAGS= ${AM_CXXFLAGS} -libcls_statelog_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) -libcls_statelog_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' - -radoslib_LTLIBRARIES += libcls_statelog.la - -# replica log class -libcls_replica_log_la_SOURCES = cls/replica_log/cls_replica_log.cc -libcls_replica_log_la_CFLAGS = ${AM_CFLAGS} -libcls_replica_log_la_CXXFLAGS= ${AM_CXXFLAGS} -libcls_replica_log_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) -libcls_replica_log_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' - -radoslib_LTLIBRARIES += libcls_replica_log.la - -# rgw: rados gateway -libcls_rgw_la_SOURCES = \ - cls/rgw/cls_rgw.cc \ - cls/rgw/cls_rgw_ops.cc \ - cls/rgw/cls_rgw_types.cc \ - common/ceph_json.cc \ - json_spirit/json_spirit_reader.cpp \ - json_spirit/json_spirit_writer.cpp \ - json_spirit/json_spirit_value.cpp -libcls_rgw_la_CFLAGS = ${AM_CFLAGS} -libcls_rgw_la_CXXFLAGS= ${AM_CXXFLAGS} -libcls_rgw_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) -libcls_rgw_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' - -radoslib_LTLIBRARIES += libcls_rgw.la - -libcls_lock_client_a_SOURCES = \ - cls/lock/cls_lock_client.cc \ - cls/lock/cls_lock_types.cc \ - cls/lock/cls_lock_ops.cc -noinst_LIBRARIES += libcls_lock_client.a - -libcls_refcount_client_a_SOURCES = \ - cls/refcount/cls_refcount_client.cc \ - cls/refcount/cls_refcount_ops.cc -noinst_LIBRARIES += libcls_refcount_client.a - -libcls_version_client_a_SOURCES = \ - cls/version/cls_version_client.cc \ - cls/version/cls_version_types.cc -noinst_LIBRARIES += libcls_version_client.a - -libcls_log_client_a_SOURCES = \ - cls/log/cls_log_client.cc -noinst_LIBRARIES += libcls_log_client.a - -libcls_statelog_client_a_SOURCES = \ - cls/statelog/cls_statelog_client.cc -noinst_LIBRARIES += libcls_statelog_client.a - -libcls_replica_log_client_a_SOURCES = \ - cls/replica_log/cls_replica_log_types.cc \ - cls/replica_log/cls_replica_log_ops.cc \ - cls/replica_log/cls_replica_log_client.cc -noinst_LIBRARIES += libcls_replica_log_client.a - -libcls_rgw_client_a_SOURCES = \ - cls/rgw/cls_rgw_client.cc \ - cls/rgw/cls_rgw_types.cc \ - cls/rgw/cls_rgw_ops.cc -noinst_LIBRARIES += libcls_rgw_client.a - -## hadoop client -if WITH_HADOOPCLIENT -JAVA_BASE = /usr/lib/jvm/java-6-sun -libhadoopcephfs_la_SOURCES = client/hadoop/CephFSInterface.cc -libhadoopcephfs_la_LIBADD = libcephfs.la -libhadoopcephfs_la_CFLAGS = ${AM_CFLAGS} -libhadoopcephfs_la_CXXFLAGS = ${AM_CXXFLAGS} -libhadoopcephfs_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex 'hadoopcephfs_.*' -lib_LTLIBRARIES += libhadoopcephfs.la - -endif - -## CephFS Java Wrappers -## - The JNI library is here -## - The Java source Makefile.am is in src/java if ENABLE_CEPHFS_JAVA libcephfs_jni_la_SOURCES = \ java/native/libcephfs_jni.cc \ java/native/ScopedLocalRef.h \ java/native/JniConstants.cpp \ java/native/JniConstants.h -libcephfs_jni_la_LIBADD = libcephfs.la $(EXTRALIBS) +libcephfs_jni_la_LIBADD = $(LIBCEPHFS) $(EXTRALIBS) libcephfs_jni_la_CPPFLAGS = $(JDK_CPPFLAGS) -libcephfs_jni_la_CXXFLAGS = ${AM_CXXFLAGS} libcephfs_jni_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 lib_LTLIBRARIES += libcephfs_jni.la endif -## key_value_store classes - -# key_value_store: key value store class -libcls_kvs_la_SOURCES = key_value_store/cls_kvs.cc -libcls_kvs_la_CFLAGS = ${AM_CFLAGS} -libcls_kvs_la_CXXFLAGS= ${AM_CXXFLAGS} -libcls_kvs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) -libcls_kvs_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' - -radoslib_LTLIBRARIES += libcls_kvs.la - -## System tests -libsystest_la_SOURCES = \ - test/system/cross_process_sem.cc \ - test/system/systest_runnable.cc \ - test/system/systest_settings.cc -libsystest_la_LIBADD = $(LIBGLOBAL_LDA) -noinst_LTLIBRARIES += libsystest.la - -ceph_test_rados_list_parallel_SOURCES = \ - test/system/rados_list_parallel.cc \ - test/system/st_rados_create_pool.cc \ - test/system/st_rados_list_objects.cc -ceph_test_rados_list_parallel_LDADD = libsystest.la librados.la $(PTHREAD_LIBS) -bin_DEBUGPROGRAMS += ceph_test_rados_list_parallel - -ceph_test_rados_open_pools_parallel_SOURCES = \ - test/system/rados_open_pools_parallel.cc \ - test/system/st_rados_create_pool.cc -ceph_test_rados_open_pools_parallel_LDADD = libsystest.la librados.la $(PTHREAD_LIBS) -bin_DEBUGPROGRAMS += ceph_test_rados_open_pools_parallel - -ceph_test_rados_delete_pools_parallel_SOURCES = \ - test/system/rados_delete_pools_parallel.cc \ - test/system/st_rados_create_pool.cc \ - test/system/st_rados_delete_pool.cc \ - test/system/st_rados_list_objects.cc -ceph_test_rados_delete_pools_parallel_LDADD = libsystest.la librados.la $(PTHREAD_LIBS) -bin_DEBUGPROGRAMS += ceph_test_rados_delete_pools_parallel - -ceph_test_rados_watch_notify_SOURCES = \ - test/system/rados_watch_notify.cc \ - test/system/st_rados_create_pool.cc \ - test/system/st_rados_delete_pool.cc \ - test/system/st_rados_delete_objs.cc \ - test/system/st_rados_watch.cc \ - test/system/st_rados_notify.cc -ceph_test_rados_watch_notify_LDADD = libsystest.la librados.la $(PTHREAD_LIBS) -bin_DEBUGPROGRAMS += ceph_test_rados_watch_notify - -ceph_bench_log_SOURCES = \ - test/bench_log.cc -ceph_bench_log_LDADD = libcommon.la $(LIBGLOBAL_LDA) -bin_DEBUGPROGRAMS += ceph_bench_log - -## unit tests - -# target to build but not run the unit tests -unittests:: $(check_PROGRAMS) - -UNITTEST_CXXFLAGS = \ - -I$(top_srcdir)/src/gtest/include \ - -I$(top_builddir)/src/gtest/include -UNITTEST_STATIC_LDADD = \ - $(top_builddir)/src/gtest/lib/libgtest.a \ - $(top_builddir)/src/gtest/lib/libgtest_main.a \ - $(PTHREAD_LIBS) -UNITTEST_LDADD = ${UNITTEST_STATIC_LDADD} - -unittest_encoding_SOURCES = test/encoding.cc -unittest_encoding_LDADD = libcephfs.la librados.la $(PTHREAD_LIBS) -lm \ - ${UNITTEST_LDADD} -unittest_encoding_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} \ - -fno-strict-aliasing -check_PROGRAMS += unittest_encoding - -unittest_addrs_SOURCES = test/test_addrs.cc -unittest_addrs_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -unittest_addrs_LDADD = $(LIBGLOBAL_LDA) ${UNITTEST_LDADD} -check_PROGRAMS += unittest_addrs - -unittest_sharedptr_registry_SOURCES = test/common/test_sharedptr_registry.cc -unittest_sharedptr_registry_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -unittest_sharedptr_registry_LDADD = libcommon.la ${LIBGLOBAL_LDA} ${UNITTEST_LDADD} -check_PROGRAMS += unittest_sharedptr_registry - -unittest_util_SOURCES = test/common/test_util.cc common/util.cc -unittest_util_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -unittest_util_LDADD = libcommon.la $(PTHREAD_LIBS) -lm ${UNITTEST_LDADD} $(CRYPTO_LIBS) $(EXTRALIBS) -check_PROGRAMS += unittest_util - -unittest_workqueue_SOURCES = test/test_workqueue.cc -unittest_workqueue_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -unittest_workqueue_LDADD = $(LIBGLOBAL_LDA) ${UNITTEST_LDADD} -check_PROGRAMS += unittest_workqueue - -unittest_striper_SOURCES = test/test_striper.cc -unittest_striper_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -unittest_striper_LDADD = libosdc.la $(LIBGLOBAL_LDA) ${UNITTEST_LDADD} -check_PROGRAMS += unittest_striper - -unittest_prebufferedstreambuf_SOURCES = test/test_prebufferedstreambuf.cc common/PrebufferedStreambuf.cc -unittest_prebufferedstreambuf_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -unittest_prebufferedstreambuf_LDADD = ${UNITTEST_LDADD} $(EXTRALIBS) -check_PROGRAMS += unittest_prebufferedstreambuf - -unittest_str_list_SOURCES = test/test_str_list.cc -unittest_str_list_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -unittest_str_list_LDADD = $(LIBGLOBAL_LDA) ${UNITTEST_LDADD} -check_PROGRAMS += unittest_str_list - -unittest_log_SOURCES = log/test.cc common/PrebufferedStreambuf.cc -unittest_log_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -unittest_log_LDADD = libcommon.la ${UNITTEST_LDADD} -unittest_log_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -O2 -check_PROGRAMS += unittest_log - -unittest_throttle_SOURCES = test/common/Throttle.cc -unittest_throttle_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -unittest_throttle_LDADD = libcommon.la ${LIBGLOBAL_LDA} ${UNITTEST_LDADD} -unittest_throttle_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -O2 -check_PROGRAMS += unittest_throttle - -unittest_base64_SOURCES = test/base64.cc -unittest_base64_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -unittest_base64_LDADD = libcephfs.la -lm ${UNITTEST_LDADD} -unittest_base64_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_base64 - -unittest_ceph_argparse_SOURCES = test/ceph_argparse.cc -unittest_ceph_argparse_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -unittest_ceph_argparse_LDADD = $(LIBGLOBAL_LDA) ${UNITTEST_LDADD} -unittest_ceph_argparse_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_ceph_argparse - -unittest_osd_types_SOURCES = test/test_osd_types.cc -unittest_osd_types_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -unittest_osd_types_LDADD = $(LIBGLOBAL_LDA) libcommon.la ${UNITTEST_LDADD} -check_PROGRAMS += unittest_osd_types - -unittest_pglog_SOURCES = test/osd/TestPGLog.cc perfglue/disabled_heap_profiler.cc -unittest_pglog_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -unittest_pglog_LDADD = libosd.a libosdc.la $(LIBOS_LDA) $(LIBGLOBAL_LDA) ${UNITTEST_LDADD} -check_PROGRAMS += unittest_pglog - -if LINUX -unittest_pglog_LDADD += -ldl -endif - -unittest_gather_SOURCES = test/gather.cc -unittest_gather_LDADD = ${LIBGLOBAL_LDA} ${UNITTEST_LDADD} -unittest_gather_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_gather - -unittest_run_cmd_SOURCES = test/run_cmd.cc -unittest_run_cmd_LDADD = libcephfs.la ${UNITTEST_LDADD} -unittest_run_cmd_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_run_cmd - -unittest_signals_SOURCES = test/signals.cc -unittest_signals_LDADD = ${LIBGLOBAL_LDA} ${UNITTEST_LDADD} -unittest_signals_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_signals - -unittest_simple_spin_SOURCES = test/simple_spin.cc -unittest_simple_spin_LDADD = libcephfs.la ${UNITTEST_LDADD} -unittest_simple_spin_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_simple_spin - -unittest_librados_SOURCES = test/librados/librados.cc -unittest_librados_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -unittest_librados_LDADD = librados.la ${UNITTEST_LDADD} -unittest_librados_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_librados - -unittest_bufferlist_SOURCES = test/bufferlist.cc -unittest_bufferlist_LDADD = ${UNITTEST_LDADD} $(LIBGLOBAL_LDA) -unittest_bufferlist_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_bufferlist - -unittest_crypto_SOURCES = test/crypto.cc -unittest_crypto_LDFLAGS = ${CRYPTO_LDFLAGS} ${AM_LDFLAGS} -unittest_crypto_LDADD = ${LIBGLOBAL_LDA} ${UNITTEST_LDADD} -unittest_crypto_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_crypto - -unittest_perf_counters_SOURCES = test/perf_counters.cc -unittest_perf_counters_LDFLAGS = ${AM_LDFLAGS} -unittest_perf_counters_LDADD = ${LIBGLOBAL_LDA} ${UNITTEST_LDADD} -unittest_perf_counters_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_perf_counters - -unittest_admin_socket_SOURCES = test/admin_socket.cc -unittest_admin_socket_LDFLAGS = ${AM_LDFLAGS} -unittest_admin_socket_LDADD = ${LIBGLOBAL_LDA} ${UNITTEST_LDADD} libcommon.la -unittest_admin_socket_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_admin_socket - -unittest_ceph_crypto_SOURCES = test/ceph_crypto.cc -unittest_ceph_crypto_LDFLAGS = ${CRYPTO_LDFLAGS} ${AM_LDFLAGS} -unittest_ceph_crypto_LDADD = ${UNITTEST_LDADD} $(LIBGLOBAL_LDA) -unittest_ceph_crypto_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_ceph_crypto - -unittest_utf8_SOURCES = test/utf8.cc -unittest_utf8_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -unittest_utf8_LDADD = ${UNITTEST_LDADD} $(LIBGLOBAL_LDA) -unittest_utf8_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_utf8 - -unittest_mime_SOURCES = test/mime.cc -unittest_mime_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -unittest_mime_LDADD = ${UNITTEST_LDADD} $(LIBGLOBAL_LDA) -unittest_mime_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_mime - -unittest_escape_SOURCES = test/escape.cc -unittest_escape_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -unittest_escape_LDADD = ${UNITTEST_LDADD} $(LIBGLOBAL_LDA) -unittest_escape_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_escape - -unittest_chain_xattr_SOURCES = test/filestore/chain_xattr.cc -unittest_chain_xattr_LDFLAGS = ${AM_LDFLAGS} -unittest_chain_xattr_LDADD = ${UNITTEST_STATIC_LDADD} $(LIBOS_LDA) $(LIBGLOBAL_LDA) -unittest_chain_xattr_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} ${CRYPTO_CXXFLAGS} -check_PROGRAMS += unittest_chain_xattr - -unittest_flatindex_SOURCES = test/os/TestFlatIndex.cc -unittest_flatindex_LDFLAGS = ${AM_LDFLAGS} -unittest_flatindex_LDADD = ${UNITTEST_STATIC_LDADD} $(LIBOS_LDA) $(LIBGLOBAL_LDA) -unittest_flatindex_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} ${CRYPTO_CXXFLAGS} -check_PROGRAMS += unittest_flatindex - -unittest_strtol_SOURCES = test/strtol.cc -unittest_strtol_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -unittest_strtol_LDADD = ${UNITTEST_LDADD} $(LIBGLOBAL_LDA) -unittest_strtol_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_strtol - -unittest_confutils_SOURCES = test/confutils.cc -unittest_confutils_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -unittest_confutils_LDADD = ${UNITTEST_LDADD} $(LIBGLOBAL_LDA) -unittest_confutils_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_confutils - -unittest_heartbeatmap_SOURCES = test/heartbeat_map.cc common/HeartbeatMap.cc -unittest_heartbeatmap_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -unittest_heartbeatmap_LDADD = ${UNITTEST_LDADD} $(LIBGLOBAL_LDA) -unittest_heartbeatmap_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_heartbeatmap - -unittest_formatter_SOURCES = test/formatter.cc rgw/rgw_formats.cc -unittest_formatter_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -unittest_formatter_LDADD = ${UNITTEST_LDADD} $(LIBGLOBAL_LDA) -unittest_formatter_CXXFLAGS = ${CRYPTO_CXXFLAGS} ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_formatter - -unittest_libcephfs_config_SOURCES = test/libcephfs_config.cc -unittest_libcephfs_config_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -unittest_libcephfs_config_LDADD = libcephfs.la ${UNITTEST_LDADD} -unittest_libcephfs_config_CXXFLAGS = ${CRYPTO_CFLAGS} ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_libcephfs_config - -unittest_lfnindex_SOURCES = test/os/TestLFNIndex.cc -unittest_lfnindex_LDFLAGS = ${AM_LDFLAGS} -unittest_lfnindex_LDADD = ${UNITTEST_STATIC_LDADD} $(LIBOS_LDA) $(LIBGLOBAL_LDA) -unittest_lfnindex_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} ${CRYPTO_CXXFLAGS} -check_PROGRAMS += unittest_lfnindex - -unittest_librados_config_SOURCES = test/librados/librados_config.cc -unittest_librados_config_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -unittest_librados_config_LDADD = librados.la ${UNITTEST_LDADD} -unittest_librados_config_CXXFLAGS = ${CRYPTO_CFLAGS} ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_librados_config - -#unittest_librgw_link_SOURCES = test/librgw_link.cc -#unittest_librgw_link_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -#unittest_librgw_link_LDADD = librgw.la ${UNITTEST_LDADD} -#unittest_librgw_link_CXXFLAGS = ${CRYPTO_CFLAGS} ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -#check_PROGRAMS += unittest_librgw_link - -unittest_daemon_config_SOURCES = test/daemon_config.cc -unittest_daemon_config_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -unittest_daemon_config_LDADD = ${UNITTEST_LDADD} ${LIBGLOBAL_LDA} -unittest_daemon_config_CXXFLAGS = ${CRYPTO_CFLAGS} ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_daemon_config - -unittest_osd_osdcap_SOURCES = test/osd/osdcap.cc osd/OSDCap.cc -unittest_osd_osdcap_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -unittest_osd_osdcap_LDADD = ${UNITTEST_LDADD} ${LIBGLOBAL_LDA} -unittest_osd_osdcap_CXXFLAGS = ${CRYPTO_CFLAGS} ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_osd_osdcap - -unittest_mon_moncap_SOURCES = test/mon/moncap.cc mon/MonCap.cc -unittest_mon_moncap_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -unittest_mon_moncap_LDADD = ${UNITTEST_LDADD} ${LIBGLOBAL_LDA} -unittest_mon_moncap_CXXFLAGS = ${CRYPTO_CFLAGS} ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_mon_moncap - -#if WITH_RADOSGW -#unittest_librgw_SOURCES = test/librgw.cc -#unittest_librgw_LDFLAGS = -lrt $(PTHREAD_CFLAGS) -lcurl ${AM_LDFLAGS} -#unittest_librgw_LDADD = librgw.la librados.la ${UNITTEST_LDADD} -lexpat $(LIBGLOBAL_LDA) -#unittest_librgw_CXXFLAGS = ${CRYPTO_CFLAGS} ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -#check_PROGRAMS += unittest_librgw -#endif - -unittest_ipaddr_SOURCES = test/test_ipaddr.cc -unittest_ipaddr_LDADD = ${UNITTEST_LDADD} $(LIBGLOBAL_LDA) -unittest_ipaddr_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_ipaddr - -unittest_texttable_SOURCES = test/test_texttable.cc common/TextTable.cc -unittest_texttable_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -unittest_texttable_LDADD = librados.la ${UNITTEST_LDADD} -unittest_texttable_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -check_PROGRAMS += unittest_texttable - -if WITH_RADOSGW -ceph_test_cors_SOURCES = test/test_cors.cc -ceph_test_cors_LDADD = librados.la librgw.a $(LIBGLOBAL_LDA) ${UNITTEST_LDADD} ${UNITTEST_STATIC_LDADD} -lcurl -luuid -lexpat -ceph_test_cors_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_cors - -ceph_test_cls_rgw_meta_SOURCES = test/test_rgw_admin_meta.cc -ceph_test_cls_rgw_meta_LDADD = librgw.a ${UNITTEST_LDADD} ${UNITTEST_STATIC_LDADD} $(LIBGLOBAL_LDA) $(CRYPTO_LIBS) -lcurl -luuid -lexpat librados.la libcls_version_client.a \ - libcls_log_client.a libcls_statelog_client.a libcls_refcount_client.a libcls_rgw_client.a libcls_lock_client.a -ceph_test_cls_rgw_meta_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_cls_rgw_meta - -ceph_test_cls_rgw_log_SOURCES = test/test_rgw_admin_log.cc -ceph_test_cls_rgw_log_LDADD = librgw.a ${UNITTEST_LDADD} ${UNITTEST_STATIC_LDADD} ${LIBGLOBAL_LDA} $(CRYPTO_LIBS) -lcurl -luuid -lexpat librados.la libcls_version_client.a \ - libcls_log_client.a libcls_statelog_client.a libcls_refcount_client.a libcls_rgw_client.a libcls_lock_client.a -ceph_test_cls_rgw_log_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_cls_rgw_log - -ceph_test_cls_rgw_opstate_SOURCES = test/test_rgw_admin_opstate.cc -ceph_test_cls_rgw_opstate_LDADD = librgw.a ${UNITTEST_LDADD} ${UNITTEST_STATIC_LDADD} ${LIBGLOBAL_LDA} $(CRYPTO_LIBS) -lcurl -luuid -lexpat librados.la libcls_version_client.a \ - libcls_log_client.a libcls_statelog_client.a libcls_refcount_client.a libcls_rgw_client.a libcls_lock_client.a -ceph_test_cls_rgw_opstate_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_cls_rgw_opstate -endif - -ceph_test_librbd_SOURCES = test/librbd/test_librbd.cc test/librados/test.cc -ceph_test_librbd_LDADD = librbd.la librados.la ${UNITTEST_STATIC_LDADD} -ceph_test_librbd_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_librbd - -ceph_test_librbd_fsx_SOURCES = test/librbd/fsx.c -ceph_test_librbd_fsx_LDADD = librbd.la librados.la -lm -ceph_test_librbd_fsx_CFLAGS = ${AM_CFLAGS} -Wno-format -bin_DEBUGPROGRAMS += ceph_test_librbd_fsx - -ceph_test_cls_rbd_SOURCES = test/cls_rbd/test_cls_rbd.cc \ - test/librados/test.cc \ - cls/rbd/cls_rbd_client.cc \ - cls/lock/cls_lock_client.cc \ - cls/lock/cls_lock_types.cc \ - cls/lock/cls_lock_ops.cc -ceph_test_cls_rbd_LDADD = librados.la ${UNITTEST_STATIC_LDADD} -ceph_test_cls_rbd_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_cls_rbd - -ceph_test_cls_refcount_SOURCES = test/cls_refcount/test_cls_refcount.cc \ - test/librados/test.cc -ceph_test_cls_refcount_LDADD = librados.la libcls_refcount_client.a ${UNITTEST_STATIC_LDADD} -ceph_test_cls_refcount_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_cls_refcount - -ceph_test_cls_version_SOURCES = test/cls_version/test_cls_version.cc \ - test/librados/test.cc -ceph_test_cls_version_LDADD = librados.la libcls_version_client.a ${UNITTEST_STATIC_LDADD} -ceph_test_cls_version_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_cls_version - -ceph_test_cls_log_SOURCES = test/cls_log/test_cls_log.cc \ - test/librados/test.cc -ceph_test_cls_log_LDADD = librados.la libcls_log_client.a ${UNITTEST_STATIC_LDADD} ${LIBGLOBAL_LDA} -ceph_test_cls_log_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_cls_log - -ceph_test_cls_statelog_SOURCES = test/cls_statelog/test_cls_statelog.cc \ - test/librados/test.cc -ceph_test_cls_statelog_LDADD = librados.la libcls_statelog_client.a ${UNITTEST_STATIC_LDADD} ${LIBGLOBAL_LDA} -ceph_test_cls_statelog_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_cls_statelog -ceph_test_cls_replica_log_SOURCES = \ - test/cls_replica_log/test_cls_replica_log.cc \ - test/librados/test.cc -ceph_test_cls_replica_log_LDADD = librados.la \ - libcls_replica_log_client.a ${UNITTEST_STATIC_LDADD} ${LIBGLOBAL_LDA} -ceph_test_cls_replica_log_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_cls_replica_log - -ceph_test_cls_lock_SOURCES = test/cls_lock/test_cls_lock.cc test/librados/test.cc -ceph_test_cls_lock_LDFLAGS = ${AM_LDFLAGS} -ceph_test_cls_lock_LDADD = libcls_lock_client.a librados.la ${UNITTEST_STATIC_LDADD} -ceph_test_cls_lock_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_cls_lock - -ceph_test_cls_hello_SOURCES = test/cls_hello/test_cls_hello.cc test/librados/test.cc -ceph_test_cls_hello_LDADD = ${UNITTEST_LDADD} ${UNITTEST_STATIC_LDADD} $(LIBGLOBAL_LDA) $(CRYPTO_LIBS) librados.la -ceph_test_cls_hello_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_cls_hello - -if WITH_RADOSGW - -ceph_test_cls_rgw_SOURCES = test/cls_rgw/test_cls_rgw.cc \ - test/librados/test.cc -ceph_test_cls_rgw_LDADD = librados.la libcls_rgw_client.a ${UNITTEST_STATIC_LDADD} -ceph_test_cls_rgw_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_cls_rgw - -endif - -ceph_test_mon_workloadgen_SOURCES = \ - test/mon/test_mon_workloadgen.cc \ - osdc/Objecter.cc \ - osdc/Striper.cc -ceph_test_mon_workloadgen_LDADD = $(LIBOS_LDA) $(LIBGLOBAL_LDA) -ceph_test_mon_workloadgen_CXXFLAGS = ${CRYPTO_CXXFLAGS} ${AM_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_mon_workloadgen - -ceph_test_rados_api_cmd_SOURCES = test/librados/cmd.cc test/librados/test.cc -ceph_test_rados_api_cmd_LDFLAGS = ${AM_LDFLAGS} -ceph_test_rados_api_cmd_LDADD = librados.la ${UNITTEST_STATIC_LDADD} -ceph_test_rados_api_cmd_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_rados_api_cmd - -ceph_test_rados_api_io_SOURCES = test/librados/io.cc test/librados/test.cc -ceph_test_rados_api_io_LDFLAGS = ${AM_LDFLAGS} -ceph_test_rados_api_io_LDADD = librados.la ${UNITTEST_STATIC_LDADD} -ceph_test_rados_api_io_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_rados_api_io - -ceph_test_rados_api_aio_SOURCES = test/librados/aio.cc test/librados/test.cc -ceph_test_rados_api_aio_LDFLAGS = ${AM_LDFLAGS} -ceph_test_rados_api_aio_LDADD = librados.la ${UNITTEST_STATIC_LDADD} -ceph_test_rados_api_aio_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_rados_api_aio - -ceph_test_rados_api_list_SOURCES = test/librados/list.cc test/librados/test.cc -ceph_test_rados_api_list_LDFLAGS = ${AM_LDFLAGS} -ceph_test_rados_api_list_LDADD = librados.la ${UNITTEST_STATIC_LDADD} -ceph_test_rados_api_list_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_rados_api_list - -ceph_test_rados_api_pool_SOURCES = test/librados/pool.cc test/librados/test.cc -ceph_test_rados_api_pool_LDFLAGS = ${AM_LDFLAGS} -ceph_test_rados_api_pool_LDADD = librados.la ${UNITTEST_STATIC_LDADD} -ceph_test_rados_api_pool_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_rados_api_pool - -ceph_test_rados_api_stat_SOURCES = test/librados/stat.cc test/librados/test.cc -ceph_test_rados_api_stat_LDFLAGS = ${AM_LDFLAGS} -ceph_test_rados_api_stat_LDADD = librados.la ${UNITTEST_STATIC_LDADD} -ceph_test_rados_api_stat_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_rados_api_stat - -ceph_test_rados_api_watch_notify_SOURCES = test/librados/watch_notify.cc test/librados/test.cc -ceph_test_rados_api_watch_notify_LDFLAGS = ${AM_LDFLAGS} -ceph_test_rados_api_watch_notify_LDADD = librados.la ${UNITTEST_STATIC_LDADD} -ceph_test_rados_api_watch_notify_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_rados_api_watch_notify - -ceph_test_rados_api_snapshots_SOURCES = test/librados/snapshots.cc test/librados/test.cc -ceph_test_rados_api_snapshots_LDFLAGS = ${AM_LDFLAGS} -ceph_test_rados_api_snapshots_LDADD = librados.la ${UNITTEST_STATIC_LDADD} -ceph_test_rados_api_snapshots_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_rados_api_snapshots - -ceph_test_rados_api_cls_SOURCES = test/librados/cls.cc test/librados/test.cc -ceph_test_rados_api_cls_LDFLAGS = ${AM_LDFLAGS} -ceph_test_rados_api_cls_LDADD = librados.la ${UNITTEST_STATIC_LDADD} -ceph_test_rados_api_cls_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_rados_api_cls - -ceph_test_rados_api_misc_SOURCES = test/librados/misc.cc test/librados/test.cc -ceph_test_rados_api_misc_LDFLAGS = ${AM_LDFLAGS} -ceph_test_rados_api_misc_LDADD = librados.la $(LIBGLOBAL_LDA) ${UNITTEST_STATIC_LDADD} -ceph_test_rados_api_misc_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_rados_api_misc - -ceph_test_rados_api_lock_SOURCES = test/librados/lock.cc test/librados/test.cc -ceph_test_rados_api_lock_LDFLAGS = ${AM_LDFLAGS} -ceph_test_rados_api_lock_LDADD = librados.la ${UNITTEST_STATIC_LDADD} -ceph_test_rados_api_lock_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_rados_api_lock - -ceph_test_libcephfs_SOURCES = test/libcephfs/test.cc test/libcephfs/readdir_r_cb.cc test/libcephfs/caps.cc test/libcephfs/multiclient.cc -ceph_test_libcephfs_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} -ceph_test_libcephfs_LDADD = ${UNITTEST_STATIC_LDADD} libcephfs.la -ceph_test_libcephfs_CXXFLAGS = $(AM_CXXFLAGS) ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_libcephfs - -ceph_test_filestore_SOURCES = test/filestore/store_test.cc -ceph_test_filestore_LDFLAGS = ${AM_LDFLAGS} -ceph_test_filestore_LDADD = ${UNITTEST_STATIC_LDADD} $(LIBOS_LDA) $(LIBGLOBAL_LDA) -ceph_test_filestore_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} ${CRYPTO_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_filestore - -ceph_test_filestore_workloadgen_SOURCES = \ - test/filestore/workload_generator.cc \ - test/filestore/TestFileStoreState.cc -ceph_test_filestore_workloadgen_LDFLAGS = ${AM_LDFLAGS} -ceph_test_filestore_workloadgen_LDADD = $(LIBOS_LDA) $(LIBGLOBAL_LDA) -ceph_test_filestore_workloadgen_CXXFLAGS = ${CRYPTO_CXXFLAGS} ${AM_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_filestore_workloadgen - -ceph_test_filestore_idempotent_SOURCES = test/filestore/test_idempotent.cc test/filestore/FileStoreTracker.cc test/common/ObjectContents.cc -ceph_test_filestore_idempotent_LDADD = $(LIBOS_LDA) $(LIBGLOBAL_LDA) -ceph_test_filestore_idempotent_CXXFLAGS = $(AM_CXXFLAGS) -bin_DEBUGPROGRAMS += ceph_test_filestore_idempotent - -ceph_test_filestore_idempotent_sequence_SOURCES = \ - test/filestore/test_idempotent_sequence.cc \ - test/filestore/DeterministicOpSequence.cc \ - test/filestore/TestFileStoreState.cc \ - test/filestore/FileStoreDiff.cc -ceph_test_filestore_idempotent_sequence_CXXFLAGS = ${CRYPTO_CXXFLAGS} ${AM_CXXFLAGS} -ceph_test_filestore_idempotent_sequence_LDADD = $(LIBOS_LDA) $(LIBGLOBAL_LDA) -bin_DEBUGPROGRAMS += ceph_test_filestore_idempotent_sequence - -ceph_xattr_bench_SOURCES = test/xattr_bench.cc -ceph_xattr_bench_LDFLAGS = ${AM_LDFLAGS} -ceph_xattr_bench_LDADD = ${UNITTEST_STATIC_LDADD} $(LIBOS_LDA) $(LIBGLOBAL_LDA) -ceph_xattr_bench_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} ${CRYPTO_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_xattr_bench - -ceph_test_filejournal_SOURCES = test/test_filejournal.cc -ceph_test_filejournal_LDFLAGS = ${AM_LDFLAGS} -ceph_test_filejournal_LDADD = ${UNITTEST_STATIC_LDADD} $(LIBOS_LDA) $(LIBGLOBAL_LDA) -ceph_test_filejournal_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_filejournal - -ceph_test_stress_watch_SOURCES = test/test_stress_watch.cc test/librados/test.cc -ceph_test_stress_watch_LDFLAGS = ${AM_LDFLAGS} -ceph_test_stress_watch_LDADD = librados.la ${UNITTEST_STATIC_LDADD} -ceph_test_stress_watch_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_stress_watch - -ceph_test_objectcacher_stress_SOURCES = test/osdc/object_cacher_stress.cc test/osdc/FakeWriteback.cc osdc/ObjectCacher.cc -ceph_test_objectcacher_stress_LDFLAGS = ${AM_LDFLAGS} -ceph_test_objectcacher_stress_LDADD = $(LIBGLOBAL_LDA) -ceph_test_objectcacher_stress_CXXFLAGS = ${AM_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_objectcacher_stress - -ceph_test_snap_mapper_SOURCES = test/test_snap_mapper.cc osd/SnapMapper.cc -ceph_test_snap_mapper_LDFLAGS = ${AM_LDFLAGS} -ceph_test_snap_mapper_LDADD = ${UNITTEST_STATIC_LDADD} $(LIBOS_LDA) $(LIBGLOBAL_LDA) -ceph_test_snap_mapper_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} $(LEVELDB_INCLUDE) ${CRYPTO_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_snap_mapper - -ceph_test_object_map_SOURCES = test/ObjectMap/test_object_map.cc test/ObjectMap/KeyValueDBMemory.cc os/DBObjectMap.cc os/LevelDBStore.cc -ceph_test_object_map_LDFLAGS = ${AM_LDFLAGS} -ceph_test_object_map_LDADD = ${UNITTEST_STATIC_LDADD} $(LIBOS_LDA) $(LIBGLOBAL_LDA) -ceph_test_object_map_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} ${CRYPTO_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_object_map - -ceph_test_keyvaluedb_atomicity_SOURCES = test/ObjectMap/test_keyvaluedb_atomicity.cc os/LevelDBStore.cc -ceph_test_keyvaluedb_atomicity_LDFLAGS = ${AM_LDFLAGS} -ceph_test_keyvaluedb_atomicity_LDADD = ${UNITTEST_STATIC_LDADD} $(LIBOS_LDA) $(LIBGLOBAL_LDA) -ceph_test_keyvaluedb_atomicity_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} ${CRYPTO_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_keyvaluedb_atomicity - -ceph_test_keyvaluedb_iterators_SOURCES = test/ObjectMap/test_keyvaluedb_iterators.cc \ - test/ObjectMap/KeyValueDBMemory.cc \ - os/LevelDBStore.cc -ceph_test_keyvaluedb_iterators_LDFLAGS = ${AM_LDFLAGS} -ceph_test_keyvaluedb_iterators_LDADD = ${UNITTEST_STATIC_LDADD} $(LIBOS_LDA) $(LIBGLOBAL_LDA) -ceph_test_keyvaluedb_iterators_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} ${CRYPTO_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_keyvaluedb_iterators - -ceph_test_store_tool_SOURCES = test/ObjectMap/test_store_tool/test_store_tool.cc \ - os/LevelDBStore.cc -ceph_test_store_tool_LDFLAGS = ${AM_LDFLAGS} -ceph_test_store_tool_LDADD = $(LIBOS_LDA) $(LIBGLOBAL_LDA) -ceph_test_store_tool_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_store_tool - -ceph_test_cfuse_cache_invalidate_SOURCES = test/test_cfuse_cache_invalidate.cc -ceph_test_cfuse_cache_invalidate_LDFLAGS = ${AM_LDFLAGS} -ceph_test_cfuse_cache_invalidate_LDADD = -ceph_test_cfuse_cache_invalidate_CXXFLAGS = ${AM_CXXFLAGS} -bin_DEBUGPROGRAMS += ceph_test_cfuse_cache_invalidate # shell scripts + editpaths = sed \ -e 's|@bindir[@]|$(bindir)|g' \ -e 's|@sbindir[@]|$(sbindir)|g' \ @@ -1290,11 +128,8 @@ editpaths = sed \ -e 's|@datadir[@]|$(pkgdatadir)|g' \ -e 's|@prefix[@]|$(prefix)|g' \ -e 's|@@GCOV_PREFIX_STRIP[@][@]|$(GCOV_PREFIX_STRIP)|g' - -shell_scripts = init-ceph mkcephfs ceph-debugpack ceph-coverage ceph-post-file - +shell_scripts = ceph-debugpack ceph-post-file $(shell_scripts): Makefile - $(shell_scripts): %: %.in rm -f $@ $@.tmp $(editpaths) '$(srcdir)/$@.in' >$@.tmp @@ -1302,82 +137,26 @@ $(shell_scripts): %: %.in chmod a-w $@.tmp mv $@.tmp $@ -BUILT_SOURCES += init-ceph -sbin_SCRIPTS += mkcephfs - -bin_SCRIPTS += ceph-coverage - +EXTRA_DIST += $(srcdir)/$(shell_scripts:%=%.in) CLEANFILES += $(shell_scripts) -CLEANFILES += ceph_ver.h sample.fetch_config - -## - -AM_COMMON_FLAGS = \ - -D__CEPH__ \ - -D_FILE_OFFSET_BITS=64 \ - -D_REENTRANT \ - -D_THREAD_SAFE \ - -D__STDC_FORMAT_MACROS \ - -D_GNU_SOURCE \ - -Wall \ - ${WARN_TYPE_LIMITS} \ - ${WARN_IGNORED_QUALIFIERS} \ - -Winit-self \ - -Wpointer-arith \ - -Werror=format-security \ - -fno-strict-aliasing \ - -fsigned-char -if !CLANG - AM_COMMON_FLAGS += -rdynamic -endif - -AM_CFLAGS = $(AM_COMMON_FLAGS) -AM_CXXFLAGS = \ - @AM_CXXFLAGS@ \ - $(AM_COMMON_FLAGS) \ - -DCEPH_LIBDIR=\"${libdir}\" \ - -Wnon-virtual-dtor \ - -Wno-invalid-offsetof - -if !CLANG - AM_CXXFLAGS += -Wstrict-null-sentinel -endif -# note: this is position dependant, it affects the -l options that -# come after it on the command line. when you use ${AM_LDFLAGS} in -# later rules, take care where you place it. for more information, see -# http://blog.flameeyes.eu/2008/11/19/relationship-between-as-needed-and-no-undefined-part-1-what-do-they-do -# http://blog.flameeyes.eu/2008/11/20/misguided-link-and-as-needed -# http://www.gentoo.org/proj/en/qa/asneeded.xml -# http://gcc.gnu.org/ml/gcc-help/2010-12/msg00338.html -# http://sigquit.wordpress.com/2011/02/16/why-asneeded-doesnt-work-as-expected-for-your-libraries-on-your-autotools-project/ -AM_LDFLAGS = -Wl,--as-needed - -if USE_BOOST_SPIRIT_OLD_HDR -AM_CXXFLAGS += -DUSE_BOOST_SPIRIT_OLD_HDR -endif - -if WITH_LIBATOMIC -AM_LDFLAGS += -latomic_ops -endif - -if ENABLE_COVERAGE -AM_CFLAGS += -fprofile-arcs -ftest-coverage -AM_CXXFLAGS += -fprofile-arcs -ftest-coverage -O0 -EXTRALIBS += -lgcov -endif # extra bits + EXTRA_DIST += \ - $(srcdir)/verify-mds-journal.sh $(srcdir)/vstart.sh $(srcdir)/stop.sh \ - ceph-run $(srcdir)/ceph_common.sh \ + $(srcdir)/verify-mds-journal.sh \ + $(srcdir)/vstart.sh \ + $(srcdir)/stop.sh \ + ceph-run \ + $(srcdir)/ceph_common.sh \ $(srcdir)/init-radosgw \ $(srcdir)/init-radosgw.sysv \ $(srcdir)/init-rbdmap \ - $(srcdir)/ceph-clsinfo $(srcdir)/make_version $(srcdir)/check_version \ + $(srcdir)/ceph-clsinfo \ + $(srcdir)/make_version \ + $(srcdir)/check_version \ $(srcdir)/.git_version \ $(srcdir)/ceph-rbdnamer \ - $(ceph_tool_gui_DATA) \ $(srcdir)/test/encoding/readable.sh \ $(srcdir)/test/encoding/check-generated.sh \ $(srcdir)/upstart/ceph-all.conf \ @@ -1400,21 +179,39 @@ EXTRA_DIST += \ ceph-disk-activate \ ceph-disk-udev \ ceph-create-keys \ - mount.fuse.ceph \ ceph-rest-api \ mount.fuse.ceph \ - rbdmap + rbdmap \ + unittest_bufferlist.sh \ + yasm-wrapper -EXTRA_DIST += $(srcdir)/$(shell_scripts:%=%.in) +EXTRA_DIST += \ + libs3/COPYING \ + libs3/ChangeLog \ + libs3/GNUmakefile \ + libs3/GNUmakefile.mingw \ + libs3/GNUmakefile.osx \ + libs3/INSTALL \ + libs3/LICENSE \ + libs3/README \ + libs3/TODO \ + libs3/archlinux \ + libs3/debian \ + libs3/doxyfile \ + libs3/inc \ + libs3/libs3.spec \ + libs3/mswin \ + libs3/src \ + libs3/test \ + unittest_bufferlist.sh # work around old versions of automake that don't define $docdir # NOTE: this won't work on suse, where docdir is /usr/share/doc/packages/$package. docdir ?= ${datadir}/doc/ceph - doc_DATA = $(srcdir)/sample.ceph.conf sample.fetch_config -sample.fetch_config: fetch_config - cp -f $(srcdir)/fetch_config ./sample.fetch_config + +# various scripts shell_commondir = $(libdir)/ceph shell_common_SCRIPTS = ceph_common.sh @@ -1425,31 +222,53 @@ bash_completion_DATA = $(srcdir)/bash_completion/ceph \ $(srcdir)/bash_completion/rbd \ $(srcdir)/bash_completion/radosgw-admin -libcephfs_includedir = $(includedir)/cephfs -libcephfs_include_DATA = $(srcdir)/include/cephfs/libcephfs.h - -librbd_includedir = $(includedir)/rbd -librbd_include_DATA = \ - $(srcdir)/include/rbd/features.h \ - $(srcdir)/include/rbd/librbd.h \ - $(srcdir)/include/rbd/librbd.hpp - -rados_includedir = $(includedir)/rados -rados_include_DATA = \ - $(srcdir)/include/rados/librados.h \ - $(srcdir)/include/rados/rados_types.h \ - $(srcdir)/include/rados/rados_types.hpp \ - $(srcdir)/include/rados/librados.hpp \ - $(srcdir)/include/buffer.h \ - $(srcdir)/include/page.h \ - $(srcdir)/include/crc32c.h - -#crush_includedir = $(includedir)/crush -#crush_include_DATA = \ -# $(srcdir)/crush/hash.h \ -# $(srcdir)/crush/crush.h \ -# $(srcdir)/crush/mapper.h \ -# $(srcdir)/crush/types.h +ceph_sbin_SCRIPTS = \ + ceph-disk \ + ceph-disk-prepare \ + ceph-disk-activate \ + ceph-disk-udev \ + ceph-create-keys + +bin_SCRIPTS += \ + ceph \ + ceph-run \ + ceph-rest-api \ + ceph-clsinfo \ + ceph-debugpack \ + ceph-rbdnamer \ + ceph-post-file + +BUILT_SOURCES += init-ceph +sbin_SCRIPTS += mkcephfs + +shell_scripts += init-ceph mkcephfs + + + + + +# tests to actually run on "make check"; if you need extra, non-test, +# executables built, you need to replace this with manual assignments +# target by target + +TESTS = $(check_PROGRAMS) unittest_bufferlist.sh + +check-local: + $(srcdir)/test/encoding/check-generated.sh + $(srcdir)/test/encoding/readable.sh ../ceph-object-corpus + + +# base targets + +core-daemons: ceph-mon ceph-osd ceph-mds radosgw +admin-tools: monmaptool osdmaptool crushtool ceph-authtool +base: core-daemons admin-tools \ + cephfs ceph-syn ceph-conf \ + rados librados-config \ + init-ceph mkcephfs ceph_mon_store_converter ceph-post-file + + +# version stuff FORCE: .git_version: FORCE @@ -1461,6 +280,15 @@ ceph_ver.c: ./ceph_ver.h common/version.cc: ./ceph_ver.h test/encoding/ceph_dencoder.cc: ./ceph_ver.h +sample.fetch_config: fetch_config + cp -f $(srcdir)/fetch_config ./sample.fetch_config + +dist-hook: + $(srcdir)/check_version $(srcdir)/.git_version + +CLEANFILES += ceph_ver.h sample.fetch_config + + # assemble Python script with global version variables # NB: depends on format of ceph_ver.h @@ -1477,288 +305,12 @@ ceph: ceph.in ./ceph_ver.h Makefile mv $@.tmp $@ # cleaning + clean-local: -rm *.so *.gcno *.gcda -# libs - -CCAS = ${srcdir}/yasm-wrapper -AM_CCASFLAGS = -f elf64 - -# crc -libcrc_la_SOURCES = \ - common/sctp_crc32.c \ - common/crc32c.cc \ - common/crc32c_intel_baseline.c \ - common/crc32c_intel_fast.c - -if WITH_GOOD_YASM_ELF64 -libcrc_la_SOURCES += common/crc32c_intel_fast_asm.S -libcrc_la_LIBTOOLFLAGS = --tag=CC -endif - -noinst_LTLIBRARIES += libcrc.la - -# common -libcommon_la_SOURCES = $(libcommon_files) -libcommon_la_CFLAGS= ${CRYPTO_CFLAGS} ${AM_CFLAGS} -libcommon_la_CXXFLAGS= ${AM_CXXFLAGS} -libcommon_la_LDFLAGS = -lrt -libcommon_la_LIBADD = libcrc.la -noinst_LTLIBRARIES += libcommon.la - -crush_files = \ - crush/builder.c \ - crush/mapper.c \ - crush/crush.c \ - crush/hash.c \ - crush/CrushWrapper.cc \ - crush/CrushCompiler.cc \ - crush/CrushTester.cc - -# this list ommits the ceph_ver.c file -libcommon_files = \ - ./ceph_ver.c \ - arch/probe.cc \ - arch/intel.c \ - auth/AuthAuthorizeHandler.cc \ - auth/AuthClientHandler.cc \ - auth/AuthSessionHandler.cc \ - auth/AuthMethodList.cc \ - auth/cephx/CephxAuthorizeHandler.cc \ - auth/cephx/CephxClientHandler.cc \ - auth/cephx/CephxProtocol.cc \ - auth/cephx/CephxSessionHandler.cc \ - auth/none/AuthNoneAuthorizeHandler.cc \ - auth/unknown/AuthUnknownAuthorizeHandler.cc \ - auth/Crypto.cc \ - auth/KeyRing.cc \ - auth/RotatingKeyRing.cc \ - common/DecayCounter.cc \ - common/LogClient.cc \ - common/LogEntry.cc \ - common/PrebufferedStreambuf.cc \ - common/BackTrace.cc \ - common/perf_counters.cc \ - common/Mutex.cc \ - common/OutputDataSocket.cc \ - common/admin_socket.cc \ - common/admin_socket_client.cc \ - common/cmdparse.cc \ - common/escape.c \ - common/Clock.cc \ - common/Throttle.cc \ - common/Timer.cc \ - common/Finisher.cc \ - common/environment.cc\ - common/assert.cc \ - common/run_cmd.cc \ - common/WorkQueue.cc \ - common/ConfUtils.cc \ - common/MemoryModel.cc \ - common/armor.c \ - common/fd.cc \ - common/xattr.c \ - common/safe_io.c \ - common/snap_types.cc \ - common/str_list.cc \ - common/errno.cc \ - json_spirit/json_spirit_reader.cpp \ - json_spirit/json_spirit_writer.cpp \ - json_spirit/json_spirit_value.cpp \ - log/Log.cc \ - log/SubsystemMap.cc \ - mon/MonCap.cc \ - mon/MonClient.cc \ - mon/MonMap.cc \ - msg/Accepter.cc \ - msg/DispatchQueue.cc \ - msg/Message.cc \ - common/RefCountedObj.cc \ - msg/Messenger.cc \ - msg/Pipe.cc \ - msg/SimpleMessenger.cc \ - msg/msg_types.cc \ - os/hobject.cc \ - osd/OSDMap.cc \ - osd/osd_types.cc \ - mds/MDSMap.cc \ - mds/inode_backtrace.cc \ - mds/mdstypes.cc \ - common/blkdev.cc \ - common/common_init.cc \ - common/pipe.c \ - common/ceph_argparse.cc \ - common/ceph_context.cc \ - common/buffer.cc \ - common/code_environment.cc \ - common/dout.cc \ - common/signal.cc \ - common/simple_spin.cc \ - common/Thread.cc \ - common/Formatter.cc \ - common/HeartbeatMap.cc \ - include/ceph_fs.cc \ - include/ceph_hash.cc \ - include/ceph_strings.cc \ - include/ceph_frag.cc \ - common/config.cc \ - common/utf8.c \ - common/mime.c \ - common/strtol.cc \ - common/page.cc \ - common/lockdep.cc \ - common/version.cc \ - common/hex.cc \ - common/entity_name.cc \ - common/ceph_crypto.cc \ - common/ceph_crypto_cms.cc \ - common/ceph_json.cc \ - common/ipaddr.cc \ - common/pick_address.cc \ - include/addr_parsing.c \ - $(crush_files) - -if WITH_PROFILER -libcommon_files += perfglue/cpu_profiler.cc -else -libcommon_files += perfglue/disabled_stubs.cc -endif - -# global -libglobal_la_SOURCES = \ - global/global_context.cc \ - global/global_init.cc \ - global/pidfile.cc \ - global/signal_handler.cc -libglobal_la_CFLAGS= ${CRYPTO_CFLAGS} ${AM_CFLAGS} -libglobal_la_CXXFLAGS= ${AM_CXXFLAGS} -libglobal_la_LIBADD= libcommon.la -noinst_LTLIBRARIES += libglobal.la - - -libmon_a_SOURCES = \ - auth/cephx/CephxKeyServer.cc \ - auth/cephx/CephxServiceHandler.cc \ - auth/cephx/CephxSessionHandler.cc \ - auth/AuthServiceHandler.cc \ - auth/AuthSessionHandler.cc \ - mon/Monitor.cc \ - mon/Paxos.cc \ - mon/PaxosService.cc \ - mon/OSDMonitor.cc \ - mon/MDSMonitor.cc \ - mon/MonmapMonitor.cc \ - mon/PGMonitor.cc \ - mon/PGMap.cc \ - mon/LogMonitor.cc \ - mon/AuthMonitor.cc \ - mon/Elector.cc \ - mon/MonitorStore.cc \ - os/LevelDBStore.cc \ - mon/HealthMonitor.cc \ - mon/DataHealthService.cc \ - mon/ConfigKeyService.cc \ - common/util.cc \ - common/TextTable.cc -libmon_a_CXXFLAGS= ${AM_CXXFLAGS} -noinst_LIBRARIES += libmon.a - -libmds_a_SOURCES = \ - mds/Anchor.cc \ - mds/Capability.cc \ - mds/Dumper.cc \ - mds/Resetter.cc \ - mds/MDS.cc \ - mds/flock.cc \ - mds/locks.c \ - mds/journal.cc \ - mds/Server.cc \ - mds/Mutation.cc \ - mds/MDCache.cc \ - mds/Locker.cc \ - mds/Migrator.cc \ - mds/MDBalancer.cc \ - mds/CDentry.cc \ - mds/CDir.cc \ - mds/CInode.cc \ - mds/LogEvent.cc \ - mds/MDSTable.cc \ - mds/InoTable.cc \ - mds/MDSTableClient.cc \ - mds/MDSTableServer.cc \ - mds/AnchorServer.cc \ - mds/AnchorClient.cc \ - mds/SnapRealm.cc \ - mds/SnapServer.cc \ - mds/snap.cc \ - mds/SessionMap.cc \ - mds/MDLog.cc \ - osdc/Journaler.cc -noinst_LIBRARIES += libmds.a - -libos_a_SOURCES = \ - os/FileJournal.cc \ - os/FileStore.cc \ - os/chain_xattr.cc \ - os/ObjectStore.cc \ - os/JournalingObjectStore.cc \ - os/LFNIndex.cc \ - os/HashIndex.cc \ - os/IndexManager.cc \ - os/FlatIndex.cc \ - os/DBObjectMap.cc \ - os/LevelDBStore.cc \ - os/WBThrottle.cc \ - os/BtrfsFileStoreBackend.cc \ - os/GenericFileStoreBackend.cc \ - os/ZFSFileStoreBackend.cc -libos_a_CXXFLAGS= ${AM_CXXFLAGS} -noinst_LIBRARIES += libos.a - -if WITH_LIBZFS -libos_zfs_a_SOURCES = os/ZFS.cc -libos_zfs_a_CXXFLAGS= ${AM_CXXFLAGS} ${LIBZFS_CFLAGS} -noinst_LIBRARIES += libos_zfs.a -endif - -libosd_a_SOURCES = \ - osd/PG.cc \ - osd/PGLog.cc \ - osd/ReplicatedPG.cc \ - osd/Ager.cc \ - osd/OSD.cc \ - osd/OSDCap.cc \ - osd/Watch.cc \ - osd/ClassHandler.cc \ - osd/OpRequest.cc \ - osd/SnapMapper.cc \ - objclass/class_api.cc -libosd_a_CXXFLAGS = ${AM_CXXFLAGS} -noinst_LIBRARIES += libosd.a - -libosdc_la_SOURCES = \ - osdc/Objecter.cc \ - osdc/ObjectCacher.cc \ - osdc/Filer.cc \ - osdc/Striper.cc -libosdc_la_CXXFLAGS= ${AM_CXXFLAGS} -libosdc_la_LIBADD = libcommon.la -noinst_LTLIBRARIES += libosdc.la - -libclient_la_SOURCES = \ - client/Client.cc \ - client/Inode.cc \ - client/Dentry.cc \ - client/MetaRequest.cc \ - client/ClientSnapRealm.cc \ - client/MetaSession.cc \ - client/Trace.cc -libclient_la_LIBADD = libosdc.la $(LIBEDIT_LIBS) -noinst_LTLIBRARIES += libclient.la -dist-hook: - $(srcdir)/check_version $(srcdir)/.git_version +# pybind python_PYTHON = pybind/rados.py \ pybind/rbd.py \ @@ -1766,597 +318,29 @@ python_PYTHON = pybind/rados.py \ pybind/ceph_argparse.py \ pybind/ceph_rest_api.py -# headers... and everything else we want to include in a 'make dist' -# that autotools doesn't magically identify. -noinst_HEADERS = \ - rados_sync.h \ - arch/probe.h \ - arch/intel.h \ - auth/cephx/CephxAuthorizeHandler.h\ - auth/cephx/CephxKeyServer.h\ - auth/cephx/CephxProtocol.h\ - auth/cephx/CephxClientHandler.h\ - auth/cephx/CephxServiceHandler.h\ - auth/cephx/CephxSessionHandler.h\ - auth/none/AuthNoneAuthorizeHandler.h\ - auth/none/AuthNoneClientHandler.h\ - auth/none/AuthNoneServiceHandler.h\ - auth/none/AuthNoneSessionHandler.h\ - auth/none/AuthNoneProtocol.h\ - auth/unknown/AuthUnknownAuthorizeHandler.h\ - auth/unknown/AuthUnknownClientHandler.h\ - auth/unknown/AuthUnknownServiceHandler.h\ - auth/unknown/AuthUnknownSessionHandler.h\ - auth/unknown/AuthUnknownProtocol.h\ - auth/Auth.h\ - auth/AuthMethodList.h\ - auth/AuthClientHandler.h\ - auth/AuthServiceHandler.h\ - auth/AuthSessionHandler.h\ - auth/AuthAuthorizeHandler.h\ - auth/KeyRing.h\ - auth/RotatingKeyRing.h\ - auth/Crypto.h\ - bash_completion/ceph\ - bash_completion/rados\ - bash_completion/rbd\ - bash_completion/radosgw-admin\ - client/Client.h\ - client/Dentry.h\ - client/Dir.h\ - client/Fh.h\ - client/Inode.h\ - client/MetaRequest.h\ - client/MetaSession.h\ - client/ClientSnapRealm.h\ - client/SyntheticClient.h\ - client/Trace.h\ - client/fuse_ll.h\ - client/ioctl.h\ - client/hadoop/CephFSInterface.h\ - client/ObjecterWriteback.h\ + +# everything else we want to include in a 'make dist' + +noinst_HEADERS += \ cls_acl.cc\ cls_crypto.cc\ - cls/lock/cls_lock_types.h\ - cls/lock/cls_lock_ops.h\ - cls/lock/cls_lock_client.h\ - cls/rbd/cls_rbd.h\ - cls/rbd/cls_rbd_client.h\ - cls/refcount/cls_refcount_ops.h\ - cls/refcount/cls_refcount_client.h\ - cls/version/cls_version_types.h\ - cls/version/cls_version_ops.h\ - cls/version/cls_version_client.h\ - cls/log/cls_log_types.h\ - cls/log/cls_log_ops.h\ - cls/log/cls_log_client.h\ - cls/statelog/cls_statelog_types.h\ - cls/statelog/cls_statelog_ops.h\ - cls/statelog/cls_statelog_client.h\ - cls/replica_log/cls_replica_log_types.h\ - cls/replica_log/cls_replica_log_ops.h\ - cls/replica_log/cls_replica_log_client.h\ - cls/rgw/cls_rgw_client.h\ - cls/rgw/cls_rgw_ops.h\ - cls/rgw/cls_rgw_types.h\ - common/BackTrace.h\ - common/RefCountedObj.h\ - common/HeartbeatMap.h\ - common/LogClient.h\ - common/LogEntry.h\ - common/Preforker.h\ - common/WorkQueue.h\ - common/PrioritizedQueue.h\ - common/ceph_argparse.h\ - common/ceph_context.h\ - common/xattr.h\ - common/blkdev.h\ - common/compiler_extensions.h\ - common/debug.h\ - common/dout.h\ - common/escape.h\ - common/fd.h\ - common/version.h\ - common/hex.h\ - common/entity_name.h\ - common/errno.h\ - common/environment.h\ - common/likely.h\ - common/lockdep.h\ - common/obj_bencher.h\ - common/snap_types.h\ - common/Clock.h\ - common/Cond.h\ - common/ConfUtils.h\ - common/DecayCounter.h\ - common/Finisher.h\ - common/Formatter.h\ - common/perf_counters.h\ - common/OutputDataSocket.h \ - common/admin_socket.h \ - common/admin_socket_client.h \ - common/shared_cache.hpp \ - common/tracked_int_ptr.hpp \ - common/simple_cache.hpp \ - common/sharedptr_registry.hpp \ - common/map_cacher.hpp \ - common/MemoryModel.h\ - common/Mutex.h\ - common/PrebufferedStreambuf.h\ - common/RWLock.h\ - common/Semaphore.h\ - common/SimpleRNG.h\ - common/TextTable.h\ - common/Thread.h\ - common/Throttle.h\ - common/Timer.h\ - common/TrackedOp.h\ - common/arch.h\ - common/armor.h\ - global/global_init.h \ - global/global_context.h \ - common/common_init.h\ - common/pipe.h\ - common/code_environment.h \ - common/signal.h\ - global/signal_handler.h\ - common/simple_spin.h\ - common/run_cmd.h\ - common/safe_io.h\ - common/config.h\ - common/config_obs.h\ - common/config_opts.h\ - common/ceph_crypto.h\ - common/ceph_crypto_cms.h\ - common/ceph_json.h\ - common/crc32c_intel_baseline.h\ - common/crc32c_intel_fast.h\ - common/lru_map.h\ - common/utf8.h\ - common/mime.h\ - common/pick_address.h\ - common/sctp_crc32.h\ - common/secret.h\ - common/strtol.h\ - common/static_assert.h\ - common/AsyncReserver.h\ - crush/CrushCompiler.h\ - crush/CrushTester.h\ - crush/CrushWrapper.h\ - crush/CrushWrapper.i\ - crush/builder.h\ - crush/crush.h\ - crush/grammar.h\ - crush/hash.h\ - crush/mapper.h\ - crush/sample.txt\ - crush/types.h\ fetch_config\ - include/bloom_filter.hpp\ - include/Context.h\ - include/CompatSet.h\ - include/Distribution.h\ - include/addr_parsing.h\ - include/assert.h\ - include/atomic.h\ - include/bitmapper.h\ - include/blobhash.h\ - include/buffer.h\ - include/byteorder.h\ - include/cephfs/libcephfs.h\ - include/ceph_features.h\ - include/ceph_frag.h\ - include/ceph_fs.h\ - include/ceph_hash.h\ - include/cmp.h\ - include/color.h\ - include/compat.h\ - include/crc32c.h\ - include/encoding.h\ - include/err.h\ - include/error.h\ - include/filepath.h\ - include/frag.h\ - include/hash.h\ - include/intarith.h\ - include/interval_set.h\ - include/inttypes.h\ - include/ipaddr.h\ - include/linux_fiemap.h\ - include/lru.h\ - include/msgr.h\ - include/object.h\ - include/page.h\ - include/rangeset.h\ - include/rados.h\ - include/rbd_types.h\ - include/statlite.h\ - include/str_list.h\ - include/stringify.h\ - include/triple.h\ - include/types.h\ - include/utime.h\ - include/dlist.h\ - include/elist.h\ - include/uuid.h\ - include/xlist.h\ - include/rados/librados.h\ - include/rados/rados_types.h\ - include/rados/rados_types.hpp\ - include/rados/librados.hpp\ - include/rados/librgw.h\ - include/rados/page.h\ - include/rados/crc32c.h\ - include/rados/buffer.h\ - include/rbd/features.h\ - include/rbd/librbd.h\ - include/rbd/librbd.hpp\ - include/util.h\ - librados/snap_set_diff.h\ - librados/AioCompletionImpl.h\ - librados/IoCtxImpl.h\ - librados/PoolAsyncCompletionImpl.h\ - librados/RadosClient.h\ - librbd/AioCompletion.h\ - librbd/AioRequest.h\ - librbd/ImageCtx.h\ - librbd/internal.h\ - librbd/LibrbdWriteback.h\ - librbd/parent_types.h\ - librbd/SnapInfo.h\ - librbd/WatchCtx.h\ logrotate.conf\ - json_spirit/json_spirit.h\ - json_spirit/json_spirit_error_position.h\ - json_spirit/json_spirit_reader.h\ - json_spirit/json_spirit_reader_template.h\ - json_spirit/json_spirit_stream_reader.h\ - json_spirit/json_spirit_utils.h\ - json_spirit/json_spirit_value.h\ - json_spirit/json_spirit_writer.h\ - json_spirit/json_spirit_writer_options.h\ - json_spirit/json_spirit_writer_template.h\ - key_value_store/key_value_structure.h\ - key_value_store/kv_flat_btree_async.h\ - key_value_store/kvs_arg_types.h\ - log/Entry.h\ - log/EntryQueue.h\ - log/Log.h\ - log/SubsystemMap.h\ - mds/inode_backtrace.h\ - mds/flock.h\ - mds/locks.c\ - mds/locks.h\ - mds/Anchor.h\ - mds/AnchorClient.h\ - mds/AnchorServer.h\ - mds/CDentry.h\ - mds/CDir.h\ - mds/CInode.h\ - mds/Capability.h\ - mds/Dumper.h\ - mds/InoTable.h\ - mds/LocalLock.h\ - mds/Locker.h\ - mds/LogEvent.h\ - mds/LogSegment.h\ - mds/MDBalancer.h\ - mds/MDCache.h\ - mds/MDLog.h\ - mds/MDS.h\ - mds/MDSMap.h\ - mds/MDSTable.h\ - mds/MDSTableServer.h\ - mds/MDSTableClient.h\ - mds/Mutation.h\ - mds/Migrator.h\ - mds/Resetter.h\ - mds/ScatterLock.h\ - mds/Server.h\ - mds/SessionMap.h\ - mds/SimpleLock.h\ - mds/SnapClient.h\ - mds/SnapRealm.h\ - mds/SnapServer.h\ - mds/events/ECommitted.h\ - mds/events/EExport.h\ - mds/events/EFragment.h\ - mds/events/EImportFinish.h\ - mds/events/EImportStart.h\ - mds/events/EMetaBlob.h\ - mds/events/EOpen.h\ - mds/events/EResetJournal.h\ - mds/events/ESession.h\ - mds/events/ESessions.h\ - mds/events/ESlaveUpdate.h\ - mds/events/ESubtreeMap.h\ - mds/events/ETableClient.h\ - mds/events/ETableServer.h\ - mds/events/EUpdate.h\ - mds/mds_table_types.h\ - mds/mdstypes.h\ - mds/snap.h\ - messages/MAuth.h\ - messages/MAuthReply.h\ - messages/MCacheExpire.h\ - messages/MClientCaps.h\ - messages/MClientCapRelease.h\ - messages/MClientLease.h\ - messages/MClientReconnect.h\ - messages/MClientReply.h\ - messages/MClientRequest.h\ - messages/MClientRequestForward.h\ - messages/MClientSession.h\ - messages/MClientSnap.h\ - messages/MCommand.h\ - messages/MCommandReply.h\ - messages/MDentryLink.h\ - messages/MDentryUnlink.h\ - messages/MDirUpdate.h\ - messages/MDiscover.h\ - messages/MDiscoverReply.h\ - messages/MExportCaps.h\ - messages/MExportCapsAck.h\ - messages/MExportDir.h\ - messages/MExportDirAck.h\ - messages/MExportDirCancel.h\ - messages/MExportDirDiscover.h\ - messages/MExportDirDiscoverAck.h\ - messages/MExportDirFinish.h\ - messages/MExportDirNotify.h\ - messages/MExportDirNotifyAck.h\ - messages/MExportDirPrep.h\ - messages/MExportDirPrepAck.h\ - messages/MGenericMessage.h\ - messages/MGetPoolStats.h\ - messages/MGetPoolStatsReply.h\ - messages/MHeartbeat.h\ - messages/MInodeFileCaps.h\ - messages/MLock.h\ - messages/MLog.h\ - messages/MLogAck.h\ - messages/MMDSBeacon.h\ - messages/MMDSCacheRejoin.h\ - messages/MMDSLoadTargets.h\ - messages/MMDSFindIno.h\ - messages/MMDSFindInoReply.h\ - messages/MMDSFragmentNotify.h\ - messages/MMDSMap.h\ - messages/MMDSOpenIno.h \ - messages/MMDSOpenInoReply.h \ - messages/MMDSResolve.h\ - messages/MMDSResolveAck.h\ - messages/MMDSSlaveRequest.h\ - messages/MMDSTableRequest.h\ - messages/MMonCommand.h\ - messages/MMonCommandAck.h\ - messages/MMonElection.h\ - messages/MMonGetMap.h\ - messages/MMonGetVersion.h\ - messages/MMonGetVersionReply.h\ - messages/MMonGlobalID.h\ - messages/MMonHealth.h\ - messages/MMonJoin.h\ - messages/MMonMap.h\ - messages/MMonPaxos.h\ - messages/MMonProbe.h\ - messages/MMonScrub.h \ - messages/MMonSubscribe.h\ - messages/MMonSubscribeAck.h\ - messages/MMonSync.h \ - messages/MOSDAlive.h\ - messages/MOSDBoot.h\ - messages/MOSDFailure.h\ - messages/MOSDMarkMeDown.h\ - messages/MOSDMap.h\ - messages/MOSDOp.h\ - messages/MOSDOpReply.h\ - messages/MOSDPGBackfill.h\ - messages/MOSDPGCreate.h\ - messages/MOSDPGPush.h\ - messages/MOSDPGPull.h\ - messages/MOSDPGPushReply.h\ - messages/MOSDPGInfo.h\ - messages/MOSDPGLog.h\ - messages/MOSDPGMissing.h\ - messages/MOSDPGNotify.h\ - messages/MOSDPGQuery.h\ - messages/MOSDPGRemove.h\ - messages/MOSDPGScan.h\ - messages/MBackfillReserve.h\ - messages/MRecoveryReserve.h\ - messages/MMonQuorumService.h\ - messages/MOSDPGTemp.h\ - messages/MOSDPGTrim.h\ - messages/MOSDPing.h\ - messages/MOSDRepScrub.h\ - messages/MOSDScrub.h\ - messages/MOSDSubOp.h\ - messages/MOSDSubOpReply.h\ - messages/MPGStats.h\ - messages/MPGStatsAck.h\ - messages/MPing.h\ - messages/MPoolOp.h\ - messages/MPoolOpReply.h\ - messages/MRemoveSnaps.h\ - messages/MRoute.h\ - messages/MForward.h\ - messages/MStatfs.h\ - messages/MStatfsReply.h\ - messages/MTimeCheck.h\ - messages/MWatchNotify.h\ - messages/PaxosServiceMessage.h\ - mon/AuthMonitor.h\ - mon/DataHealthService.h\ - mon/Elector.h\ - mon/LogMonitor.h\ - mon/ConfigKeyService.h\ - mon/HealthMonitor.h\ - mon/HealthService.h\ - mon/MDSMonitor.h\ - mon/MonmapMonitor.h\ - mon/MonCap.h\ - mon/MonClient.h\ - mon/MonCommands.h\ - mon/MonMap.h\ - mon/Monitor.h\ - mon/MonitorStore.h\ - mon/MonitorDBStore.h\ - mon/OSDMonitor.h\ - mon/PGMap.h\ - mon/PGMonitor.h\ - mon/Paxos.h\ - mon/PaxosService.h\ - mon/QuorumService.h\ - mon/Session.h\ - mon/mon_types.h\ - mount/canonicalize.c\ - mount/mtab.c\ - msg/Accepter.h\ - msg/DispatchQueue.h\ - msg/Dispatcher.h\ - msg/Message.h\ - msg/Messenger.h\ - msg/Pipe.h\ - msg/SimpleMessenger.h\ - msg/msg_types.h\ - objclass/objclass.h\ - os/btrfs_ioctl.h\ - os/ZFS.h\ - os/chain_xattr.h\ - os/hobject.h \ - os/CollectionIndex.h\ - os/FileJournal.h\ - os/FileStore.h\ - os/BtrfsFileStoreBackend.h\ - os/GenericFileStoreBackend.h\ - os/ZFSFileStoreBackend.h\ - os/FlatIndex.h\ - os/HashIndex.h\ - os/FDCache.h\ - os/WBThrottle.h\ - os/IndexManager.h\ - os/Journal.h\ - os/JournalingObjectStore.h\ - os/LFNIndex.h\ - os/ObjectStore.h\ - os/SequencerPosition.h\ - osd/Ager.h\ - osd/ClassHandler.h\ - osd/OSD.h\ - osd/OSDCap.h\ - osd/OSDMap.h\ - osd/ObjectVersioner.h\ - osd/OpRequest.h\ - osd/SnapMapper.h\ - osd/PG.h\ - osd/PGLog.h\ - osd/ReplicatedPG.h\ - osd/Watch.h\ - osd/osd_types.h\ - osdc/Blinker.h\ - osdc/Filer.h\ - osdc/Journaler.h\ - osdc/ObjectCacher.h\ - osdc/Objecter.h\ - osdc/Striper.h\ - osdc/WritebackHandler.h\ - perfglue/cpu_profiler.h\ - perfglue/heap_profiler.h\ - rgw/logrotate.conf\ - rgw/rgw_acl.h\ - rgw/rgw_acl_s3.h\ - rgw/rgw_acl_swift.h\ - rgw/rgw_client_io.h\ - rgw/rgw_fcgi.h\ - rgw/rgw_xml.h\ - rgw/rgw_cache.h\ - rgw/rgw_common.h\ - rgw/rgw_cors.h\ - rgw/rgw_cors_s3.h\ - rgw/rgw_cors_swift.h\ - rgw/rgw_string.h\ - rgw/rgw_formats.h\ - rgw/rgw_http_errors.h\ - rgw/rgw_log.h\ - rgw/rgw_multi.h\ - rgw/rgw_policy_s3.h\ - rgw/rgw_gc.h\ - rgw/rgw_metadata.h\ - rgw/rgw_multi_del.h\ - rgw/rgw_op.h\ - rgw/rgw_http_client.h\ - rgw/rgw_swift.h\ - rgw/rgw_swift_auth.h\ - rgw/rgw_rados.h\ - rgw/rgw_replica_log.h \ - rgw/rgw_resolve.h\ - rgw/rgw_rest.h\ - rgw/rgw_rest_swift.h\ - rgw/rgw_rest_s3.h\ - rgw/rgw_auth_s3.h\ - rgw/rgw_rest_admin.h\ - rgw/rgw_rest_usage.h\ - rgw/rgw_rest_user.h\ - rgw/rgw_rest_bucket.h\ - rgw/rgw_rest_client.h\ - rgw/rgw_rest_conn.h\ - rgw/rgw_tools.h\ - rgw/rgw_rest_metadata.h\ - rgw/rgw_rest_log.h\ - rgw/rgw_rest_opstate.h\ - rgw/rgw_rest_replica_log.h\ - rgw/rgw_rest_config.h\ - rgw/rgw_usage.h\ - rgw/rgw_user.h\ - rgw/rgw_bucket.h\ - rgw/rgw_keystone.h\ - sample.ceph.conf\ - tools/common.h\ - test/osd/RadosModel.h\ - test/osd/Object.h\ - test/osd/TestOpStat.h\ - global/pidfile.h\ - common/sync_filesystem.h \ - test/bench/distribution.h \ - test/bench/rados_backend.h \ - test/bench/rbd_backend.h \ - test/bench/bencher.h \ - test/bench/backend.h \ - test/bench/dumb_backend.h \ - test/bench/stat_collector.h \ - test/bench/detailed_stat_collector.h \ - test/bench/testfilestore_backend.h \ - test/common/ObjectContents.h \ - test/encoding/types.h \ - test/filestore/DeterministicOpSequence.h \ - test/filestore/FileStoreTracker.h \ - test/filestore/FileStoreDiff.h \ - test/filestore/TestFileStoreState.h \ - test/filestore/workload_generator.h \ - test/kv_store_bench.h \ - test/librados/test.h \ - test/ObjectMap/KeyValueDBMemory.h \ - test/omap_bench.h \ - test/osd/Object.h \ - test/osd/RadosModel.h \ - test/osd/TestOpStat.h \ - test/osdc/FakeWriteback.h \ - test/system/cross_process_sem.h \ - test/system/st_rados_create_pool.h \ - test/system/st_rados_list_objects.h \ - test/system/st_rados_delete_objs.h \ - test/system/st_rados_delete_pool.h \ - test/system/st_rados_notify.h \ - test/system/st_rados_watch.h \ - test/system/systest_runnable.h \ - test/system/systest_settings.h \ - test/unit.h \ - os/ObjectMap.h \ - os/DBObjectMap.h \ - os/KeyValueDB.h \ - os/LevelDBStore.h \ - common/cmdparse.h + sample.ceph.conf\ + bash_completion/ceph \ + bash_completion/rados \ + bash_completion/rbd \ + bash_completion/radosgw-admin \ + mount/canonicalize.c \ + mount/mtab.c \ + objclass/objclass.h + + +# coverage + +shell_scripts += ceph-coverage +bin_SCRIPTS += ceph-coverage + if ENABLE_COVERAGE COV_DIR = $(DESTDIR)$(libdir)/ceph/coverage @@ -2394,13 +378,6 @@ uninstall-local: uninstall-coverage -rmdir -p $(DESTDIR)$(localstatedir)/log/ceph -rmdir -p $(DESTDIR)$(localstatedir)/lib/ceph/tmp -# if we are doing a debug build, tell make to actually build the debug -# targets -if WITH_DEBUG -bin_PROGRAMS += $(bin_DEBUGPROGRAMS) -endif - - # # coverity rules expect: # - cov-build to be in the path diff --git a/src/arch/Makefile.am b/src/arch/Makefile.am new file mode 100644 index 00000000000..27342078150 --- /dev/null +++ b/src/arch/Makefile.am @@ -0,0 +1,11 @@ +libarch_la_SOURCES = \ + arch/intel.c \ + arch/neon.c \ + arch/probe.cc + +noinst_LTLIBRARIES += libarch.la + +noinst_HEADERS += \ + arch/intel.h \ + arch/neon.h \ + arch/probe.h diff --git a/src/arch/neon.c b/src/arch/neon.c new file mode 100644 index 00000000000..32c1f621ef7 --- /dev/null +++ b/src/arch/neon.c @@ -0,0 +1,51 @@ +#include "arch/probe.h" + +/* flags we export */ +int ceph_arch_neon = 0; + +#include <stdio.h> + +#if __linux__ + +#include <elf.h> +#include <link.h> // ElfW macro + +#if __arm__ +#include <asm/hwcap.h> +#endif // __arm__ + +static unsigned long get_auxval(unsigned long type) +{ + unsigned long result = 0; + FILE *f = fopen("/proc/self/auxv", "r"); + if (f) { + ElfW(auxv_t) entry; + while (fread(&entry, sizeof(entry), 1, f)) { + if (entry.a_type == type) { + result = entry.a_un.a_val; + break; + } + } + fclose(f); + } + return result; +} + +static unsigned long get_hwcap(void) +{ + return get_auxval(AT_HWCAP); +} + +#endif // __linux__ + +int ceph_arch_neon_probe(void) +{ +#if __arm__ && __linux__ + ceph_arch_neon = (get_hwcap() & HWCAP_NEON) == HWCAP_NEON; +#else + if (0) + get_hwcap(); // make compiler shut up +#endif + return 0; +} + diff --git a/src/arch/neon.h b/src/arch/neon.h new file mode 100644 index 00000000000..0c8aacf5e87 --- /dev/null +++ b/src/arch/neon.h @@ -0,0 +1,16 @@ +#ifndef CEPH_ARCH_NEON_H +#define CEPH_ARCH_NEON_H + +#ifdef __cplusplus +extern "C" { +#endif + +extern int ceph_arch_neon; /* true if we have ARM NEON abilities */ + +extern int ceph_arch_neon_probe(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/arch/probe.cc b/src/arch/probe.cc index 9f8bc9d2d0f..8648e54d945 100644 --- a/src/arch/probe.cc +++ b/src/arch/probe.cc @@ -4,6 +4,7 @@ #include "arch/probe.h" #include "arch/intel.h" +#include "arch/neon.h" int ceph_arch_probe(void) { @@ -11,6 +12,7 @@ int ceph_arch_probe(void) return 1; ceph_arch_intel_probe(); + ceph_arch_neon_probe(); ceph_arch_probed = 1; return 1; diff --git a/src/auth/AuthMethodList.h b/src/auth/AuthMethodList.h index 7b21b325aeb..b5aae0bba49 100644 --- a/src/auth/AuthMethodList.h +++ b/src/auth/AuthMethodList.h @@ -15,7 +15,8 @@ #ifndef CEPH_AUTHMETHODLIST_H #define CEPH_AUTHMETHODLIST_H -#include "include/inttypes.h" +#include "include/int_types.h" + #include <list> #include <set> #include <string> diff --git a/src/auth/Makefile.am b/src/auth/Makefile.am new file mode 100644 index 00000000000..f7f3b386df5 --- /dev/null +++ b/src/auth/Makefile.am @@ -0,0 +1,46 @@ +libauth_la_SOURCES = \ + auth/AuthAuthorizeHandler.cc \ + auth/AuthClientHandler.cc \ + auth/AuthSessionHandler.cc \ + auth/AuthServiceHandler.cc \ + auth/AuthMethodList.cc \ + auth/cephx/CephxAuthorizeHandler.cc \ + auth/cephx/CephxClientHandler.cc \ + auth/cephx/CephxProtocol.cc \ + auth/cephx/CephxServiceHandler.cc \ + auth/cephx/CephxSessionHandler.cc \ + auth/cephx/CephxKeyServer.cc \ + auth/none/AuthNoneAuthorizeHandler.cc \ + auth/unknown/AuthUnknownAuthorizeHandler.cc \ + auth/Crypto.cc \ + auth/KeyRing.cc \ + auth/RotatingKeyRing.cc +noinst_LTLIBRARIES += libauth.la + +noinst_HEADERS += \ + auth/cephx/CephxAuthorizeHandler.h \ + auth/cephx/CephxKeyServer.h \ + auth/cephx/CephxProtocol.h \ + auth/cephx/CephxClientHandler.h \ + auth/cephx/CephxServiceHandler.h \ + auth/cephx/CephxSessionHandler.h \ + auth/none/AuthNoneAuthorizeHandler.h \ + auth/none/AuthNoneClientHandler.h \ + auth/none/AuthNoneServiceHandler.h \ + auth/none/AuthNoneSessionHandler.h \ + auth/none/AuthNoneProtocol.h \ + auth/unknown/AuthUnknownAuthorizeHandler.h \ + auth/unknown/AuthUnknownClientHandler.h \ + auth/unknown/AuthUnknownServiceHandler.h \ + auth/unknown/AuthUnknownSessionHandler.h \ + auth/unknown/AuthUnknownProtocol.h \ + auth/Auth.h \ + auth/AuthMethodList.h \ + auth/AuthClientHandler.h \ + auth/AuthServiceHandler.h \ + auth/AuthSessionHandler.h \ + auth/AuthAuthorizeHandler.h \ + auth/KeyRing.h \ + auth/RotatingKeyRing.h \ + auth/Crypto.h + diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc index 120f4f28f6e..3d517da1f89 100644 --- a/src/ceph_osd.cc +++ b/src/ceph_osd.cc @@ -170,7 +170,7 @@ int main(int argc, const char **argv) if (mc.get_monmap_privately() < 0) return -1; - int err = OSD::mkfs(g_conf->osd_data, g_conf->osd_journal, mc.monmap.fsid, whoami); + int err = OSD::mkfs(g_ceph_context, g_conf->osd_data, g_conf->osd_journal, mc.monmap.fsid, whoami); if (err < 0) { derr << TEXT_RED << " ** ERROR: error creating empty object store in " << g_conf->osd_data << ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl; @@ -213,7 +213,7 @@ int main(int argc, const char **argv) exit(0); if (mkjournal) { common_init_finish(g_ceph_context); - int err = OSD::mkjournal(g_conf->osd_data, g_conf->osd_journal); + int err = OSD::mkjournal(g_ceph_context, g_conf->osd_data, g_conf->osd_journal); if (err < 0) { derr << TEXT_RED << " ** ERROR: error creating fresh journal " << g_conf->osd_journal << " for object store " << g_conf->osd_data @@ -226,7 +226,7 @@ int main(int argc, const char **argv) } if (flushjournal) { common_init_finish(g_ceph_context); - int err = OSD::flushjournal(g_conf->osd_data, g_conf->osd_journal); + int err = OSD::flushjournal(g_ceph_context, g_conf->osd_data, g_conf->osd_journal); if (err < 0) { derr << TEXT_RED << " ** ERROR: error flushing journal " << g_conf->osd_journal << " for object store " << g_conf->osd_data @@ -240,7 +240,7 @@ int main(int argc, const char **argv) } if (dump_journal) { common_init_finish(g_ceph_context); - int err = OSD::dump_journal(g_conf->osd_data, g_conf->osd_journal, cout); + int err = OSD::dump_journal(g_ceph_context, g_conf->osd_data, g_conf->osd_journal, cout); if (err < 0) { derr << TEXT_RED << " ** ERROR: error dumping journal " << g_conf->osd_journal << " for object store " << g_conf->osd_data @@ -332,7 +332,7 @@ int main(int argc, const char **argv) entity_name_t::OSD(whoami), "hb_front_server", getpid()); Messenger *ms_objecter = Messenger::create(g_ceph_context, - entity_name_t::OSD(whoami), "hbclient", + entity_name_t::OSD(whoami), "ms_objecter", getpid()); ms_cluster->set_cluster_protocol(CEPH_OSD_PROTOCOL); ms_hbclient->set_cluster_protocol(CEPH_OSD_PROTOCOL); @@ -417,6 +417,8 @@ int main(int argc, const char **argv) if (r < 0) exit(1); + ms_objecter->bind(g_conf->public_addr); + // Set up crypto, daemonize, etc. global_init_daemonize(g_ceph_context, 0); common_init_finish(g_ceph_context); @@ -435,7 +437,7 @@ int main(int argc, const char **argv) return -1; global_init_chdir(g_ceph_context); - osd = new OSD(whoami, + osd = new OSD(g_ceph_context, whoami, ms_cluster, ms_public, ms_hbclient, diff --git a/src/client/Makefile.am b/src/client/Makefile.am new file mode 100644 index 00000000000..53107eba517 --- /dev/null +++ b/src/client/Makefile.am @@ -0,0 +1,35 @@ +libclient_la_SOURCES = \ + client/Client.cc \ + client/Inode.cc \ + client/Dentry.cc \ + client/MetaRequest.cc \ + client/ClientSnapRealm.cc \ + client/MetaSession.cc \ + client/Trace.cc +libclient_la_LIBADD = $(LIBOSDC) $(LIBEDIT_LIBS) +noinst_LTLIBRARIES += libclient.la + +noinst_HEADERS += \ + client/Client.h \ + client/Dentry.h \ + client/Dir.h \ + client/Fh.h \ + client/Inode.h \ + client/MetaRequest.h \ + client/MetaSession.h \ + client/ClientSnapRealm.h \ + client/SyntheticClient.h \ + client/Trace.h \ + client/ioctl.h \ + client/ObjecterWriteback.h + +if WITH_FUSE +libclient_fuse_la_SOURCES = client/fuse_ll.cc +libclient_fuse_la_LIBADD = libclient.la -lfuse +noinst_LTLIBRARIES += libclient_fuse.la +noinst_HEADERS += client/fuse_ll.h +endif + +ceph_test_ioctls_SOURCES = client/test_ioctls.c +bin_DEBUGPROGRAMS += ceph_test_ioctls + diff --git a/src/client/SyntheticClient.cc b/src/client/SyntheticClient.cc index cb211f5461b..6b2c7b10565 100644 --- a/src/client/SyntheticClient.cc +++ b/src/client/SyntheticClient.cc @@ -267,7 +267,7 @@ void parse_syn_options(vector<const char*>& args) } -SyntheticClient::SyntheticClient(Client *client, int w) +SyntheticClient::SyntheticClient(Client *client, int w) { this->client = client; whoami = w; @@ -282,7 +282,7 @@ SyntheticClient::SyntheticClient(Client *client, int w) this->iargs = syn_iargs; this->sargs = syn_sargs; - run_start = ceph_clock_now(g_ceph_context); + run_start = ceph_clock_now(client->cct); } @@ -332,7 +332,7 @@ int SyntheticClient::run() return -1; } - //run_start = ceph_clock_now(g_ceph_context); + //run_start = ceph_clock_now(client->cct); run_until = utime_t(0,0); dout(5) << "run" << dendl; @@ -442,7 +442,7 @@ int SyntheticClient::run() iargs.pop_front(); if (iarg1 && run_me()) { dout(2) << "sleepuntil " << iarg1 << dendl; - utime_t at = ceph_clock_now(g_ceph_context) - run_start; + utime_t at = ceph_clock_now(client->cct) - run_start; if (at.sec() < iarg1) sleep(iarg1 - at.sec()); } @@ -797,14 +797,14 @@ int SyntheticClient::run() if (iarg1 == 0) iarg1 = 1; // play trace at least once! for (int i=0; i<iarg1; i++) { - utime_t start = ceph_clock_now(g_ceph_context); + utime_t start = ceph_clock_now(client->cct); if (time_to_stop()) break; play_trace(t, prefix, !playdata); if (time_to_stop()) break; if (iarg1 > 1) clean_dir(prefix); // clean only if repeat - utime_t lat = ceph_clock_now(g_ceph_context); + utime_t lat = ceph_clock_now(client->cct); lat -= start; dout(0) << " trace " << tfile << " loop " << (i+1) << "/" << iarg1 << " done in " << (double)lat << " seconds" << dendl; @@ -1012,7 +1012,7 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only) char buf[1024]; char buf2[1024]; - utime_t start = ceph_clock_now(g_ceph_context); + utime_t start = ceph_clock_now(client->cct); hash_map<int64_t, int64_t> open_files; hash_map<int64_t, dir_result_t*> open_dirs; @@ -1046,7 +1046,7 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only) Cond cond; bool ack; bool safe; - C_GatherBuilder safeg(g_ceph_context, new C_SafeCond(&lock, &cond, &safe)); + C_GatherBuilder safeg(client->cct, new C_SafeCond(&lock, &cond, &safe)); Context *safegref = safeg.new_sub(); // take a ref while (!t.end()) { @@ -1436,7 +1436,7 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only) bufferlist bl; bl.push_back(bp); SnapContext snapc; - client->objecter->write(oid, oloc, off, len, snapc, bl, ceph_clock_now(g_ceph_context), 0, + client->objecter->write(oid, oloc, off, len, snapc, bl, ceph_clock_now(client->cct), 0, new C_SafeCond(&lock, &cond, &ack), safeg.new_sub()); safeg.activate(); @@ -1452,7 +1452,7 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only) object_locator_t oloc(CEPH_DATA_RULE); lock.Lock(); SnapContext snapc; - client->objecter->zero(oid, oloc, off, len, snapc, ceph_clock_now(g_ceph_context), 0, + client->objecter->zero(oid, oloc, off, len, snapc, ceph_clock_now(client->cct), 0, new C_SafeCond(&lock, &cond, &ack), safeg.new_sub()); safeg.activate(); @@ -1762,9 +1762,9 @@ int SyntheticClient::read_dirs(const char *basedir, int dirs, int files, int dep dout(3) << "read_dirs " << basedir << " dirs " << dirs << " files " << files << " depth " << depth << dendl; list<string> contents; - utime_t s = ceph_clock_now(g_ceph_context); + utime_t s = ceph_clock_now(client->cct); int r = client->getdir(basedir, contents); - utime_t e = ceph_clock_now(g_ceph_context); + utime_t e = ceph_clock_now(client->cct); e -= s; if (r < 0) { dout(0) << "read_dirs couldn't readdir " << basedir << ", stopping" << dendl; @@ -1773,12 +1773,12 @@ int SyntheticClient::read_dirs(const char *basedir, int dirs, int files, int dep for (int i=0; i<files; i++) { snprintf(d, sizeof(d), "%s/file.%d", basedir, i); - utime_t s = ceph_clock_now(g_ceph_context); + utime_t s = ceph_clock_now(client->cct); if (client->lstat(d, &st) < 0) { dout(2) << "read_dirs failed stat on " << d << ", stopping" << dendl; return -1; } - utime_t e = ceph_clock_now(g_ceph_context); + utime_t e = ceph_clock_now(client->cct); e -= s; } @@ -1816,7 +1816,7 @@ int SyntheticClient::make_files(int num, int count, int priv, bool more) // files struct stat st; - utime_t start = ceph_clock_now(g_ceph_context); + utime_t start = ceph_clock_now(client->cct); for (int c=0; c<count; c++) { for (int n=0; n<num; n++) { snprintf(d, sizeof(d), "dir.%d.run%d/file.client%d.%d", priv ? whoami:0, c, whoami, n); @@ -1833,7 +1833,7 @@ int SyntheticClient::make_files(int num, int count, int priv, bool more) if (time_to_stop()) return 0; } } - utime_t end = ceph_clock_now(g_ceph_context); + utime_t end = ceph_clock_now(client->cct); end -= start; dout(0) << "makefiles time is " << end << " or " << ((double)end / (double)num) <<" per file" << dendl; @@ -1851,24 +1851,24 @@ int SyntheticClient::link_test() client->mkdir("orig", 0755); client->mkdir("copy", 0755); - utime_t start = ceph_clock_now(g_ceph_context); + utime_t start = ceph_clock_now(client->cct); for (int i=0; i<num; i++) { snprintf(d, sizeof(d), "orig/file.%d", i); client->mknod(d, 0755); } - utime_t end = ceph_clock_now(g_ceph_context); + utime_t end = ceph_clock_now(client->cct); end -= start; dout(0) << "orig " << end << dendl; // link - start = ceph_clock_now(g_ceph_context); + start = ceph_clock_now(client->cct); for (int i=0; i<num; i++) { snprintf(d, sizeof(d), "orig/file.%d", i); snprintf(e, sizeof(e), "copy/file.%d", i); client->link(d, e); } - end = ceph_clock_now(g_ceph_context); + end = ceph_clock_now(client->cct); end -= start; dout(0) << "copy " << end << dendl; @@ -1982,7 +1982,7 @@ int SyntheticClient::write_file(string& fn, int size, loff_t wrsize) // size i return fd; } - utime_t from = ceph_clock_now(g_ceph_context); + utime_t from = ceph_clock_now(client->cct); utime_t start = from; uint64_t bytes = 0, total = 0; @@ -2010,7 +2010,7 @@ int SyntheticClient::write_file(string& fn, int size, loff_t wrsize) // size i bytes += wrsize; total += wrsize; - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(client->cct); if (now - from >= 1.0) { double el = now - from; dout(0) << "write " << (bytes / el / 1048576.0) << " MB/sec" << dendl; @@ -2021,7 +2021,7 @@ int SyntheticClient::write_file(string& fn, int size, loff_t wrsize) // size i client->fsync(fd, true); - utime_t stop = ceph_clock_now(g_ceph_context); + utime_t stop = ceph_clock_now(client->cct); double el = stop - start; dout(0) << "write total " << (total / el / 1048576.0) << " MB/sec (" << total << " bytes in " << el << " seconds)" << dendl; @@ -2098,7 +2098,7 @@ int SyntheticClient::read_file(const std::string& fn, int size, return fd; } - utime_t from = ceph_clock_now(g_ceph_context); + utime_t from = ceph_clock_now(client->cct); utime_t start = from; uint64_t bytes = 0, total = 0; @@ -2114,7 +2114,7 @@ int SyntheticClient::read_file(const std::string& fn, int size, bytes += rdsize; total += rdsize; - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(client->cct); if (now - from >= 1.0) { double el = now - from; dout(0) << "read " << (bytes / el / 1048576.0) << " MB/sec" << dendl; @@ -2146,7 +2146,7 @@ int SyntheticClient::read_file(const std::string& fn, int size, dout(0) << " + " << (bad-1) << " other bad 16-byte bits in this block" << dendl; } - utime_t stop = ceph_clock_now(g_ceph_context); + utime_t stop = ceph_clock_now(client->cct); double el = stop - start; dout(0) << "read total " << (total / el / 1048576.0) << " MB/sec (" << total << " bytes in " << el << " seconds)" << dendl; @@ -2181,7 +2181,7 @@ public: int SyntheticClient::create_objects(int nobj, int osize, int inflight) { // divy up - int numc = g_conf->num_client ? g_conf->num_client : 1; + int numc = client->cct->_conf->num_client ? client->cct->_conf->num_client : 1; int start, inc, end; @@ -2226,9 +2226,9 @@ int SyntheticClient::create_objects(int nobj, int osize, int inflight) } dout(10) << "writing " << oid << dendl; - starts.push_back(ceph_clock_now(g_ceph_context)); + starts.push_back(ceph_clock_now(client->cct)); client->client_lock.Lock(); - client->objecter->write(oid, oloc, 0, osize, snapc, bl, ceph_clock_now(g_ceph_context), 0, + client->objecter->write(oid, oloc, 0, osize, snapc, bl, ceph_clock_now(client->cct), 0, new C_Ref(lock, cond, &unack), new C_Ref(lock, cond, &unsafe)); client->client_lock.Unlock(); @@ -2240,7 +2240,7 @@ int SyntheticClient::create_objects(int nobj, int osize, int inflight) } lock.Unlock(); - utime_t lat = ceph_clock_now(g_ceph_context); + utime_t lat = ceph_clock_now(client->cct); lat -= starts.front(); starts.pop_front(); } @@ -2323,7 +2323,7 @@ int SyntheticClient::object_rw(int nobj, int osize, int wrpc, SnapContext snapc; client->client_lock.Lock(); - utime_t start = ceph_clock_now(g_ceph_context); + utime_t start = ceph_clock_now(client->cct); if (write) { dout(10) << "write to " << oid << dendl; @@ -2339,7 +2339,7 @@ int SyntheticClient::object_rw(int nobj, int osize, int wrpc, op.op.op = CEPH_OSD_OP_STARTSYNC; m.ops.push_back(op); } - client->objecter->mutate(oid, oloc, m, snapc, ceph_clock_now(g_ceph_context), 0, + client->objecter->mutate(oid, oloc, m, snapc, ceph_clock_now(client->cct), 0, NULL, new C_Ref(lock, cond, &unack)); /*client->objecter->write(oid, layout, 0, osize, snapc, bl, 0, new C_Ref(lock, cond, &unack), @@ -2359,7 +2359,7 @@ int SyntheticClient::object_rw(int nobj, int osize, int wrpc, } lock.Unlock(); - utime_t lat = ceph_clock_now(g_ceph_context); + utime_t lat = ceph_clock_now(client->cct); lat -= start; if (client->logger) { if (write) @@ -3292,7 +3292,7 @@ void SyntheticClient::import_find(const char *base, const char *find, bool data) if (sp < 0) dirnum++; //dout(0) << "leading dir " << filename << " " << dirnum << dendl; - if (dirnum % g_conf->num_client != client->get_nodeid()) { + if (dirnum % client->cct->_conf->num_client != client->get_nodeid()) { dout(20) << "skipping leading dir " << dirnum << " " << filename << dendl; continue; } diff --git a/src/client/SyntheticClient.h b/src/client/SyntheticClient.h index 3bbcb73cb1a..f9f4e3de0fb 100644 --- a/src/client/SyntheticClient.h +++ b/src/client/SyntheticClient.h @@ -205,7 +205,7 @@ class SyntheticClient { } bool time_to_stop() { - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(client->cct); if (0) cout << "time_to_stop .. now " << now << " until " << run_until << " start " << run_start @@ -271,6 +271,7 @@ class SyntheticClient { void mksnap(const char *base, const char *name); void rmsnap(const char *base, const char *name); void mksnapfile(const char *dir); + }; #endif diff --git a/src/client/fuse_ll.cc b/src/client/fuse_ll.cc index 0c78557f041..6bf5ea3d34f 100644 --- a/src/client/fuse_ll.cc +++ b/src/client/fuse_ll.cc @@ -330,7 +330,7 @@ static void fuse_ll_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info * if (r == 0) { fi->fh = (long)fh; #if FUSE_VERSION >= FUSE_MAKE_VERSION(2, 8) - if (g_conf->fuse_use_invalidate_cb) + if (cfuse->client->cct->_conf->fuse_use_invalidate_cb) fi->keep_cache = 1; #endif fuse_reply_open(req, fi); @@ -673,24 +673,24 @@ int CephFuse::Handle::init(int argc, const char *argv[]) newargv[newargc++] = argv[0]; newargv[newargc++] = "-f"; // stay in foreground - if (g_conf->fuse_allow_other) { + if (client->cct->_conf->fuse_allow_other) { newargv[newargc++] = "-o"; newargv[newargc++] = "allow_other"; } - if (g_conf->fuse_default_permissions) { + if (client->cct->_conf->fuse_default_permissions) { newargv[newargc++] = "-o"; newargv[newargc++] = "default_permissions"; } - if (g_conf->fuse_big_writes) { + if (client->cct->_conf->fuse_big_writes) { newargv[newargc++] = "-o"; newargv[newargc++] = "big_writes"; } - if (g_conf->fuse_atomic_o_trunc) { + if (client->cct->_conf->fuse_atomic_o_trunc) { newargv[newargc++] = "-o"; newargv[newargc++] = "atomic_o_trunc"; } - if (g_conf->fuse_debug) + if (client->cct->_conf->fuse_debug) newargv[newargc++] = "-d"; for (int argctr = 1; argctr < argc; argctr++) @@ -744,7 +744,7 @@ int CephFuse::Handle::init(int argc, const char *argv[]) */ - if (g_conf->fuse_use_invalidate_cb) + if (client->cct->_conf->fuse_use_invalidate_cb) client->ll_register_ino_invalidate_cb(invalidate_cb, this); done: diff --git a/src/client/hadoop/CephFSInterface.cc b/src/client/hadoop/CephFSInterface.cc deleted file mode 100644 index d5a3c8f4fcd..00000000000 --- a/src/client/hadoop/CephFSInterface.cc +++ /dev/null @@ -1,993 +0,0 @@ -// -*- mode:c++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -#include "CephFSInterface.h" -#include "include/cephfs/libcephfs.h" -#include "common/ceph_argparse.h" -#include "common/config.h" -#include "msg/SimpleMessenger.h" - -#include <arpa/inet.h> -#include <sys/stat.h> -#include <sys/statvfs.h> - -#define dout_subsys ceph_subsys_hadoop - -union ceph_mount_union_t { - struct ceph_mount_info *cmount; - jlong cjlong; -}; - -static void set_ceph_mount_info(JNIEnv *env, jobject obj, struct ceph_mount_info *cmount) -{ - jclass cls = env->GetObjectClass(obj); - if (cls == NULL) - return; - jfieldID fid = env->GetFieldID(cls, "cluster", "J"); - if (fid == NULL) - return; - ceph_mount_union_t ceph_mount_union; - ceph_mount_union.cjlong = 0; - ceph_mount_union.cmount = cmount; - env->SetLongField(obj, fid, ceph_mount_union.cjlong); -} - -static struct ceph_mount_info *get_ceph_mount_t(JNIEnv *env, jobject obj) -{ - jclass cls = env->GetObjectClass(obj); - jfieldID fid = env->GetFieldID(cls, "cluster", "J"); - if (fid == NULL) - return NULL; - ceph_mount_union_t ceph_mount_union; - ceph_mount_union.cjlong = env->GetLongField(obj, fid); - return ceph_mount_union.cmount; -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_initializeClient - * Signature: (Ljava/lang/String;I)Z - * - * Performs any necessary setup to allow general use of the filesystem. - * Inputs: - * jstring args -- a command-line style input of Ceph config params - * jint block_size -- the size in bytes to use for blocks - * Returns: true on success, false otherwise - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1initializeClient - (JNIEnv *env, jobject obj, jstring j_args, jint block_size) -{ - // Convert Java argument string to argv - const char *c_args = env->GetStringUTFChars(j_args, 0); - if (c_args == NULL) - return false; //out of memory! - string cppargs(c_args); - char b[cppargs.length()+1]; - strcpy(b, cppargs.c_str()); - env->ReleaseStringUTFChars(j_args, c_args); - std::vector<const char*> args; - char *p = b; - while (*p) { - args.push_back(p); - while (*p && *p != ' ') - p++; - if (!*p) - break; - *p++ = 0; - while (*p && *p == ' ') - p++; - } - - // parse the arguments - bool set_local_writes = false; - std::string mount_root, val; - for (std::vector<const char*>::iterator i = args.begin(); i != args.end(); ) { - if (ceph_argparse_witharg(args, i, &val, "mount_root", (char*)NULL)) { - mount_root = val; - } else if (ceph_argparse_flag(args, i, "set_local_pg", (char*)NULL)) { - set_local_writes = true; - } else { - ++i; - } - } - - // connect to the cmount - struct ceph_mount_info *cmount; - int ret = ceph_create(&cmount, NULL); - if (ret) - return false; - ceph_conf_read_file(cmount, NULL); // read config file from the default location - ceph_conf_parse_argv(cmount, args.size(), &args[0]); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 3) << "CephFSInterface: mounting filesystem...:" << dendl; - - ret = ceph_mount(cmount, mount_root.c_str()); - if (ret) - return false; - - ceph_localize_reads(cmount, true); - ceph_set_default_file_stripe_unit(cmount, block_size); - ceph_set_default_object_size(cmount, block_size); - - if (set_local_writes) { - ceph_set_default_preferred_pg(cmount, ceph_get_local_osd(cmount)); - } - - set_ceph_mount_info(env, obj, cmount); - return true; -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_getcwd - * Signature: (J)Ljava/lang/String; - * - * Returns the current working directory.(absolute) as a jstring - */ -JNIEXPORT jstring JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1getcwd - (JNIEnv *env, jobject obj) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 10) << "CephFSInterface: In getcwd" << dendl; - jstring j_path = env->NewStringUTF(ceph_getcwd(cmount)); - return j_path; -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_setcwd - * Signature: (Ljava/lang/String;)Z - * - * Changes the working directory. - * Inputs: - * jstring j_path: The path (relative or absolute) to switch to - * Returns: true on success, false otherwise. - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1setcwd -(JNIEnv *env, jobject obj, jstring j_path) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 10) << "CephFSInterface: In setcwd" << dendl; - - const char *c_path = env->GetStringUTFChars(j_path, 0); - if (c_path == NULL) - return false; - int ret = ceph_chdir(cmount, c_path); - env->ReleaseStringUTFChars(j_path, c_path); - return ret ? JNI_FALSE : JNI_TRUE; -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_rmdir - * Signature: (Ljava/lang/String;)Z - * - * Given a path to a directory, removes the directory.if empty. - * Inputs: - * jstring j_path: The path (relative or absolute) to the directory - * Returns: true on successful delete; false otherwise - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1rmdir - (JNIEnv *env, jobject obj, jstring j_path) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 10) << "CephFSInterface: In rmdir" << dendl; - - const char *c_path = env->GetStringUTFChars(j_path, 0); - if(c_path == NULL) - return false; - int ret = ceph_rmdir(cmount, c_path); - env->ReleaseStringUTFChars(j_path, c_path); - return ret ? JNI_FALSE : JNI_TRUE; -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_unlink - * Signature: (Ljava/lang/String;)Z - * Given a path, unlinks it. - * Inputs: - * jstring j_path: The path (relative or absolute) to the file or empty dir - * Returns: true if the unlink occurred, false otherwise. - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1unlink - (JNIEnv *env, jobject obj, jstring j_path) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - const char *c_path = env->GetStringUTFChars(j_path, 0); - if (c_path == NULL) - return false; - ldout(cct, 10) << "CephFSInterface: In unlink for path " << c_path << ":" << dendl; - int ret = ceph_unlink(cmount, c_path); - env->ReleaseStringUTFChars(j_path, c_path); - return ret ? JNI_FALSE : JNI_TRUE; -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_rename - * Signature: (Ljava/lang/String;Ljava/lang/String;)Z - * Changes a given path name to a new name. - * Inputs: - * jstring j_from: The path whose name you want to change. - * jstring j_to: The new name for the path. - * Returns: true if the rename occurred, false otherwise - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1rename - (JNIEnv *env, jobject obj, jstring j_from, jstring j_to) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 10) << "CephFSInterface: In rename" << dendl; - const char *c_from = env->GetStringUTFChars(j_from, 0); - if (c_from == NULL) - return false; - const char *c_to = env->GetStringUTFChars(j_to, 0); - if (c_to == NULL) { - env->ReleaseStringUTFChars(j_from, c_from); - return false; - } - struct stat stbuf; - int ret = ceph_lstat(cmount, c_to, &stbuf); - if (ret != -ENOENT) { - // Hadoop doesn't want to overwrite files in a rename. - env->ReleaseStringUTFChars(j_from, c_from); - env->ReleaseStringUTFChars(j_to, c_to); - return JNI_FALSE; - } - - ret = ceph_rename(cmount, c_from, c_to); - return ret ? JNI_FALSE : JNI_TRUE; -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_exists - * Signature: (Ljava/lang/String;)Z - * Returns true if it the input path exists, false - * if it does not or there is an unexpected failure. - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1exists -(JNIEnv *env, jobject obj, jstring j_path) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 10) << "CephFSInterface: In exists" << dendl; - - struct stat stbuf; - - const char *c_path = env->GetStringUTFChars(j_path, 0); - if (c_path == NULL) - return false; - ldout(cct, 10) << "Attempting lstat with file " << c_path << ":" << dendl; - - int ret = ceph_lstat(cmount, c_path, &stbuf); - ldout(cct, 10) << "result is " << ret << dendl; - env->ReleaseStringUTFChars(j_path, c_path); - if (ret < 0) { - ldout(cct, 10) << "Returning false (file does not exist)" << dendl; - return JNI_FALSE; - } - else { - ldout(cct, 10) << "Returning true (file exists)" << dendl; - return JNI_TRUE; - } -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_getblocksize - * Signature: (Ljava/lang/String;)J - * Get the block size for a given path. - * Input: - * j_string j_path: The path (relative or absolute) you want - * the block size for. - * Returns: block size (as a long) if the path exists, otherwise a negative - * number corresponding to the standard C++ error codes (which are positive). - */ -JNIEXPORT jlong JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1getblocksize - (JNIEnv *env, jobject obj, jstring j_path) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 10) << "In getblocksize" << dendl; - - //struct stat stbuf; - - jlong result; - - const char *c_path = env->GetStringUTFChars(j_path, 0); - if (c_path == NULL) - return -ENOMEM; - // we need to open the file to retrieve the stripe size - ldout(cct, 10) << "CephFSInterface: getblocksize: opening file" << dendl; - int fh = ceph_open(cmount, c_path, O_RDONLY, 0); - env->ReleaseStringUTFChars(j_path, c_path); - if (fh < 0) - return fh; - - result = ceph_get_file_stripe_unit(cmount, fh); - - int close_result = ceph_close(cmount, fh); - if (close_result < 0) - return close_result; - - return result; -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_isfile - * Signature: (Ljava/lang/String;)Z - * Returns true if the given path is a file; false otherwise. - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1isfile - (JNIEnv *env, jobject obj, jstring j_path) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 10) << "In isfile" << dendl; - - struct stat stbuf; - - const char *c_path = env->GetStringUTFChars(j_path, 0); - if (c_path == NULL) - return false; - int ret = ceph_lstat(cmount, c_path, &stbuf); - env->ReleaseStringUTFChars(j_path, c_path); - - // if the stat call failed, it's definitely not a file... - if (ret < 0) - return false; - - // check the stat result - return !!S_ISREG(stbuf.st_mode); -} - - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_isdirectory - * Signature: (Ljava/lang/String;)Z - * Returns true if the given path is a directory, false otherwise. - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1isdirectory - (JNIEnv *env, jobject obj, jstring j_path) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 10) << "In isdirectory" << dendl; - - struct stat stbuf; - - const char *c_path = env->GetStringUTFChars(j_path, 0); - if (c_path == NULL) - return false; - int result = ceph_lstat(cmount, c_path, &stbuf); - env->ReleaseStringUTFChars(j_path, c_path); - - // if the stat call failed, it's definitely not a directory... - if (result < 0) - return JNI_FALSE; - - // check the stat result - return !!S_ISDIR(stbuf.st_mode); -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_getdir - * Signature: (Ljava/lang/String;)[Ljava/lang/String; - * Get the contents of a given directory. - * Inputs: - * jstring j_path: The path (relative or absolute) to the directory. - * Returns: A Java String[] of the contents of the directory, or - * NULL if there is an error (ie, path is not a dir). This listing - * will not contain . or .. entries. - */ -JNIEXPORT jobjectArray JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1getdir -(JNIEnv *env, jobject obj, jstring j_path) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 10) << "In getdir" << dendl; - - // get the directory listing - list<string> contents; - const char *c_path = env->GetStringUTFChars(j_path, 0); - if (c_path == NULL) return NULL; - struct ceph_dir_result *dirp; - int r; - r = ceph_opendir(cmount, c_path, &dirp); - if (r<0) { - env->ReleaseStringUTFChars(j_path, c_path); - return NULL; - } - int buflen = 100; //good default? - char *buf = new char[buflen]; - string *ent; - int bufpos; - while (1) { - r = ceph_getdnames(cmount, dirp, buf, buflen); - if (r==-ERANGE) { //expand the buffer - delete [] buf; - buflen *= 2; - buf = new char[buflen]; - continue; - } - if (r<=0) break; - - //if we make it here, we got at least one name - bufpos = 0; - while (bufpos<r) {//make new strings and add them to listing - ent = new string(buf+bufpos); - if (ent->compare(".") && ent->compare("..")) - //we DON'T want to include dot listings; Hadoop gets confused - contents.push_back(*ent); - bufpos+=ent->size()+1; - delete ent; - } - } - delete [] buf; - ceph_closedir(cmount, dirp); - env->ReleaseStringUTFChars(j_path, c_path); - - if (r < 0) return NULL; - - // Create a Java String array of the size of the directory listing - jclass stringClass = env->FindClass("java/lang/String"); - if (stringClass == NULL) { - ldout(cct, 0) << "ERROR: java String class not found; dying a horrible, painful death" << dendl; - assert(0); - } - jobjectArray dirListingStringArray = (jobjectArray) env->NewObjectArray(contents.size(), stringClass, NULL); - if(dirListingStringArray == NULL) return NULL; - - // populate the array with the elements of the directory list - int i = 0; - for (list<string>::iterator it = contents.begin(); - it != contents.end(); - ++it) { - env->SetObjectArrayElement(dirListingStringArray, i, - env->NewStringUTF(it->c_str())); - ++i; - } - - return dirListingStringArray; -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_mkdirs - * Signature: (Ljava/lang/String;I)I - * Create the specified directory and any required intermediate ones with the - * given mode. - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1mkdirs -(JNIEnv *env, jobject obj, jstring j_path, jint mode) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 10) << "In Hadoop mk_dirs" << dendl; - - //get c-style string and make the call, clean up the string... - jint result; - const char *c_path = env->GetStringUTFChars(j_path, 0); - if (c_path == NULL) - return -ENOMEM; - result = ceph_mkdirs(cmount, c_path, mode); - env->ReleaseStringUTFChars(j_path, c_path); - - //...and return - return result; -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_open_for_append - * Signature: (Ljava/lang/String;)I - * Open a file to append. If the file does not exist, it will be created. - * Opening a dir is possible but may have bad results. - * Inputs: - * jstring j_path: The path to open. - * Returns: a jint filehandle, or a number<0 if an error occurs. - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1open_1for_1append -(JNIEnv *env, jobject obj, jstring j_path) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 10) << "In hadoop open_for_append" << dendl; - - jint result; - - const char *c_path = env->GetStringUTFChars(j_path, 0); - if (c_path == NULL) - return -ENOMEM; - result = ceph_open(cmount, c_path, O_WRONLY|O_CREAT|O_APPEND, 0); - env->ReleaseStringUTFChars(j_path, c_path); - - return result; -} - - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_open_for_read - * Signature: (Ljava/lang/String;)I - * Open a file for reading. - * Opening a dir is possible but may have bad results. - * Inputs: - * jstring j_path: The path to open. - * Returns: a jint filehandle, or a number<0 if an error occurs. - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1open_1for_1read - (JNIEnv *env, jobject obj, jstring j_path) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 10) << "In open_for_read" << dendl; - - jint result; - - // open as read-only: flag = O_RDONLY - const char *c_path = env->GetStringUTFChars(j_path, 0); - if (c_path == NULL) - return -ENOMEM; - result = ceph_open(cmount, c_path, O_RDONLY, 0); - env->ReleaseStringUTFChars(j_path, c_path); - - // returns file handle, or -1 on failure - return result; -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_open_for_overwrite - * Signature: (Ljava/lang/String;)I - * Opens a file for overwriting; creates it if necessary. - * Opening a dir is possible but may have bad results. - * Inputs: - * jstring j_path: The path to open. - * jint mode: The mode to open with. - * Returns: a jint filehandle, or a number<0 if an error occurs. - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1open_1for_1overwrite - (JNIEnv *env, jobject obj, jstring j_path, jint mode) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 10) << "In open_for_overwrite" << dendl; - - jint result; - const char *c_path = env->GetStringUTFChars(j_path, 0); - if (c_path == NULL) - return -ENOMEM; - result = ceph_open(cmount, c_path, O_WRONLY|O_CREAT|O_TRUNC, mode); - env->ReleaseStringUTFChars(j_path, c_path); - - // returns file handle, or -1 on failure - return result; -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_close - * Signature: (I)I - * Closes a given filehandle. - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1close -(JNIEnv *env, jobject obj, jint fh) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 10) << "In CephTalker::ceph_close" << dendl; - return ceph_close(cmount, fh); -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_setPermission - * Signature: (Ljava/lang/String;I)Z - * Change the mode on a path. - * Inputs: - * jstring j_path: The path to change mode on. - * jint j_new_mode: The mode to apply. - * Returns: true if the mode is properly applied, false if there - * is any error. - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1setPermission -(JNIEnv *env, jobject obj, jstring j_path, jint j_new_mode) -{ - const char *c_path = env->GetStringUTFChars(j_path, 0); - if (c_path == NULL) - return false; - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - int result = ceph_chmod(cmount, c_path, j_new_mode); - env->ReleaseStringUTFChars(j_path, c_path); - - return (result==0); -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_kill_client - * Signature: (J)Z - * - * Closes the Ceph client. This should be called before shutting down - * (multiple times is okay but redundant). - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1kill_1client - (JNIEnv *env, jobject obj) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - if (!cmount) - return true; - ceph_shutdown(cmount); - set_ceph_mount_info(env, obj, NULL); - return true; -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_stat - * Signature: (Ljava/lang/String;Lorg/apache/hadoop/fs/ceph/CephFileSystem/Stat;)Z - * Get the statistics on a path returned in a custom format defined - * in CephTalker. - * Inputs: - * jstring j_path: The path to stat. - * jobject j_stat: The stat object to fill. - * Returns: true if the stat is successful, false otherwise. - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1stat -(JNIEnv *env, jobject obj, jstring j_path, jobject j_stat) -{ - //setup variables - struct stat st; - const char *c_path = env->GetStringUTFChars(j_path, 0); - if (c_path == NULL) return false; - - jclass cls = env->GetObjectClass(j_stat); - if (cls == NULL) return false; - jfieldID c_size_id = env->GetFieldID(cls, "size", "J"); - if (c_size_id == NULL) return false; - jfieldID c_dir_id = env->GetFieldID(cls, "is_dir", "Z"); - if (c_dir_id == NULL) return false; - jfieldID c_block_id = env->GetFieldID(cls, "block_size", "J"); - if (c_block_id == NULL) return false; - jfieldID c_mod_id = env->GetFieldID(cls, "mod_time", "J"); - if (c_mod_id == NULL) return false; - jfieldID c_access_id = env->GetFieldID(cls, "access_time", "J"); - if (c_access_id == NULL) return false; - jfieldID c_mode_id = env->GetFieldID(cls, "mode", "I"); - if (c_mode_id == NULL) return false; - //do actual lstat - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - int r = ceph_lstat(cmount, c_path, &st); - env->ReleaseStringUTFChars(j_path, c_path); - - if (r < 0) return false; //fail out; file DNE or Ceph broke - - //put variables from struct stat into Java - env->SetLongField(j_stat, c_size_id, st.st_size); - env->SetBooleanField(j_stat, c_dir_id, (0 != S_ISDIR(st.st_mode))); - env->SetLongField(j_stat, c_block_id, st.st_blksize); - - long long java_mtime(st.st_mtim.tv_sec); - java_mtime *= 1000; - java_mtime += st.st_mtim.tv_nsec / 1000; - env->SetLongField(j_stat, c_mod_id, java_mtime); - - long long java_atime(st.st_atim.tv_sec); - java_atime *= 1000; - java_atime += st.st_atim.tv_nsec / 1000; - env->SetLongField(j_stat, c_access_id, java_atime); - - env->SetIntField(j_stat, c_mode_id, (int)st.st_mode); - - //return happy - return true; -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_statfs - * Signature: (Ljava/lang/String;Lorg/apache/hadoop/fs/ceph/CephFileSystem/CephStat;)I - * Statfs a filesystem in a custom format defined in CephTalker. - * Inputs: - * jstring j_path: A path on the filesystem that you wish to stat. - * jobject j_ceph_stat: The CephStat object to fill. - * Returns: true if successful and the CephStat is filled; false otherwise. - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1statfs -(JNIEnv *env, jobject obj, jstring j_path, jobject j_cephstat) -{ - //setup variables - struct statvfs stbuf; - const char *c_path = env->GetStringUTFChars(j_path, 0); - if (c_path == NULL) - return -ENOMEM; - jclass cls = env->GetObjectClass(j_cephstat); - if (cls == NULL) - return 1; //JVM error of some kind - jfieldID c_capacity_id = env->GetFieldID(cls, "capacity", "J"); - jfieldID c_used_id = env->GetFieldID(cls, "used", "J"); - jfieldID c_remaining_id = env->GetFieldID(cls, "remaining", "J"); - - //do the statfs - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - int r = ceph_statfs(cmount, c_path, &stbuf); - env->ReleaseStringUTFChars(j_path, c_path); - if (r != 0) - return r; //something broke - - //place info into Java; convert from bytes to kilobytes - env->SetLongField(j_cephstat, c_capacity_id, - (long)stbuf.f_blocks*stbuf.f_bsize/1024); - env->SetLongField(j_cephstat, c_used_id, - (long)(stbuf.f_blocks-stbuf.f_bavail)*stbuf.f_bsize/1024); - env->SetLongField(j_cephstat, c_remaining_id, - (long)stbuf.f_bavail*stbuf.f_bsize/1024); - return r; -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_replication - * Signature: (Ljava/lang/String;)I - * Check how many times a path should be replicated (if it is - * degraded it may not actually be replicated this often). - * Inputs: - * jstring j_path: The path to check. - * Returns: an int containing the number of times replicated. - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1replication -(JNIEnv *env, jobject obj, jstring j_path) -{ - //get c-string of path, send off to libceph, release c-string, return - const char *c_path = env->GetStringUTFChars(j_path, 0); - if (c_path == NULL) - return -ENOMEM; - ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - int fh = 0; - fh = ceph_open(cmount, c_path, O_RDONLY, 0); - env->ReleaseStringUTFChars(j_path, c_path); - if (fh < 0) { - return fh; - } - int replication = ceph_get_file_replication(cmount, fh); - ceph_close(cmount, fh); - return replication; -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_hosts - * Signature: (IJ)[Ljava/lang/String; - * Find the IP:port addresses of the primary OSD for a given file and offset. - * Inputs: - * jint j_fh: The filehandle for the file. - * jlong j_offset: The offset to get the location of. - * Returns: a jstring of the location as IP, or NULL if there is an error. - */ -JNIEXPORT jobjectArray JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1hosts -(JNIEnv *env, jobject obj, jint j_fh, jlong j_offset) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - struct sockaddr_storage *ss; - char address[30]; - jobjectArray addr_array; - jclass string_cls; - jstring j_addr; - int r, n = 3; /* initial guess at # of replicas */ - - for (;;) { - ss = new struct sockaddr_storage[n]; - r = ceph_get_file_stripe_address(cmount, j_fh, j_offset, ss, n); - if (r < 0) { - if (r == -ERANGE) { - delete [] ss; - n *= 2; - continue; - } - return NULL; - } - n = r; - break; - } - - /* TODO: cache this */ - string_cls = env->FindClass("java/lang/String"); - if (!string_cls) - goto out; - - addr_array = env->NewObjectArray(n, string_cls, NULL); - if (!addr_array) - goto out; - - for (r = 0; r < n; r++) { - /* Hadoop only deals with IPv4 */ - if (ss[r].ss_family != AF_INET) - goto out; - - memset(address, 0, sizeof(address)); - - inet_ntop(ss[r].ss_family, &((struct sockaddr_in *)&ss[r])->sin_addr, - address, sizeof(address)); - - j_addr = env->NewStringUTF(address); - - env->SetObjectArrayElement(addr_array, r, j_addr); - if (env->ExceptionOccurred()) - goto out; - - env->DeleteLocalRef(j_addr); - } - - delete [] ss; - return addr_array; - -out: - delete [] ss; - return NULL; -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_setTimes - * Signature: (Ljava/lang/String;JJ)I - * Set the mtime and atime for a given path. - * Inputs: - * jstring j_path: The path to set the times for. - * jlong mtime: The mtime to set, in millis since epoch (-1 to not set). - * jlong atime: The atime to set, in millis since epoch (-1 to not set) - * Returns: 0 if successful, an error code otherwise. - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1setTimes -(JNIEnv *env, jobject obj, jstring j_path, jlong mtime, jlong atime) -{ - const char *c_path = env->GetStringUTFChars(j_path, 0); - if(c_path == NULL) return -ENOMEM; - - //build the mask for ceph_setattr - int mask = 0; - if (mtime!=-1) mask = CEPH_SETATTR_MTIME; - if (atime!=-1) mask |= CEPH_SETATTR_ATIME; - //build a struct stat and fill it in! - //remember to convert from millis to seconds and microseconds - struct stat attr; - attr.st_mtim.tv_sec = mtime / 1000; - attr.st_mtim.tv_nsec = (mtime % 1000) * 1000000; - attr.st_atim.tv_sec = atime / 1000; - attr.st_atim.tv_nsec = (atime % 1000) * 1000000; - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - return ceph_setattr(cmount, c_path, &attr, mask); -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_read - * Signature: (JI[BII)I - * Reads into the given byte array from the current position. - * Inputs: - * jint fh: the filehandle to read from - * jbyteArray j_buffer: the byte array to read into - * jint buffer_offset: where in the buffer to start writing - * jint length: how much to read. - * There'd better be enough space in the buffer to write all - * the data from the given offset! - * Returns: the number of bytes read on success (as jint), - * or an error code otherwise. - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1read - (JNIEnv *env, jobject obj, jint fh, jbyteArray j_buffer, jint buffer_offset, jint length) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 10) << "In read" << dendl; - - - // Make sure to convert the Hadoop read arguments into a - // more ceph-friendly form - jint result; - - // Step 1: get a pointer to the buffer. - jbyte *j_buffer_ptr = env->GetByteArrayElements(j_buffer, NULL); - if (j_buffer_ptr == NULL) return -ENOMEM; - char *c_buffer = (char*) j_buffer_ptr; - - // Step 2: pointer arithmetic to start in the right buffer position - c_buffer += (int)buffer_offset; - - // Step 3: do the read - result = ceph_read(cmount, (int)fh, c_buffer, length, -1); - - // Step 4: release the pointer to the buffer - env->ReleaseByteArrayElements(j_buffer, j_buffer_ptr, 0); - - return result; -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_seek_from_start - * Signature: (JIJ)J - * Seeks to the given position in the given file. - * Inputs: - * jint fh: The filehandle to seek in. - * jlong pos: The position to seek to. - * Returns: the new position (as a jlong) of the filehandle on success, - * or a negative error code on failure. - */ -JNIEXPORT jlong JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1seek_1from_1start - (JNIEnv *env, jobject obj, jint fh, jlong pos) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 10) << "In CephTalker::seek_from_start" << dendl; - return ceph_lseek(cmount, fh, pos, SEEK_SET); -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_getpos - * Signature: (I)J - * - * Get the current position in a file (as a jlong) of a given filehandle. - * Returns: jlong current file position on success, or a - * negative error code on failure. - */ -JNIEXPORT jlong JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1getpos - (JNIEnv *env, jobject obj, jint fh) -{ - // seek a distance of 0 to get current offset - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 10) << "In CephTalker::ceph_getpos" << dendl; - return ceph_lseek(cmount, fh, 0, SEEK_CUR); -} - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_write - * Signature: (I[BII)I - * Write the given buffer contents to the given filehandle. - * Inputs: - * jint fh: The filehandle to write to. - * jbyteArray j_buffer: The buffer to write from - * jint buffer_offset: The position in the buffer to write from - * jint length: The number of (sequential) bytes to write. - * Returns: jint, on success the number of bytes written, on failure - * a negative error code. - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1write - (JNIEnv *env, jobject obj, jint fh, jbyteArray j_buffer, jint buffer_offset, jint length) -{ - struct ceph_mount_info *cmount = get_ceph_mount_t(env, obj); - CephContext *cct = ceph_get_mount_context(cmount); - ldout(cct, 10) << "In write" << dendl; - - // IMPORTANT NOTE: Hadoop write arguments are a bit different from POSIX so we - // have to convert. The write is *always* from the current position in the file, - // and buffer_offset is the location in the *buffer* where we start writing. - jint result; - - // Step 1: get a pointer to the buffer. - jbyte *j_buffer_ptr = env->GetByteArrayElements(j_buffer, NULL); - if (j_buffer_ptr == NULL) - return -ENOMEM; - char *c_buffer = (char*) j_buffer_ptr; - - // Step 2: pointer arithmetic to start in the right buffer position - c_buffer += (int)buffer_offset; - - // Step 3: do the write - result = ceph_write(cmount, (int)fh, c_buffer, length, -1); - - // Step 4: release the pointer to the buffer - env->ReleaseByteArrayElements(j_buffer, j_buffer_ptr, 0); - - return result; -} diff --git a/src/client/hadoop/CephFSInterface.h b/src/client/hadoop/CephFSInterface.h deleted file mode 100644 index 6939b3a501d..00000000000 --- a/src/client/hadoop/CephFSInterface.h +++ /dev/null @@ -1,236 +0,0 @@ -// -*- mode:c++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -/* BE CAREFUL EDITING THIS FILE - it is a compilation of JNI - machine-generated headers */ - -#include <jni.h> - -#ifndef _Included_org_apache_hadoop_fs_ceph_CephTalker -#define _Included_org_apache_hadoop_fs_ceph_CephTalker -#ifdef __cplusplus -extern "C" { -#endif - //these constants are machine-generated to match Java constants in the source -#undef org_apache_hadoop_fs_ceph_CephFileSystem_DEFAULT_BLOCK_SIZE -#define org_apache_hadoop_fs_ceph_CephFileSystem_DEFAULT_BLOCK_SIZE 8388608LL -#undef org_apache_hadoop_fs_ceph_CephInputStream_SKIP_BUFFER_SIZE -#define org_apache_hadoop_fs_ceph_CephInputStream_SKIP_BUFFER_SIZE 2048L -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_initializeClient - * Signature: (Ljava/lang/String;I)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1initializeClient - (JNIEnv *, jobject, jstring, jint); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_getcwd - * Signature: ()Ljava/lang/String; - */ -JNIEXPORT jstring JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1getcwd - (JNIEnv *, jobject); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_setcwd - * Signature: (Ljava/lang/String;)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1setcwd - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_rmdir - * Signature: (Ljava/lang/String;)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1rmdir - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_unlink - * Signature: (Ljava/lang/String;)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1unlink - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_rename - * Signature: (Ljava/lang/String;Ljava/lang/String;)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1rename - (JNIEnv *, jobject, jstring, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_exists - * Signature: (Ljava/lang/String;)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1exists - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_getblocksize - * Signature: (Ljava/lang/String;)J - */ -JNIEXPORT jlong JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1getblocksize - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_isdirectory - * Signature: (Ljava/lang/String;)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1isdirectory - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_isfile - * Signature: (Ljava/lang/String;)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1isfile - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_getdir - * Signature: (Ljava/lang/String;)[Ljava/lang/String; - */ -JNIEXPORT jobjectArray JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1getdir - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_mkdirs - * Signature: (Ljava/lang/String;I)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1mkdirs - (JNIEnv *, jobject, jstring, jint); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_open_for_append - * Signature: (Ljava/lang/String;)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1open_1for_1append - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_open_for_read - * Signature: (Ljava/lang/String;)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1open_1for_1read - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_open_for_overwrite - * Signature: (Ljava/lang/String;I)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1open_1for_1overwrite - (JNIEnv *, jobject, jstring, jint); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_close - * Signature: (I)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1close - (JNIEnv *, jobject, jint); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_setPermission - * Signature: (Ljava/lang/String;I)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1setPermission - (JNIEnv *, jobject, jstring, jint); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_kill_client - * Signature: ()Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1kill_1client - (JNIEnv *, jobject); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_stat - * Signature: (Ljava/lang/String;Lorg/apache/hadoop/fs/ceph/CephFileSystem/Stat;)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1stat - (JNIEnv *, jobject, jstring, jobject); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_statfs - * Signature: (Ljava/lang/String;Lorg/apache/hadoop/fs/ceph/CephFileSystem/CephStat;)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1statfs -(JNIEnv * env, jobject obj, jstring j_path, jobject j_cephstat); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_replication - * Signature: (Ljava/lang/String;)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1replication - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_hosts - * Signature: (IJ)[Ljava/lang/String; - */ -JNIEXPORT jobjectArray JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1hosts - (JNIEnv *, jobject, jint, jlong); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_setTimes - * Signature: (Ljava/lang/String;JJ)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1setTimes - (JNIEnv *, jobject, jstring, jlong, jlong); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_read - * Signature: (I[BII)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1read - (JNIEnv *, jobject, jint, jbyteArray, jint, jint); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_seek_from_start - * Signature: (IJ)J - */ -JNIEXPORT jlong JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1seek_1from_1start - (JNIEnv *, jobject, jint, jlong); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_getpos - * Signature: (I)J - */ -JNIEXPORT jlong JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1getpos - (JNIEnv *, jobject, jint); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_write - * Signature: (I[BII)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1write - (JNIEnv *, jobject, jint, jbyteArray, jint, jint); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/src/client/hadoop/HADOOP-ceph.patch b/src/client/hadoop/HADOOP-ceph.patch deleted file mode 100644 index 84cdb370f77..00000000000 --- a/src/client/hadoop/HADOOP-ceph.patch +++ /dev/null @@ -1,2234 +0,0 @@ -diff --git a/src/core/core-default.xml b/src/core/core-default.xml -index 8bc3b99..26543bc 100644 ---- a/src/core/core-default.xml -+++ b/src/core/core-default.xml -@@ -210,6 +210,12 @@ - </property> - - <property> -+ <name>fs.ceph.impl</name> -+ <value>org.apache.hadoop.fs.ceph.CephFileSystem</value> -+ <description>The file system for ceph: uris.</description> -+</property> -+ -+<property> - <name>fs.har.impl.disable.cache</name> - <value>true</value> - <description>Don't cache 'har' filesystem instances.</description> -diff --git a/src/core/org/apache/hadoop/fs/ceph/CephFS.java b/src/core/org/apache/hadoop/fs/ceph/CephFS.java -new file mode 100644 -index 0000000..5d51eb2 ---- /dev/null -+++ b/src/core/org/apache/hadoop/fs/ceph/CephFS.java -@@ -0,0 +1,250 @@ -+// -*- mode:Java; tab-width:2; c-basic-offset:2; indent-tabs-mode:t -*- -+ -+/** -+ * -+ * Licensed under the Apache License, Version 2.0 -+ * (the "License"); you may not use this file except in compliance with -+ * the License. You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -+ * implied. See the License for the specific language governing -+ * permissions and limitations under the License. -+ * -+ * -+ * Abstract base class for communicating with a Ceph filesystem and its -+ * C++ codebase from Java, or pretending to do so (for unit testing purposes). -+ * As only the Ceph package should be using this directly, all methods -+ * are protected. -+ */ -+package org.apache.hadoop.fs.ceph; -+ -+import org.apache.hadoop.conf.Configuration; -+ -+abstract class CephFS { -+ -+ protected static final int ENOTDIR = 20; -+ protected static final int EEXIST = 17; -+ protected static final int ENOENT = 2; -+ -+ /* -+ * Performs any necessary setup to allow general use of the filesystem. -+ * Inputs: -+ * String argsuments -- a command-line style input of Ceph config params -+ * int block_size -- the size in bytes to use for blocks -+ * Returns: true on success, false otherwise -+ */ -+ abstract protected boolean ceph_initializeClient(String arguments, int block_size); -+ -+ /* -+ * Returns the current working directory (absolute) as a String -+ */ -+ abstract protected String ceph_getcwd(); -+ -+ /* -+ * Changes the working directory. -+ * Inputs: -+ * String path: The path (relative or absolute) to switch to -+ * Returns: true on success, false otherwise. -+ */ -+ abstract protected boolean ceph_setcwd(String path); -+ -+ /* -+ * Given a path to a directory, removes the directory if empty. -+ * Inputs: -+ * jstring j_path: The path (relative or absolute) to the directory -+ * Returns: true on successful delete; false otherwise -+ */ -+ abstract protected boolean ceph_rmdir(String path); -+ -+ /* -+ * Given a path, unlinks it. -+ * Inputs: -+ * String path: The path (relative or absolute) to the file or empty dir -+ * Returns: true if the unlink occurred, false otherwise. -+ */ -+ abstract protected boolean ceph_unlink(String path); -+ -+ /* -+ * Changes a given path name to a new name, assuming new_path doesn't exist. -+ * Inputs: -+ * jstring j_from: The path whose name you want to change. -+ * jstring j_to: The new name for the path. -+ * Returns: true if the rename occurred, false otherwise -+ */ -+ abstract protected boolean ceph_rename(String old_path, String new_path); -+ -+ /* -+ * Returns true if it the input path exists, false -+ * if it does not or there is an unexpected failure. -+ */ -+ abstract protected boolean ceph_exists(String path); -+ -+ /* -+ * Get the block size for a given path. -+ * Input: -+ * String path: The path (relative or absolute) you want -+ * the block size for. -+ * Returns: block size if the path exists, otherwise a negative number -+ * corresponding to the standard C++ error codes (which are positive). -+ */ -+ abstract protected long ceph_getblocksize(String path); -+ -+ /* -+ * Returns true if the given path is a directory, false otherwise. -+ */ -+ abstract protected boolean ceph_isdirectory(String path); -+ -+ /* -+ * Returns true if the given path is a file; false otherwise. -+ */ -+ abstract protected boolean ceph_isfile(String path); -+ -+ /* -+ * Get the contents of a given directory. -+ * Inputs: -+ * String path: The path (relative or absolute) to the directory. -+ * Returns: A Java String[] of the contents of the directory, or -+ * NULL if there is an error (ie, path is not a dir). This listing -+ * will not contain . or .. entries. -+ */ -+ abstract protected String[] ceph_getdir(String path); -+ -+ /* -+ * Create the specified directory and any required intermediate ones with the -+ * given mode. -+ */ -+ abstract protected int ceph_mkdirs(String path, int mode); -+ -+ /* -+ * Open a file to append. If the file does not exist, it will be created. -+ * Opening a dir is possible but may have bad results. -+ * Inputs: -+ * String path: The path to open. -+ * Returns: an int filehandle, or a number<0 if an error occurs. -+ */ -+ abstract protected int ceph_open_for_append(String path); -+ -+ /* -+ * Open a file for reading. -+ * Opening a dir is possible but may have bad results. -+ * Inputs: -+ * String path: The path to open. -+ * Returns: an int filehandle, or a number<0 if an error occurs. -+ */ -+ abstract protected int ceph_open_for_read(String path); -+ -+ /* -+ * Opens a file for overwriting; creates it if necessary. -+ * Opening a dir is possible but may have bad results. -+ * Inputs: -+ * String path: The path to open. -+ * int mode: The mode to open with. -+ * Returns: an int filehandle, or a number<0 if an error occurs. -+ */ -+ abstract protected int ceph_open_for_overwrite(String path, int mode); -+ -+ /* -+ * Closes the given file. Returns 0 on success, or a negative -+ * error code otherwise. -+ */ -+ abstract protected int ceph_close(int filehandle); -+ -+ /* -+ * Change the mode on a path. -+ * Inputs: -+ * String path: The path to change mode on. -+ * int mode: The mode to apply. -+ * Returns: true if the mode is properly applied, false if there -+ * is any error. -+ */ -+ abstract protected boolean ceph_setPermission(String path, int mode); -+ -+ /* -+ * Closes the Ceph client. This should be called before shutting down -+ * (multiple times is okay but redundant). -+ */ -+ abstract protected boolean ceph_kill_client(); -+ -+ /* -+ * Get the statistics on a path returned in a custom format defined -+ * in CephFileSystem. -+ * Inputs: -+ * String path: The path to stat. -+ * Stat fill: The stat object to fill. -+ * Returns: true if the stat is successful, false otherwise. -+ */ -+ abstract protected boolean ceph_stat(String path, CephFileSystem.Stat fill); -+ -+ /* -+ * Check how many times a file should be replicated. If it is, -+ * degraded it may not actually be replicated this often. -+ * Inputs: -+ * int fh: a file descriptor -+ * Returns: an int containing the number of times replicated. -+ */ -+ abstract protected int ceph_replication(String path); -+ -+ /* -+ * Find the IP address of the primary OSD for a given file and offset. -+ * Inputs: -+ * int fh: The filehandle for the file. -+ * long offset: The offset to get the location of. -+ * Returns: an array of String of the location as IP, or NULL if there is an error. -+ */ -+ abstract protected String[] ceph_hosts(int fh, long offset); -+ -+ /* -+ * Set the mtime and atime for a given path. -+ * Inputs: -+ * String path: The path to set the times for. -+ * long mtime: The mtime to set, in millis since epoch (-1 to not set). -+ * long atime: The atime to set, in millis since epoch (-1 to not set) -+ * Returns: 0 if successful, an error code otherwise. -+ */ -+ abstract protected int ceph_setTimes(String path, long mtime, long atime); -+ -+ /* -+ * Get the current position in a file (as a long) of a given filehandle. -+ * Returns: (long) current file position on success, or a -+ * negative error code on failure. -+ */ -+ abstract protected long ceph_getpos(int fh); -+ -+ /* -+ * Write the given buffer contents to the given filehandle. -+ * Inputs: -+ * int fh: The filehandle to write to. -+ * byte[] buffer: The buffer to write from -+ * int buffer_offset: The position in the buffer to write from -+ * int length: The number of (sequential) bytes to write. -+ * Returns: int, on success the number of bytes written, on failure -+ * a negative error code. -+ */ -+ abstract protected int ceph_write(int fh, byte[] buffer, int buffer_offset, int length); -+ -+ /* -+ * Reads into the given byte array from the current position. -+ * Inputs: -+ * int fh: the filehandle to read from -+ * byte[] buffer: the byte array to read into -+ * int buffer_offset: where in the buffer to start writing -+ * int length: how much to read. -+ * There'd better be enough space in the buffer to write all -+ * the data from the given offset! -+ * Returns: the number of bytes read on success (as an int), -+ * or an error code otherwise. */ -+ abstract protected int ceph_read(int fh, byte[] buffer, int buffer_offset, int length); -+ -+ /* -+ * Seeks to the given position in the given file. -+ * Inputs: -+ * int fh: The filehandle to seek in. -+ * long pos: The position to seek to. -+ * Returns: the new position (as a long) of the filehandle on success, -+ * or a negative error code on failure. */ -+ abstract protected long ceph_seek_from_start(int fh, long pos); -+} -diff --git a/src/core/org/apache/hadoop/fs/ceph/CephFaker.java b/src/core/org/apache/hadoop/fs/ceph/CephFaker.java -new file mode 100644 -index 0000000..c598f53 ---- /dev/null -+++ b/src/core/org/apache/hadoop/fs/ceph/CephFaker.java -@@ -0,0 +1,483 @@ -+// -*- mode:Java; tab-width:2; c-basic-offset:2; indent-tabs-mode:t -*- -+ -+/** -+ * -+ * Licensed under the Apache License, Version 2.0 -+ * (the "License"); you may not use this file except in compliance with -+ * the License. You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -+ * implied. See the License for the specific language governing -+ * permissions and limitations under the License. -+ * -+ * -+ * This uses the local Filesystem but pretends to be communicating -+ * with a Ceph deployment, for unit testing the CephFileSystem. -+ */ -+ -+package org.apache.hadoop.fs.ceph; -+ -+ -+import java.net.URI; -+import java.util.Hashtable; -+import java.io.Closeable; -+import java.io.FileNotFoundException; -+import java.io.IOException; -+ -+import org.apache.commons.logging.Log; -+import org.apache.commons.logging.LogFactory; -+import org.apache.hadoop.conf.Configuration; -+import org.apache.hadoop.fs.BlockLocation; -+import org.apache.hadoop.fs.FileStatus; -+import org.apache.hadoop.fs.FileSystem; -+import org.apache.hadoop.fs.FSDataInputStream; -+import org.apache.hadoop.fs.FSDataOutputStream; -+import org.apache.hadoop.fs.Path; -+import org.apache.hadoop.fs.permission.FsPermission; -+ -+ -+class CephFaker extends CephFS { -+ private static final Log LOG = LogFactory.getLog(CephFaker.class); -+ FileSystem localFS; -+ String localPrefix; -+ int blockSize; -+ Configuration conf; -+ Hashtable<Integer, Object> files; -+ Hashtable<Integer, String> filenames; -+ int fileCount = 0; -+ boolean initialized = false; -+ -+ public CephFaker(Configuration con, Log log) { -+ conf = con; -+ files = new Hashtable<Integer, Object>(); -+ filenames = new Hashtable<Integer, String>(); -+ } -+ -+ protected boolean ceph_initializeClient(String args, int block_size) { -+ if (!initialized) { -+ // let's remember the default block_size -+ blockSize = block_size; -+ -+ /* for a real Ceph deployment, this starts up the client, -+ * sets debugging levels, etc. We just need to get the -+ * local FileSystem to use, and we'll ignore any -+ * command-line arguments. */ -+ try { -+ localFS = FileSystem.getLocal(conf); -+ localFS.initialize(URI.create("file://localhost"), conf); -+ localFS.setVerifyChecksum(false); -+ String testDir = conf.get("hadoop.tmp.dir"); -+ -+ localPrefix = localFS.getWorkingDirectory().toString(); -+ int testDirLoc = localPrefix.indexOf(testDir) - 1; -+ -+ if (-2 == testDirLoc) { -+ testDirLoc = localPrefix.length(); -+ } -+ localPrefix = localPrefix.substring(0, testDirLoc) + "/" -+ + conf.get("hadoop.tmp.dir"); -+ -+ localFS.setWorkingDirectory( -+ new Path(localPrefix + "/user/" + System.getProperty("user.name"))); -+ // I don't know why, but the unit tests expect the default -+ // working dir to be /user/username, so satisfy them! -+ // debug("localPrefix is " + localPrefix, INFO); -+ } catch (IOException e) { -+ return false; -+ } -+ initialized = true; -+ } -+ return true; -+ } -+ -+ protected String ceph_getcwd() { -+ return sanitize_path(localFS.getWorkingDirectory().toString()); -+ } -+ -+ protected boolean ceph_setcwd(String path) { -+ localFS.setWorkingDirectory(new Path(prepare_path(path))); -+ return true; -+ } -+ -+ // the caller is responsible for ensuring empty dirs -+ protected boolean ceph_rmdir(String pth) { -+ Path path = new Path(prepare_path(pth)); -+ boolean ret = false; -+ -+ try { -+ if (localFS.listStatus(path).length <= 1) { -+ ret = localFS.delete(path, true); -+ } -+ } catch (IOException e) {} -+ return ret; -+ } -+ -+ // this needs to work on (empty) directories too -+ protected boolean ceph_unlink(String path) { -+ path = prepare_path(path); -+ boolean ret = false; -+ -+ if (ceph_isdirectory(path)) { -+ ret = ceph_rmdir(path); -+ } else { -+ try { -+ ret = localFS.delete(new Path(path), false); -+ } catch (IOException e) {} -+ } -+ return ret; -+ } -+ -+ protected boolean ceph_rename(String oldName, String newName) { -+ oldName = prepare_path(oldName); -+ newName = prepare_path(newName); -+ try { -+ Path parent = new Path(newName).getParent(); -+ Path newPath = new Path(newName); -+ -+ if (localFS.exists(parent) && !localFS.exists(newPath)) { -+ return localFS.rename(new Path(oldName), newPath); -+ } -+ return false; -+ } catch (IOException e) { -+ return false; -+ } -+ } -+ -+ protected boolean ceph_exists(String path) { -+ path = prepare_path(path); -+ boolean ret = false; -+ -+ try { -+ ret = localFS.exists(new Path(path)); -+ } catch (IOException e) {} -+ return ret; -+ } -+ -+ protected long ceph_getblocksize(String path) { -+ path = prepare_path(path); -+ try { -+ FileStatus status = localFS.getFileStatus(new Path(path)); -+ -+ return status.getBlockSize(); -+ } catch (FileNotFoundException e) { -+ return -CephFS.ENOENT; -+ } catch (IOException e) { -+ return -1; // just fail generically -+ } -+ } -+ -+ protected boolean ceph_isdirectory(String path) { -+ path = prepare_path(path); -+ try { -+ FileStatus status = localFS.getFileStatus(new Path(path)); -+ -+ return status.isDir(); -+ } catch (IOException e) { -+ return false; -+ } -+ } -+ -+ protected boolean ceph_isfile(String path) { -+ path = prepare_path(path); -+ boolean ret = false; -+ -+ try { -+ FileStatus status = localFS.getFileStatus(new Path(path)); -+ -+ ret = !status.isDir(); -+ } catch (Exception e) {} -+ return ret; -+ } -+ -+ protected String[] ceph_getdir(String path) { -+ path = prepare_path(path); -+ if (!ceph_isdirectory(path)) { -+ return null; -+ } -+ try { -+ FileStatus[] stats = localFS.listStatus(new Path(path)); -+ String[] names = new String[stats.length]; -+ String name; -+ -+ for (int i = 0; i < stats.length; ++i) { -+ name = stats[i].getPath().toString(); -+ names[i] = name.substring(name.lastIndexOf(Path.SEPARATOR) + 1); -+ } -+ return names; -+ } catch (IOException e) {} -+ return null; -+ } -+ -+ protected int ceph_mkdirs(String path, int mode) { -+ path = prepare_path(path); -+ // debug("ceph_mkdirs on " + path, INFO); -+ try { -+ if (localFS.mkdirs(new Path(path), new FsPermission((short) mode))) { -+ return 0; -+ } -+ } catch (IOException e) {} -+ if (ceph_isdirectory(path)) { // apparently it already existed -+ return -EEXIST; -+ } else if (ceph_isfile(path)) { -+ return -ENOTDIR; -+ } -+ return -1; -+ } -+ -+ /* -+ * Unlike a real Ceph deployment, you can't do opens on a directory. -+ * Since that has unpredictable behavior and you shouldn't do it anyway, -+ * it's okay. -+ */ -+ protected int ceph_open_for_append(String path) { -+ path = prepare_path(path); -+ FSDataOutputStream stream; -+ -+ try { -+ stream = localFS.append(new Path(path)); -+ files.put(new Integer(fileCount), stream); -+ filenames.put(new Integer(fileCount), path); -+ return fileCount++; -+ } catch (IOException e) {} -+ return -1; // failure -+ } -+ -+ protected int ceph_open_for_read(String path) { -+ path = prepare_path(path); -+ FSDataInputStream stream; -+ -+ try { -+ stream = localFS.open(new Path(path)); -+ files.put(new Integer(fileCount), stream); -+ filenames.put(new Integer(fileCount), path); -+ LOG.info("ceph_open_for_read fh:" + fileCount + ", pathname:" + path); -+ return fileCount++; -+ } catch (IOException e) {} -+ return -1; // failure -+ } -+ -+ protected int ceph_open_for_overwrite(String path, int mode) { -+ path = prepare_path(path); -+ FSDataOutputStream stream; -+ -+ try { -+ stream = localFS.create(new Path(path)); -+ files.put(new Integer(fileCount), stream); -+ filenames.put(new Integer(fileCount), path); -+ LOG.info("ceph_open_for_overwrite fh:" + fileCount + ", pathname:" + path); -+ return fileCount++; -+ } catch (IOException e) {} -+ return -1; // failure -+ } -+ -+ protected int ceph_close(int filehandle) { -+ LOG.info("ceph_close(filehandle " + filehandle + ")"); -+ try { -+ ((Closeable) files.get(new Integer(filehandle))).close(); -+ if (null == files.get(new Integer(filehandle))) { -+ return -ENOENT; // this isn't quite the right error code, -+ // but the important part is it's negative -+ } -+ return 0; // hurray, success -+ } catch (NullPointerException ne) { -+ LOG.warn("ceph_close caught NullPointerException!" + ne); -+ } // err, how? -+ catch (IOException ie) { -+ LOG.warn("ceph_close caught IOException!" + ie); -+ } -+ return -1; // failure -+ } -+ -+ protected boolean ceph_setPermission(String pth, int mode) { -+ pth = prepare_path(pth); -+ Path path = new Path(pth); -+ boolean ret = false; -+ -+ try { -+ localFS.setPermission(path, new FsPermission((short) mode)); -+ ret = true; -+ } catch (IOException e) {} -+ return ret; -+ } -+ -+ // rather than try and match a Ceph deployment's behavior exactly, -+ // just make bad things happen if they try and call methods after this -+ protected boolean ceph_kill_client() { -+ // debug("ceph_kill_client", INFO); -+ localFS.setWorkingDirectory(new Path(localPrefix)); -+ // debug("working dir is now " + localFS.getWorkingDirectory(), INFO); -+ try { -+ localFS.close(); -+ } catch (Exception e) {} -+ localFS = null; -+ files = null; -+ filenames = null; -+ return true; -+ } -+ -+ protected boolean ceph_stat(String pth, CephFileSystem.Stat fill) { -+ pth = prepare_path(pth); -+ Path path = new Path(pth); -+ boolean ret = false; -+ -+ try { -+ FileStatus status = localFS.getFileStatus(path); -+ -+ fill.size = status.getLen(); -+ fill.is_dir = status.isDir(); -+ fill.block_size = status.getBlockSize(); -+ fill.mod_time = status.getModificationTime(); -+ fill.access_time = status.getAccessTime(); -+ fill.mode = status.getPermission().toShort(); -+ ret = true; -+ } catch (IOException e) {} -+ return ret; -+ } -+ -+ protected int ceph_replication(String path) { -+ path = prepare_path(path); -+ int ret = -1; // -1 for failure -+ -+ try { -+ ret = localFS.getFileStatus(new Path(path)).getReplication(); -+ } catch (IOException e) {} -+ return ret; -+ } -+ -+ protected String[] ceph_hosts(int fh, long offset) { -+ String[] ret = null; -+ -+ try { -+ BlockLocation[] locs = localFS.getFileBlockLocations( -+ localFS.getFileStatus(new Path(filenames.get(new Integer(fh)))), -+ offset, 1); -+ -+ ret = locs[0].getNames(); -+ } catch (IOException e) {} catch (NullPointerException f) {} -+ return ret; -+ } -+ -+ protected int ceph_setTimes(String pth, long mtime, long atime) { -+ pth = prepare_path(pth); -+ Path path = new Path(pth); -+ int ret = -1; // generic fail -+ -+ try { -+ localFS.setTimes(path, mtime, atime); -+ ret = 0; -+ } catch (IOException e) {} -+ return ret; -+ } -+ -+ protected long ceph_getpos(int fh) { -+ long ret = -1; // generic fail -+ -+ try { -+ Object stream = files.get(new Integer(fh)); -+ -+ if (stream instanceof FSDataInputStream) { -+ ret = ((FSDataInputStream) stream).getPos(); -+ } else if (stream instanceof FSDataOutputStream) { -+ ret = ((FSDataOutputStream) stream).getPos(); -+ } -+ } catch (IOException e) {} catch (NullPointerException f) {} -+ return ret; -+ } -+ -+ protected int ceph_write(int fh, byte[] buffer, -+ int buffer_offset, int length) { -+ LOG.info( -+ "ceph_write fh:" + fh + ", buffer_offset:" + buffer_offset + ", length:" -+ + length); -+ long ret = -1; // generic fail -+ -+ try { -+ FSDataOutputStream os = (FSDataOutputStream) files.get(new Integer(fh)); -+ -+ LOG.info("ceph_write got outputstream"); -+ long startPos = os.getPos(); -+ -+ os.write(buffer, buffer_offset, length); -+ ret = os.getPos() - startPos; -+ } catch (IOException e) { -+ LOG.warn("ceph_write caught IOException!"); -+ } catch (NullPointerException f) { -+ LOG.warn("ceph_write caught NullPointerException!"); -+ } -+ return (int) ret; -+ } -+ -+ protected int ceph_read(int fh, byte[] buffer, -+ int buffer_offset, int length) { -+ long ret = -1; // generic fail -+ -+ try { -+ FSDataInputStream is = (FSDataInputStream) files.get(new Integer(fh)); -+ long startPos = is.getPos(); -+ -+ is.read(buffer, buffer_offset, length); -+ ret = is.getPos() - startPos; -+ } catch (IOException e) {} catch (NullPointerException f) {} -+ return (int) ret; -+ } -+ -+ protected long ceph_seek_from_start(int fh, long pos) { -+ LOG.info("ceph_seek_from_start(fh " + fh + ", pos " + pos + ")"); -+ long ret = -1; // generic fail -+ -+ try { -+ LOG.info("ceph_seek_from_start filename is " + filenames.get(new Integer(fh))); -+ if (null == files.get(new Integer(fh))) { -+ LOG.warn("ceph_seek_from_start: is is null!"); -+ } -+ FSDataInputStream is = (FSDataInputStream) files.get(new Integer(fh)); -+ -+ LOG.info("ceph_seek_from_start retrieved is!"); -+ is.seek(pos); -+ ret = is.getPos(); -+ } catch (IOException e) { -+ LOG.warn("ceph_seek_from_start caught IOException!"); -+ } catch (NullPointerException f) { -+ LOG.warn("ceph_seek_from_start caught NullPointerException!"); -+ } -+ return (int) ret; -+ } -+ -+ /* -+ * We need to remove the localFS file prefix before returning to Ceph -+ */ -+ private String sanitize_path(String path) { -+ // debug("sanitize_path(" + path + ")", INFO); -+ /* if (path.startsWith("file:")) -+ path = path.substring("file:".length()); */ -+ if (path.startsWith(localPrefix)) { -+ path = path.substring(localPrefix.length()); -+ if (path.length() == 0) { // it was a root path -+ path = "/"; -+ } -+ } -+ // debug("sanitize_path returning " + path, INFO); -+ return path; -+ } -+ -+ /* -+ * If it's an absolute path we need to shove the -+ * test dir onto the front as a prefix. -+ */ -+ private String prepare_path(String path) { -+ // debug("prepare_path(" + path + ")", INFO); -+ if (path.startsWith("/")) { -+ path = localPrefix + path; -+ } else if (path.equals("..")) { -+ if (ceph_getcwd().equals("/")) { -+ path = "."; -+ } // you can't go up past root! -+ } -+ // debug("prepare_path returning" + path, INFO); -+ return path; -+ } -+} -diff --git a/src/core/org/apache/hadoop/fs/ceph/CephFileSystem.java b/src/core/org/apache/hadoop/fs/ceph/CephFileSystem.java -new file mode 100644 -index 0000000..95f2223 ---- /dev/null -+++ b/src/core/org/apache/hadoop/fs/ceph/CephFileSystem.java -@@ -0,0 +1,804 @@ -+// -*- mode:Java; tab-width:2; c-basic-offset:2; indent-tabs-mode:t -*- -+ -+/** -+ * -+ * Licensed under the Apache License, Version 2.0 -+ * (the "License"); you may not use this file except in compliance with -+ * the License. You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -+ * implied. See the License for the specific language governing -+ * permissions and limitations under the License. -+ * -+ * -+ * Implements the Hadoop FS interfaces to allow applications to store -+ * files in Ceph. -+ */ -+package org.apache.hadoop.fs.ceph; -+ -+ -+import java.io.IOException; -+import java.io.FileNotFoundException; -+import java.io.OutputStream; -+import java.net.URI; -+import java.net.InetAddress; -+import java.util.EnumSet; -+import java.lang.Math; -+import java.util.ArrayList; -+ -+import org.apache.commons.logging.Log; -+import org.apache.commons.logging.LogFactory; -+import org.apache.hadoop.conf.Configuration; -+import org.apache.hadoop.fs.BlockLocation; -+import org.apache.hadoop.fs.FSDataInputStream; -+import org.apache.hadoop.fs.FSInputStream; -+import org.apache.hadoop.fs.FSDataOutputStream; -+import org.apache.hadoop.fs.FileSystem; -+import org.apache.hadoop.fs.FileUtil; -+import org.apache.hadoop.fs.Path; -+import org.apache.hadoop.fs.permission.FsPermission; -+import org.apache.hadoop.util.Progressable; -+import org.apache.hadoop.fs.FileStatus; -+import org.apache.hadoop.net.DNS; -+ -+ -+/** -+ * <p> -+ * A {@link FileSystem} backed by <a href="http://ceph.newdream.net">Ceph.</a>. -+ * This will not start a Ceph instance; one must already be running. -+ * </p> -+ * Configuration of the CephFileSystem is handled via a few Hadoop -+ * Configuration properties: <br> -+ * fs.ceph.monAddr -- the ip address/port of the monitor to connect to. <br> -+ * fs.ceph.libDir -- the directory that libcephfs and libhadoopceph are -+ * located in. This assumes Hadoop is being run on a linux-style machine -+ * with names like libcephfs.so. -+ * fs.ceph.commandLine -- if you prefer you can fill in this property -+ * just as you would when starting Ceph up from the command line. Specific -+ * properties override any configuration specified here. -+ * <p> -+ * You can also enable debugging of the CephFileSystem and Ceph itself: <br> -+ * fs.ceph.debug -- if 'true' will print out method enter/exit messages, -+ * plus a little more. -+ * fs.ceph.clientDebug/fs.ceph.messengerDebug -- will print out debugging -+ * from the respective Ceph system of at least that importance. -+ */ -+public class CephFileSystem extends FileSystem { -+ private static final Log LOG = LogFactory.getLog(CephFileSystem.class); -+ private URI uri; -+ -+ private Path workingDir; -+ private final Path root; -+ private CephFS ceph = null; -+ -+ private static String CEPH_NAMESERVER; -+ private static final String CEPH_NAMESERVER_KEY = "fs.ceph.nameserver"; -+ private static final String CEPH_NAMESERVER_DEFAULT = "localhost"; -+ -+ /** -+ * Create a new CephFileSystem. -+ */ -+ public CephFileSystem() { -+ root = new Path("/"); -+ } -+ -+ /** -+ * Used for testing purposes, this constructor -+ * sets the given CephFS instead of defaulting to a -+ * CephTalker (with its assumed real Ceph instance to talk to). -+ */ -+ public CephFileSystem(CephFS ceph_fs) { -+ super(); -+ root = new Path("/"); -+ ceph = ceph_fs; -+ } -+ -+ /** -+ * Lets you get the URI of this CephFileSystem. -+ * @return the URI. -+ */ -+ public URI getUri() { -+ LOG.debug("getUri:exit with return " + uri); -+ return uri; -+ } -+ -+ /** -+ * Should be called after constructing a CephFileSystem but before calling -+ * any other methods. -+ * Starts up the connection to Ceph, reads in configuraton options, etc. -+ * @param uri The URI for this filesystem. -+ * @param conf The Hadoop Configuration to retrieve properties from. -+ * @throws IOException if necessary properties are unset. -+ */ -+ @Override -+ public void initialize(URI uri, Configuration conf) throws IOException { -+ super.initialize(uri, conf); -+ setConf(conf); -+ this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority()); -+ if (ceph == null) { -+ ceph = new CephTalker(conf, LOG); -+ } -+ -+ CEPH_NAMESERVER = conf.get(CEPH_NAMESERVER_KEY, CEPH_NAMESERVER_DEFAULT); -+ -+ // build up the arguments for Ceph -+ String arguments = "CephFSInterface"; -+ -+ arguments += conf.get("fs.ceph.commandLine", ""); -+ if (conf.get("fs.ceph.clientDebug") != null) { -+ arguments += " --debug_client "; -+ arguments += conf.get("fs.ceph.clientDebug"); -+ } -+ if (conf.get("fs.ceph.messengerDebug") != null) { -+ arguments += " --debug_ms "; -+ arguments += conf.get("fs.ceph.messengerDebug"); -+ } -+ if (conf.get("fs.ceph.monAddr") != null) { -+ arguments += " -m "; -+ arguments += conf.get("fs.ceph.monAddr"); -+ } -+ arguments += " --client-readahead-max-periods=" -+ + conf.get("fs.ceph.readahead", "1"); -+ // make sure they gave us a ceph monitor address or conf file -+ LOG.info("initialize:Ceph initialization arguments: " + arguments); -+ if ((conf.get("fs.ceph.monAddr") == null) && (arguments.indexOf("-m") == -1) -+ && (arguments.indexOf("-c") == -1)) { -+ LOG.fatal("initialize:You need to specify a Ceph monitor address."); -+ throw new IOException( -+ "You must specify a Ceph monitor address or config file!"); -+ } -+ // Initialize the client -+ if (!ceph.ceph_initializeClient(arguments, -+ conf.getInt("fs.ceph.blockSize", 1 << 26))) { -+ LOG.fatal("initialize:Ceph initialization failed!"); -+ throw new IOException("Ceph initialization failed!"); -+ } -+ LOG.info("initialize:Ceph initialized client. Setting cwd to /"); -+ ceph.ceph_setcwd("/"); -+ LOG.debug("initialize:exit"); -+ -+ this.workingDir = getHomeDirectory(); -+ } -+ -+ /** -+ * Close down the CephFileSystem. Runs the base-class close method -+ * and then kills the Ceph client itself. -+ */ -+ @Override -+ public void close() throws IOException { -+ LOG.debug("close:enter"); -+ super.close(); // this method does stuff, make sure it's run! -+ LOG.trace("close: Calling ceph_kill_client from Java"); -+ ceph.ceph_kill_client(); -+ LOG.debug("close:exit"); -+ } -+ -+ /** -+ * Get an FSDataOutputStream to append onto a file. -+ * @param file The File you want to append onto -+ * @param bufferSize Ceph does internal buffering but you can buffer in the Java code as well if you like. -+ * @param progress The Progressable to report progress to. -+ * Reporting is limited but exists. -+ * @return An FSDataOutputStream that connects to the file on Ceph. -+ * @throws IOException If the file cannot be found or appended to. -+ */ -+ public FSDataOutputStream append(Path file, int bufferSize, -+ Progressable progress) throws IOException { -+ LOG.debug("append:enter with path " + file + " bufferSize " + bufferSize); -+ Path abs_path = makeAbsolute(file); -+ -+ if (progress != null) { -+ progress.progress(); -+ } -+ LOG.trace("append: Entering ceph_open_for_append from Java"); -+ int fd = ceph.ceph_open_for_append(getCephPath(abs_path)); -+ -+ LOG.trace("append: Returned to Java"); -+ if (progress != null) { -+ progress.progress(); -+ } -+ if (fd < 0) { // error in open -+ throw new IOException( -+ "append: Open for append failed on path \"" + abs_path.toString() -+ + "\""); -+ } -+ CephOutputStream cephOStream = new CephOutputStream(getConf(), ceph, fd, -+ bufferSize); -+ -+ LOG.debug("append:exit"); -+ return new FSDataOutputStream(cephOStream, statistics); -+ } -+ -+ /** -+ * Get the current working directory for the given file system -+ * @return the directory Path -+ */ -+ public Path getWorkingDirectory() { -+ return workingDir; -+ } -+ -+ /** -+ * Set the current working directory for the given file system. All relative -+ * paths will be resolved relative to it. -+ * -+ * @param dir The directory to change to. -+ */ -+ @Override -+ public void setWorkingDirectory(Path dir) { -+ workingDir = makeAbsolute(dir); -+ } -+ -+ /** -+ * Return only the path component from a potentially fully qualified path. -+ */ -+ private String getCephPath(Path path) { -+ if (!path.isAbsolute()) { -+ throw new IllegalArgumentException("Path must be absolute: " + path); -+ } -+ return path.toUri().getPath(); -+ } -+ -+ /** -+ * Check if a path exists. -+ * Overriden because it's moderately faster than the generic implementation. -+ * @param path The file to check existence on. -+ * @return true if the file exists, false otherwise. -+ */ -+ @Override -+ public boolean exists(Path path) throws IOException { -+ LOG.debug("exists:enter with path " + path); -+ boolean result; -+ Path abs_path = makeAbsolute(path); -+ -+ if (abs_path.equals(root)) { -+ result = true; -+ } else { -+ LOG.trace( -+ "exists:Calling ceph_exists from Java on path " + abs_path.toString()); -+ result = ceph.ceph_exists(getCephPath(abs_path)); -+ LOG.trace("exists:Returned from ceph_exists to Java"); -+ } -+ LOG.debug("exists:exit with value " + result); -+ return result; -+ } -+ -+ /** -+ * Create a directory and any nonexistent parents. Any portion -+ * of the directory tree can exist without error. -+ * @param path The directory path to create -+ * @param perms The permissions to apply to the created directories. -+ * @return true if successful, false otherwise -+ * @throws IOException if the path is a child of a file. -+ */ -+ @Override -+ public boolean mkdirs(Path path, FsPermission perms) throws IOException { -+ LOG.debug("mkdirs:enter with path " + path); -+ Path abs_path = makeAbsolute(path); -+ -+ LOG.trace("mkdirs:calling ceph_mkdirs from Java"); -+ int result = ceph.ceph_mkdirs(getCephPath(abs_path), (int) perms.toShort()); -+ -+ if (result != 0) { -+ LOG.warn( -+ "mkdirs: make directory " + abs_path + "Failing with result " + result); -+ if (-ceph.ENOTDIR == result) { -+ throw new IOException("Parent path is not a directory"); -+ } -+ return false; -+ } else { -+ LOG.debug("mkdirs:exiting succesfully"); -+ return true; -+ } -+ } -+ -+ /** -+ * Check if a path is a file. This is moderately faster than the -+ * generic implementation. -+ * @param path The path to check. -+ * @return true if the path is definitely a file, false otherwise. -+ */ -+ @Override -+ public boolean isFile(Path path) throws IOException { -+ LOG.debug("isFile:enter with path " + path); -+ Path abs_path = makeAbsolute(path); -+ boolean result; -+ -+ if (abs_path.equals(root)) { -+ result = false; -+ } else { -+ LOG.trace("isFile:entering ceph_isfile from Java"); -+ result = ceph.ceph_isfile(getCephPath(abs_path)); -+ } -+ LOG.debug("isFile:exit with result " + result); -+ return result; -+ } -+ -+ /** -+ * Get stat information on a file. This does not fill owner or group, as -+ * Ceph's support for these is a bit different than HDFS'. -+ * @param path The path to stat. -+ * @return FileStatus object containing the stat information. -+ * @throws FileNotFoundException if the path could not be resolved. -+ */ -+ public FileStatus getFileStatus(Path path) throws IOException { -+ LOG.debug("getFileStatus:enter with path " + path); -+ Path abs_path = makeAbsolute(path); -+ // sadly, Ceph doesn't really do uids/gids just yet, but -+ // everything else is filled -+ FileStatus status; -+ Stat lstat = new Stat(); -+ -+ LOG.trace("getFileStatus: calling ceph_stat from Java"); -+ if (ceph.ceph_stat(getCephPath(abs_path), lstat)) { -+ status = new FileStatus(lstat.size, lstat.is_dir, -+ ceph.ceph_replication(getCephPath(abs_path)), lstat.block_size, -+ lstat.mod_time, lstat.access_time, -+ new FsPermission((short) lstat.mode), System.getProperty("user.name"), null, -+ path.makeQualified(this)); -+ } else { // fail out -+ throw new FileNotFoundException( -+ "org.apache.hadoop.fs.ceph.CephFileSystem: File " + path -+ + " does not exist or could not be accessed"); -+ } -+ -+ LOG.debug("getFileStatus:exit"); -+ return status; -+ } -+ -+ /** -+ * Get the FileStatus for each listing in a directory. -+ * @param path The directory to get listings from. -+ * @return FileStatus[] containing one FileStatus for each directory listing; -+ * null if path does not exist. -+ */ -+ public FileStatus[] listStatus(Path path) throws IOException { -+ LOG.debug("listStatus:enter with path " + path); -+ Path abs_path = makeAbsolute(path); -+ Path[] paths = listPaths(abs_path); -+ -+ if (paths != null) { -+ FileStatus[] statuses = new FileStatus[paths.length]; -+ -+ for (int i = 0; i < paths.length; ++i) { -+ statuses[i] = getFileStatus(paths[i]); -+ } -+ LOG.debug("listStatus:exit"); -+ return statuses; -+ } -+ -+ if (isFile(path)) { -+ return new FileStatus[] { getFileStatus(path) }; -+ } -+ -+ return null; -+ } -+ -+ @Override -+ public void setPermission(Path p, FsPermission permission) throws IOException { -+ LOG.debug( -+ "setPermission:enter with path " + p + " and permissions " + permission); -+ Path abs_path = makeAbsolute(p); -+ -+ LOG.trace("setPermission:calling ceph_setpermission from Java"); -+ ceph.ceph_setPermission(getCephPath(abs_path), permission.toShort()); -+ LOG.debug("setPermission:exit"); -+ } -+ -+ /** -+ * Set access/modification times of a file. -+ * @param p The path -+ * @param mtime Set modification time in number of millis since Jan 1, 1970. -+ * @param atime Set access time in number of millis since Jan 1, 1970. -+ */ -+ @Override -+ public void setTimes(Path p, long mtime, long atime) throws IOException { -+ LOG.debug( -+ "setTimes:enter with path " + p + " mtime:" + mtime + " atime:" + atime); -+ Path abs_path = makeAbsolute(p); -+ -+ LOG.trace("setTimes:calling ceph_setTimes from Java"); -+ int r = ceph.ceph_setTimes(getCephPath(abs_path), mtime, atime); -+ -+ if (r < 0) { -+ throw new IOException( -+ "Failed to set times on path " + abs_path.toString() + " Error code: " -+ + r); -+ } -+ LOG.debug("setTimes:exit"); -+ } -+ -+ /** -+ * Create a new file and open an FSDataOutputStream that's connected to it. -+ * @param path The file to create. -+ * @param permission The permissions to apply to the file. -+ * @param overwrite If true, overwrite any existing file with -+ * this name; otherwise don't. -+ * @param bufferSize Ceph does internal buffering, but you can buffer -+ * in the Java code too if you like. -+ * @param replication Ignored by Ceph. This can be -+ * configured via Ceph configuration. -+ * @param blockSize Ignored by Ceph. You can set client-wide block sizes -+ * via the fs.ceph.blockSize param if you like. -+ * @param progress A Progressable to report back to. -+ * Reporting is limited but exists. -+ * @return An FSDataOutputStream pointing to the created file. -+ * @throws IOException if the path is an -+ * existing directory, or the path exists but overwrite is false, or there is a -+ * failure in attempting to open for append with Ceph. -+ */ -+ public FSDataOutputStream create(Path path, -+ FsPermission permission, -+ boolean overwrite, -+ int bufferSize, -+ short replication, -+ long blockSize, -+ Progressable progress) throws IOException { -+ LOG.debug("create:enter with path " + path); -+ Path abs_path = makeAbsolute(path); -+ -+ if (progress != null) { -+ progress.progress(); -+ } -+ // We ignore replication since that's not configurable here, and -+ // progress reporting is quite limited. -+ // Required semantics: if the file exists, overwrite if 'overwrite' is set; -+ // otherwise, throw an exception -+ -+ // Step 1: existence test -+ boolean exists = exists(abs_path); -+ -+ if (exists) { -+ if (getFileStatus(abs_path).isDir()) { -+ throw new IOException( -+ "create: Cannot overwrite existing directory \"" + path.toString() -+ + "\" with a file"); -+ } -+ if (!overwrite) { -+ throw new IOException( -+ "createRaw: Cannot open existing file \"" + abs_path.toString() -+ + "\" for writing without overwrite flag"); -+ } -+ } -+ -+ if (progress != null) { -+ progress.progress(); -+ } -+ -+ // Step 2: create any nonexistent directories in the path -+ if (!exists) { -+ Path parent = abs_path.getParent(); -+ -+ if (parent != null) { // if parent is root, we're done -+ int r = ceph.ceph_mkdirs(getCephPath(parent), permission.toShort()); -+ -+ if (!(r == 0 || r == -ceph.EEXIST)) { -+ throw new IOException("Error creating parent directory; code: " + r); -+ } -+ } -+ if (progress != null) { -+ progress.progress(); -+ } -+ } -+ // Step 3: open the file -+ LOG.trace("calling ceph_open_for_overwrite from Java"); -+ int fh = ceph.ceph_open_for_overwrite(getCephPath(abs_path), -+ (int) permission.toShort()); -+ -+ if (progress != null) { -+ progress.progress(); -+ } -+ LOG.trace("Returned from ceph_open_for_overwrite to Java with fh " + fh); -+ if (fh < 0) { -+ throw new IOException( -+ "create: Open for overwrite failed on path \"" + path.toString() -+ + "\""); -+ } -+ -+ // Step 4: create the stream -+ OutputStream cephOStream = new CephOutputStream(getConf(), ceph, fh, -+ bufferSize); -+ -+ LOG.debug("create:exit"); -+ return new FSDataOutputStream(cephOStream, statistics); -+ } -+ -+ /** -+ * Open a Ceph file and attach the file handle to an FSDataInputStream. -+ * @param path The file to open -+ * @param bufferSize Ceph does internal buffering; but you can buffer in -+ * the Java code too if you like. -+ * @return FSDataInputStream reading from the given path. -+ * @throws IOException if the path DNE or is a -+ * directory, or there is an error getting data to set up the FSDataInputStream. -+ */ -+ public FSDataInputStream open(Path path, int bufferSize) throws IOException { -+ LOG.debug("open:enter with path " + path); -+ Path abs_path = makeAbsolute(path); -+ -+ int fh = ceph.ceph_open_for_read(getCephPath(abs_path)); -+ -+ if (fh < 0) { // uh-oh, something's bad! -+ if (fh == -ceph.ENOENT) { // well that was a stupid open -+ throw new IOException( -+ "open: absolute path \"" + abs_path.toString() -+ + "\" does not exist"); -+ } else { // hrm...the file exists but we can't open it :( -+ throw new IOException("open: Failed to open file " + abs_path.toString()); -+ } -+ } -+ -+ if (getFileStatus(abs_path).isDir()) { // yes, it is possible to open Ceph directories -+ // but that doesn't mean you should in Hadoop! -+ ceph.ceph_close(fh); -+ throw new IOException( -+ "open: absolute path \"" + abs_path.toString() + "\" is a directory!"); -+ } -+ Stat lstat = new Stat(); -+ -+ LOG.trace("open:calling ceph_stat from Java"); -+ ceph.ceph_stat(getCephPath(abs_path), lstat); -+ LOG.trace("open:returned to Java"); -+ long size = lstat.size; -+ -+ if (size < 0) { -+ throw new IOException( -+ "Failed to get file size for file " + abs_path.toString() -+ + " but succeeded in opening file. Something bizarre is going on."); -+ } -+ FSInputStream cephIStream = new CephInputStream(getConf(), ceph, fh, size, -+ bufferSize); -+ -+ LOG.debug("open:exit"); -+ return new FSDataInputStream(cephIStream); -+ } -+ -+ /** -+ * Rename a file or directory. -+ * @param src The current path of the file/directory -+ * @param dst The new name for the path. -+ * @return true if the rename succeeded, false otherwise. -+ */ -+ @Override -+ public boolean rename(Path src, Path dst) throws IOException { -+ LOG.debug("rename:enter with src:" + src + " and dest:" + dst); -+ Path abs_src = makeAbsolute(src); -+ Path abs_dst = makeAbsolute(dst); -+ -+ LOG.trace("calling ceph_rename from Java"); -+ boolean result = ceph.ceph_rename(getCephPath(abs_src), getCephPath(abs_dst)); -+ -+ if (!result) { -+ boolean isDir = false; -+ try { -+ isDir = getFileStatus(abs_dst).isDir(); -+ } catch (FileNotFoundException e) {} -+ if (isDir) { // move the srcdir into destdir -+ LOG.debug("ceph_rename failed but dst is a directory!"); -+ Path new_dst = new Path(abs_dst, abs_src.getName()); -+ -+ result = rename(abs_src, new_dst); -+ LOG.debug( -+ "attempt to move " + abs_src.toString() + " to " -+ + new_dst.toString() + "has result:" + result); -+ } -+ } -+ LOG.debug("rename:exit with result: " + result); -+ return result; -+ } -+ -+ /* -+ * Attempt to convert an IP into its hostname -+ */ -+ private String[] ips2Hosts(String[] ips) { -+ ArrayList<String> hosts = new ArrayList<String>(); -+ for (String ip : ips) { -+ try { -+ String host = DNS.reverseDns(InetAddress.getByName(ip), CEPH_NAMESERVER); -+ if (host.charAt(host.length()-1) == '.') { -+ host = host.substring(0, host.length()-1); -+ } -+ hosts.add(host); /* append */ -+ } catch (Exception e) { -+ LOG.error("reverseDns ["+ip+"] failed: "+ e); -+ } -+ } -+ return hosts.toArray(new String[hosts.size()]); -+ } -+ -+ /** -+ * Get a BlockLocation object for each block in a file. -+ * -+ * Note that this doesn't include port numbers in the name field as -+ * Ceph handles slow/down servers internally. This data should be used -+ * only for selecting which servers to run which jobs on. -+ * -+ * @param file A FileStatus object corresponding to the file you want locations for. -+ * @param start The offset of the first part of the file you are interested in. -+ * @param len The amount of the file past the offset you are interested in. -+ * @return A BlockLocation[] where each object corresponds to a block within -+ * the given range. -+ */ -+ @Override -+ public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException { -+ Path abs_path = makeAbsolute(file.getPath()); -+ -+ int fh = ceph.ceph_open_for_read(getCephPath(abs_path)); -+ if (fh < 0) { -+ LOG.error("getFileBlockLocations:got error " + fh + ", exiting and returning null!"); -+ return null; -+ } -+ -+ long blockSize = ceph.ceph_getblocksize(getCephPath(abs_path)); -+ BlockLocation[] locations = new BlockLocation[(int) Math.ceil(len / (float) blockSize)]; -+ -+ for (int i = 0; i < locations.length; ++i) { -+ long offset = start + i * blockSize; -+ long blockStart = start + i * blockSize - (start % blockSize); -+ String ips[] = ceph.ceph_hosts(fh, offset); -+ String hosts[] = ips2Hosts(ips); -+ locations[i] = new BlockLocation(null, hosts, blockStart, blockSize); -+ LOG.debug("getFileBlockLocations: location[" + i + "]: " + locations[i]); -+ } -+ -+ ceph.ceph_close(fh); -+ return locations; -+ } -+ -+ @Deprecated -+ public boolean delete(Path path) throws IOException { -+ return delete(path, false); -+ } -+ -+ /** -+ * Delete the given path, and optionally its children. -+ * @param path the path to delete. -+ * @param recursive If the path is a non-empty directory and this is false, -+ * delete will throw an IOException. If path is a file this is ignored. -+ * @return true if the delete succeeded, false otherwise (including if -+ * path doesn't exist). -+ * @throws IOException if you attempt to non-recursively delete a directory, -+ * or you attempt to delete the root directory. -+ */ -+ public boolean delete(Path path, boolean recursive) throws IOException { -+ LOG.debug("delete:enter with path " + path + " and recursive=" + recursive); -+ Path abs_path = makeAbsolute(path); -+ -+ // sanity check -+ if (abs_path.equals(root)) { -+ throw new IOException("Error: deleting the root directory is a Bad Idea."); -+ } -+ if (!exists(abs_path)) { -+ return false; -+ } -+ -+ // if the path is a file, try to delete it. -+ if (isFile(abs_path)) { -+ LOG.trace("delete:calling ceph_unlink from Java with path " + abs_path); -+ boolean result = ceph.ceph_unlink(getCephPath(abs_path)); -+ -+ if (!result) { -+ LOG.error( -+ "delete: failed to delete file \"" + abs_path.toString() + "\"."); -+ } -+ LOG.debug("delete:exit with success=" + result); -+ return result; -+ } -+ -+ /* The path is a directory, so recursively try to delete its contents, -+ and then delete the directory. */ -+ // get the entries; listPaths will remove . and .. for us -+ Path[] contents = listPaths(abs_path); -+ -+ if (contents == null) { -+ LOG.error( -+ "delete: Failed to read contents of directory \"" -+ + abs_path.toString() + "\" while trying to delete it, BAILING"); -+ return false; -+ } -+ if (!recursive && contents.length > 0) { -+ throw new IOException("Directories must be deleted recursively!"); -+ } -+ // delete the entries -+ LOG.debug("delete: recursively calling delete on contents of " + abs_path); -+ for (Path p : contents) { -+ if (!delete(p, true)) { -+ LOG.error( -+ "delete: Failed to delete file \"" + p.toString() -+ + "\" while recursively deleting \"" + abs_path.toString() -+ + "\", BAILING"); -+ return false; -+ } -+ } -+ // if we've come this far it's a now-empty directory, so delete it! -+ boolean result = ceph.ceph_rmdir(getCephPath(abs_path)); -+ -+ if (!result) { -+ LOG.error( -+ "delete: failed to delete \"" + abs_path.toString() + "\", BAILING"); -+ } -+ LOG.debug("delete:exit"); -+ return result; -+ } -+ -+ /** -+ * Returns the default replication value of 1. This may -+ * NOT be the actual value, as replication is controlled -+ * by a separate Ceph configuration. -+ */ -+ @Override -+ public short getDefaultReplication() { -+ return 1; -+ } -+ -+ /** -+ * Get the default block size. -+ * @return the default block size, in bytes, as a long. -+ */ -+ @Override -+ public long getDefaultBlockSize() { -+ return getConf().getInt("fs.ceph.blockSize", 1 << 26); -+ } -+ -+ /** -+ * Adds the working directory to path if path is not already -+ * an absolute path. The URI scheme is not removed here. It -+ * is removed only when users (e.g. ceph native calls) need -+ * the path-only portion. -+ */ -+ private Path makeAbsolute(Path path) { -+ if (path.isAbsolute()) { -+ return path; -+ } -+ return new Path(workingDir, path); -+ } -+ -+ private Path[] listPaths(Path path) throws IOException { -+ LOG.debug("listPaths:enter with path " + path); -+ String dirlist[]; -+ -+ Path abs_path = makeAbsolute(path); -+ -+ // If it's a directory, get the listing. Otherwise, complain and give up. -+ LOG.debug("calling ceph_getdir from Java with path " + abs_path); -+ dirlist = ceph.ceph_getdir(getCephPath(abs_path)); -+ LOG.debug("returning from ceph_getdir to Java"); -+ -+ if (dirlist == null) { -+ return null; -+ } -+ -+ // convert the strings to Paths -+ Path[] paths = new Path[dirlist.length]; -+ -+ for (int i = 0; i < dirlist.length; ++i) { -+ LOG.trace( -+ "Raw enumeration of paths in \"" + abs_path.toString() + "\": \"" -+ + dirlist[i] + "\""); -+ // convert each listing to an absolute path -+ Path raw_path = new Path(dirlist[i]); -+ -+ if (raw_path.isAbsolute()) { -+ paths[i] = raw_path; -+ } else { -+ paths[i] = new Path(abs_path, raw_path); -+ } -+ } -+ LOG.debug("listPaths:exit"); -+ return paths; -+ } -+ -+ static class Stat { -+ public long size; -+ public boolean is_dir; -+ public long block_size; -+ public long mod_time; -+ public long access_time; -+ public int mode; -+ -+ public Stat() {} -+ } -+} -diff --git a/src/core/org/apache/hadoop/fs/ceph/CephInputStream.java b/src/core/org/apache/hadoop/fs/ceph/CephInputStream.java -new file mode 100644 -index 0000000..d9668d0 ---- /dev/null -+++ b/src/core/org/apache/hadoop/fs/ceph/CephInputStream.java -@@ -0,0 +1,254 @@ -+// -*- mode:Java; tab-width:2; c-basic-offset:2; indent-tabs-mode:t -*- -+ -+/** -+ * -+ * Licensed under the Apache License, Version 2.0 -+ * (the "License"); you may not use this file except in compliance with -+ * the License. You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -+ * implied. See the License for the specific language governing -+ * permissions and limitations under the License. -+ * -+ * -+ * Implements the Hadoop FS interfaces to allow applications to store -+ * files in Ceph. -+ */ -+package org.apache.hadoop.fs.ceph; -+ -+ -+import java.io.IOException; -+ -+import org.apache.commons.logging.Log; -+import org.apache.commons.logging.LogFactory; -+import org.apache.hadoop.conf.Configuration; -+import org.apache.hadoop.fs.FSInputStream; -+ -+ -+/** -+ * <p> -+ * An {@link FSInputStream} for a CephFileSystem and corresponding -+ * Ceph instance. -+ */ -+public class CephInputStream extends FSInputStream { -+ private static final Log LOG = LogFactory.getLog(CephInputStream.class); -+ private boolean closed; -+ -+ private int fileHandle; -+ -+ private long fileLength; -+ -+ private CephFS ceph; -+ -+ private byte[] buffer; -+ private int bufPos = 0; -+ private int bufValid = 0; -+ private long cephPos = 0; -+ -+ /** -+ * Create a new CephInputStream. -+ * @param conf The system configuration. Unused. -+ * @param fh The filehandle provided by Ceph to reference. -+ * @param flength The current length of the file. If the length changes -+ * you will need to close and re-open it to access the new data. -+ */ -+ public CephInputStream(Configuration conf, CephFS cephfs, -+ int fh, long flength, int bufferSize) { -+ // Whoever's calling the constructor is responsible for doing the actual ceph_open -+ // call and providing the file handle. -+ fileLength = flength; -+ fileHandle = fh; -+ closed = false; -+ ceph = cephfs; -+ buffer = new byte[bufferSize]; -+ LOG.debug( -+ "CephInputStream constructor: initializing stream with fh " + fh -+ + " and file length " + flength); -+ -+ } -+ -+ /** Ceph likes things to be closed before it shuts down, -+ * so closing the IOStream stuff voluntarily in a finalizer is good -+ */ -+ protected void finalize() throws Throwable { -+ try { -+ if (!closed) { -+ close(); -+ } -+ } finally { -+ super.finalize(); -+ } -+ } -+ -+ private synchronized boolean fillBuffer() throws IOException { -+ bufValid = ceph.ceph_read(fileHandle, buffer, 0, buffer.length); -+ bufPos = 0; -+ if (bufValid < 0) { -+ int err = bufValid; -+ -+ bufValid = 0; -+ // attempt to reset to old position. If it fails, too bad. -+ ceph.ceph_seek_from_start(fileHandle, cephPos); -+ throw new IOException("Failed to fill read buffer! Error code:" + err); -+ } -+ cephPos += bufValid; -+ return (bufValid != 0); -+ } -+ -+ /* -+ * Get the current position of the stream. -+ */ -+ public synchronized long getPos() throws IOException { -+ return cephPos - bufValid + bufPos; -+ } -+ -+ /** -+ * Find the number of bytes remaining in the file. -+ */ -+ @Override -+ public synchronized int available() throws IOException { -+ return (int) (fileLength - getPos()); -+ } -+ -+ public synchronized void seek(long targetPos) throws IOException { -+ LOG.trace( -+ "CephInputStream.seek: Seeking to position " + targetPos + " on fd " -+ + fileHandle); -+ if (targetPos > fileLength) { -+ throw new IOException( -+ "CephInputStream.seek: failed seek to position " + targetPos -+ + " on fd " + fileHandle + ": Cannot seek after EOF " + fileLength); -+ } -+ long oldPos = cephPos; -+ -+ cephPos = ceph.ceph_seek_from_start(fileHandle, targetPos); -+ bufValid = 0; -+ bufPos = 0; -+ if (cephPos < 0) { -+ cephPos = oldPos; -+ throw new IOException("Ceph failed to seek to new position!"); -+ } -+ } -+ -+ /** -+ * Failovers are handled by the Ceph code at a very low level; -+ * if there are issues that can be solved by changing sources -+ * they'll be dealt with before anybody even tries to call this method! -+ * @return false. -+ */ -+ public synchronized boolean seekToNewSource(long targetPos) { -+ return false; -+ } -+ -+ /** -+ * Read a byte from the file. -+ * @return the next byte. -+ */ -+ @Override -+ public synchronized int read() throws IOException { -+ LOG.trace( -+ "CephInputStream.read: Reading a single byte from fd " + fileHandle -+ + " by calling general read function"); -+ -+ byte result[] = new byte[1]; -+ -+ if (getPos() >= fileLength) { -+ return -1; -+ } -+ if (-1 == read(result, 0, 1)) { -+ return -1; -+ } -+ if (result[0] < 0) { -+ return 256 + (int) result[0]; -+ } else { -+ return result[0]; -+ } -+ } -+ -+ /** -+ * Read a specified number of bytes from the file into a byte[]. -+ * @param buf the byte array to read into. -+ * @param off the offset to start at in the file -+ * @param len the number of bytes to read -+ * @return 0 if successful, otherwise an error code. -+ * @throws IOException on bad input. -+ */ -+ @Override -+ public synchronized int read(byte buf[], int off, int len) -+ throws IOException { -+ LOG.trace( -+ "CephInputStream.read: Reading " + len + " bytes from fd " + fileHandle); -+ -+ if (closed) { -+ throw new IOException( -+ "CephInputStream.read: cannot read " + len + " bytes from fd " -+ + fileHandle + ": stream closed"); -+ } -+ -+ // ensure we're not past the end of the file -+ if (getPos() >= fileLength) { -+ LOG.debug( -+ "CephInputStream.read: cannot read " + len + " bytes from fd " -+ + fileHandle + ": current position is " + getPos() -+ + " and file length is " + fileLength); -+ -+ return -1; -+ } -+ -+ int totalRead = 0; -+ int initialLen = len; -+ int read; -+ -+ do { -+ read = Math.min(len, bufValid - bufPos); -+ try { -+ System.arraycopy(buffer, bufPos, buf, off, read); -+ } catch (IndexOutOfBoundsException ie) { -+ throw new IOException( -+ "CephInputStream.read: Indices out of bounds:" + "read length is " -+ + len + ", buffer offset is " + off + ", and buffer size is " -+ + buf.length); -+ } catch (ArrayStoreException ae) { -+ throw new IOException( -+ "Uh-oh, CephInputStream failed to do an array" -+ + "copy due to type mismatch..."); -+ } catch (NullPointerException ne) { -+ throw new IOException( -+ "CephInputStream.read: cannot read " + len + "bytes from fd:" -+ + fileHandle + ": buf is null"); -+ } -+ bufPos += read; -+ len -= read; -+ off += read; -+ totalRead += read; -+ } while (len > 0 && fillBuffer()); -+ -+ LOG.trace( -+ "CephInputStream.read: Reading " + initialLen + " bytes from fd " -+ + fileHandle + ": succeeded in reading " + totalRead + " bytes"); -+ return totalRead; -+ } -+ -+ /** -+ * Close the CephInputStream and release the associated filehandle. -+ */ -+ @Override -+ public void close() throws IOException { -+ LOG.trace("CephOutputStream.close:enter"); -+ if (!closed) { -+ int result = ceph.ceph_close(fileHandle); -+ -+ closed = true; -+ if (result != 0) { -+ throw new IOException( -+ "Close somehow failed!" -+ + "Don't try and use this stream again, though"); -+ } -+ LOG.trace("CephOutputStream.close:exit"); -+ } -+ } -+} -diff --git a/src/core/org/apache/hadoop/fs/ceph/CephOutputStream.java b/src/core/org/apache/hadoop/fs/ceph/CephOutputStream.java -new file mode 100644 -index 0000000..4c50f88 ---- /dev/null -+++ b/src/core/org/apache/hadoop/fs/ceph/CephOutputStream.java -@@ -0,0 +1,219 @@ -+// -*- mode:Java; tab-width:2; c-basic-offset:2; indent-tabs-mode:t -*- -+ -+/** -+ * -+ * Licensed under the Apache License, Version 2.0 -+ * (the "License"); you may not use this file except in compliance with -+ * the License. You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -+ * implied. See the License for the specific language governing -+ * permissions and limitations under the License. -+ * -+ * -+ * Implements the Hadoop FS interfaces to allow applications to store -+ * files in Ceph. -+ */ -+ -+package org.apache.hadoop.fs.ceph; -+ -+ -+import java.io.IOException; -+import java.io.OutputStream; -+ -+import org.apache.commons.logging.Log; -+import org.apache.commons.logging.LogFactory; -+import org.apache.hadoop.conf.Configuration; -+import org.apache.hadoop.util.Progressable; -+ -+ -+/** -+ * <p> -+ * An {@link OutputStream} for a CephFileSystem and corresponding -+ * Ceph instance. -+ */ -+public class CephOutputStream extends OutputStream { -+ private static final Log LOG = LogFactory.getLog(CephOutputStream.class); -+ private boolean closed; -+ -+ private CephFS ceph; -+ -+ private int fileHandle; -+ -+ private byte[] buffer; -+ private int bufUsed = 0; -+ -+ /** -+ * Construct the CephOutputStream. -+ * @param conf The FileSystem configuration. -+ * @param fh The Ceph filehandle to connect to. -+ */ -+ public CephOutputStream(Configuration conf, CephFS cephfs, -+ int fh, int bufferSize) { -+ ceph = cephfs; -+ fileHandle = fh; -+ closed = false; -+ buffer = new byte[bufferSize]; -+ } -+ -+ /** Ceph likes things to be closed before it shuts down, -+ *so closing the IOStream stuff voluntarily is good -+ */ -+ protected void finalize() throws Throwable { -+ try { -+ if (!closed) { -+ close(); -+ } -+ } finally { -+ super.finalize(); -+ } -+ } -+ -+ /** -+ * Get the current position in the file. -+ * @return The file offset in bytes. -+ */ -+ public long getPos() throws IOException { -+ return ceph.ceph_getpos(fileHandle); -+ } -+ -+ /** -+ * Write a byte. -+ * @param b The byte to write. -+ * @throws IOException If you have closed the CephOutputStream or the -+ * write fails. -+ */ -+ @Override -+ public synchronized void write(int b) throws IOException { -+ LOG.trace( -+ "CephOutputStream.write: writing a single byte to fd " + fileHandle); -+ -+ if (closed) { -+ throw new IOException( -+ "CephOutputStream.write: cannot write " + "a byte to fd " + fileHandle -+ + ": stream closed"); -+ } -+ // Stick the byte in a buffer and write it -+ byte buf[] = new byte[1]; -+ -+ buf[0] = (byte) b; -+ write(buf, 0, 1); -+ return; -+ } -+ -+ /** -+ * Write a byte buffer into the Ceph file. -+ * @param buf the byte array to write from -+ * @param off the position in the file to start writing at. -+ * @param len The number of bytes to actually write. -+ * @throws IOException if you have closed the CephOutputStream, or -+ * if buf is null or off + len > buf.length, or -+ * if the write fails due to a Ceph error. -+ */ -+ @Override -+ public synchronized void write(byte buf[], int off, int len) throws IOException { -+ LOG.trace( -+ "CephOutputStream.write: writing " + len + " bytes to fd " + fileHandle); -+ // make sure stream is open -+ if (closed) { -+ throw new IOException( -+ "CephOutputStream.write: cannot write " + len + "bytes to fd " -+ + fileHandle + ": stream closed"); -+ } -+ -+ int result; -+ int write; -+ -+ while (len > 0) { -+ write = Math.min(len, buffer.length - bufUsed); -+ try { -+ System.arraycopy(buf, off, buffer, bufUsed, write); -+ } catch (IndexOutOfBoundsException ie) { -+ throw new IOException( -+ "CephOutputStream.write: Indices out of bounds: " -+ + "write length is " + len + ", buffer offset is " + off -+ + ", and buffer size is " + buf.length); -+ } catch (ArrayStoreException ae) { -+ throw new IOException( -+ "Uh-oh, CephOutputStream failed to do an array" -+ + " copy due to type mismatch..."); -+ } catch (NullPointerException ne) { -+ throw new IOException( -+ "CephOutputStream.write: cannot write " + len + "bytes to fd " -+ + fileHandle + ": buffer is null"); -+ } -+ bufUsed += write; -+ len -= write; -+ off += write; -+ if (bufUsed == buffer.length) { -+ result = ceph.ceph_write(fileHandle, buffer, 0, bufUsed); -+ if (result < 0) { -+ throw new IOException( -+ "CephOutputStream.write: Buffered write of " + bufUsed -+ + " bytes failed!"); -+ } -+ if (result != bufUsed) { -+ throw new IOException( -+ "CephOutputStream.write: Wrote only " + result + " bytes of " -+ + bufUsed + " in buffer! Data may be lost or written" -+ + " twice to Ceph!"); -+ } -+ bufUsed = 0; -+ } -+ -+ } -+ return; -+ } -+ -+ /** -+ * Flush the buffered data. -+ * @throws IOException if you've closed the stream or the write fails. -+ */ -+ @Override -+ public synchronized void flush() throws IOException { -+ if (!closed) { -+ if (bufUsed == 0) { -+ return; -+ } -+ int result = ceph.ceph_write(fileHandle, buffer, 0, bufUsed); -+ -+ if (result < 0) { -+ throw new IOException( -+ "CephOutputStream.write: Write of " + bufUsed + "bytes to fd " -+ + fileHandle + " failed"); -+ } -+ if (result != bufUsed) { -+ throw new IOException( -+ "CephOutputStream.write: Write of " + bufUsed + "bytes to fd " -+ + fileHandle + "was incomplete: only " + result + " of " + bufUsed -+ + " bytes were written."); -+ } -+ bufUsed = 0; -+ return; -+ } -+ } -+ -+ /** -+ * Close the CephOutputStream. -+ * @throws IOException if Ceph somehow returns an error. In current code it can't. -+ */ -+ @Override -+ public synchronized void close() throws IOException { -+ LOG.trace("CephOutputStream.close:enter"); -+ if (!closed) { -+ flush(); -+ int result = ceph.ceph_close(fileHandle); -+ -+ if (result != 0) { -+ throw new IOException("Close failed!"); -+ } -+ -+ closed = true; -+ LOG.trace("CephOutputStream.close:exit"); -+ } -+ } -+} -diff --git a/src/core/org/apache/hadoop/fs/ceph/CephTalker.java b/src/core/org/apache/hadoop/fs/ceph/CephTalker.java -new file mode 100644 -index 0000000..569652f ---- /dev/null -+++ b/src/core/org/apache/hadoop/fs/ceph/CephTalker.java -@@ -0,0 +1,91 @@ -+// -*- mode:Java; tab-width:2; c-basic-offset:2; indent-tabs-mode:t -*- -+ -+/** -+ * -+ * Licensed under the Apache License, Version 2.0 -+ * (the "License"); you may not use this file except in compliance with -+ * the License. You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -+ * implied. See the License for the specific language governing -+ * permissions and limitations under the License. -+ * -+ * -+ * Wraps a number of native function calls to communicate with the Ceph -+ * filesystem. -+ */ -+package org.apache.hadoop.fs.ceph; -+ -+ -+import org.apache.hadoop.conf.Configuration; -+import org.apache.commons.logging.Log; -+ -+ -+class CephTalker extends CephFS { -+ // JNI doesn't give us any way to store pointers, so use a long. -+ // Here we're assuming pointers aren't longer than 8 bytes. -+ long cluster; -+ -+ // we write a constructor so we can load the libraries -+ public CephTalker(Configuration conf, Log log) { -+ System.load(conf.get("fs.ceph.libDir") + "/libcephfs.so"); -+ System.load(conf.get("fs.ceph.libDir") + "/libhadoopcephfs.so"); -+ cluster = 0; -+ } -+ -+ protected native boolean ceph_initializeClient(String arguments, int block_size); -+ -+ protected native String ceph_getcwd(); -+ -+ protected native boolean ceph_setcwd(String path); -+ -+ protected native boolean ceph_rmdir(String path); -+ -+ protected native boolean ceph_unlink(String path); -+ -+ protected native boolean ceph_rename(String old_path, String new_path); -+ -+ protected native boolean ceph_exists(String path); -+ -+ protected native long ceph_getblocksize(String path); -+ -+ protected native boolean ceph_isdirectory(String path); -+ -+ protected native boolean ceph_isfile(String path); -+ -+ protected native String[] ceph_getdir(String path); -+ -+ protected native int ceph_mkdirs(String path, int mode); -+ -+ protected native int ceph_open_for_append(String path); -+ -+ protected native int ceph_open_for_read(String path); -+ -+ protected native int ceph_open_for_overwrite(String path, int mode); -+ -+ protected native int ceph_close(int filehandle); -+ -+ protected native boolean ceph_setPermission(String path, int mode); -+ -+ protected native boolean ceph_kill_client(); -+ -+ protected native boolean ceph_stat(String path, CephFileSystem.Stat fill); -+ -+ protected native int ceph_replication(String Path); -+ -+ protected native String[] ceph_hosts(int fh, long offset); -+ -+ protected native int ceph_setTimes(String path, long mtime, long atime); -+ -+ protected native long ceph_getpos(int fh); -+ -+ protected native int ceph_write(int fh, byte[] buffer, int buffer_offset, int length); -+ -+ protected native int ceph_read(int fh, byte[] buffer, int buffer_offset, int length); -+ -+ protected native long ceph_seek_from_start(int fh, long pos); -+} -diff --git a/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java b/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java -index 9e22f1f..cd55361 100644 ---- a/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java -+++ b/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java -@@ -386,10 +386,12 @@ public class TrackerDistributedCacheManager { - if (modifiedTime != desiredTimestamp) { - DateFormat df = DateFormat.getDateTimeInstance(DateFormat.SHORT, - DateFormat.SHORT); -+ /* - throw new IOException("The distributed cache object " + source + - " changed during the job from " + - df.format(new Date(desiredTimestamp)) + " to " + - df.format(new Date(modifiedTime))); -+ */ - } - - Path parchive = null; -diff --git a/src/test/commit-tests b/src/test/commit-tests -index 1148c8b..85fa53d 100644 ---- a/src/test/commit-tests -+++ b/src/test/commit-tests -@@ -53,6 +53,7 @@ - **/TestRPC.java - **/TestS3Credentials.java - **/TestS3FileSystem.java -+**/TestCeph.java - **/TestSaslRPC.java - **/TestScriptBasedMapping.java - **/TestSequenceFileSerialization.java -diff --git a/src/test/org/apache/hadoop/fs/ceph/TestCeph.java b/src/test/org/apache/hadoop/fs/ceph/TestCeph.java -new file mode 100644 -index 0000000..e46b0ee ---- /dev/null -+++ b/src/test/org/apache/hadoop/fs/ceph/TestCeph.java -@@ -0,0 +1,45 @@ -+// -*- mode:Java; tab-width:2; c-basic-offset:2; indent-tabs-mode:t -*- -+ -+/** -+ * Licensed to the Apache Software Foundation (ASF) under one -+ * or more contributor license agreements. See the NOTICE file -+ * distributed with this work for additional information -+ * regarding copyright ownership. The ASF licenses this file -+ * to you under the Apache License, Version 2.0 (the -+ * "License"); you may not use this file except in compliance -+ * with the License. You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+ * See the License for the specific language governing permissions and -+ * limitations under the License. -+ * -+ * Unit tests for the CephFileSystem API implementation. -+ */ -+ -+package org.apache.hadoop.fs.ceph; -+ -+ -+import java.io.IOException; -+import java.net.URI; -+import org.apache.hadoop.conf.Configuration; -+import org.apache.hadoop.fs.FileSystemContractBaseTest; -+import org.apache.hadoop.fs.FileSystem; -+import org.apache.hadoop.fs.Path; -+ -+ -+public class TestCeph extends FileSystemContractBaseTest { -+ -+ @Override -+ protected void setUp() throws IOException { -+ Configuration conf = new Configuration(); -+ CephFaker cephfaker = new CephFaker(conf, FileSystem.LOG); -+ CephFileSystem cephfs = new CephFileSystem(cephfaker); -+ -+ cephfs.initialize(URI.create("ceph://null"), conf); -+ fs = cephfs; -+ } -+} diff --git a/src/client/hadoop/Readme b/src/client/hadoop/Readme deleted file mode 100644 index 2967b96cf5a..00000000000 --- a/src/client/hadoop/Readme +++ /dev/null @@ -1,17 +0,0 @@ -This directory contains: -CephFSInterface.cc/h: A C++ JNI library used by the Hadoop Java code. -ceph: A directory containing all the Java source files for a -Hadoop-compliant CephFileSystem. -HADOOP-ceph.patch: A patch for Hadoop. It should apply fine to one of the -.20 branches. (It was generated against .20.205.0) This -patch adds in all the files contained in the ceph dir as well as making -some changes so that Hadoop's configuration code will recognize the -CephFileSystem properties and classes. It is possible that this will be -out-of-date compared to the files contained in the ceph dir, so you -should apply this patch and then copy ceph/* into the appropriate Hadoop -dir. - -There are also a number of javah-generated C header files which are used -in writing CephFSInterface but can be safely ignored otherwise. - -Configuration instructions are included in Javadoc format in the ceph dir. diff --git a/src/client/hadoop/ceph/CephFS.java b/src/client/hadoop/ceph/CephFS.java deleted file mode 100644 index 5d51eb21600..00000000000 --- a/src/client/hadoop/ceph/CephFS.java +++ /dev/null @@ -1,250 +0,0 @@ -// -*- mode:Java; tab-width:2; c-basic-offset:2; indent-tabs-mode:t -*- - -/** - * - * Licensed under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. - * - * - * Abstract base class for communicating with a Ceph filesystem and its - * C++ codebase from Java, or pretending to do so (for unit testing purposes). - * As only the Ceph package should be using this directly, all methods - * are protected. - */ -package org.apache.hadoop.fs.ceph; - -import org.apache.hadoop.conf.Configuration; - -abstract class CephFS { - - protected static final int ENOTDIR = 20; - protected static final int EEXIST = 17; - protected static final int ENOENT = 2; - - /* - * Performs any necessary setup to allow general use of the filesystem. - * Inputs: - * String argsuments -- a command-line style input of Ceph config params - * int block_size -- the size in bytes to use for blocks - * Returns: true on success, false otherwise - */ - abstract protected boolean ceph_initializeClient(String arguments, int block_size); - - /* - * Returns the current working directory (absolute) as a String - */ - abstract protected String ceph_getcwd(); - - /* - * Changes the working directory. - * Inputs: - * String path: The path (relative or absolute) to switch to - * Returns: true on success, false otherwise. - */ - abstract protected boolean ceph_setcwd(String path); - - /* - * Given a path to a directory, removes the directory if empty. - * Inputs: - * jstring j_path: The path (relative or absolute) to the directory - * Returns: true on successful delete; false otherwise - */ - abstract protected boolean ceph_rmdir(String path); - - /* - * Given a path, unlinks it. - * Inputs: - * String path: The path (relative or absolute) to the file or empty dir - * Returns: true if the unlink occurred, false otherwise. - */ - abstract protected boolean ceph_unlink(String path); - - /* - * Changes a given path name to a new name, assuming new_path doesn't exist. - * Inputs: - * jstring j_from: The path whose name you want to change. - * jstring j_to: The new name for the path. - * Returns: true if the rename occurred, false otherwise - */ - abstract protected boolean ceph_rename(String old_path, String new_path); - - /* - * Returns true if it the input path exists, false - * if it does not or there is an unexpected failure. - */ - abstract protected boolean ceph_exists(String path); - - /* - * Get the block size for a given path. - * Input: - * String path: The path (relative or absolute) you want - * the block size for. - * Returns: block size if the path exists, otherwise a negative number - * corresponding to the standard C++ error codes (which are positive). - */ - abstract protected long ceph_getblocksize(String path); - - /* - * Returns true if the given path is a directory, false otherwise. - */ - abstract protected boolean ceph_isdirectory(String path); - - /* - * Returns true if the given path is a file; false otherwise. - */ - abstract protected boolean ceph_isfile(String path); - - /* - * Get the contents of a given directory. - * Inputs: - * String path: The path (relative or absolute) to the directory. - * Returns: A Java String[] of the contents of the directory, or - * NULL if there is an error (ie, path is not a dir). This listing - * will not contain . or .. entries. - */ - abstract protected String[] ceph_getdir(String path); - - /* - * Create the specified directory and any required intermediate ones with the - * given mode. - */ - abstract protected int ceph_mkdirs(String path, int mode); - - /* - * Open a file to append. If the file does not exist, it will be created. - * Opening a dir is possible but may have bad results. - * Inputs: - * String path: The path to open. - * Returns: an int filehandle, or a number<0 if an error occurs. - */ - abstract protected int ceph_open_for_append(String path); - - /* - * Open a file for reading. - * Opening a dir is possible but may have bad results. - * Inputs: - * String path: The path to open. - * Returns: an int filehandle, or a number<0 if an error occurs. - */ - abstract protected int ceph_open_for_read(String path); - - /* - * Opens a file for overwriting; creates it if necessary. - * Opening a dir is possible but may have bad results. - * Inputs: - * String path: The path to open. - * int mode: The mode to open with. - * Returns: an int filehandle, or a number<0 if an error occurs. - */ - abstract protected int ceph_open_for_overwrite(String path, int mode); - - /* - * Closes the given file. Returns 0 on success, or a negative - * error code otherwise. - */ - abstract protected int ceph_close(int filehandle); - - /* - * Change the mode on a path. - * Inputs: - * String path: The path to change mode on. - * int mode: The mode to apply. - * Returns: true if the mode is properly applied, false if there - * is any error. - */ - abstract protected boolean ceph_setPermission(String path, int mode); - - /* - * Closes the Ceph client. This should be called before shutting down - * (multiple times is okay but redundant). - */ - abstract protected boolean ceph_kill_client(); - - /* - * Get the statistics on a path returned in a custom format defined - * in CephFileSystem. - * Inputs: - * String path: The path to stat. - * Stat fill: The stat object to fill. - * Returns: true if the stat is successful, false otherwise. - */ - abstract protected boolean ceph_stat(String path, CephFileSystem.Stat fill); - - /* - * Check how many times a file should be replicated. If it is, - * degraded it may not actually be replicated this often. - * Inputs: - * int fh: a file descriptor - * Returns: an int containing the number of times replicated. - */ - abstract protected int ceph_replication(String path); - - /* - * Find the IP address of the primary OSD for a given file and offset. - * Inputs: - * int fh: The filehandle for the file. - * long offset: The offset to get the location of. - * Returns: an array of String of the location as IP, or NULL if there is an error. - */ - abstract protected String[] ceph_hosts(int fh, long offset); - - /* - * Set the mtime and atime for a given path. - * Inputs: - * String path: The path to set the times for. - * long mtime: The mtime to set, in millis since epoch (-1 to not set). - * long atime: The atime to set, in millis since epoch (-1 to not set) - * Returns: 0 if successful, an error code otherwise. - */ - abstract protected int ceph_setTimes(String path, long mtime, long atime); - - /* - * Get the current position in a file (as a long) of a given filehandle. - * Returns: (long) current file position on success, or a - * negative error code on failure. - */ - abstract protected long ceph_getpos(int fh); - - /* - * Write the given buffer contents to the given filehandle. - * Inputs: - * int fh: The filehandle to write to. - * byte[] buffer: The buffer to write from - * int buffer_offset: The position in the buffer to write from - * int length: The number of (sequential) bytes to write. - * Returns: int, on success the number of bytes written, on failure - * a negative error code. - */ - abstract protected int ceph_write(int fh, byte[] buffer, int buffer_offset, int length); - - /* - * Reads into the given byte array from the current position. - * Inputs: - * int fh: the filehandle to read from - * byte[] buffer: the byte array to read into - * int buffer_offset: where in the buffer to start writing - * int length: how much to read. - * There'd better be enough space in the buffer to write all - * the data from the given offset! - * Returns: the number of bytes read on success (as an int), - * or an error code otherwise. */ - abstract protected int ceph_read(int fh, byte[] buffer, int buffer_offset, int length); - - /* - * Seeks to the given position in the given file. - * Inputs: - * int fh: The filehandle to seek in. - * long pos: The position to seek to. - * Returns: the new position (as a long) of the filehandle on success, - * or a negative error code on failure. */ - abstract protected long ceph_seek_from_start(int fh, long pos); -} diff --git a/src/client/hadoop/ceph/CephFaker.java b/src/client/hadoop/ceph/CephFaker.java deleted file mode 100644 index c598f536039..00000000000 --- a/src/client/hadoop/ceph/CephFaker.java +++ /dev/null @@ -1,483 +0,0 @@ -// -*- mode:Java; tab-width:2; c-basic-offset:2; indent-tabs-mode:t -*- - -/** - * - * Licensed under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. - * - * - * This uses the local Filesystem but pretends to be communicating - * with a Ceph deployment, for unit testing the CephFileSystem. - */ - -package org.apache.hadoop.fs.ceph; - - -import java.net.URI; -import java.util.Hashtable; -import java.io.Closeable; -import java.io.FileNotFoundException; -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.BlockLocation; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; - - -class CephFaker extends CephFS { - private static final Log LOG = LogFactory.getLog(CephFaker.class); - FileSystem localFS; - String localPrefix; - int blockSize; - Configuration conf; - Hashtable<Integer, Object> files; - Hashtable<Integer, String> filenames; - int fileCount = 0; - boolean initialized = false; - - public CephFaker(Configuration con, Log log) { - conf = con; - files = new Hashtable<Integer, Object>(); - filenames = new Hashtable<Integer, String>(); - } - - protected boolean ceph_initializeClient(String args, int block_size) { - if (!initialized) { - // let's remember the default block_size - blockSize = block_size; - - /* for a real Ceph deployment, this starts up the client, - * sets debugging levels, etc. We just need to get the - * local FileSystem to use, and we'll ignore any - * command-line arguments. */ - try { - localFS = FileSystem.getLocal(conf); - localFS.initialize(URI.create("file://localhost"), conf); - localFS.setVerifyChecksum(false); - String testDir = conf.get("hadoop.tmp.dir"); - - localPrefix = localFS.getWorkingDirectory().toString(); - int testDirLoc = localPrefix.indexOf(testDir) - 1; - - if (-2 == testDirLoc) { - testDirLoc = localPrefix.length(); - } - localPrefix = localPrefix.substring(0, testDirLoc) + "/" - + conf.get("hadoop.tmp.dir"); - - localFS.setWorkingDirectory( - new Path(localPrefix + "/user/" + System.getProperty("user.name"))); - // I don't know why, but the unit tests expect the default - // working dir to be /user/username, so satisfy them! - // debug("localPrefix is " + localPrefix, INFO); - } catch (IOException e) { - return false; - } - initialized = true; - } - return true; - } - - protected String ceph_getcwd() { - return sanitize_path(localFS.getWorkingDirectory().toString()); - } - - protected boolean ceph_setcwd(String path) { - localFS.setWorkingDirectory(new Path(prepare_path(path))); - return true; - } - - // the caller is responsible for ensuring empty dirs - protected boolean ceph_rmdir(String pth) { - Path path = new Path(prepare_path(pth)); - boolean ret = false; - - try { - if (localFS.listStatus(path).length <= 1) { - ret = localFS.delete(path, true); - } - } catch (IOException e) {} - return ret; - } - - // this needs to work on (empty) directories too - protected boolean ceph_unlink(String path) { - path = prepare_path(path); - boolean ret = false; - - if (ceph_isdirectory(path)) { - ret = ceph_rmdir(path); - } else { - try { - ret = localFS.delete(new Path(path), false); - } catch (IOException e) {} - } - return ret; - } - - protected boolean ceph_rename(String oldName, String newName) { - oldName = prepare_path(oldName); - newName = prepare_path(newName); - try { - Path parent = new Path(newName).getParent(); - Path newPath = new Path(newName); - - if (localFS.exists(parent) && !localFS.exists(newPath)) { - return localFS.rename(new Path(oldName), newPath); - } - return false; - } catch (IOException e) { - return false; - } - } - - protected boolean ceph_exists(String path) { - path = prepare_path(path); - boolean ret = false; - - try { - ret = localFS.exists(new Path(path)); - } catch (IOException e) {} - return ret; - } - - protected long ceph_getblocksize(String path) { - path = prepare_path(path); - try { - FileStatus status = localFS.getFileStatus(new Path(path)); - - return status.getBlockSize(); - } catch (FileNotFoundException e) { - return -CephFS.ENOENT; - } catch (IOException e) { - return -1; // just fail generically - } - } - - protected boolean ceph_isdirectory(String path) { - path = prepare_path(path); - try { - FileStatus status = localFS.getFileStatus(new Path(path)); - - return status.isDir(); - } catch (IOException e) { - return false; - } - } - - protected boolean ceph_isfile(String path) { - path = prepare_path(path); - boolean ret = false; - - try { - FileStatus status = localFS.getFileStatus(new Path(path)); - - ret = !status.isDir(); - } catch (Exception e) {} - return ret; - } - - protected String[] ceph_getdir(String path) { - path = prepare_path(path); - if (!ceph_isdirectory(path)) { - return null; - } - try { - FileStatus[] stats = localFS.listStatus(new Path(path)); - String[] names = new String[stats.length]; - String name; - - for (int i = 0; i < stats.length; ++i) { - name = stats[i].getPath().toString(); - names[i] = name.substring(name.lastIndexOf(Path.SEPARATOR) + 1); - } - return names; - } catch (IOException e) {} - return null; - } - - protected int ceph_mkdirs(String path, int mode) { - path = prepare_path(path); - // debug("ceph_mkdirs on " + path, INFO); - try { - if (localFS.mkdirs(new Path(path), new FsPermission((short) mode))) { - return 0; - } - } catch (IOException e) {} - if (ceph_isdirectory(path)) { // apparently it already existed - return -EEXIST; - } else if (ceph_isfile(path)) { - return -ENOTDIR; - } - return -1; - } - - /* - * Unlike a real Ceph deployment, you can't do opens on a directory. - * Since that has unpredictable behavior and you shouldn't do it anyway, - * it's okay. - */ - protected int ceph_open_for_append(String path) { - path = prepare_path(path); - FSDataOutputStream stream; - - try { - stream = localFS.append(new Path(path)); - files.put(new Integer(fileCount), stream); - filenames.put(new Integer(fileCount), path); - return fileCount++; - } catch (IOException e) {} - return -1; // failure - } - - protected int ceph_open_for_read(String path) { - path = prepare_path(path); - FSDataInputStream stream; - - try { - stream = localFS.open(new Path(path)); - files.put(new Integer(fileCount), stream); - filenames.put(new Integer(fileCount), path); - LOG.info("ceph_open_for_read fh:" + fileCount + ", pathname:" + path); - return fileCount++; - } catch (IOException e) {} - return -1; // failure - } - - protected int ceph_open_for_overwrite(String path, int mode) { - path = prepare_path(path); - FSDataOutputStream stream; - - try { - stream = localFS.create(new Path(path)); - files.put(new Integer(fileCount), stream); - filenames.put(new Integer(fileCount), path); - LOG.info("ceph_open_for_overwrite fh:" + fileCount + ", pathname:" + path); - return fileCount++; - } catch (IOException e) {} - return -1; // failure - } - - protected int ceph_close(int filehandle) { - LOG.info("ceph_close(filehandle " + filehandle + ")"); - try { - ((Closeable) files.get(new Integer(filehandle))).close(); - if (null == files.get(new Integer(filehandle))) { - return -ENOENT; // this isn't quite the right error code, - // but the important part is it's negative - } - return 0; // hurray, success - } catch (NullPointerException ne) { - LOG.warn("ceph_close caught NullPointerException!" + ne); - } // err, how? - catch (IOException ie) { - LOG.warn("ceph_close caught IOException!" + ie); - } - return -1; // failure - } - - protected boolean ceph_setPermission(String pth, int mode) { - pth = prepare_path(pth); - Path path = new Path(pth); - boolean ret = false; - - try { - localFS.setPermission(path, new FsPermission((short) mode)); - ret = true; - } catch (IOException e) {} - return ret; - } - - // rather than try and match a Ceph deployment's behavior exactly, - // just make bad things happen if they try and call methods after this - protected boolean ceph_kill_client() { - // debug("ceph_kill_client", INFO); - localFS.setWorkingDirectory(new Path(localPrefix)); - // debug("working dir is now " + localFS.getWorkingDirectory(), INFO); - try { - localFS.close(); - } catch (Exception e) {} - localFS = null; - files = null; - filenames = null; - return true; - } - - protected boolean ceph_stat(String pth, CephFileSystem.Stat fill) { - pth = prepare_path(pth); - Path path = new Path(pth); - boolean ret = false; - - try { - FileStatus status = localFS.getFileStatus(path); - - fill.size = status.getLen(); - fill.is_dir = status.isDir(); - fill.block_size = status.getBlockSize(); - fill.mod_time = status.getModificationTime(); - fill.access_time = status.getAccessTime(); - fill.mode = status.getPermission().toShort(); - ret = true; - } catch (IOException e) {} - return ret; - } - - protected int ceph_replication(String path) { - path = prepare_path(path); - int ret = -1; // -1 for failure - - try { - ret = localFS.getFileStatus(new Path(path)).getReplication(); - } catch (IOException e) {} - return ret; - } - - protected String[] ceph_hosts(int fh, long offset) { - String[] ret = null; - - try { - BlockLocation[] locs = localFS.getFileBlockLocations( - localFS.getFileStatus(new Path(filenames.get(new Integer(fh)))), - offset, 1); - - ret = locs[0].getNames(); - } catch (IOException e) {} catch (NullPointerException f) {} - return ret; - } - - protected int ceph_setTimes(String pth, long mtime, long atime) { - pth = prepare_path(pth); - Path path = new Path(pth); - int ret = -1; // generic fail - - try { - localFS.setTimes(path, mtime, atime); - ret = 0; - } catch (IOException e) {} - return ret; - } - - protected long ceph_getpos(int fh) { - long ret = -1; // generic fail - - try { - Object stream = files.get(new Integer(fh)); - - if (stream instanceof FSDataInputStream) { - ret = ((FSDataInputStream) stream).getPos(); - } else if (stream instanceof FSDataOutputStream) { - ret = ((FSDataOutputStream) stream).getPos(); - } - } catch (IOException e) {} catch (NullPointerException f) {} - return ret; - } - - protected int ceph_write(int fh, byte[] buffer, - int buffer_offset, int length) { - LOG.info( - "ceph_write fh:" + fh + ", buffer_offset:" + buffer_offset + ", length:" - + length); - long ret = -1; // generic fail - - try { - FSDataOutputStream os = (FSDataOutputStream) files.get(new Integer(fh)); - - LOG.info("ceph_write got outputstream"); - long startPos = os.getPos(); - - os.write(buffer, buffer_offset, length); - ret = os.getPos() - startPos; - } catch (IOException e) { - LOG.warn("ceph_write caught IOException!"); - } catch (NullPointerException f) { - LOG.warn("ceph_write caught NullPointerException!"); - } - return (int) ret; - } - - protected int ceph_read(int fh, byte[] buffer, - int buffer_offset, int length) { - long ret = -1; // generic fail - - try { - FSDataInputStream is = (FSDataInputStream) files.get(new Integer(fh)); - long startPos = is.getPos(); - - is.read(buffer, buffer_offset, length); - ret = is.getPos() - startPos; - } catch (IOException e) {} catch (NullPointerException f) {} - return (int) ret; - } - - protected long ceph_seek_from_start(int fh, long pos) { - LOG.info("ceph_seek_from_start(fh " + fh + ", pos " + pos + ")"); - long ret = -1; // generic fail - - try { - LOG.info("ceph_seek_from_start filename is " + filenames.get(new Integer(fh))); - if (null == files.get(new Integer(fh))) { - LOG.warn("ceph_seek_from_start: is is null!"); - } - FSDataInputStream is = (FSDataInputStream) files.get(new Integer(fh)); - - LOG.info("ceph_seek_from_start retrieved is!"); - is.seek(pos); - ret = is.getPos(); - } catch (IOException e) { - LOG.warn("ceph_seek_from_start caught IOException!"); - } catch (NullPointerException f) { - LOG.warn("ceph_seek_from_start caught NullPointerException!"); - } - return (int) ret; - } - - /* - * We need to remove the localFS file prefix before returning to Ceph - */ - private String sanitize_path(String path) { - // debug("sanitize_path(" + path + ")", INFO); - /* if (path.startsWith("file:")) - path = path.substring("file:".length()); */ - if (path.startsWith(localPrefix)) { - path = path.substring(localPrefix.length()); - if (path.length() == 0) { // it was a root path - path = "/"; - } - } - // debug("sanitize_path returning " + path, INFO); - return path; - } - - /* - * If it's an absolute path we need to shove the - * test dir onto the front as a prefix. - */ - private String prepare_path(String path) { - // debug("prepare_path(" + path + ")", INFO); - if (path.startsWith("/")) { - path = localPrefix + path; - } else if (path.equals("..")) { - if (ceph_getcwd().equals("/")) { - path = "."; - } // you can't go up past root! - } - // debug("prepare_path returning" + path, INFO); - return path; - } -} diff --git a/src/client/hadoop/ceph/CephFileSystem.java b/src/client/hadoop/ceph/CephFileSystem.java deleted file mode 100644 index 95f22238b4d..00000000000 --- a/src/client/hadoop/ceph/CephFileSystem.java +++ /dev/null @@ -1,804 +0,0 @@ -// -*- mode:Java; tab-width:2; c-basic-offset:2; indent-tabs-mode:t -*- - -/** - * - * Licensed under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. - * - * - * Implements the Hadoop FS interfaces to allow applications to store - * files in Ceph. - */ -package org.apache.hadoop.fs.ceph; - - -import java.io.IOException; -import java.io.FileNotFoundException; -import java.io.OutputStream; -import java.net.URI; -import java.net.InetAddress; -import java.util.EnumSet; -import java.lang.Math; -import java.util.ArrayList; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.BlockLocation; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.util.Progressable; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.net.DNS; - - -/** - * <p> - * A {@link FileSystem} backed by <a href="http://ceph.newdream.net">Ceph.</a>. - * This will not start a Ceph instance; one must already be running. - * </p> - * Configuration of the CephFileSystem is handled via a few Hadoop - * Configuration properties: <br> - * fs.ceph.monAddr -- the ip address/port of the monitor to connect to. <br> - * fs.ceph.libDir -- the directory that libcephfs and libhadoopceph are - * located in. This assumes Hadoop is being run on a linux-style machine - * with names like libcephfs.so. - * fs.ceph.commandLine -- if you prefer you can fill in this property - * just as you would when starting Ceph up from the command line. Specific - * properties override any configuration specified here. - * <p> - * You can also enable debugging of the CephFileSystem and Ceph itself: <br> - * fs.ceph.debug -- if 'true' will print out method enter/exit messages, - * plus a little more. - * fs.ceph.clientDebug/fs.ceph.messengerDebug -- will print out debugging - * from the respective Ceph system of at least that importance. - */ -public class CephFileSystem extends FileSystem { - private static final Log LOG = LogFactory.getLog(CephFileSystem.class); - private URI uri; - - private Path workingDir; - private final Path root; - private CephFS ceph = null; - - private static String CEPH_NAMESERVER; - private static final String CEPH_NAMESERVER_KEY = "fs.ceph.nameserver"; - private static final String CEPH_NAMESERVER_DEFAULT = "localhost"; - - /** - * Create a new CephFileSystem. - */ - public CephFileSystem() { - root = new Path("/"); - } - - /** - * Used for testing purposes, this constructor - * sets the given CephFS instead of defaulting to a - * CephTalker (with its assumed real Ceph instance to talk to). - */ - public CephFileSystem(CephFS ceph_fs) { - super(); - root = new Path("/"); - ceph = ceph_fs; - } - - /** - * Lets you get the URI of this CephFileSystem. - * @return the URI. - */ - public URI getUri() { - LOG.debug("getUri:exit with return " + uri); - return uri; - } - - /** - * Should be called after constructing a CephFileSystem but before calling - * any other methods. - * Starts up the connection to Ceph, reads in configuraton options, etc. - * @param uri The URI for this filesystem. - * @param conf The Hadoop Configuration to retrieve properties from. - * @throws IOException if necessary properties are unset. - */ - @Override - public void initialize(URI uri, Configuration conf) throws IOException { - super.initialize(uri, conf); - setConf(conf); - this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority()); - if (ceph == null) { - ceph = new CephTalker(conf, LOG); - } - - CEPH_NAMESERVER = conf.get(CEPH_NAMESERVER_KEY, CEPH_NAMESERVER_DEFAULT); - - // build up the arguments for Ceph - String arguments = "CephFSInterface"; - - arguments += conf.get("fs.ceph.commandLine", ""); - if (conf.get("fs.ceph.clientDebug") != null) { - arguments += " --debug_client "; - arguments += conf.get("fs.ceph.clientDebug"); - } - if (conf.get("fs.ceph.messengerDebug") != null) { - arguments += " --debug_ms "; - arguments += conf.get("fs.ceph.messengerDebug"); - } - if (conf.get("fs.ceph.monAddr") != null) { - arguments += " -m "; - arguments += conf.get("fs.ceph.monAddr"); - } - arguments += " --client-readahead-max-periods=" - + conf.get("fs.ceph.readahead", "1"); - // make sure they gave us a ceph monitor address or conf file - LOG.info("initialize:Ceph initialization arguments: " + arguments); - if ((conf.get("fs.ceph.monAddr") == null) && (arguments.indexOf("-m") == -1) - && (arguments.indexOf("-c") == -1)) { - LOG.fatal("initialize:You need to specify a Ceph monitor address."); - throw new IOException( - "You must specify a Ceph monitor address or config file!"); - } - // Initialize the client - if (!ceph.ceph_initializeClient(arguments, - conf.getInt("fs.ceph.blockSize", 1 << 26))) { - LOG.fatal("initialize:Ceph initialization failed!"); - throw new IOException("Ceph initialization failed!"); - } - LOG.info("initialize:Ceph initialized client. Setting cwd to /"); - ceph.ceph_setcwd("/"); - LOG.debug("initialize:exit"); - - this.workingDir = getHomeDirectory(); - } - - /** - * Close down the CephFileSystem. Runs the base-class close method - * and then kills the Ceph client itself. - */ - @Override - public void close() throws IOException { - LOG.debug("close:enter"); - super.close(); // this method does stuff, make sure it's run! - LOG.trace("close: Calling ceph_kill_client from Java"); - ceph.ceph_kill_client(); - LOG.debug("close:exit"); - } - - /** - * Get an FSDataOutputStream to append onto a file. - * @param file The File you want to append onto - * @param bufferSize Ceph does internal buffering but you can buffer in the Java code as well if you like. - * @param progress The Progressable to report progress to. - * Reporting is limited but exists. - * @return An FSDataOutputStream that connects to the file on Ceph. - * @throws IOException If the file cannot be found or appended to. - */ - public FSDataOutputStream append(Path file, int bufferSize, - Progressable progress) throws IOException { - LOG.debug("append:enter with path " + file + " bufferSize " + bufferSize); - Path abs_path = makeAbsolute(file); - - if (progress != null) { - progress.progress(); - } - LOG.trace("append: Entering ceph_open_for_append from Java"); - int fd = ceph.ceph_open_for_append(getCephPath(abs_path)); - - LOG.trace("append: Returned to Java"); - if (progress != null) { - progress.progress(); - } - if (fd < 0) { // error in open - throw new IOException( - "append: Open for append failed on path \"" + abs_path.toString() - + "\""); - } - CephOutputStream cephOStream = new CephOutputStream(getConf(), ceph, fd, - bufferSize); - - LOG.debug("append:exit"); - return new FSDataOutputStream(cephOStream, statistics); - } - - /** - * Get the current working directory for the given file system - * @return the directory Path - */ - public Path getWorkingDirectory() { - return workingDir; - } - - /** - * Set the current working directory for the given file system. All relative - * paths will be resolved relative to it. - * - * @param dir The directory to change to. - */ - @Override - public void setWorkingDirectory(Path dir) { - workingDir = makeAbsolute(dir); - } - - /** - * Return only the path component from a potentially fully qualified path. - */ - private String getCephPath(Path path) { - if (!path.isAbsolute()) { - throw new IllegalArgumentException("Path must be absolute: " + path); - } - return path.toUri().getPath(); - } - - /** - * Check if a path exists. - * Overriden because it's moderately faster than the generic implementation. - * @param path The file to check existence on. - * @return true if the file exists, false otherwise. - */ - @Override - public boolean exists(Path path) throws IOException { - LOG.debug("exists:enter with path " + path); - boolean result; - Path abs_path = makeAbsolute(path); - - if (abs_path.equals(root)) { - result = true; - } else { - LOG.trace( - "exists:Calling ceph_exists from Java on path " + abs_path.toString()); - result = ceph.ceph_exists(getCephPath(abs_path)); - LOG.trace("exists:Returned from ceph_exists to Java"); - } - LOG.debug("exists:exit with value " + result); - return result; - } - - /** - * Create a directory and any nonexistent parents. Any portion - * of the directory tree can exist without error. - * @param path The directory path to create - * @param perms The permissions to apply to the created directories. - * @return true if successful, false otherwise - * @throws IOException if the path is a child of a file. - */ - @Override - public boolean mkdirs(Path path, FsPermission perms) throws IOException { - LOG.debug("mkdirs:enter with path " + path); - Path abs_path = makeAbsolute(path); - - LOG.trace("mkdirs:calling ceph_mkdirs from Java"); - int result = ceph.ceph_mkdirs(getCephPath(abs_path), (int) perms.toShort()); - - if (result != 0) { - LOG.warn( - "mkdirs: make directory " + abs_path + "Failing with result " + result); - if (-ceph.ENOTDIR == result) { - throw new IOException("Parent path is not a directory"); - } - return false; - } else { - LOG.debug("mkdirs:exiting succesfully"); - return true; - } - } - - /** - * Check if a path is a file. This is moderately faster than the - * generic implementation. - * @param path The path to check. - * @return true if the path is definitely a file, false otherwise. - */ - @Override - public boolean isFile(Path path) throws IOException { - LOG.debug("isFile:enter with path " + path); - Path abs_path = makeAbsolute(path); - boolean result; - - if (abs_path.equals(root)) { - result = false; - } else { - LOG.trace("isFile:entering ceph_isfile from Java"); - result = ceph.ceph_isfile(getCephPath(abs_path)); - } - LOG.debug("isFile:exit with result " + result); - return result; - } - - /** - * Get stat information on a file. This does not fill owner or group, as - * Ceph's support for these is a bit different than HDFS'. - * @param path The path to stat. - * @return FileStatus object containing the stat information. - * @throws FileNotFoundException if the path could not be resolved. - */ - public FileStatus getFileStatus(Path path) throws IOException { - LOG.debug("getFileStatus:enter with path " + path); - Path abs_path = makeAbsolute(path); - // sadly, Ceph doesn't really do uids/gids just yet, but - // everything else is filled - FileStatus status; - Stat lstat = new Stat(); - - LOG.trace("getFileStatus: calling ceph_stat from Java"); - if (ceph.ceph_stat(getCephPath(abs_path), lstat)) { - status = new FileStatus(lstat.size, lstat.is_dir, - ceph.ceph_replication(getCephPath(abs_path)), lstat.block_size, - lstat.mod_time, lstat.access_time, - new FsPermission((short) lstat.mode), System.getProperty("user.name"), null, - path.makeQualified(this)); - } else { // fail out - throw new FileNotFoundException( - "org.apache.hadoop.fs.ceph.CephFileSystem: File " + path - + " does not exist or could not be accessed"); - } - - LOG.debug("getFileStatus:exit"); - return status; - } - - /** - * Get the FileStatus for each listing in a directory. - * @param path The directory to get listings from. - * @return FileStatus[] containing one FileStatus for each directory listing; - * null if path does not exist. - */ - public FileStatus[] listStatus(Path path) throws IOException { - LOG.debug("listStatus:enter with path " + path); - Path abs_path = makeAbsolute(path); - Path[] paths = listPaths(abs_path); - - if (paths != null) { - FileStatus[] statuses = new FileStatus[paths.length]; - - for (int i = 0; i < paths.length; ++i) { - statuses[i] = getFileStatus(paths[i]); - } - LOG.debug("listStatus:exit"); - return statuses; - } - - if (isFile(path)) { - return new FileStatus[] { getFileStatus(path) }; - } - - return null; - } - - @Override - public void setPermission(Path p, FsPermission permission) throws IOException { - LOG.debug( - "setPermission:enter with path " + p + " and permissions " + permission); - Path abs_path = makeAbsolute(p); - - LOG.trace("setPermission:calling ceph_setpermission from Java"); - ceph.ceph_setPermission(getCephPath(abs_path), permission.toShort()); - LOG.debug("setPermission:exit"); - } - - /** - * Set access/modification times of a file. - * @param p The path - * @param mtime Set modification time in number of millis since Jan 1, 1970. - * @param atime Set access time in number of millis since Jan 1, 1970. - */ - @Override - public void setTimes(Path p, long mtime, long atime) throws IOException { - LOG.debug( - "setTimes:enter with path " + p + " mtime:" + mtime + " atime:" + atime); - Path abs_path = makeAbsolute(p); - - LOG.trace("setTimes:calling ceph_setTimes from Java"); - int r = ceph.ceph_setTimes(getCephPath(abs_path), mtime, atime); - - if (r < 0) { - throw new IOException( - "Failed to set times on path " + abs_path.toString() + " Error code: " - + r); - } - LOG.debug("setTimes:exit"); - } - - /** - * Create a new file and open an FSDataOutputStream that's connected to it. - * @param path The file to create. - * @param permission The permissions to apply to the file. - * @param overwrite If true, overwrite any existing file with - * this name; otherwise don't. - * @param bufferSize Ceph does internal buffering, but you can buffer - * in the Java code too if you like. - * @param replication Ignored by Ceph. This can be - * configured via Ceph configuration. - * @param blockSize Ignored by Ceph. You can set client-wide block sizes - * via the fs.ceph.blockSize param if you like. - * @param progress A Progressable to report back to. - * Reporting is limited but exists. - * @return An FSDataOutputStream pointing to the created file. - * @throws IOException if the path is an - * existing directory, or the path exists but overwrite is false, or there is a - * failure in attempting to open for append with Ceph. - */ - public FSDataOutputStream create(Path path, - FsPermission permission, - boolean overwrite, - int bufferSize, - short replication, - long blockSize, - Progressable progress) throws IOException { - LOG.debug("create:enter with path " + path); - Path abs_path = makeAbsolute(path); - - if (progress != null) { - progress.progress(); - } - // We ignore replication since that's not configurable here, and - // progress reporting is quite limited. - // Required semantics: if the file exists, overwrite if 'overwrite' is set; - // otherwise, throw an exception - - // Step 1: existence test - boolean exists = exists(abs_path); - - if (exists) { - if (getFileStatus(abs_path).isDir()) { - throw new IOException( - "create: Cannot overwrite existing directory \"" + path.toString() - + "\" with a file"); - } - if (!overwrite) { - throw new IOException( - "createRaw: Cannot open existing file \"" + abs_path.toString() - + "\" for writing without overwrite flag"); - } - } - - if (progress != null) { - progress.progress(); - } - - // Step 2: create any nonexistent directories in the path - if (!exists) { - Path parent = abs_path.getParent(); - - if (parent != null) { // if parent is root, we're done - int r = ceph.ceph_mkdirs(getCephPath(parent), permission.toShort()); - - if (!(r == 0 || r == -ceph.EEXIST)) { - throw new IOException("Error creating parent directory; code: " + r); - } - } - if (progress != null) { - progress.progress(); - } - } - // Step 3: open the file - LOG.trace("calling ceph_open_for_overwrite from Java"); - int fh = ceph.ceph_open_for_overwrite(getCephPath(abs_path), - (int) permission.toShort()); - - if (progress != null) { - progress.progress(); - } - LOG.trace("Returned from ceph_open_for_overwrite to Java with fh " + fh); - if (fh < 0) { - throw new IOException( - "create: Open for overwrite failed on path \"" + path.toString() - + "\""); - } - - // Step 4: create the stream - OutputStream cephOStream = new CephOutputStream(getConf(), ceph, fh, - bufferSize); - - LOG.debug("create:exit"); - return new FSDataOutputStream(cephOStream, statistics); - } - - /** - * Open a Ceph file and attach the file handle to an FSDataInputStream. - * @param path The file to open - * @param bufferSize Ceph does internal buffering; but you can buffer in - * the Java code too if you like. - * @return FSDataInputStream reading from the given path. - * @throws IOException if the path DNE or is a - * directory, or there is an error getting data to set up the FSDataInputStream. - */ - public FSDataInputStream open(Path path, int bufferSize) throws IOException { - LOG.debug("open:enter with path " + path); - Path abs_path = makeAbsolute(path); - - int fh = ceph.ceph_open_for_read(getCephPath(abs_path)); - - if (fh < 0) { // uh-oh, something's bad! - if (fh == -ceph.ENOENT) { // well that was a stupid open - throw new IOException( - "open: absolute path \"" + abs_path.toString() - + "\" does not exist"); - } else { // hrm...the file exists but we can't open it :( - throw new IOException("open: Failed to open file " + abs_path.toString()); - } - } - - if (getFileStatus(abs_path).isDir()) { // yes, it is possible to open Ceph directories - // but that doesn't mean you should in Hadoop! - ceph.ceph_close(fh); - throw new IOException( - "open: absolute path \"" + abs_path.toString() + "\" is a directory!"); - } - Stat lstat = new Stat(); - - LOG.trace("open:calling ceph_stat from Java"); - ceph.ceph_stat(getCephPath(abs_path), lstat); - LOG.trace("open:returned to Java"); - long size = lstat.size; - - if (size < 0) { - throw new IOException( - "Failed to get file size for file " + abs_path.toString() - + " but succeeded in opening file. Something bizarre is going on."); - } - FSInputStream cephIStream = new CephInputStream(getConf(), ceph, fh, size, - bufferSize); - - LOG.debug("open:exit"); - return new FSDataInputStream(cephIStream); - } - - /** - * Rename a file or directory. - * @param src The current path of the file/directory - * @param dst The new name for the path. - * @return true if the rename succeeded, false otherwise. - */ - @Override - public boolean rename(Path src, Path dst) throws IOException { - LOG.debug("rename:enter with src:" + src + " and dest:" + dst); - Path abs_src = makeAbsolute(src); - Path abs_dst = makeAbsolute(dst); - - LOG.trace("calling ceph_rename from Java"); - boolean result = ceph.ceph_rename(getCephPath(abs_src), getCephPath(abs_dst)); - - if (!result) { - boolean isDir = false; - try { - isDir = getFileStatus(abs_dst).isDir(); - } catch (FileNotFoundException e) {} - if (isDir) { // move the srcdir into destdir - LOG.debug("ceph_rename failed but dst is a directory!"); - Path new_dst = new Path(abs_dst, abs_src.getName()); - - result = rename(abs_src, new_dst); - LOG.debug( - "attempt to move " + abs_src.toString() + " to " - + new_dst.toString() + "has result:" + result); - } - } - LOG.debug("rename:exit with result: " + result); - return result; - } - - /* - * Attempt to convert an IP into its hostname - */ - private String[] ips2Hosts(String[] ips) { - ArrayList<String> hosts = new ArrayList<String>(); - for (String ip : ips) { - try { - String host = DNS.reverseDns(InetAddress.getByName(ip), CEPH_NAMESERVER); - if (host.charAt(host.length()-1) == '.') { - host = host.substring(0, host.length()-1); - } - hosts.add(host); /* append */ - } catch (Exception e) { - LOG.error("reverseDns ["+ip+"] failed: "+ e); - } - } - return hosts.toArray(new String[hosts.size()]); - } - - /** - * Get a BlockLocation object for each block in a file. - * - * Note that this doesn't include port numbers in the name field as - * Ceph handles slow/down servers internally. This data should be used - * only for selecting which servers to run which jobs on. - * - * @param file A FileStatus object corresponding to the file you want locations for. - * @param start The offset of the first part of the file you are interested in. - * @param len The amount of the file past the offset you are interested in. - * @return A BlockLocation[] where each object corresponds to a block within - * the given range. - */ - @Override - public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException { - Path abs_path = makeAbsolute(file.getPath()); - - int fh = ceph.ceph_open_for_read(getCephPath(abs_path)); - if (fh < 0) { - LOG.error("getFileBlockLocations:got error " + fh + ", exiting and returning null!"); - return null; - } - - long blockSize = ceph.ceph_getblocksize(getCephPath(abs_path)); - BlockLocation[] locations = new BlockLocation[(int) Math.ceil(len / (float) blockSize)]; - - for (int i = 0; i < locations.length; ++i) { - long offset = start + i * blockSize; - long blockStart = start + i * blockSize - (start % blockSize); - String ips[] = ceph.ceph_hosts(fh, offset); - String hosts[] = ips2Hosts(ips); - locations[i] = new BlockLocation(null, hosts, blockStart, blockSize); - LOG.debug("getFileBlockLocations: location[" + i + "]: " + locations[i]); - } - - ceph.ceph_close(fh); - return locations; - } - - @Deprecated - public boolean delete(Path path) throws IOException { - return delete(path, false); - } - - /** - * Delete the given path, and optionally its children. - * @param path the path to delete. - * @param recursive If the path is a non-empty directory and this is false, - * delete will throw an IOException. If path is a file this is ignored. - * @return true if the delete succeeded, false otherwise (including if - * path doesn't exist). - * @throws IOException if you attempt to non-recursively delete a directory, - * or you attempt to delete the root directory. - */ - public boolean delete(Path path, boolean recursive) throws IOException { - LOG.debug("delete:enter with path " + path + " and recursive=" + recursive); - Path abs_path = makeAbsolute(path); - - // sanity check - if (abs_path.equals(root)) { - throw new IOException("Error: deleting the root directory is a Bad Idea."); - } - if (!exists(abs_path)) { - return false; - } - - // if the path is a file, try to delete it. - if (isFile(abs_path)) { - LOG.trace("delete:calling ceph_unlink from Java with path " + abs_path); - boolean result = ceph.ceph_unlink(getCephPath(abs_path)); - - if (!result) { - LOG.error( - "delete: failed to delete file \"" + abs_path.toString() + "\"."); - } - LOG.debug("delete:exit with success=" + result); - return result; - } - - /* The path is a directory, so recursively try to delete its contents, - and then delete the directory. */ - // get the entries; listPaths will remove . and .. for us - Path[] contents = listPaths(abs_path); - - if (contents == null) { - LOG.error( - "delete: Failed to read contents of directory \"" - + abs_path.toString() + "\" while trying to delete it, BAILING"); - return false; - } - if (!recursive && contents.length > 0) { - throw new IOException("Directories must be deleted recursively!"); - } - // delete the entries - LOG.debug("delete: recursively calling delete on contents of " + abs_path); - for (Path p : contents) { - if (!delete(p, true)) { - LOG.error( - "delete: Failed to delete file \"" + p.toString() - + "\" while recursively deleting \"" + abs_path.toString() - + "\", BAILING"); - return false; - } - } - // if we've come this far it's a now-empty directory, so delete it! - boolean result = ceph.ceph_rmdir(getCephPath(abs_path)); - - if (!result) { - LOG.error( - "delete: failed to delete \"" + abs_path.toString() + "\", BAILING"); - } - LOG.debug("delete:exit"); - return result; - } - - /** - * Returns the default replication value of 1. This may - * NOT be the actual value, as replication is controlled - * by a separate Ceph configuration. - */ - @Override - public short getDefaultReplication() { - return 1; - } - - /** - * Get the default block size. - * @return the default block size, in bytes, as a long. - */ - @Override - public long getDefaultBlockSize() { - return getConf().getInt("fs.ceph.blockSize", 1 << 26); - } - - /** - * Adds the working directory to path if path is not already - * an absolute path. The URI scheme is not removed here. It - * is removed only when users (e.g. ceph native calls) need - * the path-only portion. - */ - private Path makeAbsolute(Path path) { - if (path.isAbsolute()) { - return path; - } - return new Path(workingDir, path); - } - - private Path[] listPaths(Path path) throws IOException { - LOG.debug("listPaths:enter with path " + path); - String dirlist[]; - - Path abs_path = makeAbsolute(path); - - // If it's a directory, get the listing. Otherwise, complain and give up. - LOG.debug("calling ceph_getdir from Java with path " + abs_path); - dirlist = ceph.ceph_getdir(getCephPath(abs_path)); - LOG.debug("returning from ceph_getdir to Java"); - - if (dirlist == null) { - return null; - } - - // convert the strings to Paths - Path[] paths = new Path[dirlist.length]; - - for (int i = 0; i < dirlist.length; ++i) { - LOG.trace( - "Raw enumeration of paths in \"" + abs_path.toString() + "\": \"" - + dirlist[i] + "\""); - // convert each listing to an absolute path - Path raw_path = new Path(dirlist[i]); - - if (raw_path.isAbsolute()) { - paths[i] = raw_path; - } else { - paths[i] = new Path(abs_path, raw_path); - } - } - LOG.debug("listPaths:exit"); - return paths; - } - - static class Stat { - public long size; - public boolean is_dir; - public long block_size; - public long mod_time; - public long access_time; - public int mode; - - public Stat() {} - } -} diff --git a/src/client/hadoop/ceph/CephInputStream.java b/src/client/hadoop/ceph/CephInputStream.java deleted file mode 100644 index d9668d031ba..00000000000 --- a/src/client/hadoop/ceph/CephInputStream.java +++ /dev/null @@ -1,254 +0,0 @@ -// -*- mode:Java; tab-width:2; c-basic-offset:2; indent-tabs-mode:t -*- - -/** - * - * Licensed under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. - * - * - * Implements the Hadoop FS interfaces to allow applications to store - * files in Ceph. - */ -package org.apache.hadoop.fs.ceph; - - -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSInputStream; - - -/** - * <p> - * An {@link FSInputStream} for a CephFileSystem and corresponding - * Ceph instance. - */ -public class CephInputStream extends FSInputStream { - private static final Log LOG = LogFactory.getLog(CephInputStream.class); - private boolean closed; - - private int fileHandle; - - private long fileLength; - - private CephFS ceph; - - private byte[] buffer; - private int bufPos = 0; - private int bufValid = 0; - private long cephPos = 0; - - /** - * Create a new CephInputStream. - * @param conf The system configuration. Unused. - * @param fh The filehandle provided by Ceph to reference. - * @param flength The current length of the file. If the length changes - * you will need to close and re-open it to access the new data. - */ - public CephInputStream(Configuration conf, CephFS cephfs, - int fh, long flength, int bufferSize) { - // Whoever's calling the constructor is responsible for doing the actual ceph_open - // call and providing the file handle. - fileLength = flength; - fileHandle = fh; - closed = false; - ceph = cephfs; - buffer = new byte[bufferSize]; - LOG.debug( - "CephInputStream constructor: initializing stream with fh " + fh - + " and file length " + flength); - - } - - /** Ceph likes things to be closed before it shuts down, - * so closing the IOStream stuff voluntarily in a finalizer is good - */ - protected void finalize() throws Throwable { - try { - if (!closed) { - close(); - } - } finally { - super.finalize(); - } - } - - private synchronized boolean fillBuffer() throws IOException { - bufValid = ceph.ceph_read(fileHandle, buffer, 0, buffer.length); - bufPos = 0; - if (bufValid < 0) { - int err = bufValid; - - bufValid = 0; - // attempt to reset to old position. If it fails, too bad. - ceph.ceph_seek_from_start(fileHandle, cephPos); - throw new IOException("Failed to fill read buffer! Error code:" + err); - } - cephPos += bufValid; - return (bufValid != 0); - } - - /* - * Get the current position of the stream. - */ - public synchronized long getPos() throws IOException { - return cephPos - bufValid + bufPos; - } - - /** - * Find the number of bytes remaining in the file. - */ - @Override - public synchronized int available() throws IOException { - return (int) (fileLength - getPos()); - } - - public synchronized void seek(long targetPos) throws IOException { - LOG.trace( - "CephInputStream.seek: Seeking to position " + targetPos + " on fd " - + fileHandle); - if (targetPos > fileLength) { - throw new IOException( - "CephInputStream.seek: failed seek to position " + targetPos - + " on fd " + fileHandle + ": Cannot seek after EOF " + fileLength); - } - long oldPos = cephPos; - - cephPos = ceph.ceph_seek_from_start(fileHandle, targetPos); - bufValid = 0; - bufPos = 0; - if (cephPos < 0) { - cephPos = oldPos; - throw new IOException("Ceph failed to seek to new position!"); - } - } - - /** - * Failovers are handled by the Ceph code at a very low level; - * if there are issues that can be solved by changing sources - * they'll be dealt with before anybody even tries to call this method! - * @return false. - */ - public synchronized boolean seekToNewSource(long targetPos) { - return false; - } - - /** - * Read a byte from the file. - * @return the next byte. - */ - @Override - public synchronized int read() throws IOException { - LOG.trace( - "CephInputStream.read: Reading a single byte from fd " + fileHandle - + " by calling general read function"); - - byte result[] = new byte[1]; - - if (getPos() >= fileLength) { - return -1; - } - if (-1 == read(result, 0, 1)) { - return -1; - } - if (result[0] < 0) { - return 256 + (int) result[0]; - } else { - return result[0]; - } - } - - /** - * Read a specified number of bytes from the file into a byte[]. - * @param buf the byte array to read into. - * @param off the offset to start at in the file - * @param len the number of bytes to read - * @return 0 if successful, otherwise an error code. - * @throws IOException on bad input. - */ - @Override - public synchronized int read(byte buf[], int off, int len) - throws IOException { - LOG.trace( - "CephInputStream.read: Reading " + len + " bytes from fd " + fileHandle); - - if (closed) { - throw new IOException( - "CephInputStream.read: cannot read " + len + " bytes from fd " - + fileHandle + ": stream closed"); - } - - // ensure we're not past the end of the file - if (getPos() >= fileLength) { - LOG.debug( - "CephInputStream.read: cannot read " + len + " bytes from fd " - + fileHandle + ": current position is " + getPos() - + " and file length is " + fileLength); - - return -1; - } - - int totalRead = 0; - int initialLen = len; - int read; - - do { - read = Math.min(len, bufValid - bufPos); - try { - System.arraycopy(buffer, bufPos, buf, off, read); - } catch (IndexOutOfBoundsException ie) { - throw new IOException( - "CephInputStream.read: Indices out of bounds:" + "read length is " - + len + ", buffer offset is " + off + ", and buffer size is " - + buf.length); - } catch (ArrayStoreException ae) { - throw new IOException( - "Uh-oh, CephInputStream failed to do an array" - + "copy due to type mismatch..."); - } catch (NullPointerException ne) { - throw new IOException( - "CephInputStream.read: cannot read " + len + "bytes from fd:" - + fileHandle + ": buf is null"); - } - bufPos += read; - len -= read; - off += read; - totalRead += read; - } while (len > 0 && fillBuffer()); - - LOG.trace( - "CephInputStream.read: Reading " + initialLen + " bytes from fd " - + fileHandle + ": succeeded in reading " + totalRead + " bytes"); - return totalRead; - } - - /** - * Close the CephInputStream and release the associated filehandle. - */ - @Override - public void close() throws IOException { - LOG.trace("CephOutputStream.close:enter"); - if (!closed) { - int result = ceph.ceph_close(fileHandle); - - closed = true; - if (result != 0) { - throw new IOException( - "Close somehow failed!" - + "Don't try and use this stream again, though"); - } - LOG.trace("CephOutputStream.close:exit"); - } - } -} diff --git a/src/client/hadoop/ceph/CephOutputStream.java b/src/client/hadoop/ceph/CephOutputStream.java deleted file mode 100644 index 4c50f88467d..00000000000 --- a/src/client/hadoop/ceph/CephOutputStream.java +++ /dev/null @@ -1,219 +0,0 @@ -// -*- mode:Java; tab-width:2; c-basic-offset:2; indent-tabs-mode:t -*- - -/** - * - * Licensed under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. - * - * - * Implements the Hadoop FS interfaces to allow applications to store - * files in Ceph. - */ - -package org.apache.hadoop.fs.ceph; - - -import java.io.IOException; -import java.io.OutputStream; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.util.Progressable; - - -/** - * <p> - * An {@link OutputStream} for a CephFileSystem and corresponding - * Ceph instance. - */ -public class CephOutputStream extends OutputStream { - private static final Log LOG = LogFactory.getLog(CephOutputStream.class); - private boolean closed; - - private CephFS ceph; - - private int fileHandle; - - private byte[] buffer; - private int bufUsed = 0; - - /** - * Construct the CephOutputStream. - * @param conf The FileSystem configuration. - * @param fh The Ceph filehandle to connect to. - */ - public CephOutputStream(Configuration conf, CephFS cephfs, - int fh, int bufferSize) { - ceph = cephfs; - fileHandle = fh; - closed = false; - buffer = new byte[bufferSize]; - } - - /** Ceph likes things to be closed before it shuts down, - *so closing the IOStream stuff voluntarily is good - */ - protected void finalize() throws Throwable { - try { - if (!closed) { - close(); - } - } finally { - super.finalize(); - } - } - - /** - * Get the current position in the file. - * @return The file offset in bytes. - */ - public long getPos() throws IOException { - return ceph.ceph_getpos(fileHandle); - } - - /** - * Write a byte. - * @param b The byte to write. - * @throws IOException If you have closed the CephOutputStream or the - * write fails. - */ - @Override - public synchronized void write(int b) throws IOException { - LOG.trace( - "CephOutputStream.write: writing a single byte to fd " + fileHandle); - - if (closed) { - throw new IOException( - "CephOutputStream.write: cannot write " + "a byte to fd " + fileHandle - + ": stream closed"); - } - // Stick the byte in a buffer and write it - byte buf[] = new byte[1]; - - buf[0] = (byte) b; - write(buf, 0, 1); - return; - } - - /** - * Write a byte buffer into the Ceph file. - * @param buf the byte array to write from - * @param off the position in the file to start writing at. - * @param len The number of bytes to actually write. - * @throws IOException if you have closed the CephOutputStream, or - * if buf is null or off + len > buf.length, or - * if the write fails due to a Ceph error. - */ - @Override - public synchronized void write(byte buf[], int off, int len) throws IOException { - LOG.trace( - "CephOutputStream.write: writing " + len + " bytes to fd " + fileHandle); - // make sure stream is open - if (closed) { - throw new IOException( - "CephOutputStream.write: cannot write " + len + "bytes to fd " - + fileHandle + ": stream closed"); - } - - int result; - int write; - - while (len > 0) { - write = Math.min(len, buffer.length - bufUsed); - try { - System.arraycopy(buf, off, buffer, bufUsed, write); - } catch (IndexOutOfBoundsException ie) { - throw new IOException( - "CephOutputStream.write: Indices out of bounds: " - + "write length is " + len + ", buffer offset is " + off - + ", and buffer size is " + buf.length); - } catch (ArrayStoreException ae) { - throw new IOException( - "Uh-oh, CephOutputStream failed to do an array" - + " copy due to type mismatch..."); - } catch (NullPointerException ne) { - throw new IOException( - "CephOutputStream.write: cannot write " + len + "bytes to fd " - + fileHandle + ": buffer is null"); - } - bufUsed += write; - len -= write; - off += write; - if (bufUsed == buffer.length) { - result = ceph.ceph_write(fileHandle, buffer, 0, bufUsed); - if (result < 0) { - throw new IOException( - "CephOutputStream.write: Buffered write of " + bufUsed - + " bytes failed!"); - } - if (result != bufUsed) { - throw new IOException( - "CephOutputStream.write: Wrote only " + result + " bytes of " - + bufUsed + " in buffer! Data may be lost or written" - + " twice to Ceph!"); - } - bufUsed = 0; - } - - } - return; - } - - /** - * Flush the buffered data. - * @throws IOException if you've closed the stream or the write fails. - */ - @Override - public synchronized void flush() throws IOException { - if (!closed) { - if (bufUsed == 0) { - return; - } - int result = ceph.ceph_write(fileHandle, buffer, 0, bufUsed); - - if (result < 0) { - throw new IOException( - "CephOutputStream.write: Write of " + bufUsed + "bytes to fd " - + fileHandle + " failed"); - } - if (result != bufUsed) { - throw new IOException( - "CephOutputStream.write: Write of " + bufUsed + "bytes to fd " - + fileHandle + "was incomplete: only " + result + " of " + bufUsed - + " bytes were written."); - } - bufUsed = 0; - return; - } - } - - /** - * Close the CephOutputStream. - * @throws IOException if Ceph somehow returns an error. In current code it can't. - */ - @Override - public synchronized void close() throws IOException { - LOG.trace("CephOutputStream.close:enter"); - if (!closed) { - flush(); - int result = ceph.ceph_close(fileHandle); - - if (result != 0) { - throw new IOException("Close failed!"); - } - - closed = true; - LOG.trace("CephOutputStream.close:exit"); - } - } -} diff --git a/src/client/hadoop/ceph/CephTalker.java b/src/client/hadoop/ceph/CephTalker.java deleted file mode 100644 index 569652fdd0b..00000000000 --- a/src/client/hadoop/ceph/CephTalker.java +++ /dev/null @@ -1,91 +0,0 @@ -// -*- mode:Java; tab-width:2; c-basic-offset:2; indent-tabs-mode:t -*- - -/** - * - * Licensed under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. - * - * - * Wraps a number of native function calls to communicate with the Ceph - * filesystem. - */ -package org.apache.hadoop.fs.ceph; - - -import org.apache.hadoop.conf.Configuration; -import org.apache.commons.logging.Log; - - -class CephTalker extends CephFS { - // JNI doesn't give us any way to store pointers, so use a long. - // Here we're assuming pointers aren't longer than 8 bytes. - long cluster; - - // we write a constructor so we can load the libraries - public CephTalker(Configuration conf, Log log) { - System.load(conf.get("fs.ceph.libDir") + "/libcephfs.so"); - System.load(conf.get("fs.ceph.libDir") + "/libhadoopcephfs.so"); - cluster = 0; - } - - protected native boolean ceph_initializeClient(String arguments, int block_size); - - protected native String ceph_getcwd(); - - protected native boolean ceph_setcwd(String path); - - protected native boolean ceph_rmdir(String path); - - protected native boolean ceph_unlink(String path); - - protected native boolean ceph_rename(String old_path, String new_path); - - protected native boolean ceph_exists(String path); - - protected native long ceph_getblocksize(String path); - - protected native boolean ceph_isdirectory(String path); - - protected native boolean ceph_isfile(String path); - - protected native String[] ceph_getdir(String path); - - protected native int ceph_mkdirs(String path, int mode); - - protected native int ceph_open_for_append(String path); - - protected native int ceph_open_for_read(String path); - - protected native int ceph_open_for_overwrite(String path, int mode); - - protected native int ceph_close(int filehandle); - - protected native boolean ceph_setPermission(String path, int mode); - - protected native boolean ceph_kill_client(); - - protected native boolean ceph_stat(String path, CephFileSystem.Stat fill); - - protected native int ceph_replication(String Path); - - protected native String[] ceph_hosts(int fh, long offset); - - protected native int ceph_setTimes(String path, long mtime, long atime); - - protected native long ceph_getpos(int fh); - - protected native int ceph_write(int fh, byte[] buffer, int buffer_offset, int length); - - protected native int ceph_read(int fh, byte[] buffer, int buffer_offset, int length); - - protected native long ceph_seek_from_start(int fh, long pos); -} diff --git a/src/client/hadoop/ceph/LICENSE b/src/client/hadoop/ceph/LICENSE deleted file mode 100644 index 7a0decda573..00000000000 --- a/src/client/hadoop/ceph/LICENSE +++ /dev/null @@ -1,4 +0,0 @@ -Unlike the rest of the code in this repository, this -directory (src/client/hadoop) is licensed under the Apache License 2.0. This -is for the obvious reason that we want to integrate it into the Apache Hadoop -project.
\ No newline at end of file diff --git a/src/client/hadoop/ceph/TestCeph.java b/src/client/hadoop/ceph/TestCeph.java deleted file mode 100644 index e46b0eed3a1..00000000000 --- a/src/client/hadoop/ceph/TestCeph.java +++ /dev/null @@ -1,45 +0,0 @@ -// -*- mode:Java; tab-width:2; c-basic-offset:2; indent-tabs-mode:t -*- - -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Unit tests for the CephFileSystem API implementation. - */ - -package org.apache.hadoop.fs.ceph; - - -import java.io.IOException; -import java.net.URI; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystemContractBaseTest; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; - - -public class TestCeph extends FileSystemContractBaseTest { - - @Override - protected void setUp() throws IOException { - Configuration conf = new Configuration(); - CephFaker cephfaker = new CephFaker(conf, FileSystem.LOG); - CephFileSystem cephfs = new CephFileSystem(cephfaker); - - cephfs.initialize(URI.create("ceph://null"), conf); - fs = cephfs; - } -} diff --git a/src/client/hadoop/ceph/package.html b/src/client/hadoop/ceph/package.html deleted file mode 100644 index 8167b1dde92..00000000000 --- a/src/client/hadoop/ceph/package.html +++ /dev/null @@ -1,101 +0,0 @@ -<html> - -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<head></head> -<body> -<h1>A client for the Ceph filesystem</h1> - -<h3>Introduction</h3> - -This page describes how to use <a href="http://ceph.newdream.net">Ceph</a> -as a backing store with Hadoop. This page assumes that you have downloaded -the Ceph software and installed necessary binaries as outlined in the Ceph -documentation. - -<h3>Steps</h3> -<ul> - <li>In the Hadoop conf directory edit core-site.xml, - adding the following (with appropriate substitutions). Note that - different nodes can connect to different monitors in the same cluster - without issue (the Ceph client will automatically redirect as necessary). -<pre> -<property> - <name>fs.default.name</name> - <value>ceph://null</value> -</property> - -<property> - <name>fs.ceph.monAddr</name> - <value><serverIP:port></value> - <description>The location of the Ceph monitor to connect to. - This should be an IP address, not a domain-based web address.</description> -</property> - -<property> - <name>fs.ceph.libDir</name> - <value>/usr/local/lib</value> - <description>The folder holding libcephfs and libhadoopceph</description> - </property> -</pre> - <li>There are also a number of optional Ceph configuration options. -<pre> -<property> - <name>fs.ceph.blockSize</name> - <value>67108864</value> - <description>Defaulting to 64MB, this is the size (in bytes) you want Ceph to use in striping data internally and presenting it to Hadoop.</description> -</property> - -<property> - <name>fs.ceph.debug</name> - <value>true</value> - <description>If true, the Java-based code will print debugging information to standard error. This is useful if attempting to debug a Ceph issue as it puts both outputs in the same place.</description> -</property> - -<property> - <name>fs.ceph.clientDebug</name> - <value>1</value> - <description>If non-zero, the Ceph client will print debugging information to standard error (a higher number=more debugging).</description> -</property> - -<property> - <name>fs.ceph.messengerDebug</name> - <value>1</value> - <description>If non-zero, the Ceph messenger will print debugging information to standard error(a higher number=more debugging)</description> -</property> - -<property> - <name>fs.ceph.readahead</name> - <value>1</value> - <description>Sets the number of object periods to read ahead in prefetching. This should probably be left at the default of 1.</description> -</property> - -<property> - <name>fs.ceph.commandLine</name> - <value>a string</value> - <description>If you prefer, you may enter any of Ceph's command-line configuration here and it will get passed to the C client. Note that any filled-in configuration options will override what you put here. <br> -By default, Ceph performs writes across the network rather than locally. To force local writes, add "set_local_pg" in this property.</description> -</property> -</pre> - - <li>Start up your Ceph instance according to the Ceph documentation.</li> - <li>Do not use the bin/start-all.sh commands, as they will attempt to start - up an hdfs instance. Just start whatever systems you need and they will - automatically make use of the Ceph filesystem once configured as above.</li> -</body> -</html> diff --git a/src/client/hadoop/org_apache_hadoop_fs_ceph_CephFS.h b/src/client/hadoop/org_apache_hadoop_fs_ceph_CephFS.h deleted file mode 100644 index 0c07fd56e37..00000000000 --- a/src/client/hadoop/org_apache_hadoop_fs_ceph_CephFS.h +++ /dev/null @@ -1,13 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include <jni.h> -/* Header for class org_apache_hadoop_fs_ceph_CephFS */ - -#ifndef _Included_org_apache_hadoop_fs_ceph_CephFS -#define _Included_org_apache_hadoop_fs_ceph_CephFS -#ifdef __cplusplus -extern "C" { -#endif -#ifdef __cplusplus -} -#endif -#endif diff --git a/src/client/hadoop/org_apache_hadoop_fs_ceph_CephFileSystem.h b/src/client/hadoop/org_apache_hadoop_fs_ceph_CephFileSystem.h deleted file mode 100644 index 6f2bc93926c..00000000000 --- a/src/client/hadoop/org_apache_hadoop_fs_ceph_CephFileSystem.h +++ /dev/null @@ -1,31 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include <jni.h> -/* Header for class org_apache_hadoop_fs_ceph_CephFileSystem */ - -#ifndef _Included_org_apache_hadoop_fs_ceph_CephFileSystem -#define _Included_org_apache_hadoop_fs_ceph_CephFileSystem -#ifdef __cplusplus -extern "C" { -#endif -#undef org_apache_hadoop_fs_ceph_CephFileSystem_EEXIST -#define org_apache_hadoop_fs_ceph_CephFileSystem_EEXIST 17L -#undef org_apache_hadoop_fs_ceph_CephFileSystem_ENOENT -#define org_apache_hadoop_fs_ceph_CephFileSystem_ENOENT 2L -#undef org_apache_hadoop_fs_ceph_CephFileSystem_FATAL -#define org_apache_hadoop_fs_ceph_CephFileSystem_FATAL 0L -#undef org_apache_hadoop_fs_ceph_CephFileSystem_ERROR -#define org_apache_hadoop_fs_ceph_CephFileSystem_ERROR 1L -#undef org_apache_hadoop_fs_ceph_CephFileSystem_WARN -#define org_apache_hadoop_fs_ceph_CephFileSystem_WARN 2L -#undef org_apache_hadoop_fs_ceph_CephFileSystem_INFO -#define org_apache_hadoop_fs_ceph_CephFileSystem_INFO 3L -#undef org_apache_hadoop_fs_ceph_CephFileSystem_DEBUG -#define org_apache_hadoop_fs_ceph_CephFileSystem_DEBUG 4L -#undef org_apache_hadoop_fs_ceph_CephFileSystem_TRACE -#define org_apache_hadoop_fs_ceph_CephFileSystem_TRACE 5L -#undef org_apache_hadoop_fs_ceph_CephFileSystem_NOLOG -#define org_apache_hadoop_fs_ceph_CephFileSystem_NOLOG 6L -#ifdef __cplusplus -} -#endif -#endif diff --git a/src/client/hadoop/org_apache_hadoop_fs_ceph_CephFileSystem_CephStat.h b/src/client/hadoop/org_apache_hadoop_fs_ceph_CephFileSystem_CephStat.h deleted file mode 100644 index 5ab70f7dc66..00000000000 --- a/src/client/hadoop/org_apache_hadoop_fs_ceph_CephFileSystem_CephStat.h +++ /dev/null @@ -1,13 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include <jni.h> -/* Header for class org_apache_hadoop_fs_ceph_CephFileSystem_CephStat */ - -#ifndef _Included_org_apache_hadoop_fs_ceph_CephFileSystem_CephStat -#define _Included_org_apache_hadoop_fs_ceph_CephFileSystem_CephStat -#ifdef __cplusplus -extern "C" { -#endif -#ifdef __cplusplus -} -#endif -#endif diff --git a/src/client/hadoop/org_apache_hadoop_fs_ceph_CephFileSystem_Stat.h b/src/client/hadoop/org_apache_hadoop_fs_ceph_CephFileSystem_Stat.h deleted file mode 100644 index e9ade0e4504..00000000000 --- a/src/client/hadoop/org_apache_hadoop_fs_ceph_CephFileSystem_Stat.h +++ /dev/null @@ -1,13 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include <jni.h> -/* Header for class org_apache_hadoop_fs_ceph_CephFileSystem_Stat */ - -#ifndef _Included_org_apache_hadoop_fs_ceph_CephFileSystem_Stat -#define _Included_org_apache_hadoop_fs_ceph_CephFileSystem_Stat -#ifdef __cplusplus -extern "C" { -#endif -#ifdef __cplusplus -} -#endif -#endif diff --git a/src/client/hadoop/org_apache_hadoop_fs_ceph_CephInputStream.h b/src/client/hadoop/org_apache_hadoop_fs_ceph_CephInputStream.h deleted file mode 100644 index 4ec903294b7..00000000000 --- a/src/client/hadoop/org_apache_hadoop_fs_ceph_CephInputStream.h +++ /dev/null @@ -1,47 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include <jni.h> -/* Header for class org_apache_hadoop_fs_ceph_CephInputStream */ - -#ifndef _Included_org_apache_hadoop_fs_ceph_CephInputStream -#define _Included_org_apache_hadoop_fs_ceph_CephInputStream -#ifdef __cplusplus -extern "C" { -#endif -#undef org_apache_hadoop_fs_ceph_CephInputStream_SKIP_BUFFER_SIZE -#define org_apache_hadoop_fs_ceph_CephInputStream_SKIP_BUFFER_SIZE 2048L -/* - * Class: org_apache_hadoop_fs_ceph_CephInputStream - * Method: ceph_read - * Signature: (I[BII)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephInputStream_ceph_1read - (JNIEnv *, jobject, jint, jbyteArray, jint, jint); - -/* - * Class: org_apache_hadoop_fs_ceph_CephInputStream - * Method: ceph_seek_from_start - * Signature: (IJ)J - */ -JNIEXPORT jlong JNICALL Java_org_apache_hadoop_fs_ceph_CephInputStream_ceph_1seek_1from_1start - (JNIEnv *, jobject, jint, jlong); - -/* - * Class: org_apache_hadoop_fs_ceph_CephInputStream - * Method: ceph_getpos - * Signature: (I)J - */ -JNIEXPORT jlong JNICALL Java_org_apache_hadoop_fs_ceph_CephInputStream_ceph_1getpos - (JNIEnv *, jobject, jint); - -/* - * Class: org_apache_hadoop_fs_ceph_CephInputStream - * Method: ceph_close - * Signature: (I)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephInputStream_ceph_1close - (JNIEnv *, jobject, jint); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/src/client/hadoop/org_apache_hadoop_fs_ceph_CephOutputStream.h b/src/client/hadoop/org_apache_hadoop_fs_ceph_CephOutputStream.h deleted file mode 100644 index 676b137c9f9..00000000000 --- a/src/client/hadoop/org_apache_hadoop_fs_ceph_CephOutputStream.h +++ /dev/null @@ -1,37 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include <jni.h> -/* Header for class org_apache_hadoop_fs_ceph_CephOutputStream */ - -#ifndef _Included_org_apache_hadoop_fs_ceph_CephOutputStream -#define _Included_org_apache_hadoop_fs_ceph_CephOutputStream -#ifdef __cplusplus -extern "C" { -#endif -/* - * Class: org_apache_hadoop_fs_ceph_CephOutputStream - * Method: ceph_getpos - * Signature: (I)J - */ -JNIEXPORT jlong JNICALL Java_org_apache_hadoop_fs_ceph_CephOutputStream_ceph_1getpos - (JNIEnv *, jobject, jint); - -/* - * Class: org_apache_hadoop_fs_ceph_CephOutputStream - * Method: ceph_close - * Signature: (I)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephOutputStream_ceph_1close - (JNIEnv *, jobject, jint); - -/* - * Class: org_apache_hadoop_fs_ceph_CephOutputStream - * Method: ceph_write - * Signature: (I[BII)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephOutputStream_ceph_1write - (JNIEnv *, jobject, jint, jbyteArray, jint, jint); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/src/client/hadoop/org_apache_hadoop_fs_ceph_CephTalker.h b/src/client/hadoop/org_apache_hadoop_fs_ceph_CephTalker.h deleted file mode 100644 index 55854549b8c..00000000000 --- a/src/client/hadoop/org_apache_hadoop_fs_ceph_CephTalker.h +++ /dev/null @@ -1,197 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include <jni.h> -/* Header for class org_apache_hadoop_fs_ceph_CephTalker */ - -#ifndef _Included_org_apache_hadoop_fs_ceph_CephTalker -#define _Included_org_apache_hadoop_fs_ceph_CephTalker -#ifdef __cplusplus -extern "C" { -#endif -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_initializeClient - * Signature: (Ljava/lang/String;I)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1initializeClient - (JNIEnv *, jobject, jstring, jint); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_getcwd - * Signature: ()Ljava/lang/String; - */ -JNIEXPORT jstring JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1getcwd - (JNIEnv *, jobject); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_setcwd - * Signature: (Ljava/lang/String;)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1setcwd - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_rmdir - * Signature: (Ljava/lang/String;)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1rmdir - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_unlink - * Signature: (Ljava/lang/String;)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1unlink - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_rename - * Signature: (Ljava/lang/String;Ljava/lang/String;)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1rename - (JNIEnv *, jobject, jstring, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_exists - * Signature: (Ljava/lang/String;)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1exists - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_getblocksize - * Signature: (Ljava/lang/String;)J - */ -JNIEXPORT jlong JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1getblocksize - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_isdirectory - * Signature: (Ljava/lang/String;)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1isdirectory - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_isfile - * Signature: (Ljava/lang/String;)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1isfile - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_getdir - * Signature: (Ljava/lang/String;)[Ljava/lang/String; - */ -JNIEXPORT jobjectArray JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1getdir - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_mkdirs - * Signature: (Ljava/lang/String;I)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1mkdirs - (JNIEnv *, jobject, jstring, jint); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_open_for_append - * Signature: (Ljava/lang/String;)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1open_1for_1append - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_open_for_read - * Signature: (Ljava/lang/String;)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1open_1for_1read - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_open_for_overwrite - * Signature: (Ljava/lang/String;I)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1open_1for_1overwrite - (JNIEnv *, jobject, jstring, jint); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_close - * Signature: (I)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1close - (JNIEnv *, jobject, jint); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_setPermission - * Signature: (Ljava/lang/String;I)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1setPermission - (JNIEnv *, jobject, jstring, jint); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_kill_client - * Signature: ()Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1kill_1client - (JNIEnv *, jobject); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_stat - * Signature: (Ljava/lang/String;Lorg/apache/hadoop/fs/ceph/CephFileSystem/Stat;)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1stat - (JNIEnv *, jobject, jstring, jobject); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_statfs - * Signature: (Ljava/lang/String;Lorg/apache/hadoop/fs/ceph/CephFileSystem/CephStat;)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1statfs - (JNIEnv *, jobject, jstring, jobject); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_replication - * Signature: (Ljava/lang/String;)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1replication - (JNIEnv *, jobject, jstring); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_hosts - * Signature: (IJ)Ljava/lang/String; - */ -JNIEXPORT jstring JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1hosts - (JNIEnv *, jobject, jint, jlong); - -/* - * Class: org_apache_hadoop_fs_ceph_CephTalker - * Method: ceph_setTimes - * Signature: (Ljava/lang/String;JJ)I - */ -JNIEXPORT jint JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1setTimes - (JNIEnv *, jobject, jstring, jlong, jlong); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/src/client/ioctl.h b/src/client/ioctl.h index c15d3340a76..35b4ce4ac0f 100644 --- a/src/client/ioctl.h +++ b/src/client/ioctl.h @@ -1,13 +1,14 @@ #ifndef FS_CEPH_IOCTL_H #define FS_CEPH_IOCTL_H +#include "include/int_types.h" + #if defined(__linux__) #include <linux/ioctl.h> #include <linux/types.h> #elif defined(__FreeBSD__) #include <sys/ioctl.h> #include <sys/types.h> -#include "include/inttypes.h" #endif #define CEPH_IOCTL_MAGIC 0x97 diff --git a/src/cls/Makefile.am b/src/cls/Makefile.am new file mode 100644 index 00000000000..2d3d43cb1e3 --- /dev/null +++ b/src/cls/Makefile.am @@ -0,0 +1,122 @@ +## Rados object classes + +libcls_hello_la_SOURCES = cls/hello/cls_hello.cc +libcls_hello_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_hello_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' +radoslib_LTLIBRARIES += libcls_hello.la + +libcls_rbd_la_SOURCES = cls/rbd/cls_rbd.cc +libcls_rbd_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_rbd_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' +radoslib_LTLIBRARIES += libcls_rbd.la + +libcls_lock_la_SOURCES = cls/lock/cls_lock.cc +libcls_lock_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_lock_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' +radoslib_LTLIBRARIES += libcls_lock.la + +libcls_refcount_la_SOURCES = \ + cls/refcount/cls_refcount.cc \ + cls/refcount/cls_refcount_ops.cc \ + common/ceph_json.cc +libcls_refcount_la_LIBADD = libjson_spirit.la $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_refcount_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' +radoslib_LTLIBRARIES += libcls_refcount.la + +libcls_version_la_SOURCES = cls/version/cls_version.cc +libcls_version_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_version_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' +radoslib_LTLIBRARIES += libcls_version.la + +libcls_log_la_SOURCES = cls/log/cls_log.cc +libcls_log_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_log_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' +radoslib_LTLIBRARIES += libcls_log.la + +libcls_statelog_la_SOURCES = cls/statelog/cls_statelog.cc +libcls_statelog_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_statelog_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' +radoslib_LTLIBRARIES += libcls_statelog.la + +libcls_replica_log_la_SOURCES = cls/replica_log/cls_replica_log.cc +libcls_replica_log_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_replica_log_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' +radoslib_LTLIBRARIES += libcls_replica_log.la + +libcls_rgw_la_SOURCES = \ + cls/rgw/cls_rgw.cc \ + cls/rgw/cls_rgw_ops.cc \ + cls/rgw/cls_rgw_types.cc \ + common/ceph_json.cc +libcls_rgw_la_LIBADD = libjson_spirit.la $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_rgw_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' +radoslib_LTLIBRARIES += libcls_rgw.la + +## Rados object client classes + +libcls_lock_client_la_SOURCES = \ + cls/lock/cls_lock_client.cc \ + cls/lock/cls_lock_types.cc \ + cls/lock/cls_lock_ops.cc +noinst_LTLIBRARIES += libcls_lock_client.la +DENCODER_DEPS += libcls_lock_client.la + +libcls_refcount_client_la_SOURCES = \ + cls/refcount/cls_refcount_client.cc \ + cls/refcount/cls_refcount_ops.cc +noinst_LTLIBRARIES += libcls_refcount_client.la +DENCODER_DEPS += libcls_refcount_client.la + +libcls_version_client_a_SOURCES = \ + cls/version/cls_version_client.cc \ + cls/version/cls_version_types.cc +noinst_LIBRARIES += libcls_version_client.a + +libcls_log_client_a_SOURCES = cls/log/cls_log_client.cc +noinst_LIBRARIES += libcls_log_client.a + +libcls_statelog_client_a_SOURCES = cls/statelog/cls_statelog_client.cc +noinst_LIBRARIES += libcls_statelog_client.a + +libcls_replica_log_client_a_SOURCES = \ + cls/replica_log/cls_replica_log_types.cc \ + cls/replica_log/cls_replica_log_ops.cc \ + cls/replica_log/cls_replica_log_client.cc +noinst_LIBRARIES += libcls_replica_log_client.a +DENCODER_DEPS += libcls_replica_log_client.a + +libcls_rgw_client_la_SOURCES = \ + cls/rgw/cls_rgw_client.cc \ + cls/rgw/cls_rgw_types.cc \ + cls/rgw/cls_rgw_ops.cc +noinst_LTLIBRARIES += libcls_rgw_client.la +DENCODER_DEPS += libcls_rgw_client.la + +libcls_rbd_client_la_SOURCES = cls/rbd/cls_rbd_client.cc +noinst_LTLIBRARIES += libcls_rbd_client.la + + +noinst_HEADERS += \ + cls/lock/cls_lock_types.h \ + cls/lock/cls_lock_ops.h \ + cls/lock/cls_lock_client.h \ + cls/rbd/cls_rbd.h \ + cls/rbd/cls_rbd_client.h \ + cls/refcount/cls_refcount_ops.h \ + cls/refcount/cls_refcount_client.h \ + cls/version/cls_version_types.h \ + cls/version/cls_version_ops.h \ + cls/version/cls_version_client.h \ + cls/log/cls_log_types.h \ + cls/log/cls_log_ops.h \ + cls/log/cls_log_client.h \ + cls/statelog/cls_statelog_types.h \ + cls/statelog/cls_statelog_ops.h \ + cls/statelog/cls_statelog_client.h \ + cls/replica_log/cls_replica_log_types.h \ + cls/replica_log/cls_replica_log_ops.h \ + cls/replica_log/cls_replica_log_client.h \ + cls/rgw/cls_rgw_client.h \ + cls/rgw/cls_rgw_ops.h \ + cls/rgw/cls_rgw_types.h + diff --git a/src/cls/rbd/cls_rbd.cc b/src/cls/rbd/cls_rbd.cc index 420de514ee6..12947a08540 100644 --- a/src/cls/rbd/cls_rbd.cc +++ b/src/cls/rbd/cls_rbd.cc @@ -25,6 +25,8 @@ * parameters as the client sees them - it would be silly to mention * in each one that they take an input and an output bufferlist. */ +#include "include/int_types.h" +#include "include/types.h" #include <algorithm> #include <cstring> @@ -35,10 +37,8 @@ #include <sstream> #include <vector> -#include "include/types.h" #include "objclass/objclass.h" #include "include/rbd_types.h" -#include <inttypes.h> #include "cls/rbd/cls_rbd.h" diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc index 6cda4cba5c3..2f5711ecb32 100644 --- a/src/cls/rgw/cls_rgw.cc +++ b/src/cls/rgw/cls_rgw.cc @@ -1,14 +1,14 @@ // -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab -#include <iostream> +#include "include/int_types.h" +#include "include/types.h" +#include <iostream> #include <string.h> #include <stdlib.h> #include <errno.h> -#include <inttypes.h> -#include "include/types.h" #include "include/utime.h" #include "objclass/objclass.h" #include "cls/rgw/cls_rgw_ops.h" diff --git a/src/common/Formatter.cc b/src/common/Formatter.cc index c08ea5b9a20..4ef833a45f8 100644 --- a/src/common/Formatter.cc +++ b/src/common/Formatter.cc @@ -14,11 +14,12 @@ #define LARGE_SIZE 1024 +#include "include/int_types.h" + #include "assert.h" #include "Formatter.h" #include "common/escape.h" -#include <inttypes.h> #include <iostream> #include <sstream> #include <stdarg.h> diff --git a/src/common/Formatter.h b/src/common/Formatter.h index da730103f41..27089ce04f2 100644 --- a/src/common/Formatter.h +++ b/src/common/Formatter.h @@ -1,8 +1,9 @@ #ifndef CEPH_FORMATTER_H #define CEPH_FORMATTER_H +#include "include/int_types.h" + #include <deque> -#include <inttypes.h> #include <iostream> #include <list> #include <ostream> diff --git a/src/common/Makefile.am b/src/common/Makefile.am new file mode 100644 index 00000000000..4c027909b4d --- /dev/null +++ b/src/common/Makefile.am @@ -0,0 +1,195 @@ +libcommon_la_SOURCES = \ + ceph_ver.c \ + common/DecayCounter.cc \ + common/LogClient.cc \ + common/LogEntry.cc \ + common/PrebufferedStreambuf.cc \ + common/BackTrace.cc \ + common/perf_counters.cc \ + common/Mutex.cc \ + common/OutputDataSocket.cc \ + common/admin_socket.cc \ + common/admin_socket_client.cc \ + common/cmdparse.cc \ + common/escape.c \ + common/Clock.cc \ + common/Throttle.cc \ + common/Timer.cc \ + common/Finisher.cc \ + common/environment.cc\ + common/assert.cc \ + common/run_cmd.cc \ + common/WorkQueue.cc \ + common/ConfUtils.cc \ + common/MemoryModel.cc \ + common/armor.c \ + common/fd.cc \ + common/xattr.c \ + common/safe_io.c \ + common/snap_types.cc \ + common/str_list.cc \ + common/errno.cc \ + common/RefCountedObj.cc \ + common/blkdev.cc \ + common/common_init.cc \ + common/pipe.c \ + common/ceph_argparse.cc \ + common/ceph_context.cc \ + common/buffer.cc \ + common/code_environment.cc \ + common/dout.cc \ + common/signal.cc \ + common/simple_spin.cc \ + common/Thread.cc \ + common/Formatter.cc \ + common/HeartbeatMap.cc \ + common/config.cc \ + common/utf8.c \ + common/mime.c \ + common/strtol.cc \ + common/page.cc \ + common/lockdep.cc \ + common/version.cc \ + common/hex.cc \ + common/entity_name.cc \ + common/ceph_crypto.cc \ + common/ceph_crypto_cms.cc \ + common/ceph_json.cc \ + common/ipaddr.cc \ + common/pick_address.cc \ + common/util.cc \ + common/TextTable.cc \ + common/secret.c \ + common/ceph_fs.cc \ + common/ceph_hash.cc \ + common/ceph_strings.cc \ + common/ceph_frag.cc \ + common/addr_parsing.c \ + common/hobject.cc + +# these should go out of libcommon +libcommon_la_SOURCES += \ + mon/MonCap.cc \ + mon/MonClient.cc \ + mon/MonMap.cc \ + osd/OSDMap.cc \ + osd/osd_types.cc \ + mds/MDSMap.cc \ + mds/inode_backtrace.cc \ + mds/mdstypes.cc + +# inject crc in common +libcommon_crc_la_SOURCES = \ + common/sctp_crc32.c \ + common/crc32c.cc \ + common/crc32c_intel_baseline.c \ + common/crc32c_intel_fast.c + +if WITH_GOOD_YASM_ELF64 +libcommon_crc_la_SOURCES += common/crc32c_intel_fast_asm.S +libcommon_crc_la_LIBTOOLFLAGS = --tag=CC +endif +LIBCOMMON_DEPS += libcommon_crc.la +noinst_LTLIBRARIES += libcommon_crc.la + +noinst_HEADERS += \ + common/sctp_crc32.h \ + common/crc32c_intel_baseline.h \ + common/crc32c_intel_fast.h + + +# important; libmsg before libauth! +LIBCOMMON_DEPS += \ + $(LIBMSG) $(LIBAUTH) \ + $(LIBCRUSH) $(LIBJSON_SPIRIT) $(LIBLOG) $(LIBARCH) \ + -lkeyutils + +if LINUX +LIBCOMMON_DEPS += -lrt +endif # LINUX + +libcommon_la_LIBADD = $(LIBCOMMON_DEPS) + +noinst_HEADERS += \ + common/BackTrace.h \ + common/RefCountedObj.h \ + common/HeartbeatMap.h \ + common/LogClient.h \ + common/LogEntry.h \ + common/Preforker.h \ + common/WorkQueue.h \ + common/PrioritizedQueue.h \ + common/ceph_argparse.h \ + common/ceph_context.h \ + common/xattr.h \ + common/blkdev.h \ + common/compiler_extensions.h \ + common/debug.h \ + common/dout.h \ + common/escape.h \ + common/fd.h \ + common/version.h \ + common/hex.h \ + common/entity_name.h \ + common/errno.h \ + common/environment.h \ + common/likely.h \ + common/lockdep.h \ + common/obj_bencher.h \ + common/snap_types.h \ + common/Clock.h \ + common/Cond.h \ + common/ConfUtils.h \ + common/DecayCounter.h \ + common/Finisher.h \ + common/Formatter.h \ + common/perf_counters.h \ + common/OutputDataSocket.h \ + common/admin_socket.h \ + common/admin_socket_client.h \ + common/shared_cache.hpp \ + common/tracked_int_ptr.hpp \ + common/simple_cache.hpp \ + common/sharedptr_registry.hpp \ + common/map_cacher.hpp \ + common/MemoryModel.h \ + common/Mutex.h \ + common/PrebufferedStreambuf.h \ + common/RWLock.h \ + common/Semaphore.h \ + common/SimpleRNG.h \ + common/TextTable.h \ + common/Thread.h \ + common/Throttle.h \ + common/Timer.h \ + common/TrackedOp.h \ + common/arch.h \ + common/armor.h \ + common/common_init.h \ + common/pipe.h \ + common/code_environment.h \ + common/signal.h \ + common/simple_spin.h \ + common/run_cmd.h \ + common/safe_io.h \ + common/config.h \ + common/config_obs.h \ + common/config_opts.h \ + common/ceph_crypto.h \ + common/ceph_crypto_cms.h \ + common/ceph_json.h \ + common/lru_map.h \ + common/utf8.h \ + common/mime.h \ + common/pick_address.h \ + common/secret.h \ + common/strtol.h \ + common/static_assert.h \ + common/AsyncReserver.h \ + common/sync_filesystem.h \ + common/cmdparse.h \ + common/hobject.h + +noinst_LTLIBRARIES += libcommon.la + + diff --git a/src/common/OutputDataSocket.cc b/src/common/OutputDataSocket.cc index e4d21fe13ee..3051ca02dbe 100644 --- a/src/common/OutputDataSocket.cc +++ b/src/common/OutputDataSocket.cc @@ -12,6 +12,8 @@ * */ +#include "include/int_types.h" + #include "common/Thread.h" #include "common/OutputDataSocket.h" #include "common/config.h" @@ -25,7 +27,6 @@ #include <errno.h> #include <fcntl.h> -#include <inttypes.h> #include <map> #include <poll.h> #include <set> diff --git a/src/include/addr_parsing.c b/src/common/addr_parsing.c index c01f817772c..c01f817772c 100644 --- a/src/include/addr_parsing.c +++ b/src/common/addr_parsing.c diff --git a/src/common/admin_socket.cc b/src/common/admin_socket.cc index 1a507e606bf..12e5868b409 100644 --- a/src/common/admin_socket.cc +++ b/src/common/admin_socket.cc @@ -12,6 +12,8 @@ * */ +#include "include/int_types.h" + #include "common/Thread.h" #include "common/admin_socket.h" #include "common/config.h" @@ -26,7 +28,6 @@ #include <errno.h> #include <fcntl.h> -#include <inttypes.h> #include <map> #include <poll.h> #include <set> diff --git a/src/common/admin_socket_client.cc b/src/common/admin_socket_client.cc index 782e808ad86..335695f9b4b 100644 --- a/src/common/admin_socket_client.cc +++ b/src/common/admin_socket_client.cc @@ -12,6 +12,8 @@ * */ +#include "include/int_types.h" + #include "common/admin_socket.h" #include "common/ceph_context.h" #include "common/errno.h" @@ -21,7 +23,6 @@ #include <arpa/inet.h> #include <errno.h> #include <fcntl.h> -#include <inttypes.h> #include <map> #include <poll.h> #include <sstream> diff --git a/src/common/blkdev.cc b/src/common/blkdev.cc index 22b0aa67b3e..b0dc0a54e9e 100644 --- a/src/common/blkdev.cc +++ b/src/common/blkdev.cc @@ -1,8 +1,8 @@ -#include <inttypes.h> +#include "include/int_types.h" + #include <fcntl.h> #include <sys/ioctl.h> #include <errno.h> - #include <sys/types.h> #include <sys/stat.h> #include <sys/mount.h> diff --git a/src/include/ceph_frag.cc b/src/common/ceph_frag.cc index ab6cf35c409..444b910c24e 100644 --- a/src/include/ceph_frag.cc +++ b/src/common/ceph_frag.cc @@ -1,7 +1,7 @@ /* * Ceph 'frag' type */ -#include "types.h" +#include "include/types.h" int ceph_frag_compare(__u32 a, __u32 b) { diff --git a/src/include/ceph_fs.cc b/src/common/ceph_fs.cc index 6e3c143361e..3172c577162 100644 --- a/src/include/ceph_fs.cc +++ b/src/common/ceph_fs.cc @@ -9,7 +9,7 @@ /* * Some non-inline ceph helpers */ -#include "types.h" +#include "include/types.h" /* * return true if @layout appears to be valid diff --git a/src/include/ceph_hash.cc b/src/common/ceph_hash.cc index d6081dfa97d..c581806d83d 100644 --- a/src/include/ceph_hash.cc +++ b/src/common/ceph_hash.cc @@ -1,5 +1,5 @@ -#include "types.h" +#include "include/types.h" /* * Robert Jenkin's hash function. diff --git a/src/include/ceph_strings.cc b/src/common/ceph_strings.cc index e86aae4fd50..cd08083967a 100644 --- a/src/include/ceph_strings.cc +++ b/src/common/ceph_strings.cc @@ -1,7 +1,7 @@ /* * Ceph string constants */ -#include "types.h" +#include "include/types.h" const char *ceph_entity_type_name(int type) { diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 2fa72d4ce0f..f6283239660 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -91,7 +91,6 @@ SUBSYS(finisher, 1, 1) SUBSYS(heartbeatmap, 1, 5) SUBSYS(perfcounter, 1, 5) SUBSYS(rgw, 1, 5) // log level for the Rados gateway -SUBSYS(hadoop, 1, 5) SUBSYS(javaclient, 1, 5) SUBSYS(asok, 1, 5) SUBSYS(throttle, 1, 1) diff --git a/src/common/crc32c_intel_baseline.c b/src/common/crc32c_intel_baseline.c index cfcfec624ae..3a92c77b63c 100644 --- a/src/common/crc32c_intel_baseline.c +++ b/src/common/crc32c_intel_baseline.c @@ -34,7 +34,8 @@ * OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include <inttypes.h> +#include "include/int_types.h" + #include <stdlib.h> #define MAX_ITER 8 diff --git a/src/common/crc32c_intel_baseline.h b/src/common/crc32c_intel_baseline.h index 5b14ddfc07e..e463575e28c 100644 --- a/src/common/crc32c_intel_baseline.h +++ b/src/common/crc32c_intel_baseline.h @@ -1,6 +1,8 @@ #ifndef CEPH_COMMON_CRC32C_INTEL_BASELINE_H #define CEPH_COMMON_CRC32C_INTEL_BASELINE_H +#include "include/int_types.h" + #ifdef __cplusplus extern "C" { #endif diff --git a/src/common/crc32c_intel_fast.c b/src/common/crc32c_intel_fast.c index 10b3c1c5c27..0532dd261cf 100644 --- a/src/common/crc32c_intel_fast.c +++ b/src/common/crc32c_intel_fast.c @@ -1,13 +1,30 @@ -#include <inttypes.h> #include "acconfig.h" +#include "include/int_types.h" + +#include "common/crc32c_intel_baseline.h" extern unsigned int crc32_iscsi_00(unsigned char const *buffer, int len, unsigned int crc); -#ifdef WITH_GOOD_YASM_ELF64 +#ifdef HAVE_GOOD_YASM_ELF64 uint32_t ceph_crc32c_intel_fast(uint32_t crc, unsigned char const *buffer, unsigned len) { - return crc32_iscsi_00(buffer, len, crc); + uint32_t v; + unsigned left; + + /* + * the crc32_iscsi_00 method reads past buffer+len (because it + * reads full words) which makes valgrind unhappy. don't do + * that. + */ + if (len < 16) + return ceph_crc32c_intel_baseline(crc, buffer, len); + left = ((unsigned long)buffer + len) & 7; + len -= left; + v = crc32_iscsi_00(buffer, len, crc); + if (left) + v = ceph_crc32c_intel_baseline(v, buffer + len, left); + return v; } int ceph_crc32c_intel_fast_exists(void) diff --git a/src/os/hobject.cc b/src/common/hobject.cc index d6273693c62..d6273693c62 100644 --- a/src/os/hobject.cc +++ b/src/common/hobject.cc diff --git a/src/os/hobject.h b/src/common/hobject.h index 633e471dffc..633e471dffc 100644 --- a/src/os/hobject.h +++ b/src/common/hobject.h diff --git a/src/common/obj_bencher.cc b/src/common/obj_bencher.cc index 6490b4f5932..599cbbc62fb 100644 --- a/src/common/obj_bencher.cc +++ b/src/common/obj_bencher.cc @@ -79,7 +79,7 @@ ostream& ObjBencher::out(ostream& os, utime_t& t) ostream& ObjBencher::out(ostream& os) { - utime_t cur_time = ceph_clock_now(g_ceph_context); + utime_t cur_time = ceph_clock_now(cct); return out(os, cur_time); } @@ -95,7 +95,7 @@ void *ObjBencher::status_printer(void *_bencher) { ONE_SECOND.set_from_double(1.0); bencher->lock.Lock(); while(!data.done) { - utime_t cur_time = ceph_clock_now(g_ceph_context); + utime_t cur_time = ceph_clock_now(bencher->cct); if (i % 20 == 0) { if (i > 0) @@ -158,7 +158,7 @@ void *ObjBencher::status_printer(void *_bencher) { } ++i; ++cycleSinceChange; - cond.WaitInterval(g_ceph_context, bencher->lock, ONE_SECOND); + cond.WaitInterval(bencher->cct, bencher->lock, ONE_SECOND); } bencher->lock.Unlock(); return NULL; @@ -339,10 +339,10 @@ int ObjBencher::write_bench(int secondsToRun, int maxObjectsToCreate, pthread_create(&print_thread, NULL, ObjBencher::status_printer, (void *)this); lock.Lock(); - data.start_time = ceph_clock_now(g_ceph_context); + data.start_time = ceph_clock_now(cct); lock.Unlock(); for (int i = 0; i<concurrentios; ++i) { - start_times[i] = ceph_clock_now(g_ceph_context); + start_times[i] = ceph_clock_now(cct); r = create_completion(i, _aio_cb, (void *)&lc); if (r < 0) goto ERR; @@ -365,7 +365,7 @@ int ObjBencher::write_bench(int secondsToRun, int maxObjectsToCreate, stopTime = data.start_time + runtime; slot = 0; lock.Lock(); - while( ceph_clock_now(g_ceph_context) < stopTime && + while( ceph_clock_now(cct) < stopTime && (!maxObjectsToCreate || data.started < maxObjectsToCreate)) { bool found = false; while (1) { @@ -397,7 +397,7 @@ int ObjBencher::write_bench(int secondsToRun, int maxObjectsToCreate, lock.Unlock(); goto ERR; } - data.cur_latency = ceph_clock_now(g_ceph_context) - start_times[slot]; + data.cur_latency = ceph_clock_now(cct) - start_times[slot]; data.history.latency.push_back(data.cur_latency); total_latency += data.cur_latency; if( data.cur_latency > data.max_latency) data.max_latency = data.cur_latency; @@ -407,11 +407,11 @@ int ObjBencher::write_bench(int secondsToRun, int maxObjectsToCreate, --data.in_flight; lock.Unlock(); release_completion(slot); - timePassed = ceph_clock_now(g_ceph_context) - data.start_time; + timePassed = ceph_clock_now(cct) - data.start_time; //write new stuff to backend, then delete old stuff //and save locations of new stuff for later deletion - start_times[slot] = ceph_clock_now(g_ceph_context); + start_times[slot] = ceph_clock_now(cct); r = create_completion(slot, _aio_cb, &lc); if (r < 0) goto ERR; @@ -438,7 +438,7 @@ int ObjBencher::write_bench(int secondsToRun, int maxObjectsToCreate, lock.Unlock(); goto ERR; } - data.cur_latency = ceph_clock_now(g_ceph_context) - start_times[slot]; + data.cur_latency = ceph_clock_now(cct) - start_times[slot]; data.history.latency.push_back(data.cur_latency); total_latency += data.cur_latency; if (data.cur_latency > data.max_latency) data.max_latency = data.cur_latency; @@ -451,7 +451,7 @@ int ObjBencher::write_bench(int secondsToRun, int maxObjectsToCreate, delete contents[slot]; } - timePassed = ceph_clock_now(g_ceph_context) - data.start_time; + timePassed = ceph_clock_now(cct) - data.start_time; lock.Lock(); data.done = true; lock.Unlock(); @@ -529,7 +529,7 @@ int ObjBencher::seq_read_bench(int seconds_to_run, int num_objects, int concurre lock.Lock(); data.finished = 0; - data.start_time = ceph_clock_now(g_ceph_context); + data.start_time = ceph_clock_now(cct); lock.Unlock(); pthread_t print_thread; @@ -539,7 +539,7 @@ int ObjBencher::seq_read_bench(int seconds_to_run, int num_objects, int concurre //start initial reads for (int i = 0; i < concurrentios; ++i) { index[i] = i; - start_times[i] = ceph_clock_now(g_ceph_context); + start_times[i] = ceph_clock_now(cct); create_completion(i, _aio_cb, (void *)&lc); r = aio_read(name[i], i, contents[i], data.object_size); if (r < 0) { //naughty, doesn't clean up heap -- oh, or handle the print thread! @@ -557,7 +557,7 @@ int ObjBencher::seq_read_bench(int seconds_to_run, int num_objects, int concurre bufferlist *cur_contents; slot = 0; - while (seconds_to_run && (ceph_clock_now(g_ceph_context) < finish_time) && + while (seconds_to_run && (ceph_clock_now(cct) < finish_time) && num_objects > data.started) { lock.Lock(); int old_slot = slot; @@ -590,7 +590,7 @@ int ObjBencher::seq_read_bench(int seconds_to_run, int num_objects, int concurre lock.Unlock(); goto ERR; } - data.cur_latency = ceph_clock_now(g_ceph_context) - start_times[slot]; + data.cur_latency = ceph_clock_now(cct) - start_times[slot]; total_latency += data.cur_latency; if( data.cur_latency > data.max_latency) data.max_latency = data.cur_latency; if (data.cur_latency < data.min_latency) data.min_latency = data.cur_latency; @@ -602,7 +602,7 @@ int ObjBencher::seq_read_bench(int seconds_to_run, int num_objects, int concurre cur_contents = contents[slot]; //start new read and check data if requested - start_times[slot] = ceph_clock_now(g_ceph_context); + start_times[slot] = ceph_clock_now(cct); contents[slot] = new bufferlist(); create_completion(slot, _aio_cb, (void *)&lc); r = aio_read(newName, slot, contents[slot], data.object_size); @@ -633,7 +633,7 @@ int ObjBencher::seq_read_bench(int seconds_to_run, int num_objects, int concurre lock.Unlock(); goto ERR; } - data.cur_latency = ceph_clock_now(g_ceph_context) - start_times[slot]; + data.cur_latency = ceph_clock_now(cct) - start_times[slot]; total_latency += data.cur_latency; if (data.cur_latency > data.max_latency) data.max_latency = data.cur_latency; if (data.cur_latency < data.min_latency) data.min_latency = data.cur_latency; @@ -650,7 +650,7 @@ int ObjBencher::seq_read_bench(int seconds_to_run, int num_objects, int concurre delete contents[slot]; } - runtime = ceph_clock_now(g_ceph_context) - data.start_time; + runtime = ceph_clock_now(cct) - data.start_time; lock.Lock(); data.done = true; lock.Unlock(); diff --git a/src/common/obj_bencher.h b/src/common/obj_bencher.h index d626eda376d..c8f671f8c90 100644 --- a/src/common/obj_bencher.h +++ b/src/common/obj_bencher.h @@ -16,6 +16,7 @@ #include "common/config.h" #include "common/Cond.h" +#include "common/ceph_context.h" struct bench_interval_data { double min_bandwidth; @@ -51,6 +52,8 @@ const int OP_RAND_READ = 3; class ObjBencher { bool show_time; +public: + CephContext *cct; protected: Mutex lock; @@ -89,7 +92,7 @@ protected: ostream& out(ostream& os); ostream& out(ostream& os, utime_t& t); public: - ObjBencher() : show_time(false), lock("ObjBencher::lock") {} + ObjBencher(CephContext *cct_) : show_time(false), cct(cct_), lock("ObjBencher::lock") {} virtual ~ObjBencher() {} int aio_bench( int operation, int secondsToRun, int maxObjectsToCreate, diff --git a/src/common/perf_counters.cc b/src/common/perf_counters.cc index 339ff6a372b..4fe1354fa63 100644 --- a/src/common/perf_counters.cc +++ b/src/common/perf_counters.cc @@ -12,13 +12,14 @@ * */ +#include "include/int_types.h" + #include "common/perf_counters.h" #include "common/dout.h" #include "common/errno.h" #include "common/Formatter.h" #include <errno.h> -#include <inttypes.h> #include <map> #include <sstream> #include <stdint.h> diff --git a/src/crush/CrushCompiler.cc b/src/crush/CrushCompiler.cc index aee621d8e32..5f92bf7e4ec 100644 --- a/src/crush/CrushCompiler.cc +++ b/src/crush/CrushCompiler.cc @@ -527,9 +527,17 @@ int CrushCompiler::parse_bucket(iter_t const& i) item_id[name] = id; item_weight[id] = bucketweight; - crush.add_bucket(id, alg, hash, type, size, &items[0], &weights[0]); - crush.set_item_name(id, name.c_str()); - return 0; + assert(id != 0); + int r = crush.add_bucket(id, alg, hash, type, size, &items[0], &weights[0], NULL); + if (r < 0) { + if (r == -EEXIST) + err << "Duplicate bucket id " << id << std::endl; + else + err << "add_bucket failed " << strerror(-r) << std::endl; + return r; + } + r = crush.set_item_name(id, name.c_str()); + return r; } int CrushCompiler::parse_rule(iter_t const& i) diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc index e96e6123aab..bab9f9a817e 100644 --- a/src/crush/CrushWrapper.cc +++ b/src/crush/CrushWrapper.cc @@ -363,9 +363,15 @@ int CrushWrapper::insert_item(CephContext *cct, int item, float weight, string n if (!name_exists(q->second)) { ldout(cct, 5) << "insert_item creating bucket " << q->second << dendl; - int empty = 0; - cur = add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_DEFAULT, p->first, 1, &cur, &empty); - set_item_name(cur, q->second); + int empty = 0, newid; + int r = add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_DEFAULT, p->first, 1, &cur, &empty, &newid); + if (r < 0) { + char buf[128]; + ldout(cct, 1) << "add_bucket failure error: " << strerror_r(-r, buf, sizeof(buf)) << dendl; + return r; + } + set_item_name(newid, q->second); + cur = newid; continue; } diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h index 3d07a281956..80906e4fe18 100644 --- a/src/crush/CrushWrapper.h +++ b/src/crush/CrushWrapper.h @@ -720,10 +720,10 @@ public: /* modifiers */ int add_bucket(int bucketno, int alg, int hash, int type, int size, - int *items, int *weights) { + int *items, int *weights, int *idout) { crush_bucket *b = crush_make_bucket(alg, hash, type, size, items, weights); assert(b); - return crush_add_bucket(crush, bucketno, b); + return crush_add_bucket(crush, bucketno, b, idout); } void finalize() { diff --git a/src/crush/Makefile.am b/src/crush/Makefile.am new file mode 100644 index 00000000000..3d2f45b80e6 --- /dev/null +++ b/src/crush/Makefile.am @@ -0,0 +1,30 @@ +libcrush_la_SOURCES = \ + crush/builder.c \ + crush/mapper.c \ + crush/crush.c \ + crush/hash.c \ + crush/CrushWrapper.cc \ + crush/CrushCompiler.cc \ + crush/CrushTester.cc +noinst_LTLIBRARIES += libcrush.la + +noinst_HEADERS += \ + crush/CrushCompiler.h \ + crush/CrushTester.h \ + crush/CrushWrapper.h \ + crush/CrushWrapper.i \ + crush/builder.h \ + crush/crush.h \ + crush/grammar.h \ + crush/hash.h \ + crush/mapper.h \ + crush/sample.txt \ + crush/types.h + +#crush_includedir = $(includedir)/crush +#crush_include_DATA = \ +# $(srcdir)/crush/hash.h \ +# $(srcdir)/crush/crush.h \ +# $(srcdir)/crush/mapper.h \ +# $(srcdir)/crush/types.h + diff --git a/src/crush/builder.c b/src/crush/builder.c index 2eb6ff5fc1e..9bfde0bd8e2 100644 --- a/src/crush/builder.c +++ b/src/crush/builder.c @@ -123,7 +123,8 @@ int crush_get_next_bucket_id(struct crush_map *map) int crush_add_bucket(struct crush_map *map, int id, - struct crush_bucket *bucket) + struct crush_bucket *bucket, + int *idout) { int pos; @@ -148,13 +149,16 @@ int crush_add_bucket(struct crush_map *map, memset(map->buckets + oldsize, 0, (map->max_buckets-oldsize) * sizeof(map->buckets[0])); } - assert(map->buckets[pos] == 0); + if (map->buckets[pos] != 0) { + return -EEXIST; + } /* add it */ bucket->id = id; map->buckets[pos] = bucket; - return id; + if (idout) *idout = id; + return 0; } int crush_remove_bucket(struct crush_map *map, struct crush_bucket *bucket) diff --git a/src/crush/builder.h b/src/crush/builder.h index 7d30c882343..1003c353e60 100644 --- a/src/crush/builder.h +++ b/src/crush/builder.h @@ -15,7 +15,7 @@ extern int crush_add_rule(struct crush_map *map, struct crush_rule *rule, int ru extern int crush_get_next_bucket_id(struct crush_map *map); extern int crush_add_bucket(struct crush_map *map, int bucketno, - struct crush_bucket *bucket); + struct crush_bucket *bucket, int *idout); struct crush_bucket *crush_make_bucket(int alg, int hash, int type, int size, int *items, int *weights); extern int crush_bucket_add_item(struct crush_bucket *bucket, int item, int weight); extern int crush_bucket_adjust_item_weight(struct crush_bucket *bucket, int item, int weight); diff --git a/src/crush/crush.h b/src/crush/crush.h index 82d032879d9..4adabcbf331 100644 --- a/src/crush/crush.h +++ b/src/crush/crush.h @@ -1,11 +1,12 @@ #ifndef CEPH_CRUSH_CRUSH_H #define CEPH_CRUSH_CRUSH_H +#include "include/int_types.h" + #if defined(__linux__) #include <linux/types.h> #elif defined(__FreeBSD__) #include <sys/types.h> -#include "include/inttypes.h" #endif /* diff --git a/src/crush/hash.c b/src/crush/hash.c index 9f7f3257ebd..9b15321d783 100644 --- a/src/crush/hash.c +++ b/src/crush/hash.c @@ -1,10 +1,11 @@ +#include "include/int_types.h" #if defined(__linux__) #include <linux/types.h> #elif defined(__FreeBSD__) #include <sys/types.h> -#include "include/inttypes.h" #endif + #include "hash.h" /* diff --git a/src/crush/mapper.c b/src/crush/mapper.c index 3215564172a..ce23ef7c711 100644 --- a/src/crush/mapper.c +++ b/src/crush/mapper.c @@ -562,7 +562,7 @@ int crush_do_rule(const struct crush_map *map, /* copy final _leaf_ values to output set */ memcpy(o, c, osize*sizeof(*o)); - /* swap t and w arrays */ + /* swap o and w arrays */ tmp = o; o = w; w = tmp; diff --git a/src/global/Makefile.am b/src/global/Makefile.am new file mode 100644 index 00000000000..79a7ffff689 --- /dev/null +++ b/src/global/Makefile.am @@ -0,0 +1,14 @@ +libglobal_la_SOURCES = \ + global/global_context.cc \ + global/global_init.cc \ + global/pidfile.cc \ + global/signal_handler.cc +libglobal_la_LIBADD = $(LIBCOMMON) +noinst_LTLIBRARIES += libglobal.la + +noinst_HEADERS += \ + global/pidfile.h \ + global/global_init.h \ + global/global_context.h \ + global/signal_handler.h + diff --git a/src/include/Makefile.am b/src/include/Makefile.am new file mode 100644 index 00000000000..d702ebd2795 --- /dev/null +++ b/src/include/Makefile.am @@ -0,0 +1,80 @@ +libcephfs_includedir = $(includedir)/cephfs +libcephfs_include_DATA = $(srcdir)/include/cephfs/libcephfs.h + +librbd_includedir = $(includedir)/rbd +librbd_include_DATA = \ + $(srcdir)/include/rbd/features.h \ + $(srcdir)/include/rbd/librbd.h \ + $(srcdir)/include/rbd/librbd.hpp + +rados_includedir = $(includedir)/rados +rados_include_DATA = \ + $(srcdir)/include/rados/librados.h \ + $(srcdir)/include/rados/rados_types.h \ + $(srcdir)/include/rados/rados_types.hpp \ + $(srcdir)/include/rados/librados.hpp \ + $(srcdir)/include/buffer.h \ + $(srcdir)/include/page.h \ + $(srcdir)/include/crc32c.h + +noinst_HEADERS += \ + include/bloom_filter.hpp \ + include/Context.h \ + include/CompatSet.h \ + include/Distribution.h \ + include/addr_parsing.h \ + include/assert.h \ + include/atomic.h \ + include/bitmapper.h \ + include/blobhash.h \ + include/buffer.h \ + include/byteorder.h \ + include/cephfs/libcephfs.h \ + include/ceph_features.h \ + include/ceph_frag.h \ + include/ceph_fs.h \ + include/ceph_hash.h \ + include/cmp.h \ + include/color.h \ + include/compat.h \ + include/crc32c.h \ + include/encoding.h \ + include/err.h \ + include/error.h \ + include/filepath.h \ + include/frag.h \ + include/hash.h \ + include/intarith.h \ + include/interval_set.h \ + include/int_types.h \ + include/ipaddr.h \ + include/linux_fiemap.h \ + include/lru.h \ + include/msgr.h \ + include/object.h \ + include/page.h \ + include/rangeset.h \ + include/rados.h \ + include/rbd_types.h \ + include/statlite.h \ + include/str_list.h \ + include/stringify.h \ + include/triple.h \ + include/types.h \ + include/utime.h \ + include/dlist.h \ + include/elist.h \ + include/uuid.h \ + include/xlist.h \ + include/rados/librados.h \ + include/rados/rados_types.h \ + include/rados/rados_types.hpp \ + include/rados/librados.hpp \ + include/rados/librgw.h \ + include/rados/page.h \ + include/rados/crc32c.h \ + include/rados/buffer.h \ + include/rbd/features.h \ + include/rbd/librbd.h \ + include/rbd/librbd.hpp\ + include/util.h diff --git a/src/include/buffer.h b/src/include/buffer.h index 8e637d658c5..077cf0d9b0b 100644 --- a/src/include/buffer.h +++ b/src/include/buffer.h @@ -11,16 +11,16 @@ * Foundation. See file COPYING. * */ - #ifndef CEPH_BUFFER_H #define CEPH_BUFFER_H +#include "include/int_types.h" + #if defined(__linux__) #include <stdlib.h> #include <linux/types.h> #elif defined(__FreeBSD__) #include <sys/types.h> -#include "include/inttypes.h" #include <stdlib.h> #endif diff --git a/src/include/crc32c.h b/src/include/crc32c.h index d5f7388be56..8e22c624636 100644 --- a/src/include/crc32c.h +++ b/src/include/crc32c.h @@ -1,7 +1,8 @@ #ifndef CEPH_CRC32C_H #define CEPH_CRC32C_H -#include "include/inttypes.h" +#include "include/int_types.h" + #include <string.h> typedef uint32_t (*ceph_crc32c_func_t)(uint32_t crc, unsigned char const *data, unsigned length); diff --git a/src/include/encoding.h b/src/include/encoding.h index a091f7f69e9..fedc8b31299 100644 --- a/src/include/encoding.h +++ b/src/include/encoding.h @@ -11,13 +11,13 @@ * Foundation. See file COPYING. * */ - #ifndef CEPH_ENCODING_H #define CEPH_ENCODING_H +#include "include/int_types.h" + #include <tr1/memory> -#include "inttypes.h" #include "byteorder.h" #include "buffer.h" #include "assert.h" diff --git a/src/include/int_types.h b/src/include/int_types.h new file mode 100644 index 00000000000..f290f9d8d23 --- /dev/null +++ b/src/include/int_types.h @@ -0,0 +1,75 @@ +#ifndef CEPH_INTTYPES_H +#define CEPH_INTTYPES_H + +#include "acconfig.h" + +#if defined(__linux__) +#include <linux/types.h> +#endif + +/* + * Get 64b integers either from inttypes.h or glib.h + */ +#ifdef HAVE_INTTYPES_H +# include <inttypes.h> +//#else +//# ifdef HAVE_GLIB +//# include <glib.h> +//# endif +#endif + +/* + * C99 says inttypes.h includes stdint.h, but that's not true on all + * systems. If it's there, include it always - just in case. + */ +#ifdef HAVE_STDINT_H +#include <stdint.h> +#endif + +/* + * Emergency replacements for PRI*64 modifiers. Some systems have + * an inttypes.h that doesn't define all the PRI[doxu]64 macros. + */ +#if !defined(PRIu64) +# if defined(HAVE_INTTYPES_H) || defined(HAVE_GLIB) +/* If we have inttypes or glib, assume we have 64-bit long long int */ +# define PRIu64 "llu" +# define PRIi64 "lli" +# define PRIx64 "llx" +# define PRIX64 "llX" +# define PRIo64 "llo" +# define PRId64 "lld" +# else +/* Assume that we don't have long long, so use long int modifiers */ +# define PRIu64 "lu" +# define PRIi64 "li" +# define PRIx64 "lx" +# define PRIX64 "lX" +# define PRIo64 "lo" +# define PRId64 "ld" +# endif +#endif + +#if defined(__FreeBSD__) +#include <sys/types.h> + +typedef int8_t __s8; +typedef uint8_t __u8; +typedef int16_t __s16; +typedef uint16_t __u16; +typedef int32_t __s32; +typedef uint32_t __u32; +typedef int64_t __s64; +typedef uint64_t __u64; + +#define __bitwise__ + +typedef __u16 __bitwise__ __le16; +typedef __u16 __bitwise__ __be16; +typedef __u32 __bitwise__ __le32; +typedef __u32 __bitwise__ __be32; +typedef __u64 __bitwise__ __le64; +typedef __u64 __bitwise__ __be64; + +#endif +#endif diff --git a/src/include/inttypes.h b/src/include/inttypes.h deleted file mode 100644 index 656e2bb0a0f..00000000000 --- a/src/include/inttypes.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef CEPH_INTTYPES_H -#define CEPH_INTTYPES_H - -#include <stdint.h> -#if defined(__linux__) -#include <linux/types.h> -#elif defined(__FreeBSD__) -#include <sys/types.h> -typedef int8_t __s8; -typedef uint8_t __u8; -typedef int16_t __s16; -typedef uint16_t __u16; -typedef int32_t __s32; -typedef uint32_t __u32; -typedef int64_t __s64; -typedef uint64_t __u64; - -#define __bitwise__ - -typedef __u16 __bitwise__ __le16; -typedef __u16 __bitwise__ __be16; -typedef __u32 __bitwise__ __le32; -typedef __u32 __bitwise__ __be32; -typedef __u64 __bitwise__ __le64; -typedef __u64 __bitwise__ __be64; - -#endif -#endif diff --git a/src/include/linux_fiemap.h b/src/include/linux_fiemap.h index ba5cb6657fa..352208b2207 100644 --- a/src/include/linux_fiemap.h +++ b/src/include/linux_fiemap.h @@ -7,15 +7,15 @@ * Kalpak Shah <kalpak.shah@sun.com> * Andreas Dilger <adilger@sun.com> */ - #ifndef _LINUX_FIEMAP_H #define _LINUX_FIEMAP_H +#include "include/int_types.h" + #if defined(__linux__) #include <linux/types.h> #elif defined(__FreeBSD_) #include <sys/types.h> -#include "include/inttypes.h" #endif struct fiemap_extent { diff --git a/src/include/rados/librados.h b/src/include/rados/librados.h index 6f5e454c8e8..a85ef3057bc 100644 --- a/src/include/rados/librados.h +++ b/src/include/rados/librados.h @@ -1,6 +1,8 @@ #ifndef CEPH_LIBRADOS_H #define CEPH_LIBRADOS_H +#include "include/int_types.h" + #ifdef __cplusplus extern "C" { #endif @@ -10,7 +12,6 @@ extern "C" { #include <linux/types.h> #elif defined(__FreeBSD__) #include <sys/types.h> -#include "include/inttypes.h" #endif #include <string.h> #include "rados_types.h" diff --git a/src/include/rados/librados.hpp b/src/include/rados/librados.hpp index 5a750cbc0d1..94d3d23a824 100644 --- a/src/include/rados/librados.hpp +++ b/src/include/rados/librados.hpp @@ -439,8 +439,23 @@ namespace librados int create(const std::string& oid, bool exclusive); int create(const std::string& oid, bool exclusive, const std::string& category); + /** + * write bytes to an object at a specified offset + * + * NOTE: this call steals the contents of @param bl. + */ int write(const std::string& oid, bufferlist& bl, size_t len, uint64_t off); + /** + * append bytes to an object + * + * NOTE: this call steals the contents of @param bl. + */ int append(const std::string& oid, bufferlist& bl, size_t len); + /** + * replace object contents with provided data + * + * NOTE: this call steals the contents of @param bl. + */ int write_full(const std::string& oid, bufferlist& bl); int clone_range(const std::string& dst_oid, uint64_t dst_off, const std::string& src_oid, uint64_t src_off, @@ -457,7 +472,17 @@ namespace librados int stat(const std::string& oid, uint64_t *psize, time_t *pmtime); int exec(const std::string& oid, const char *cls, const char *method, bufferlist& inbl, bufferlist& outbl); + /** + * modify object tmap based on encoded update sequence + * + * NOTE: this call steals the contents of @param bl + */ int tmap_update(const std::string& oid, bufferlist& cmdbl); + /** + * replace object contents with provided encoded tmap data + * + * NOTE: this call steals the contents of @param bl + */ int tmap_put(const std::string& oid, bufferlist& bl); int tmap_get(const std::string& oid, bufferlist& bl); diff --git a/src/include/types.h b/src/include/types.h index 7e6ddb7117e..1f9756b22c7 100644 --- a/src/include/types.h +++ b/src/include/types.h @@ -11,12 +11,11 @@ * Foundation. See file COPYING. * */ - #ifndef CEPH_TYPES_H #define CEPH_TYPES_H // this is needed for ceph_fs to compile in userland -#include "inttypes.h" +#include "int_types.h" #include "byteorder.h" #include "uuid.h" diff --git a/src/java/Makefile.am b/src/java/Makefile.am index ac7e86cf9fa..8b28f839e46 100644 --- a/src/java/Makefile.am +++ b/src/java/Makefile.am @@ -64,7 +64,6 @@ BUILT_SOURCES = $(JAVA_H) if HAVE_JUNIT4 JAVA_TEST_CLASSES = $(JAVA_TEST_SRC:test/%.java=%.class) -ESCAPED_JAVA_TEST_CLASSES = com/ceph/fs/CephAllTests\$$1.class CEPH_TEST_PROXY=test/com/ceph/fs/CephMountTest.class @@ -73,7 +72,7 @@ $(CEPH_TEST_PROXY): $(JAVA_TEST_SRC) $(CEPH_PROXY) $(JAVAC) -source 1.5 -target 1.5 -Xlint:-options test/com/ceph/fs/*.java libcephfs-test.jar: $(CEPH_TEST_PROXY) - $(JAR) cf $@ $(JAVA_TEST_CLASSES:%=-C test %) $(ESCAPED_JAVA_TEST_CLASSES:%=-C test %) + $(JAR) cf $@ $(JAVA_TEST_CLASSES:%=-C test %) java_DATA += libcephfs-test.jar diff --git a/src/java/test/com/ceph/fs/CephAllTests.java b/src/java/test/com/ceph/fs/CephAllTests.java index 71c2ddfee96..039ad6da3b7 100644 --- a/src/java/test/com/ceph/fs/CephAllTests.java +++ b/src/java/test/com/ceph/fs/CephAllTests.java @@ -23,7 +23,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.UUID; import org.junit.*; -import org.junit.rules.ExternalResource; import org.junit.runners.Suite; import org.junit.runner.RunWith; import static org.junit.Assert.*; @@ -42,16 +41,4 @@ import static org.junit.Assert.*; */ public class CephAllTests{ - @Rule - public static ExternalResource testRule = new ExternalResource(){ - @Override - protected void before() throws Throwable{ - // Add debugging messages or setup code here - }; - - @Override - protected void after(){ - // Add debugging messages or cleanup code here - }; - }; } diff --git a/src/json_spirit/Makefile.am b/src/json_spirit/Makefile.am new file mode 100644 index 00000000000..43025be0e71 --- /dev/null +++ b/src/json_spirit/Makefile.am @@ -0,0 +1,18 @@ +libjson_spirit_la_SOURCES = \ + json_spirit/json_spirit_reader.cpp \ + json_spirit/json_spirit_writer.cpp \ + json_spirit/json_spirit_value.cpp +noinst_LTLIBRARIES += libjson_spirit.la + +noinst_HEADERS += \ + json_spirit/json_spirit.h \ + json_spirit/json_spirit_error_position.h \ + json_spirit/json_spirit_reader.h \ + json_spirit/json_spirit_reader_template.h \ + json_spirit/json_spirit_stream_reader.h \ + json_spirit/json_spirit_utils.h \ + json_spirit/json_spirit_value.h \ + json_spirit/json_spirit_writer.h \ + json_spirit/json_spirit_writer_options.h \ + json_spirit/json_spirit_writer_template.h + diff --git a/src/key_value_store/Makefile.am b/src/key_value_store/Makefile.am new file mode 100644 index 00000000000..e5bae00155f --- /dev/null +++ b/src/key_value_store/Makefile.am @@ -0,0 +1,10 @@ +libcls_kvs_la_SOURCES = key_value_store/cls_kvs.cc +libcls_kvs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_kvs_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' +radoslib_LTLIBRARIES += libcls_kvs.la + +noinst_HEADERS += \ + key_value_store/key_value_structure.h \ + key_value_store/kv_flat_btree_async.h \ + key_value_store/kvs_arg_types.h + diff --git a/src/librados/Makefile.am b/src/librados/Makefile.am new file mode 100644 index 00000000000..570aa91af93 --- /dev/null +++ b/src/librados/Makefile.am @@ -0,0 +1,20 @@ +librados_la_SOURCES = \ + librados/librados.cc \ + librados/RadosClient.cc \ + librados/IoCtxImpl.cc \ + librados/snap_set_diff.cc + +# We need this to avoid basename conflicts with the librados build tests in test/Makefile.am +librados_la_CFLAGS = ${AM_CFLAGS} + +LIBRADOS_DEPS += libcls_lock_client.la $(LIBOSDC) $(LIBCOMMON) +librados_la_LIBADD = $(LIBRADOS_DEPS) $(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS) +librados_la_LDFLAGS = ${AM_LDFLAGS} -version-info 2:0:0 -export-symbols-regex '^rados_.*' +lib_LTLIBRARIES += librados.la + +noinst_HEADERS += \ + librados/snap_set_diff.h \ + librados/AioCompletionImpl.h \ + librados/IoCtxImpl.h \ + librados/PoolAsyncCompletionImpl.h \ + librados/RadosClient.h diff --git a/src/librbd/AioRequest.h b/src/librbd/AioRequest.h index 7625bdd7ff3..cf50ee2c049 100644 --- a/src/librbd/AioRequest.h +++ b/src/librbd/AioRequest.h @@ -3,9 +3,9 @@ #ifndef CEPH_LIBRBD_AIOREQUEST_H #define CEPH_LIBRBD_AIOREQUEST_H -#include <map> +#include "include/int_types.h" -#include "inttypes.h" +#include <map> #include "common/snap_types.h" #include "include/buffer.h" diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h index 29ca2f197ea..c9e74393e13 100644 --- a/src/librbd/ImageCtx.h +++ b/src/librbd/ImageCtx.h @@ -3,7 +3,7 @@ #ifndef CEPH_LIBRBD_IMAGECTX_H #define CEPH_LIBRBD_IMAGECTX_H -#include <inttypes.h> +#include "include/int_types.h" #include <map> #include <set> diff --git a/src/librbd/Makefile.am b/src/librbd/Makefile.am new file mode 100644 index 00000000000..d4e2455c8c1 --- /dev/null +++ b/src/librbd/Makefile.am @@ -0,0 +1,24 @@ +librbd_la_SOURCES = \ + librbd/librbd.cc \ + librbd/AioCompletion.cc \ + librbd/AioRequest.cc \ + librbd/ImageCtx.cc \ + librbd/internal.cc \ + librbd/LibrbdWriteback.cc \ + librbd/WatchCtx.cc +librbd_la_LIBADD = \ + $(LIBRADOS) $(LIBOSDC) \ + libcls_rbd_client.la libcls_lock_client.la \ + $(PTHREAD_LIBS) $(EXTRALIBS) +librbd_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '^rbd_.*' +lib_LTLIBRARIES += librbd.la + +noinst_HEADERS += \ + librbd/AioCompletion.h \ + librbd/AioRequest.h \ + librbd/ImageCtx.h \ + librbd/internal.h \ + librbd/LibrbdWriteback.h \ + librbd/parent_types.h \ + librbd/SnapInfo.h \ + librbd/WatchCtx.h diff --git a/src/librbd/SnapInfo.h b/src/librbd/SnapInfo.h index 5a7852c3c66..44dd4cf208e 100644 --- a/src/librbd/SnapInfo.h +++ b/src/librbd/SnapInfo.h @@ -3,7 +3,7 @@ #ifndef CEPH_LIBRBD_SNAPINFO_H #define CEPH_LIBRBD_SNAPINFO_H -#include <inttypes.h> +#include "include/int_types.h" #include "include/rados/librados.hpp" diff --git a/src/librbd/WatchCtx.h b/src/librbd/WatchCtx.h index 0c0802a4035..9872c84307b 100644 --- a/src/librbd/WatchCtx.h +++ b/src/librbd/WatchCtx.h @@ -3,7 +3,7 @@ #ifndef CEPH_LIBRBD_WATCHCTX_H #define CEPH_LIBRBD_WATCHCTX_H -#include <inttypes.h> +#include "include/int_types.h" #include "common/Mutex.h" #include "include/buffer.h" diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc index abc6ff92a28..b8a757ab333 100644 --- a/src/librbd/internal.cc +++ b/src/librbd/internal.cc @@ -1,5 +1,7 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab +#include "include/int_types.h" + #include <errno.h> #include <limits.h> @@ -8,7 +10,6 @@ #include "common/errno.h" #include "common/Throttle.h" #include "cls/lock/cls_lock_client.h" -#include "include/inttypes.h" #include "include/stringify.h" #include "cls/rbd/cls_rbd.h" diff --git a/src/librbd/internal.h b/src/librbd/internal.h index 33f5191706d..43458886b43 100644 --- a/src/librbd/internal.h +++ b/src/librbd/internal.h @@ -3,7 +3,7 @@ #ifndef CEPH_LIBRBD_INTERNAL_H #define CEPH_LIBRBD_INTERNAL_H -#include <inttypes.h> +#include "include/int_types.h" #include <map> #include <set> diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc index af413dda04f..cad0c5eb172 100644 --- a/src/librbd/librbd.cc +++ b/src/librbd/librbd.cc @@ -11,9 +11,9 @@ * Foundation. See file COPYING. * */ +#include "include/int_types.h" #include <errno.h> -#include <inttypes.h> #include "common/Cond.h" #include "common/dout.h" diff --git a/src/log/Makefile.am b/src/log/Makefile.am new file mode 100644 index 00000000000..b66e6cf84ce --- /dev/null +++ b/src/log/Makefile.am @@ -0,0 +1,11 @@ +liblog_la_SOURCES = \ + log/Log.cc \ + log/SubsystemMap.cc +noinst_LTLIBRARIES += liblog.la + +noinst_HEADERS += \ + log/Entry.h \ + log/EntryQueue.h \ + log/Log.h \ + log/SubsystemMap.h + diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc index dd483263b6d..46f8d33cfd8 100644 --- a/src/mds/CInode.cc +++ b/src/mds/CInode.cc @@ -12,7 +12,8 @@ * */ -#include <inttypes.h> +#include "include/int_types.h" + #include <string> #include <stdio.h> diff --git a/src/mds/Makefile.am b/src/mds/Makefile.am new file mode 100644 index 00000000000..05d0e1a2429 --- /dev/null +++ b/src/mds/Makefile.am @@ -0,0 +1,92 @@ +libmds_la_SOURCES = \ + mds/Anchor.cc \ + mds/Capability.cc \ + mds/Dumper.cc \ + mds/Resetter.cc \ + mds/MDS.cc \ + mds/flock.cc \ + mds/locks.c \ + mds/journal.cc \ + mds/Server.cc \ + mds/Mutation.cc \ + mds/MDCache.cc \ + mds/Locker.cc \ + mds/Migrator.cc \ + mds/MDBalancer.cc \ + mds/CDentry.cc \ + mds/CDir.cc \ + mds/CInode.cc \ + mds/LogEvent.cc \ + mds/MDSTable.cc \ + mds/InoTable.cc \ + mds/MDSTableClient.cc \ + mds/MDSTableServer.cc \ + mds/AnchorServer.cc \ + mds/AnchorClient.cc \ + mds/SnapRealm.cc \ + mds/SnapServer.cc \ + mds/snap.cc \ + mds/SessionMap.cc \ + mds/MDLog.cc +libmds_la_LIBADD = $(LIBOSDC) +noinst_LTLIBRARIES += libmds.la + +noinst_HEADERS += \ + mds/inode_backtrace.h \ + mds/flock.h \ + mds/locks.c \ + mds/locks.h \ + mds/Anchor.h \ + mds/AnchorClient.h \ + mds/AnchorServer.h \ + mds/CDentry.h \ + mds/CDir.h \ + mds/CInode.h \ + mds/Capability.h \ + mds/Dumper.h \ + mds/InoTable.h \ + mds/LocalLock.h \ + mds/Locker.h \ + mds/LogEvent.h \ + mds/LogSegment.h \ + mds/MDBalancer.h \ + mds/MDCache.h \ + mds/MDLog.h \ + mds/MDS.h \ + mds/MDSMap.h \ + mds/MDSTable.h \ + mds/MDSTableServer.h \ + mds/MDSTableClient.h \ + mds/Mutation.h \ + mds/Migrator.h \ + mds/Resetter.h \ + mds/ScatterLock.h \ + mds/Server.h \ + mds/SessionMap.h \ + mds/SimpleLock.h \ + mds/SnapClient.h \ + mds/SnapRealm.h \ + mds/SnapServer.h \ + mds/inode_backtrace.h \ + mds/mds_table_types.h \ + mds/mdstypes.h \ + mds/snap.h + +noinst_HEADERS += \ + mds/events/ECommitted.h \ + mds/events/EExport.h \ + mds/events/EFragment.h \ + mds/events/EImportFinish.h \ + mds/events/EImportStart.h \ + mds/events/EMetaBlob.h \ + mds/events/EOpen.h \ + mds/events/EResetJournal.h \ + mds/events/ESession.h \ + mds/events/ESessions.h \ + mds/events/ESlaveUpdate.h \ + mds/events/ESubtreeMap.h \ + mds/events/ETableClient.h \ + mds/events/ETableServer.h \ + mds/events/EUpdate.h + + diff --git a/src/mds/locks.c b/src/mds/locks.c index 37e3f5ea764..f367eda29d2 100644 --- a/src/mds/locks.c +++ b/src/mds/locks.c @@ -1,15 +1,15 @@ - // there must be a better way? typedef char bool; #define false 0 #define true 1 +#include "include/int_types.h" + #include <netinet/in.h> #if defined(__linux__) #include <linux/types.h> #elif defined(__FreeBSD__) #include <sys/types.h> -#include "include/inttypes.h" #endif #include <string.h> #include <fcntl.h> diff --git a/src/mds/mdstypes.h b/src/mds/mdstypes.h index 4fcf584fb8c..902e3104aa8 100644 --- a/src/mds/mdstypes.h +++ b/src/mds/mdstypes.h @@ -3,7 +3,8 @@ #ifndef CEPH_MDSTYPES_H #define CEPH_MDSTYPES_H -#include <inttypes.h> +#include "include/int_types.h" + #include <math.h> #include <ostream> #include <set> diff --git a/src/messages/MOSDOpReply.h b/src/messages/MOSDOpReply.h index 5bf5ab4ef62..c0e989f7c3a 100644 --- a/src/messages/MOSDOpReply.h +++ b/src/messages/MOSDOpReply.h @@ -31,7 +31,7 @@ class MOSDOpReply : public Message { - static const int HEAD_VERSION = 5; + static const int HEAD_VERSION = 6; static const int COMPAT_VERSION = 2; object_t oid; @@ -44,6 +44,7 @@ class MOSDOpReply : public Message { version_t user_version; epoch_t osdmap_epoch; int32_t retry_attempt; + request_redirect_t redirect; public: object_t get_oid() const { return oid; } @@ -87,6 +88,10 @@ public: bad_replay_version = v; } + void set_redirect(const request_redirect_t& redir) { redirect = redir; } + const request_redirect_t& get_redirect() const { return redirect; } + bool is_redirect_reply() const { return !redirect.empty(); } + void add_flags(int f) { flags |= f; } void claim_op_out_data(vector<OSDOp>& o) { @@ -109,7 +114,7 @@ public: } // osdmap - epoch_t get_map_epoch() { return osdmap_epoch; } + epoch_t get_map_epoch() const { return osdmap_epoch; } /*osd_reqid_t get_reqid() { return osd_reqid_t(get_dest(), head.client_inc, @@ -180,6 +185,7 @@ public: ::encode(replay_version, payload); ::encode(user_version, payload); + ::encode(redirect, payload); } } virtual void decode_payload() { @@ -232,6 +238,9 @@ public: replay_version = bad_replay_version; user_version = replay_version.version; } + + if (header.version >= 6) + ::decode(redirect, p); } } @@ -253,6 +262,9 @@ public: char buf[80]; out << " (" << strerror_r(-get_result(), buf, sizeof(buf)) << ")"; } + if (is_redirect_reply()) { + out << " redirect: { " << redirect << " }"; + } out << ")"; } diff --git a/src/messages/MOSDSubOp.h b/src/messages/MOSDSubOp.h index 50b1a926957..4169e01325e 100644 --- a/src/messages/MOSDSubOp.h +++ b/src/messages/MOSDSubOp.h @@ -25,7 +25,7 @@ class MOSDSubOp : public Message { - static const int HEAD_VERSION = 7; + static const int HEAD_VERSION = 8; static const int COMPAT_VERSION = 1; public: @@ -86,6 +86,9 @@ public: // indicates that we must fix hobject_t encoding bool hobject_incorrect_pool; + hobject_t new_temp_oid; ///< new temp object that we must now start tracking + hobject_t discard_temp_oid; ///< previously used temp object that we can now stop tracking + int get_cost() const { if (ops.size() == 1 && ops[0].op.op == CEPH_OSD_OP_PULL) return ops[0].op.extent.length; @@ -150,6 +153,11 @@ public: poid.pool = pgid.pool(); hobject_incorrect_pool = true; } + + if (header.version >= 8) { + ::decode(new_temp_oid, p); + ::decode(discard_temp_oid, p); + } } virtual void encode_payload(uint64_t features) { @@ -194,6 +202,8 @@ public: ::encode(current_progress, payload); ::encode(omap_entries, payload); ::encode(omap_header, payload); + ::encode(new_temp_oid, payload); + ::encode(discard_temp_oid, payload); } MOSDSubOp() diff --git a/src/messages/Makefile.am b/src/messages/Makefile.am new file mode 100644 index 00000000000..c503d3fca9b --- /dev/null +++ b/src/messages/Makefile.am @@ -0,0 +1,113 @@ + +noinst_HEADERS += \ + messages/MAuth.h \ + messages/MAuthReply.h \ + messages/MCacheExpire.h \ + messages/MClientCaps.h \ + messages/MClientCapRelease.h \ + messages/MClientLease.h \ + messages/MClientReconnect.h \ + messages/MClientReply.h \ + messages/MClientRequest.h \ + messages/MClientRequestForward.h \ + messages/MClientSession.h \ + messages/MClientSnap.h \ + messages/MCommand.h \ + messages/MCommandReply.h \ + messages/MDentryLink.h \ + messages/MDentryUnlink.h \ + messages/MDirUpdate.h \ + messages/MDiscover.h \ + messages/MDiscoverReply.h \ + messages/MExportCaps.h \ + messages/MExportCapsAck.h \ + messages/MExportDir.h \ + messages/MExportDirAck.h \ + messages/MExportDirCancel.h \ + messages/MExportDirDiscover.h \ + messages/MExportDirDiscoverAck.h \ + messages/MExportDirFinish.h \ + messages/MExportDirNotify.h \ + messages/MExportDirNotifyAck.h \ + messages/MExportDirPrep.h \ + messages/MExportDirPrepAck.h \ + messages/MGenericMessage.h \ + messages/MGetPoolStats.h \ + messages/MGetPoolStatsReply.h \ + messages/MHeartbeat.h \ + messages/MInodeFileCaps.h \ + messages/MLock.h \ + messages/MLog.h \ + messages/MLogAck.h \ + messages/MMDSBeacon.h \ + messages/MMDSCacheRejoin.h \ + messages/MMDSLoadTargets.h \ + messages/MMDSFindIno.h \ + messages/MMDSFindInoReply.h \ + messages/MMDSFragmentNotify.h \ + messages/MMDSMap.h \ + messages/MMDSOpenIno.h \ + messages/MMDSOpenInoReply.h \ + messages/MMDSResolve.h \ + messages/MMDSResolveAck.h \ + messages/MMDSSlaveRequest.h \ + messages/MMDSTableRequest.h \ + messages/MMonCommand.h \ + messages/MMonCommandAck.h \ + messages/MMonElection.h \ + messages/MMonGetMap.h \ + messages/MMonGetVersion.h \ + messages/MMonGetVersionReply.h \ + messages/MMonGlobalID.h \ + messages/MMonHealth.h \ + messages/MMonJoin.h \ + messages/MMonMap.h \ + messages/MMonPaxos.h \ + messages/MMonProbe.h \ + messages/MMonScrub.h \ + messages/MMonSubscribe.h \ + messages/MMonSubscribeAck.h \ + messages/MMonSync.h \ + messages/MOSDAlive.h \ + messages/MOSDBoot.h \ + messages/MOSDFailure.h \ + messages/MOSDMarkMeDown.h \ + messages/MOSDMap.h \ + messages/MOSDOp.h \ + messages/MOSDOpReply.h \ + messages/MOSDPGBackfill.h \ + messages/MOSDPGCreate.h \ + messages/MOSDPGPush.h \ + messages/MOSDPGPull.h \ + messages/MOSDPGPushReply.h \ + messages/MOSDPGInfo.h \ + messages/MOSDPGLog.h \ + messages/MOSDPGMissing.h \ + messages/MOSDPGNotify.h \ + messages/MOSDPGQuery.h \ + messages/MOSDPGRemove.h \ + messages/MOSDPGScan.h \ + messages/MBackfillReserve.h \ + messages/MRecoveryReserve.h \ + messages/MMonQuorumService.h \ + messages/MOSDPGTemp.h \ + messages/MOSDPGTrim.h \ + messages/MOSDPing.h \ + messages/MOSDRepScrub.h \ + messages/MOSDScrub.h \ + messages/MOSDSubOp.h \ + messages/MOSDSubOpReply.h \ + messages/MPGStats.h \ + messages/MPGStatsAck.h \ + messages/MPing.h \ + messages/MPoolOp.h \ + messages/MPoolOpReply.h \ + messages/MRemoveSnaps.h \ + messages/MRoute.h \ + messages/MForward.h \ + messages/MStatfs.h \ + messages/MStatfsReply.h \ + messages/MTimeCheck.h \ + messages/MWatchNotify.h \ + messages/PaxosServiceMessage.h + diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc index 9988d8c8402..b2273274521 100644 --- a/src/mon/MDSMonitor.cc +++ b/src/mon/MDSMonitor.cc @@ -947,6 +947,7 @@ bool MDSMonitor::prepare_command(MMonCommand *m) ss << "this is DANGEROUS and will wipe out the mdsmap's fs, and may clobber data in the new pools you specify. add --yes-i-really-mean-it if you do."; r = -EPERM; } else { + newmap.inc = pending_mdsmap.inc; pending_mdsmap = newmap; pending_mdsmap.epoch = mdsmap.epoch + 1; create_new_fs(pending_mdsmap, metadata, data); diff --git a/src/mon/Makefile.am b/src/mon/Makefile.am new file mode 100644 index 00000000000..d9094a81b0e --- /dev/null +++ b/src/mon/Makefile.am @@ -0,0 +1,45 @@ +libmon_la_SOURCES = \ + mon/Monitor.cc \ + mon/Paxos.cc \ + mon/PaxosService.cc \ + mon/OSDMonitor.cc \ + mon/MDSMonitor.cc \ + mon/MonmapMonitor.cc \ + mon/PGMonitor.cc \ + mon/PGMap.cc \ + mon/LogMonitor.cc \ + mon/AuthMonitor.cc \ + mon/Elector.cc \ + mon/MonitorStore.cc \ + mon/HealthMonitor.cc \ + mon/DataHealthService.cc \ + mon/ConfigKeyService.cc +libmon_la_LIBADD = $(LIBAUTH) $(LIBCOMMON) $(LIBOS) +noinst_LTLIBRARIES += libmon.la + +noinst_HEADERS += \ + mon/AuthMonitor.h \ + mon/DataHealthService.h \ + mon/Elector.h \ + mon/LogMonitor.h \ + mon/ConfigKeyService.h \ + mon/HealthMonitor.h \ + mon/HealthService.h \ + mon/MDSMonitor.h \ + mon/MonmapMonitor.h \ + mon/MonCap.h \ + mon/MonClient.h \ + mon/MonCommands.h \ + mon/MonMap.h \ + mon/Monitor.h \ + mon/MonitorStore.h \ + mon/MonitorDBStore.h \ + mon/OSDMonitor.h \ + mon/PGMap.h \ + mon/PGMonitor.h \ + mon/Paxos.h \ + mon/PaxosService.h \ + mon/QuorumService.h \ + mon/Session.h \ + mon/mon_types.h + diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index baef4ea50a5..365fd28b64e 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -479,7 +479,8 @@ COMMAND("osd pool rmsnap " \ COMMAND("osd pool create " \ "name=pool,type=CephPoolname " \ "name=pg_num,type=CephInt,range=0 " \ - "name=pgp_num,type=CephInt,range=0,req=false", \ + "name=pgp_num,type=CephInt,range=0,req=false " \ + "name=properties,type=CephString,n=N,req=false,goodchars=[A-Za-z0-9-_.=]", \ "create pool", "osd", "rw", "cli,rest") COMMAND("osd pool delete " \ "name=pool,type=CephPoolname " \ diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index 1bbf628fe7e..df614eedb92 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -2473,10 +2473,11 @@ int OSDMonitor::prepare_new_pool(MPoolOp *m) MonSession *session = m->get_session(); if (!session) return -EPERM; + vector<string> properties; if (m->auid) - return prepare_new_pool(m->name, m->auid, m->crush_rule, 0, 0); + return prepare_new_pool(m->name, m->auid, m->crush_rule, 0, 0, properties); else - return prepare_new_pool(m->name, session->auid, m->crush_rule, 0, 0); + return prepare_new_pool(m->name, session->auid, m->crush_rule, 0, 0, properties); } /** @@ -2485,11 +2486,13 @@ int OSDMonitor::prepare_new_pool(MPoolOp *m) * @param crush_rule The crush rule to use. If <0, will use the system default * @param pg_num The pg_num to use. If set to 0, will use the system default * @param pgp_num The pgp_num to use. If set to 0, will use the system default + * @param properties An opaque list of key[=value] pairs for pool configuration * * @return 0 in all cases. That's silly. */ int OSDMonitor::prepare_new_pool(string& name, uint64_t auid, int crush_rule, - unsigned pg_num, unsigned pgp_num) + unsigned pg_num, unsigned pgp_num, + const vector<string> &properties) { for (map<int64_t,string>::iterator p = pending_inc.new_pool_names.begin(); p != pending_inc.new_pool_names.end(); @@ -2519,6 +2522,18 @@ int OSDMonitor::prepare_new_pool(string& name, uint64_t auid, int crush_rule, pi->set_pgp_num(pgp_num ? pgp_num : g_conf->osd_pool_default_pgp_num); pi->last_change = pending_inc.epoch; pi->auid = auid; + for (vector<string>::const_iterator i = properties.begin(); + i != properties.end(); + i++) { + size_t equal = i->find('='); + if (equal == string::npos) + pi->properties[*i] = string(); + else { + const string key = i->substr(0, equal); + const string value = i->substr(equal); + pi->properties[key] = value; + } + } pending_inc.new_pool_names[pool] = name; return 0; } @@ -2699,9 +2714,15 @@ bool OSDMonitor::prepare_command(MMonCommand *m) err = -EINVAL; goto reply; } - int bucketno = newcrush.add_bucket(0, CRUSH_BUCKET_STRAW, + int bucketno; + err = newcrush.add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_DEFAULT, type, 0, NULL, - NULL); + NULL, &bucketno); + if (err < 0) { + char buf[128]; + ss << "add_bucket error: '" << strerror_r(-err, buf, sizeof(buf)) << "'"; + goto reply; + } err = newcrush.set_item_name(bucketno, name); if (err < 0) { ss << "error setting bucket name to '" << name << "'"; @@ -3382,7 +3403,11 @@ done: string snapname; cmd_getval(g_ceph_context, cmdmap, "snap", snapname); const pg_pool_t *p = osdmap.get_pg_pool(pool); - if (p->snap_exists(snapname.c_str())) { + if (p->is_unmanaged_snaps_mode()) { + ss << "pool " << poolstr << " is in unmanaged snaps mode"; + err = -EINVAL; + goto reply; + } else if (p->snap_exists(snapname.c_str())) { ss << "pool " << poolstr << " snap " << snapname << " already exists"; err = 0; goto reply; @@ -3416,7 +3441,11 @@ done: string snapname; cmd_getval(g_ceph_context, cmdmap, "snap", snapname); const pg_pool_t *p = osdmap.get_pg_pool(pool); - if (!p->snap_exists(snapname.c_str())) { + if (p->is_unmanaged_snaps_mode()) { + ss << "pool " << poolstr << " is in unmanaged snaps mode"; + err = -EINVAL; + goto reply; + } else if (!p->snap_exists(snapname.c_str())) { ss << "pool " << poolstr << " snap " << snapname << " does not exist"; err = 0; goto reply; @@ -3467,9 +3496,13 @@ done: goto reply; } + vector<string> properties; + cmd_getval(g_ceph_context, cmdmap, "properties", properties); + err = prepare_new_pool(poolstr, 0, // auid=0 for admin created pool -1, // default crush rule - pg_num, pgp_num); + pg_num, pgp_num, + properties); if (err < 0 && err != -EEXIST) { goto reply; } @@ -3566,7 +3599,7 @@ done: ss << "specified pg_num " << n << " <= current " << p->get_pg_num(); err = -EINVAL; } else if (!mon->pgmon()->pg_map.creating_pgs.empty()) { - ss << "currently creating pgs, wait"; + ss << "busy creating pgs; try again later"; err = -EAGAIN; } else { pending_inc.get_new_pool(pool, p)->set_pg_num(n); @@ -3576,7 +3609,7 @@ done: if (n > p->get_pg_num()) { ss << "specified pgp_num " << n << " > pg_num " << p->get_pg_num(); } else if (!mon->pgmon()->pg_map.creating_pgs.empty()) { - ss << "still creating pgs, wait"; + ss << "busy creating pgs; try again later"; err = -EAGAIN; } else { pending_inc.get_new_pool(pool, p)->set_pgp_num(n); @@ -3867,7 +3900,7 @@ bool OSDMonitor::preprocess_pool_op(MPoolOp *m) _pool_op_reply(m, 0, osdmap.get_epoch()); return true; } - + // check if the snap and snapname exists bool snap_exists = false; const pg_pool_t *p = osdmap.get_pg_pool(m->pool); @@ -3963,6 +3996,38 @@ bool OSDMonitor::prepare_pool_op(MPoolOp *m) int ret = 0; bool changed = false; + if (!osdmap.have_pg_pool(m->pool)) { + _pool_op_reply(m, -ENOENT, osdmap.get_epoch()); + return false; + } + + const pg_pool_t *pool = osdmap.get_pg_pool(m->pool); + + switch (m->op) { + case POOL_OP_CREATE_SNAP: + case POOL_OP_DELETE_SNAP: + if (!pool->is_unmanaged_snaps_mode()) { + bool snap_exists = pool->snap_exists(m->name.c_str()); + if ((m->op == POOL_OP_CREATE_SNAP && snap_exists) + || (m->op == POOL_OP_DELETE_SNAP && !snap_exists)) { + ret = 0; + } else { + break; + } + } else { + ret = -EINVAL; + } + _pool_op_reply(m, ret, osdmap.get_epoch()); + return false; + + case POOL_OP_CREATE_UNMANAGED_SNAP: + case POOL_OP_DELETE_UNMANAGED_SNAP: + if (pool->is_pool_snaps_mode()) { + _pool_op_reply(m, -EINVAL, osdmap.get_epoch()); + return false; + } + } + // projected pool info pg_pool_t pp; if (pending_inc.new_pools.count(m->pool)) @@ -3989,7 +4054,7 @@ bool OSDMonitor::prepare_pool_op(MPoolOp *m) goto out; } } - + switch (m->op) { case POOL_OP_CREATE_SNAP: if (!pp.snap_exists(m->name.c_str())) { @@ -4044,8 +4109,7 @@ bool OSDMonitor::prepare_pool_op(MPoolOp *m) out: wait_for_finished_proposal(new OSDMonitor::C_PoolOp(this, m, ret, pending_inc.epoch, &reply_data)); - propose_pending(); - return false; + return true; } bool OSDMonitor::prepare_pool_op_create(MPoolOp *m) diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h index 04f7cf5b196..304f9c4f609 100644 --- a/src/mon/OSDMonitor.h +++ b/src/mon/OSDMonitor.h @@ -235,7 +235,8 @@ private: bool prepare_pool_op_create (MPoolOp *m); bool prepare_pool_op_delete(MPoolOp *m); int prepare_new_pool(string& name, uint64_t auid, int crush_rule, - unsigned pg_num, unsigned pgp_num); + unsigned pg_num, unsigned pgp_num, + const vector<string> &properties); int prepare_new_pool(MPoolOp *m); void update_pool_flags(int64_t pool_id, uint64_t flags); diff --git a/src/mon/Paxos.h b/src/mon/Paxos.h index 1bd3a3c1f51..764c4fee404 100644 --- a/src/mon/Paxos.h +++ b/src/mon/Paxos.h @@ -131,7 +131,7 @@ class Paxos; * This libary is based on the Paxos algorithm, but varies in a few key ways: * 1- Only a single new value is generated at a time, simplifying the recovery logic. * 2- Nodes track "committed" values, and share them generously (and trustingly) - * 3- A 'leasing' mechism is built-in, allowing nodes to determine when it is + * 3- A 'leasing' mechanism is built-in, allowing nodes to determine when it is * safe to "read" their copy of the last committed value. * * This provides a simple replication substrate that services can be built on top of. @@ -325,8 +325,7 @@ private: * * Instead of performing a full commit each time a read is requested, we * keep leases. Each lease will have an expiration date, which may or may - * not be extended. This member variable will keep when is the lease - * expiring. + * not be extended. */ utime_t lease_expire; /** @@ -1091,7 +1090,7 @@ public: * onto paxos-related keys), and then we will decode those same bufferlists * we just wrote and apply the transactions they hold. We will also update * our first and last committed values to point to the new values, if need - * be. All all this is done tightly wrapped in a transaction to ensure we + * be. All this is done tightly wrapped in a transaction to ensure we * enjoy the atomicity guarantees given by our awesome k/v store. * * @param m A message diff --git a/src/msg/Makefile.am b/src/msg/Makefile.am new file mode 100644 index 00000000000..a849a1ca26a --- /dev/null +++ b/src/msg/Makefile.am @@ -0,0 +1,20 @@ +libmsg_la_SOURCES = \ + msg/Accepter.cc \ + msg/DispatchQueue.cc \ + msg/Message.cc \ + msg/Messenger.cc \ + msg/Pipe.cc \ + msg/SimpleMessenger.cc \ + msg/msg_types.cc + +noinst_HEADERS += \ + msg/Accepter.h \ + msg/DispatchQueue.h \ + msg/Dispatcher.h \ + msg/Message.h \ + msg/Messenger.h \ + msg/Pipe.h \ + msg/SimpleMessenger.h \ + msg/msg_types.h + +noinst_LTLIBRARIES += libmsg.la diff --git a/src/os/BtrfsFileStoreBackend.cc b/src/os/BtrfsFileStoreBackend.cc index ac7d1014ac7..bb11a45e10b 100644 --- a/src/os/BtrfsFileStoreBackend.cc +++ b/src/os/BtrfsFileStoreBackend.cc @@ -12,7 +12,9 @@ * */ -#include <inttypes.h> +#include "include/int_types.h" +#include "include/types.h" + #include <unistd.h> #include <fcntl.h> #include <errno.h> @@ -22,7 +24,6 @@ #include <sys/ioctl.h> #include "include/compat.h" #include "include/linux_fiemap.h" -#include "include/types.h" #include "include/color.h" #include "include/buffer.h" #include "include/assert.h" diff --git a/src/os/DBObjectMap.cc b/src/os/DBObjectMap.cc index 5142f4d7420..90c840bbe9c 100644 --- a/src/os/DBObjectMap.cc +++ b/src/os/DBObjectMap.cc @@ -1,13 +1,13 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -#include <iostream> -#include <inttypes.h> +#include "include/int_types.h" #include "include/buffer.h" + +#include <iostream> #include <set> #include <map> #include <string> #include <tr1/memory> - #include <vector> #include "ObjectMap.h" diff --git a/src/os/FDCache.h b/src/os/FDCache.h index f0f40e7bbf4..00e632f3e0f 100644 --- a/src/os/FDCache.h +++ b/src/os/FDCache.h @@ -18,7 +18,7 @@ #include <memory> #include <errno.h> #include <cstdio> -#include "hobject.h" +#include "common/hobject.h" #include "common/Mutex.h" #include "common/Cond.h" #include "common/shared_cache.hpp" diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc index d4d540df876..ac51f95006f 100644 --- a/src/os/FileStore.cc +++ b/src/os/FileStore.cc @@ -11,8 +11,8 @@ * Foundation. See file COPYING. * */ +#include "include/int_types.h" -#include <inttypes.h> #include <unistd.h> #include <stdlib.h> #include <sys/types.h> @@ -30,10 +30,6 @@ #include <iostream> #include <map> -#if defined(__FreeBSD__) -#include "include/inttypes.h" -#endif - #include "include/compat.h" #include "include/linux_fiemap.h" @@ -250,14 +246,14 @@ void FileStore::lfn_close(FDRef fd) { } -int FileStore::lfn_link(coll_t c, coll_t cid, const hobject_t& o) +int FileStore::lfn_link(coll_t c, coll_t newcid, const hobject_t& o, const hobject_t& newoid) { Index index_new, index_old; IndexedPath path_new, path_old; int exist; int r; - if (c < cid) { - r = get_index(cid, &index_new); + if (c < newcid) { + r = get_index(newcid, &index_new); if (r < 0) return r; r = get_index(c, &index_old); @@ -267,7 +263,7 @@ int FileStore::lfn_link(coll_t c, coll_t cid, const hobject_t& o) r = get_index(c, &index_old); if (r < 0) return r; - r = get_index(cid, &index_new); + r = get_index(newcid, &index_new); if (r < 0) return r; } @@ -280,7 +276,7 @@ int FileStore::lfn_link(coll_t c, coll_t cid, const hobject_t& o) if (!exist) return -ENOENT; - r = index_new->lookup(o, &path_new, &exist); + r = index_new->lookup(newoid, &path_new, &exist); if (r < 0) { assert(!m_filestore_fail_eio || r != -EIO); return r; @@ -294,7 +290,7 @@ int FileStore::lfn_link(coll_t c, coll_t cid, const hobject_t& o) if (r < 0) return -errno; - r = index_new->created(o, path_new->path()); + r = index_new->created(newoid, path_new->path()); if (r < 0) { assert(!m_filestore_fail_eio || r != -EIO); return r; @@ -303,7 +299,8 @@ int FileStore::lfn_link(coll_t c, coll_t cid, const hobject_t& o) } int FileStore::lfn_unlink(coll_t cid, const hobject_t& o, - const SequencerPosition &spos) + const SequencerPosition &spos, + bool force_clear_omap) { Index index; int r = get_index(cid, &index); @@ -319,14 +316,18 @@ int FileStore::lfn_unlink(coll_t cid, const hobject_t& o, return r; } - struct stat st; - r = ::stat(path->path(), &st); - if (r < 0) { - r = -errno; - assert(!m_filestore_fail_eio || r != -EIO); - return r; + if (!force_clear_omap) { + struct stat st; + r = ::stat(path->path(), &st); + if (r < 0) { + r = -errno; + assert(!m_filestore_fail_eio || r != -EIO); + return r; + } + if (st.st_nlink == 1) + force_clear_omap = true; } - if (st.st_nlink == 1) { + if (force_clear_omap) { dout(20) << __func__ << ": clearing omap on " << o << " in cid " << cid << dendl; r = object_map->clear(o, &spos); @@ -2180,6 +2181,16 @@ unsigned FileStore::_do_transaction(Transaction& t, uint64_t op_seq, int trans_n } break; + case Transaction::OP_COLL_MOVE_RENAME: + { + coll_t oldcid = i.get_cid(); + hobject_t oldoid = i.get_oid(); + coll_t newcid = i.get_cid(); + hobject_t newoid = i.get_oid(); + r = _collection_move_rename(oldcid, oldoid, newcid, newoid, spos); + } + break; + case Transaction::OP_COLL_SETATTR: { coll_t cid = i.get_cid(); @@ -4090,7 +4101,7 @@ int FileStore::_destroy_collection(coll_t c) int FileStore::_collection_add(coll_t c, coll_t oldcid, const hobject_t& o, - const SequencerPosition& spos) + const SequencerPosition& spos) { dout(15) << "collection_add " << c << "/" << o << " from " << oldcid << "/" << o << dendl; @@ -4120,7 +4131,7 @@ int FileStore::_collection_add(coll_t c, coll_t oldcid, const hobject_t& o, _set_replay_guard(**fd, spos, &o, true); } - r = lfn_link(oldcid, c, o); + r = lfn_link(oldcid, c, o, o); if (replaying && !backend->can_checkpoint() && r == -EEXIST) // crashed between link() and set_replay_guard() r = 0; @@ -4137,6 +4148,73 @@ int FileStore::_collection_add(coll_t c, coll_t oldcid, const hobject_t& o, return r; } +int FileStore::_collection_move_rename(coll_t oldcid, const hobject_t& oldoid, + coll_t c, const hobject_t& o, + const SequencerPosition& spos) +{ + dout(15) << __func__ << " " << c << "/" << o << " from " << oldcid << "/" << oldoid << dendl; + int r = 0; + int dstcmp, srccmp; + + dstcmp = _check_replay_guard(c, o, spos); + if (dstcmp < 0) + goto out_rm_src; + + // check the src name too; it might have a newer guard, and we don't + // want to clobber it + srccmp = _check_replay_guard(oldcid, oldoid, spos); + if (srccmp < 0) + return 0; + + { + // open guard on object so we don't any previous operations on the + // new name that will modify the source inode. + FDRef fd; + r = lfn_open(oldcid, oldoid, 0, &fd); + if (r < 0) { + // the source collection/object does not exist. If we are replaying, we + // should be safe, so just return 0 and move on. + assert(replaying); + dout(10) << __func__ << " " << c << "/" << o << " from " + << oldcid << "/" << oldoid << " (dne, continue replay) " << dendl; + return 0; + } + if (dstcmp > 0) { // if dstcmp == 0 the guard already says "in-progress" + _set_replay_guard(**fd, spos, &o, true); + } + + r = lfn_link(oldcid, c, oldoid, o); + if (replaying && !backend->can_checkpoint() && + r == -EEXIST) // crashed between link() and set_replay_guard() + r = 0; + + _inject_failure(); + + // the name changed; link the omap content + r = object_map->clone(oldoid, o, &spos); + if (r == -ENOENT) + r = 0; + + _inject_failure(); + + // close guard on object so we don't do this again + if (r == 0) { + _close_replay_guard(**fd, spos); + } + lfn_close(fd); + } + + out_rm_src: + // remove source + if (_check_replay_guard(oldcid, oldoid, spos) > 0) { + r = lfn_unlink(oldcid, oldoid, spos, true); + } + + dout(10) << __func__ << " " << c << "/" << o << " from " << oldcid << "/" << oldoid + << " = " << r << dendl; + return r; +} + void FileStore::_inject_failure() { if (m_filestore_kill_at.read()) { diff --git a/src/os/FileStore.h b/src/os/FileStore.h index c603949b399..4f58df4d698 100644 --- a/src/os/FileStore.h +++ b/src/os/FileStore.h @@ -298,8 +298,9 @@ public: IndexedPath *path = 0, Index *index = 0); void lfn_close(FDRef fd); - int lfn_link(coll_t c, coll_t cid, const hobject_t& o) ; - int lfn_unlink(coll_t cid, const hobject_t& o, const SequencerPosition &spos); + int lfn_link(coll_t c, coll_t newcid, const hobject_t& o, const hobject_t& newoid) ; + int lfn_unlink(coll_t cid, const hobject_t& o, const SequencerPosition &spos, + bool force_clear_omap=false); public: FileStore(const std::string &base, const std::string &jdev, const char *internal_name = "filestore", bool update_to=false); @@ -499,6 +500,9 @@ public: int _destroy_collection(coll_t c); int _collection_add(coll_t c, coll_t ocid, const hobject_t& o, const SequencerPosition& spos); + int _collection_move_rename(coll_t oldcid, const hobject_t& oldoid, + coll_t c, const hobject_t& o, + const SequencerPosition& spos); void dump_start(const std::string& file); void dump_stop(); void dump_transactions(list<ObjectStore::Transaction*>& ls, uint64_t seq, OpSequencer *osr); diff --git a/src/os/GenericFileStoreBackend.cc b/src/os/GenericFileStoreBackend.cc index d0b3df5f6f2..461158fdfab 100644 --- a/src/os/GenericFileStoreBackend.cc +++ b/src/os/GenericFileStoreBackend.cc @@ -12,7 +12,9 @@ * */ -#include <inttypes.h> +#include "include/int_types.h" +#include "include/types.h" + #include <unistd.h> #include <fcntl.h> #include <errno.h> @@ -27,7 +29,6 @@ #include "include/compat.h" #include "include/linux_fiemap.h" -#include "include/types.h" #include <iostream> #include <fstream> diff --git a/src/os/Makefile.am b/src/os/Makefile.am new file mode 100644 index 00000000000..b7fef8dd209 --- /dev/null +++ b/src/os/Makefile.am @@ -0,0 +1,49 @@ +libos_la_SOURCES = \ + os/FileJournal.cc \ + os/FileStore.cc \ + os/chain_xattr.cc \ + os/ObjectStore.cc \ + os/JournalingObjectStore.cc \ + os/LFNIndex.cc \ + os/HashIndex.cc \ + os/IndexManager.cc \ + os/FlatIndex.cc \ + os/DBObjectMap.cc \ + os/LevelDBStore.cc \ + os/WBThrottle.cc \ + os/BtrfsFileStoreBackend.cc \ + os/GenericFileStoreBackend.cc \ + os/ZFSFileStoreBackend.cc +noinst_LTLIBRARIES += libos.la + +noinst_HEADERS += \ + os/btrfs_ioctl.h \ + os/chain_xattr.h \ + os/CollectionIndex.h \ + os/FileJournal.h \ + os/FileStore.h \ + os/BtrfsFileStoreBackend.h \ + os/GenericFileStoreBackend.h \ + os/ZFSFileStoreBackend.h \ + os/FlatIndex.h \ + os/HashIndex.h \ + os/FDCache.h \ + os/WBThrottle.h \ + os/IndexManager.h \ + os/Journal.h \ + os/JournalingObjectStore.h \ + os/LFNIndex.h \ + os/ObjectStore.h \ + os/SequencerPosition.h \ + os/ObjectMap.h \ + os/DBObjectMap.h \ + os/KeyValueDB.h \ + os/LevelDBStore.h + +if WITH_LIBZFS +libos_zfs_a_SOURCES = os/ZFS.cc +libos_zfs_a_CXXFLAGS = ${AM_CXXFLAGS} ${LIBZFS_CFLAGS} +noinst_LIBRARIES += libos_zfs.a +noinst_HEADERS += os/ZFS.h +endif + diff --git a/src/os/ObjectStore.h b/src/os/ObjectStore.h index 655afee004f..7e8f6ce43bf 100644 --- a/src/os/ObjectStore.h +++ b/src/os/ObjectStore.h @@ -159,6 +159,7 @@ public: OP_SPLIT_COLLECTION2 = 36, /* cid, bits, destination doesn't create the destination */ OP_OMAP_RMKEYRANGE = 37, // cid, oid, firstkey, lastkey + OP_COLL_MOVE_RENAME = 38, // oldcid, oldoid, newcid, newoid }; private: @@ -554,6 +555,15 @@ public: collection_remove(oldcid, oid); return; } + void collection_move_rename(coll_t oldcid, const hobject_t& oldoid, + coll_t cid, const hobject_t& oid) { + __u32 op = OP_COLL_MOVE_RENAME; + ::encode(op, tbl); + ::encode(oldcid, tbl); + ::encode(oldoid, tbl); + ::encode(cid, tbl); + ::encode(oid, tbl); + } void collection_setattr(coll_t cid, const char* name, bufferlist& val) { string n(name); diff --git a/src/os/WBThrottle.h b/src/os/WBThrottle.h index 070de08e123..d480a6b751c 100644 --- a/src/os/WBThrottle.h +++ b/src/os/WBThrottle.h @@ -20,7 +20,7 @@ #include <tr1/memory> #include "include/buffer.h" #include "common/Formatter.h" -#include "os/hobject.h" +#include "common/hobject.h" #include "include/interval_set.h" #include "FDCache.h" #include "common/Thread.h" diff --git a/src/os/ZFSFileStoreBackend.cc b/src/os/ZFSFileStoreBackend.cc index 0f01bd20ee7..aa52b8d2933 100644 --- a/src/os/ZFSFileStoreBackend.cc +++ b/src/os/ZFSFileStoreBackend.cc @@ -1,7 +1,9 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab -#include <inttypes.h> +#include "include/int_types.h" +#include "include/types.h" + #include <unistd.h> #include <fcntl.h> #include <errno.h> @@ -12,7 +14,6 @@ #include "include/compat.h" #include "include/linux_fiemap.h" -#include "include/types.h" #include "include/color.h" #include "include/buffer.h" #include "include/assert.h" diff --git a/src/os/chain_xattr.cc b/src/os/chain_xattr.cc index 96f334f8d00..62733e390d3 100644 --- a/src/os/chain_xattr.cc +++ b/src/os/chain_xattr.cc @@ -3,7 +3,8 @@ #include "chain_xattr.h" -#include <inttypes.h> +#include "include/int_types.h" + #include <unistd.h> #include <stdlib.h> #include <sys/types.h> @@ -21,10 +22,6 @@ #include <linux/fs.h> #endif -#if defined(__FreeBSD__) -#include "include/inttypes.h" -#endif - #include "common/xattr.h" /* diff --git a/src/osd/Ager.cc b/src/osd/Ager.cc index a90b1c1855d..f94da1f4218 100644 --- a/src/osd/Ager.cc +++ b/src/osd/Ager.cc @@ -8,7 +8,6 @@ #include "common/Clock.h" #include "common/debug.h" -#include "global/global_context.h" // ick #include <sys/types.h> @@ -59,7 +58,7 @@ uint64_t Ager::age_fill(float pc, utime_t until) { bl.push_back(bp); uint64_t wrote = 0; while (1) { - if (ceph_clock_now(g_ceph_context) > until) break; + if (ceph_clock_now(cct) > until) break; struct statfs st; store->statfs(&st); @@ -176,7 +175,7 @@ void Ager::age(int time, srand(0); - utime_t start = ceph_clock_now(g_ceph_context); + utime_t start = ceph_clock_now(cct); utime_t until = start; until.sec_ref() += time; @@ -223,7 +222,7 @@ void Ager::age(int time, uint64_t wrote = 0; for (int c=1; c<=count; c++) { - if (ceph_clock_now(g_ceph_context) > until) break; + if (ceph_clock_now(cct) > until) break; //if (c == 7) start_debug = true; @@ -253,7 +252,7 @@ void Ager::age(int time, // dump freelist? /* - if (ceph_clock_now(g_ceph_context) > nextfl) { + if (ceph_clock_now(cct) > nextfl) { elapsed += freelist_inc; save_freelist(elapsed); nextfl.sec_ref() += freelist_inc; diff --git a/src/osd/Ager.h b/src/osd/Ager.h index 55db507993f..face0a62be4 100644 --- a/src/osd/Ager.h +++ b/src/osd/Ager.h @@ -7,12 +7,14 @@ #include "include/Distribution.h" #include "os/ObjectStore.h" #include "common/Clock.h" +#include "common/ceph_context.h" #include <list> #include <vector> using namespace std; class Ager { + CephContext *cct; ObjectStore *store; private: @@ -28,7 +30,7 @@ class Ager { file_object_t age_get_oid(); public: - Ager(ObjectStore *s) : store(s), did_distn(false) {} + Ager(CephContext *cct_, ObjectStore *s) : cct(cct_), store(s), did_distn(false) {} void age(int time, float high_water, // fill to this % diff --git a/src/osd/ClassHandler.cc b/src/osd/ClassHandler.cc index a9a920ba078..b1afe1e0626 100644 --- a/src/osd/ClassHandler.cc +++ b/src/osd/ClassHandler.cc @@ -39,7 +39,7 @@ int ClassHandler::open_class(const string& cname, ClassData **pcls) int ClassHandler::open_all_classes() { dout(10) << __func__ << dendl; - DIR *dir = ::opendir(g_conf->osd_class_dir.c_str()); + DIR *dir = ::opendir(cct->_conf->osd_class_dir.c_str()); if (!dir) return -errno; @@ -101,7 +101,7 @@ int ClassHandler::_load_class(ClassData *cls) cls->status == ClassData::CLASS_MISSING) { char fname[PATH_MAX]; snprintf(fname, sizeof(fname), "%s/" CLS_PREFIX "%s" CLS_SUFFIX, - g_conf->osd_class_dir.c_str(), + cct->_conf->osd_class_dir.c_str(), cls->name.c_str()); dout(10) << "_load_class " << cls->name << " from " << fname << dendl; diff --git a/src/osd/ClassHandler.h b/src/osd/ClassHandler.h index ae416bc6bfc..93cf3c07fbc 100644 --- a/src/osd/ClassHandler.h +++ b/src/osd/ClassHandler.h @@ -7,11 +7,14 @@ #include "common/Cond.h" #include "common/Mutex.h" +#include "common/ceph_context.h" class ClassHandler { public: + CephContext *cct; + struct ClassData; struct ClassMethod { @@ -76,7 +79,7 @@ private: int _load_class(ClassData *cls); public: - ClassHandler() : mutex("ClassHandler") {} + ClassHandler(CephContext *cct_) : cct(cct_), mutex("ClassHandler") {} int open_all_classes(); diff --git a/src/osd/ErasureCodeInterface.h b/src/osd/ErasureCodeInterface.h new file mode 100644 index 00000000000..5ce2842d562 --- /dev/null +++ b/src/osd/ErasureCodeInterface.h @@ -0,0 +1,210 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com> + * + * Author: Loic Dachary <loic@dachary.org> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#ifndef CEPH_ERASURE_CODE_INTERFACE_H +#define CEPH_ERASURE_CODE_INTERFACE_H + +/*! @file ErasureCodeInterface.h + @brief Interface provided by erasure code plugins + + The erasure coded pools rely on plugins implementing + **ErasureCodeInterface** to encode and decode content. All codes + are systematic (i.e. the data is not mangled and can be + reconstructed by concatenating chunks ). + + All methods returns **0** on success and a negative value on + error. If the value returned on error is not explained in + **ErasureCodeInterface**, the sources or the documentation of the + interface implementer must be read to figure out what it means. It + is recommended that each error code matches an *errno* value that + relates to the cause of the error. + + Assuming the interface implementer provides three data chunks ( K + = 3 ) and two coding chunks ( M = 2 ), a buffer can be encoded as + follows: + + ~~~~~~~~~~~~~~~~{.c} + set<int> want_to_encode(0, 1, 2, // data chunks + 3, 4 // coding chunks + ); + bufferlist in = "ABCDEF"; + map<int, bufferlist> encoded + encode(want_to_encode, in, &encoded); + encoded[0] == "AB" // data chunk 0 + encoded[1] == "CD" // data chunk 1 + encoded[2] == "EF" // data chunk 2 + encoded[3] // coding chunk 0 + encoded[4] // coding chunk 1 + ~~~~~~~~~~~~~~~~ + + If encoded[2] ( which contains **EF** ) is missing and accessing + encoded[3] ( the first coding chunk ) is more expensive than + accessing encoded[4] ( the second coding chunk ), the + **minimum_to_decode_with_cost** method can be called as follows: + + ~~~~~~~~~~~~~~~~{.c} + set<int> want_to_read(2); // want the chunk containing "EF" + map<int,int> available( + 0 => 1, // data chunk 0 : available and costs 1 + 1 => 1, // data chunk 1 : available and costs 1 + 3 => 9, // coding chunk 1 : available and costs 9 + 4 => 1, // coding chunk 2 : available and costs 1 + ); + set<int> minimum; + minimum_to_decode_with_cost(want_to_read, + available, + &minimum); + minimum == set<int>(0, 1, 4); + ~~~~~~~~~~~~~~~~ + + It sets **minimum** with three chunks to reconstruct the desired + data chunk and will pick the second coding chunk ( 4 ) because it + is less expensive ( 1 < 9 ) to retrieve than the first coding + chunk ( 3 ). The caller is responsible for retrieving the chunks + and call **decode** to reconstruct the second data chunk content. + + ~~~~~~~~~~~~~~~~{.c} + map<int,bufferlist> chunks; + for i in minimum.keys(): + chunks[i] = fetch_chunk(i); // get chunk from storage + map<int, bufferlist> decoded; + decode(want_to_read, chunks, &decoded); + decoded[2] == "EF" + ~~~~~~~~~~~~~~~~ + + */ + +#include <map> +#include <set> +#include <tr1/memory> +#include "include/buffer.h" + +using namespace std; + +namespace ceph { + + class ErasureCodeInterface { + public: + virtual ~ErasureCodeInterface() {} + + /** + * Compute the smallest subset of **available** chunks that needs + * to be retrieved in order to successfully decode + * **want_to_read** chunks. + * + * It is strictly equivalent to calling + * **minimum_to_decode_with_cost** where each **available** chunk + * has the same cost. + * + * @see minimum_to_decode_with_cost + * + * @param [in] want_to_read chunk indexes to be decoded + * @param [in] available chunk indexes containing valid data + * @param [out] minimum chunk indexes to retrieve for decode + * @return **0** on success or a negative errno on error. + */ + virtual int minimum_to_decode(const set<int> &want_to_read, + const set<int> &available, + set<int> *minimum) = 0; + + /** + * Compute the smallest subset of **available** chunks that needs + * to be retrieved in order to successfully decode + * **want_to_read** chunks. If there are more than one possible + * subset, select the subset that contains the chunks with the + * lowest cost. + * + * The **available** parameter maps chunk indexes to their + * retrieval cost. The higher the cost value, the more costly it + * is to retrieve the chunk content. + * + * Returns -EIO if there are not enough chunk indexes in + * **available** to decode **want_to_read**. + * + * Returns 0 on success. + * + * The **minimum** argument must be a pointer to an empty set. + * + * @param [in] want_to_read chunk indexes to be decoded + * @param [in] available map chunk indexes containing valid data + * to their retrieval cost + * @param [out] minimum chunk indexes to retrieve for decode + * @return **0** on success or a negative errno on error. + */ + virtual int minimum_to_decode_with_cost(const set<int> &want_to_read, + const map<int, int> &available, + set<int> *minimum) = 0; + + /** + * Encode the content of **in** and store the result in + * **encoded**. The **encoded** map contains at least all + * chunk indexes found in the **want_to_encode** set. + * + * The **encoded** map is expected to be a pointer to an empty + * map. + * + * The **encoded** map may contain more chunks than required by + * **want_to_encode** and the caller is expected to permanently + * store all of them, not just the chunks from **want_to_encode**. + * + * Returns 0 on success. + * + * @param [in] want_to_encode chunk indexes to be encoded + * @param [in] in data to be encoded + * @param [out] encoded map chunk indexes to chunk data + * @return **0** on success or a negative errno on error. + */ + virtual int encode(const set<int> &want_to_encode, + const bufferlist &in, + map<int, bufferlist> *encoded) = 0; + + /** + * Decode the **chunks** and store at least **want_to_read** chunks + * in **decoded**. + * + * There must be enough **chunks** ( as returned by + * **minimum_to_decode** or **minimum_to_decode_with_cost** ) to + * perform a successfull decoding of all chunks found in + * **want_to_read**. + * + * The **decoded** map is expected to be a pointer to an empty + * map. + * + * The **decoded** map may contain more chunks than required by + * **want_to_read** and they can safely be used by the caller. + * + * If a chunk is listed in **want_to_read** and there is + * corresponding **bufferlist** in **chunks**, it will be copied + * verbatim into **decoded**. If not it will be reconstructed from + * the existing chunks. + * + * Returns 0 on success. + * + * @param [in] want_to_read chunk indexes to be decoded + * @param [in] chunks map chunk indexes to chunk data + * @param [out] decoded map chunk indexes to chunk data + * @return **0** on success or a negative errno on error. + */ + virtual int decode(const set<int> &want_to_read, + const map<int, bufferlist> &chunks, + map<int, bufferlist> *decoded) = 0; + }; + + typedef std::tr1::shared_ptr<ErasureCodeInterface> ErasureCodeInterfaceRef; + +} + +#endif diff --git a/src/osd/ErasureCodePlugin.cc b/src/osd/ErasureCodePlugin.cc new file mode 100644 index 00000000000..10b65b2604b --- /dev/null +++ b/src/osd/ErasureCodePlugin.cc @@ -0,0 +1,134 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com> + * + * Author: Loic Dachary <loic@dachary.org> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include "common/debug.h" + +#include <dlfcn.h> + +#include "ErasureCodePlugin.h" + +#define dout_subsys ceph_subsys_osd +#undef dout_prefix +#define dout_prefix _prefix(_dout) + +static ostream& _prefix(std::ostream* _dout) +{ + return *_dout << "ErasureCodePlugin: "; +} + +#define PLUGIN_PREFIX "libec_" +#define PLUGIN_SUFFIX ".so" +#define PLUGIN_INIT_FUNCTION "__erasure_code_init" + +ErasureCodePluginRegistry ErasureCodePluginRegistry::singleton; + +ErasureCodePluginRegistry::ErasureCodePluginRegistry() : + lock("ErasureCodePluginRegistry::lock") +{ +} + +ErasureCodePluginRegistry::~ErasureCodePluginRegistry() +{ + for (std::map<std::string,ErasureCodePlugin*>::iterator i = plugins.begin(); + i != plugins.end(); + i++) { + void *library = i->second->library; + delete i->second; + dlclose(library); + } +} + +int ErasureCodePluginRegistry::add(const std::string &name, + ErasureCodePlugin* plugin) +{ + if (plugins.find(name) != plugins.end()) + return -EEXIST; + plugins[name] = plugin; + return 0; +} + +ErasureCodePlugin *ErasureCodePluginRegistry::get(const std::string &name) +{ + if (plugins.find(name) != plugins.end()) + return plugins[name]; + else + return 0; +} + +int ErasureCodePluginRegistry::factory(const std::string &plugin_name, + const map<std::string,std::string> ¶meters, + ErasureCodeInterfaceRef *erasure_code) +{ + Mutex::Locker l(lock); + int r = 0; + ErasureCodePlugin *plugin = get(plugin_name); + if (plugin == 0) { + r = load(plugin_name, parameters, &plugin); + if (r != 0) + return r; + } + + return plugin->factory(parameters, erasure_code); +} + +int ErasureCodePluginRegistry::load(const std::string &plugin_name, + const map<std::string,std::string> ¶meters, + ErasureCodePlugin **plugin) +{ + assert(parameters.count("erasure-code-directory") != 0); + std::string fname = parameters.find("erasure-code-directory")->second + + "/" PLUGIN_PREFIX + + plugin_name + PLUGIN_SUFFIX; + dout(10) << "load " << plugin_name << " from " << fname << dendl; + + void *library = dlopen(fname.c_str(), RTLD_NOW); + if (!library) { + derr << "load dlopen(" << fname + << "): " << dlerror() << dendl; + return -EIO; + } + + int (*erasure_code_init)(const char *) = + (int (*)(const char *))dlsym(library, PLUGIN_INIT_FUNCTION); + if (erasure_code_init) { + std::string name = plugin_name; + int r = erasure_code_init(name.c_str()); + if (r != 0) { + derr << "erasure_code_init(" << plugin_name + << "): " << strerror(-r) << dendl; + return r; + } + } else { + derr << "load dlsym(" << fname + << ", " << PLUGIN_INIT_FUNCTION + << "): " << dlerror() << dendl; + dlclose(library); + return -ENOENT; + } + + *plugin = get(plugin_name); + if (*plugin == 0) { + derr << "load " << PLUGIN_INIT_FUNCTION << "()" + << "did not register " << plugin_name << dendl; + dlclose(library); + return -EBADF; + } + + (*plugin)->library = library; + + return 0; +} + diff --git a/src/osd/ErasureCodePlugin.h b/src/osd/ErasureCodePlugin.h new file mode 100644 index 00000000000..f1c1ccb31b3 --- /dev/null +++ b/src/osd/ErasureCodePlugin.h @@ -0,0 +1,69 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com> + * + * Author: Loic Dachary <loic@dachary.org> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#ifndef CEPH_ERASURE_CODE_PLUGIN_H +#define CEPH_ERASURE_CODE_PLUGIN_H + +#include "common/Mutex.h" +#include "ErasureCodeInterface.h" + +extern "C" { + int __erasure_code_init(char *plugin_name); +} + +namespace ceph { + + class ErasureCodePlugin { + public: + void *library; + + ErasureCodePlugin() : + library(0) {} + virtual ~ErasureCodePlugin() {} + + virtual int factory(const map<std::string,std::string> ¶meters, + ErasureCodeInterfaceRef *erasure_code) = 0; + }; + + class ErasureCodePluginRegistry { + public: + Mutex lock; + std::map<std::string,ErasureCodePlugin*> plugins; + + static ErasureCodePluginRegistry singleton; + + ErasureCodePluginRegistry(); + ~ErasureCodePluginRegistry(); + + static ErasureCodePluginRegistry &instance() { + return singleton; + } + + int factory(const std::string &plugin, + const map<std::string,std::string> ¶meters, + ErasureCodeInterfaceRef *erasure_code); + + int add(const std::string &name, ErasureCodePlugin *plugin); + ErasureCodePlugin *get(const std::string &name); + + int load(const std::string &plugin_name, + const map<std::string,std::string> ¶meters, + ErasureCodePlugin **plugin); + + }; +} + +#endif diff --git a/src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc b/src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc new file mode 100644 index 00000000000..25821274081 --- /dev/null +++ b/src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc @@ -0,0 +1,389 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com> + * + * Author: Loic Dachary <loic@dachary.org> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include <errno.h> +#include "common/debug.h" +#include "ErasureCodeJerasure.h" +extern "C" { +#include "jerasure.h" +#include "reed_sol.h" +#include "galois.h" +#include "cauchy.h" +#include "liberation.h" +} + +#define dout_subsys ceph_subsys_osd +#undef dout_prefix +#define dout_prefix _prefix(_dout) + +static ostream& _prefix(std::ostream* _dout) +{ + return *_dout << "ErasureCodeJerasure: "; +} + +void ErasureCodeJerasure::init(const map<std::string,std::string> ¶meters) { + dout(10) << "technique=" << technique << dendl; + parse(parameters); + prepare(); +} + +int ErasureCodeJerasure::minimum_to_decode(const set<int> &want_to_read, + const set<int> &available_chunks, + set<int> *minimum) { + if (available_chunks.size() < (unsigned)k) + return -EIO; + set<int>::iterator i; + unsigned j; + for (i = available_chunks.begin(), j = 0; j < (unsigned)k; i++, j++) + minimum->insert(*i); + return 0; +} + +int ErasureCodeJerasure::minimum_to_decode_with_cost(const set<int> &want_to_read, + const map<int, int> &available, + set<int> *minimum) { + set <int> available_chunks; + for (map<int, int>::const_iterator i = available.begin(); + i != available.end(); + i++) + available_chunks.insert(i->first); + return minimum_to_decode(want_to_read, available_chunks, minimum); +} + +int ErasureCodeJerasure::encode(const set<int> &want_to_encode, + const bufferlist &in, + map<int, bufferlist> *encoded) { + unsigned in_length = pad_in_length(in.length()); + dout(10) << "encode adjusted buffer length from " << in.length() << " to " << in_length << dendl; + assert(in_length % k == 0); + unsigned blocksize = in_length / k; + unsigned length = blocksize * ( k + m ); + bufferlist out(in); + bufferptr pad(length - in.length()); + pad.zero(0, k); + out.push_back(pad); + char *p = out.c_str(); + char *data[k]; + for (int i = 0; i < k; i++) { + data[i] = p + i * blocksize; + } + char *coding[m]; + for (int i = 0; i < m; i++) { + coding[i] = p + ( k + i ) * blocksize; + } + jerasure_encode(data, coding, blocksize); + const bufferptr ptr = out.buffers().front(); + for (set<int>::iterator j = want_to_encode.begin(); + j != want_to_encode.end(); + j++) { + bufferptr chunk(ptr, (*j) * blocksize, blocksize); + (*encoded)[*j].push_front(chunk); + } + return 0; +} + +int ErasureCodeJerasure::decode(const set<int> &want_to_read, + const map<int, bufferlist> &chunks, + map<int, bufferlist> *decoded) { + unsigned blocksize = (*chunks.begin()).second.length(); + int erasures[k + m + 1]; + int erasures_count = 0; + char *data[k]; + char *coding[m]; + for (int i = 0; i < k + m; i++) { + if (chunks.find(i) == chunks.end()) { + erasures[erasures_count] = i; + erasures_count++; + bufferptr ptr(blocksize); + (*decoded)[i].push_front(ptr); + } else { + (*decoded)[i] = chunks.find(i)->second; + } + if (i < k) + data[i] = (*decoded)[i].c_str(); + else + coding[i - k] = (*decoded)[i].c_str(); + } + erasures[erasures_count] = -1; + + if (erasures_count > 0) + return jerasure_decode(erasures, data, coding, blocksize); + else + return 0; +} + +int ErasureCodeJerasure::to_int(const std::string &name, + const map<std::string,std::string> ¶meters, + int default_value) { + if (parameters.find(name) == parameters.end() || + parameters.find(name)->second.size() == 0) { + dout(10) << name << " defaults to " << default_value << dendl; + return default_value; + } + const std::string value = parameters.find(name)->second; + std::string p = value; + std::string err; + int r = strict_strtol(p.c_str(), 10, &err); + if (!err.empty()) { + derr << "could not convert " << name << "=" << value + << " to int because " << err + << ", set to default " << default_value << dendl; + return default_value; + } + dout(10) << name << " set to " << r << dendl; + return r; +} + +bool ErasureCodeJerasure::is_prime(int value) { + int prime55[] = { + 2,3,5,7,11,13,17,19,23,29,31,37,41,43,47,53,59,61,67,71, + 73,79,83,89,97,101,103,107,109,113,127,131,137,139,149, + 151,157,163,167,173,179, + 181,191,193,197,199,211,223,227,229,233,239,241,251,257 + }; + int i; + for (i = 0; i < 55; i++) + if (value == prime55[i]) + return true; + return false; +} + +// +// ErasureCodeJerasureReedSolomonVandermonde +// +void ErasureCodeJerasureReedSolomonVandermonde::jerasure_encode(char **data, + char **coding, + int blocksize) { + jerasure_matrix_encode(k, m, w, matrix, data, coding, blocksize); +} + +int ErasureCodeJerasureReedSolomonVandermonde::jerasure_decode(int *erasures, + char **data, + char **coding, + int blocksize) { + return jerasure_matrix_decode(k, m, w, matrix, 1, erasures, data, coding, blocksize); +} + +unsigned ErasureCodeJerasureReedSolomonVandermonde::pad_in_length(unsigned in_length) { + while (in_length%(k*w*sizeof(int)) != 0) + in_length++; + return in_length; +} + +void ErasureCodeJerasureReedSolomonVandermonde::parse(const map<std::string,std::string> ¶meters) { + k = to_int("erasure-code-k", parameters, DEFAULT_K); + m = to_int("erasure-code-m", parameters, DEFAULT_M); + w = to_int("erasure-code-w", parameters, DEFAULT_W); + if (w != 8 && w != 16 && w != 32) { + derr << "ReedSolomonVandermonde: w=" << w << " must be one of {8, 16, 32} : revert to 8 " << dendl; + w = 8; + } +} + +void ErasureCodeJerasureReedSolomonVandermonde::prepare() { + matrix = reed_sol_vandermonde_coding_matrix(k, m, w); +} + +// +// ErasureCodeJerasureReedSolomonRAID6 +// +void ErasureCodeJerasureReedSolomonRAID6::jerasure_encode(char **data, + char **coding, + int blocksize) { + reed_sol_r6_encode(k, w, data, coding, blocksize); +} + +int ErasureCodeJerasureReedSolomonRAID6::jerasure_decode(int *erasures, + char **data, + char **coding, + int blocksize) { + return jerasure_matrix_decode(k, m, w, matrix, 1, erasures, data, coding, blocksize); +} + +unsigned ErasureCodeJerasureReedSolomonRAID6::pad_in_length(unsigned in_length) { + while (in_length%(k*w*sizeof(int)) != 0) + in_length++; + return in_length; +} + +void ErasureCodeJerasureReedSolomonRAID6::parse(const map<std::string,std::string> ¶meters) { + k = to_int("erasure-code-k", parameters, DEFAULT_K); + m = 2; + w = to_int("erasure-code-w", parameters, DEFAULT_W); + if (w != 8 && w != 16 && w != 32) { + derr << "ReedSolomonRAID6: w=" << w << " must be one of {8, 16, 32} : revert to 8 " << dendl; + w = 8; + } +} + +void ErasureCodeJerasureReedSolomonRAID6::prepare() { + matrix = reed_sol_r6_coding_matrix(k, w); +} + +// +// ErasureCodeJerasureCauchy +// +void ErasureCodeJerasureCauchy::jerasure_encode(char **data, + char **coding, + int blocksize) { + jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize); +} + +int ErasureCodeJerasureCauchy::jerasure_decode(int *erasures, + char **data, + char **coding, + int blocksize) { + return jerasure_schedule_decode_lazy(k, m, w, bitmatrix, erasures, data, coding, blocksize, packetsize, 1); +} + +unsigned ErasureCodeJerasureCauchy::pad_in_length(unsigned in_length) { + while (in_length%(k*w*packetsize*sizeof(int)) != 0) + in_length++; + return in_length; +} + +void ErasureCodeJerasureCauchy::parse(const map<std::string,std::string> ¶meters) { + k = to_int("erasure-code-k", parameters, DEFAULT_K); + m = to_int("erasure-code-m", parameters, DEFAULT_M); + w = to_int("erasure-code-w", parameters, DEFAULT_W); + packetsize = to_int("erasure-code-packetsize", parameters, DEFAULT_PACKETSIZE); +} + +void ErasureCodeJerasureCauchy::prepare_schedule(int *matrix) { + bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix); + schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); +} + +// +// ErasureCodeJerasureCauchyOrig +// +void ErasureCodeJerasureCauchyOrig::prepare() { + int *matrix = cauchy_original_coding_matrix(k, m, w); + prepare_schedule(matrix); + free(matrix); +} + +// +// ErasureCodeJerasureCauchyGood +// +void ErasureCodeJerasureCauchyGood::prepare() { + int *matrix = cauchy_good_general_coding_matrix(k, m, w); + prepare_schedule(matrix); + free(matrix); +} + +// +// ErasureCodeJerasureLiberation +// +ErasureCodeJerasureLiberation::~ErasureCodeJerasureLiberation() { + if (bitmatrix) + free(bitmatrix); + if (schedule) + jerasure_free_schedule(schedule); +} + +void ErasureCodeJerasureLiberation::jerasure_encode(char **data, + char **coding, + int blocksize) { + jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize); +} + +int ErasureCodeJerasureLiberation::jerasure_decode(int *erasures, + char **data, + char **coding, + int blocksize) { + return jerasure_schedule_decode_lazy(k, m, w, bitmatrix, erasures, data, coding, blocksize, packetsize, 1); +} + +unsigned ErasureCodeJerasureLiberation::pad_in_length(unsigned in_length) { + while (in_length%(k*w*packetsize*sizeof(int)) != 0) + in_length++; + return in_length; +} + +void ErasureCodeJerasureLiberation::parse(const map<std::string,std::string> ¶meters) { + k = to_int("erasure-code-k", parameters, DEFAULT_K); + m = to_int("erasure-code-m", parameters, DEFAULT_M); + w = to_int("erasure-code-w", parameters, DEFAULT_W); + packetsize = to_int("erasure-code-packetsize", parameters, DEFAULT_PACKETSIZE); + + bool error = false; + if (k > w) { + derr << "k=" << k << " must be less than or equal to w=" << w << dendl; + error = true; + } + if (w <= 2 || !is_prime(w)) { + derr << "w=" << w << " must be greater than two and be prime" << dendl; + error = true; + } + if (packetsize == 0) { + derr << "packetsize=" << packetsize << " must be set" << dendl; + error = true; + } + if ((packetsize%(sizeof(int))) != 0) { + derr << "packetsize=" << packetsize << " must be a multiple of sizeof(int) = " << sizeof(int) << dendl; + error = true; + } + if (error) { + derr << "reverting to k=" << DEFAULT_K << ", w=" << DEFAULT_W << ", packetsize=" << DEFAULT_PACKETSIZE << dendl; + k = DEFAULT_K; + w = DEFAULT_W; + packetsize = DEFAULT_PACKETSIZE; + } +} + +void ErasureCodeJerasureLiberation::prepare() { + bitmatrix = liberation_coding_bitmatrix(k, w); + schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); +} + +// +// ErasureCodeJerasureBlaumRoth +// +void ErasureCodeJerasureBlaumRoth::prepare() { + bitmatrix = blaum_roth_coding_bitmatrix(k, w); + schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); +} + +// +// ErasureCodeJerasureLiber8tion +// +void ErasureCodeJerasureLiber8tion::parse(const map<std::string,std::string> ¶meters) { + k = to_int("erasure-code-k", parameters, DEFAULT_K); + m = DEFAULT_M; + w = DEFAULT_W; + packetsize = to_int("erasure-code-packetsize", parameters, DEFAULT_PACKETSIZE); + + bool error = false; + if (k > w) { + derr << "k=" << k << " must be less than or equal to w=" << w << dendl; + error = true; + } + if (packetsize == 0) { + derr << "packetsize=" << packetsize << " must be set" << dendl; + error = true; + } + if (error) { + derr << "reverting to k=" << DEFAULT_K << ", packetsize=" << DEFAULT_PACKETSIZE << dendl; + k = DEFAULT_K; + packetsize = DEFAULT_PACKETSIZE; + } +} + +void ErasureCodeJerasureLiber8tion::prepare() { + bitmatrix = liber8tion_coding_bitmatrix(k); + schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); +} diff --git a/src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h b/src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h new file mode 100644 index 00000000000..7728751c383 --- /dev/null +++ b/src/osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h @@ -0,0 +1,227 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com> + * + * Author: Loic Dachary <loic@dachary.org> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#ifndef CEPH_ERASURE_CODE_JERASURE_H +#define CEPH_ERASURE_CODE_JERASURE_H + +#include "osd/ErasureCodeInterface.h" + +class ErasureCodeJerasure : public ErasureCodeInterface { +public: + int k; + int m; + int w; + const char *technique; + + ErasureCodeJerasure(const char *_technique) : + technique(_technique) + {} + + virtual ~ErasureCodeJerasure() {} + + virtual int minimum_to_decode(const set<int> &want_to_read, + const set<int> &available_chunks, + set<int> *minimum); + + virtual int minimum_to_decode_with_cost(const set<int> &want_to_read, + const map<int, int> &available, + set<int> *minimum); + + virtual int encode(const set<int> &want_to_encode, + const bufferlist &in, + map<int, bufferlist> *encoded); + + virtual int decode(const set<int> &want_to_read, + const map<int, bufferlist> &chunks, + map<int, bufferlist> *decoded); + + void init(const map<std::string,std::string> ¶meters); + virtual void jerasure_encode(char **data, + char **coding, + int blocksize) = 0; + virtual int jerasure_decode(int *erasures, + char **data, + char **coding, + int blocksize) = 0; + virtual unsigned pad_in_length(unsigned in_length) = 0; + virtual void parse(const map<std::string,std::string> ¶meters) = 0; + virtual void prepare() = 0; + static int to_int(const std::string &name, + const map<std::string,std::string> ¶meters, + int default_value); + static bool is_prime(int value); +}; + +class ErasureCodeJerasureReedSolomonVandermonde : public ErasureCodeJerasure { +public: + static const int DEFAULT_K = 7; + static const int DEFAULT_M = 3; + static const int DEFAULT_W = 8; + int *matrix; + + ErasureCodeJerasureReedSolomonVandermonde() : + ErasureCodeJerasure("reed_sol_van"), + matrix(0) + { } + virtual ~ErasureCodeJerasureReedSolomonVandermonde() { + if (matrix) + free(matrix); + } + + virtual void jerasure_encode(char **data, + char **coding, + int blocksize); + virtual int jerasure_decode(int *erasures, + char **data, + char **coding, + int blocksize); + virtual unsigned pad_in_length(unsigned in_length); + virtual void parse(const map<std::string,std::string> ¶meters); + virtual void prepare(); +}; + +class ErasureCodeJerasureReedSolomonRAID6 : public ErasureCodeJerasure { +public: + static const int DEFAULT_K = 7; + static const int DEFAULT_W = 8; + int *matrix; + + ErasureCodeJerasureReedSolomonRAID6() : + ErasureCodeJerasure("reed_sol_r6_op"), + matrix(0) + { } + virtual ~ErasureCodeJerasureReedSolomonRAID6() { + if (matrix) + free(matrix); + } + + virtual void jerasure_encode(char **data, + char **coding, + int blocksize); + virtual int jerasure_decode(int *erasures, + char **data, + char **coding, + int blocksize); + virtual unsigned pad_in_length(unsigned in_length); + virtual void parse(const map<std::string,std::string> ¶meters); + virtual void prepare(); +}; + +class ErasureCodeJerasureCauchy : public ErasureCodeJerasure { +public: + static const int DEFAULT_K = 7; + static const int DEFAULT_M = 3; + static const int DEFAULT_W = 8; + static const int DEFAULT_PACKETSIZE = 8; + int *bitmatrix; + int **schedule; + int packetsize; + + ErasureCodeJerasureCauchy(const char *technique) : + ErasureCodeJerasure(technique), + bitmatrix(0), + schedule(0) + { } + virtual ~ErasureCodeJerasureCauchy() { + if (bitmatrix) + free(bitmatrix); + if (schedule) + free(schedule); + } + + virtual void jerasure_encode(char **data, + char **coding, + int blocksize); + virtual int jerasure_decode(int *erasures, + char **data, + char **coding, + int blocksize); + virtual unsigned pad_in_length(unsigned in_length); + virtual void parse(const map<std::string,std::string> ¶meters); + void prepare_schedule(int *matrix); +}; + +class ErasureCodeJerasureCauchyOrig : public ErasureCodeJerasureCauchy { +public: + ErasureCodeJerasureCauchyOrig() : + ErasureCodeJerasureCauchy("cauchy_orig") + {} + + virtual void prepare(); +}; + +class ErasureCodeJerasureCauchyGood : public ErasureCodeJerasureCauchy { +public: + ErasureCodeJerasureCauchyGood() : + ErasureCodeJerasureCauchy("cauchy_good") + {} + + virtual void prepare(); +}; + +class ErasureCodeJerasureLiberation : public ErasureCodeJerasure { +public: + static const int DEFAULT_K = 2; + static const int DEFAULT_M = 2; + static const int DEFAULT_W = 7; + static const int DEFAULT_PACKETSIZE = 8; + int *bitmatrix; + int **schedule; + int packetsize; + + ErasureCodeJerasureLiberation(const char *technique = "liberation") : + ErasureCodeJerasure(technique), + bitmatrix(0), + schedule(0) + { } + virtual ~ErasureCodeJerasureLiberation(); + + virtual void jerasure_encode(char **data, + char **coding, + int blocksize); + virtual int jerasure_decode(int *erasures, + char **data, + char **coding, + int blocksize); + virtual unsigned pad_in_length(unsigned in_length); + virtual void parse(const map<std::string,std::string> ¶meters); + virtual void prepare(); +}; + +class ErasureCodeJerasureBlaumRoth : public ErasureCodeJerasureLiberation { +public: + ErasureCodeJerasureBlaumRoth() : + ErasureCodeJerasureLiberation("blaum_roth") + {} + + virtual void prepare(); +}; + +class ErasureCodeJerasureLiber8tion : public ErasureCodeJerasureLiberation { +public: + static const int DEFAULT_K = 2; + static const int DEFAULT_M = 2; + static const int DEFAULT_W = 8; + + ErasureCodeJerasureLiber8tion() : + ErasureCodeJerasureLiberation("liber8tion") + {} + + virtual void parse(const map<std::string,std::string> ¶meters); + virtual void prepare(); +}; + +#endif diff --git a/src/osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc b/src/osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc new file mode 100644 index 00000000000..d5cb1cd6c93 --- /dev/null +++ b/src/osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc @@ -0,0 +1,70 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com> + * + * Author: Loic Dachary <loic@dachary.org> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include "common/debug.h" +#include "osd/ErasureCodePlugin.h" +#include "ErasureCodeJerasure.h" + +#define dout_subsys ceph_subsys_osd +#undef dout_prefix +#define dout_prefix _prefix(_dout) + +static ostream& _prefix(std::ostream* _dout) +{ + return *_dout << "ErasureCodePluginJerasure: "; +} + +class ErasureCodePluginJerasure : public ErasureCodePlugin { +public: + virtual int factory(const map<std::string,std::string> ¶meters, + ErasureCodeInterfaceRef *erasure_code) { + ErasureCodeJerasure *interface; + std::string t; + if (parameters.find("erasure-code-technique") != parameters.end()) + t = parameters.find("erasure-code-technique")->second; + if (t == "reed_sol_van") { + interface = new ErasureCodeJerasureReedSolomonVandermonde(); + } else if (t == "reed_sol_r6_op") { + interface = new ErasureCodeJerasureReedSolomonRAID6(); + } else if (t == "cauchy_orig") { + interface = new ErasureCodeJerasureCauchyOrig(); + } else if (t == "cauchy_good") { + interface = new ErasureCodeJerasureCauchyGood(); + } else if (t == "liberation") { + interface = new ErasureCodeJerasureLiberation(); + } else if (t == "blaum_roth") { + interface = new ErasureCodeJerasureBlaumRoth(); + } else if (t == "liber8tion") { + interface = new ErasureCodeJerasureLiber8tion(); + } else { + derr << "technique=" << t << " is not a valid coding technique. " + << " Choose one of the following: " + << "reed_sol_van, reed_sol_r6_op, cauchy_orig, " + << "cauchy_good, liberation, blaum_roth, liber8tion" + << dendl; + return -ENOENT; + } + interface->init(parameters); + *erasure_code = ErasureCodeInterfaceRef(interface); + return 0; + } +}; + +int __erasure_code_init(char *plugin_name) +{ + ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); + return instance.add(plugin_name, new ErasureCodePluginJerasure()); +} diff --git a/src/osd/ErasureCodePluginJerasure/Makefile.am b/src/osd/ErasureCodePluginJerasure/Makefile.am new file mode 100644 index 00000000000..b31fb1c0785 --- /dev/null +++ b/src/osd/ErasureCodePluginJerasure/Makefile.am @@ -0,0 +1,22 @@ +# jerasure plugin +libec_jerasure_la_SOURCES = \ + osd/ErasureCodePluginJerasure/ErasureCodePluginJerasure.cc \ + osd/ErasureCodePluginJerasure/ErasureCodeJerasure.cc \ + osd/ErasureCodePluginJerasure/cauchy.c \ + osd/ErasureCodePluginJerasure/galois.c \ + osd/ErasureCodePluginJerasure/jerasure.c \ + osd/ErasureCodePluginJerasure/liberation.c \ + osd/ErasureCodePluginJerasure/reed_sol.c +noinst_HEADERS += \ + osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h \ + osd/ErasureCodePluginJerasure/cauchy.h \ + osd/ErasureCodePluginJerasure/galois.h \ + osd/ErasureCodePluginJerasure/jerasure.h \ + osd/ErasureCodePluginJerasure/liberation.h \ + osd/ErasureCodePluginJerasure/reed_sol.h +libec_jerasure_la_CFLAGS = ${AM_CFLAGS} +libec_jerasure_la_CXXFLAGS= ${AM_CXXFLAGS} +libec_jerasure_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libec_jerasure_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__erasure_code_.*' + +erasure_codelib_LTLIBRARIES += libec_jerasure.la diff --git a/src/osd/ErasureCodePluginJerasure/cauchy.c b/src/osd/ErasureCodePluginJerasure/cauchy.c new file mode 100755 index 00000000000..c6bb3c01abf --- /dev/null +++ b/src/osd/ErasureCodePluginJerasure/cauchy.c @@ -0,0 +1,408 @@ +/* cauchy.c + * James S. Plank + +Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure Coding Techniques + +Revision 1.2A +May 24, 2011 + +James S. Plank +Department of Electrical Engineering and Computer Science +University of Tennessee +Knoxville, TN 37996 +plank@cs.utk.edu + +Copyright (c) 2011, James S. Plank +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + - Neither the name of the University of Tennessee nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "galois.h" +#include "jerasure.h" +#include "cauchy.h" + +static int PPs[33] = { -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1 }; +static int NOs[33]; +static int ONEs[33][33]; + +static int *cbest_0; +static int *cbest_1; +static int cbest_2[3]; +static int cbest_3[7]; +static int cbest_4[15]; +static int cbest_5[31]; +static int cbest_6[63]; +static int cbest_7[127]; +static int cbest_8[255]; +static int cbest_9[511]; +static int cbest_10[1023]; +static int cbest_11[1023]; +static int *cbest_12, *cbest_13, *cbest_14, *cbest_15, *cbest_16, *cbest_17, *cbest_18, *cbest_19, *cbest_20, + *cbest_21, *cbest_22, *cbest_23, *cbest_24, *cbest_25, *cbest_26, *cbest_27, *cbest_28, *cbest_29, *cbest_30, + *cbest_31, *cbest_32; + +static int cbest_max_k[33] = { -1, -1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 1023, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1 }; + +static int cbest_init = 0; + +static int *cbest_all[33]; + + +#define talloc(type, num) (type *) malloc(sizeof(type)*(num)) + +int cauchy_n_ones(int n, int w) +{ + int no; + int cno; + int nones; + int i, j; + int highbit; + + highbit = (1 << (w-1)); + + if (PPs[w] == -1) { + nones = 0; + PPs[w] = galois_single_multiply(highbit, 2, w); + for (i = 0; i < w; i++) { + if (PPs[w] & (1 << i)) { + ONEs[w][nones] = (1 << i); + nones++; + } + } + NOs[w] = nones; + } + + no = 0; + for (i = 0; i < w; i++) if (n & (1 << i)) no++; + cno = no; + for (i = 1; i < w; i++) { + if (n & highbit) { + n ^= highbit; + n <<= 1; + n ^= PPs[w]; + cno--; + for (j = 0; j < NOs[w]; j++) { + cno += (n & ONEs[w][j]) ? 1 : -1; + } + } else { + n <<= 1; + } + no += cno; + } + return no; +} + +int *cauchy_original_coding_matrix(int k, int m, int w) +{ + int *matrix; + int i, j, index; + + if (w < 31 && (k+m) > (1 << w)) return NULL; + matrix = talloc(int, k*m); + if (matrix == NULL) return NULL; + index = 0; + for (i = 0; i < m; i++) { + for (j = 0; j < k; j++) { + matrix[index] = galois_single_divide(1, (i ^ (m+j)), w); + index++; + } + } + return matrix; +} + +int *cauchy_xy_coding_matrix(int k, int m, int w, int *X, int *Y) +{ + int index, i, j; + int *matrix; + + matrix = talloc(int, k*m); + if (matrix == NULL) { return NULL; } + index = 0; + for (i = 0; i < m; i++) { + for (j = 0; j < k; j++) { + matrix[index] = galois_single_divide(1, (X[i] ^ Y[j]), w); + index++; + } + } + return matrix; +} + +void cauchy_improve_coding_matrix(int k, int m, int w, int *matrix) +{ + int index, i, j, x; + int tmp; + int bno, tno, bno_index; + + for (j = 0; j < k; j++) { + if (matrix[j] != 1) { + tmp = galois_single_divide(1, matrix[j], w); + index = j; + for (i = 0; i < m; i++) { + matrix[index] = galois_single_multiply(matrix[index], tmp, w); + index += k; + } + } + } + for (i = 1; i < m; i++) { + bno = 0; + index = i*k; + for (j = 0; j < k; j++) bno += cauchy_n_ones(matrix[index+j], w); + bno_index = -1; + for (j = 0; j < k; j++) { + if (matrix[index+j] != 1) { + tmp = galois_single_divide(1, matrix[index+j], w); + tno = 0; + for (x = 0; x < k; x++) { + tno += cauchy_n_ones(galois_single_multiply(matrix[index+x], tmp, w), w); + } + if (tno < bno) { + bno = tno; + bno_index = j; + } + } + } + if (bno_index != -1) { + tmp = galois_single_divide(1, matrix[index+bno_index], w); + for (j = 0; j < k; j++) { + matrix[index+j] = galois_single_multiply(matrix[index+j], tmp, w); + } + } + } +} + +int *cauchy_good_general_coding_matrix(int k, int m, int w) +{ + int *matrix, i; + + if (m == 2 && k <= cbest_max_k[w]) { + matrix = talloc(int, k*m); + if (matrix == NULL) return NULL; + if (!cbest_init) { + cbest_init = 1; + cbest_all[0] = cbest_0; cbest_all[1] = cbest_1; cbest_all[2] = cbest_2; cbest_all[3] = cbest_3; cbest_all[4] = + cbest_4; cbest_all[5] = cbest_5; cbest_all[6] = cbest_6; cbest_all[7] = cbest_7; cbest_all[8] = cbest_8; + cbest_all[9] = cbest_9; cbest_all[10] = cbest_10; cbest_all[11] = cbest_11; cbest_all[12] = cbest_12; + cbest_all[13] = cbest_13; cbest_all[14] = cbest_14; cbest_all[15] = cbest_15; cbest_all[16] = cbest_16; + cbest_all[17] = cbest_17; cbest_all[18] = cbest_18; cbest_all[19] = cbest_19; cbest_all[20] = cbest_20; + cbest_all[21] = cbest_21; cbest_all[22] = cbest_22; cbest_all[23] = cbest_23; cbest_all[24] = cbest_24; + cbest_all[25] = cbest_25; cbest_all[26] = cbest_26; cbest_all[27] = cbest_27; cbest_all[28] = cbest_28; + cbest_all[29] = cbest_29; cbest_all[30] = cbest_30; cbest_all[31] = cbest_31; cbest_all[32] = (int *) cbest_32; + } + for (i = 0; i < k; i++) { + matrix[i] = 1; + matrix[i+k] = cbest_all[w][i]; + } + return matrix; + } else { + matrix = cauchy_original_coding_matrix(k, m, w); + if (matrix == NULL) return NULL; + cauchy_improve_coding_matrix(k, m, w, matrix); + return matrix; + } +} + +static int cbest_2[3] = { 1, 2, 3 }; +static int cbest_3[7] = { 1, 2, 5, 4, 7, 3, 6 }; + +static int cbest_4[15] = { 1, 2, 9, 4, 8, 13, 3, 6, 12, 5, 11, 15, 10, 14, 7 }; + +static int cbest_5[31] = { 1, 2, 18, 4, 9, 8, 22, 16, 3, 11, 19, 5, 10, 6, 20, 27, 13, 23, 26, 12, + 17, 25, 24, 31, 30, 7, 15, 21, 29, 14, 28 }; + +static int cbest_6[63] = { 1, 2, 33, 4, 8, 49, 16, 32, 57, 3, 6, 12, 24, 48, 5, 35, 9, 37, 10, 17, + 41, 51, 56, 61, 18, 28, 53, 14, 20, 34, 7, 13, 25, 36, 59, 26, 39, 40, 45, 50, 60, 52, 63, + 11, 30, 55, 19, 22, 29, 43, 58, 15, 21, 38, 44, 47, 62, 27, 54, 42, 31, 23, 46 }; + +static int cbest_7[127] = { 1, 2, 68, 4, 34, 8, 17, 16, 76, 32, 38, 3, 64, 69, 5, 19, 35, 70, 6, 9, + 18, 102, 10, 36, 85, 12, 21, 42, 51, 72, 77, 84, 20, 25, 33, 50, 78, 98, 24, 39, 49, 100, 110 + , 48, 65, 93, 40, 66, 71, 92, 7, 46, 55, 87, 96, 103, 106, 11, 23, 37, 54, 81, 86, 108, 13, + 22, 27, 43, 53, 73, 80, 14, 26, 52, 74, 79, 99, 119, 44, 95, 101, 104, 111, 118, 29, 59, 89, + 94, 117, 28, 41, 58, 67, 88, 115, 116, 47, 57, 83, 97, 107, 114, 127, 56, 82, 109, 113, 126, + 112, 125, 15, 63, 75, 123, 124, 31, 45, 62, 91, 105, 122, 30, 61, 90, 121, 60, 120 }; + +static int cbest_8[255] = { 1, 2, 142, 4, 71, 8, 70, 173, 3, 35, 143, 16, 17, 67, 134, 140, 172, 6, 34 + , 69, 201, 216, 5, 33, 86, 12, 65, 138, 158, 159, 175, 10, 32, 43, 66, 108, 130, 193, 234, 9, + 24, 25, 50, 68, 79, 100, 132, 174, 200, 217, 20, 21, 42, 48, 87, 169, 41, 54, 64, 84, 96, 117 + , 154, 155, 165, 226, 77, 82, 135, 136, 141, 168, 192, 218, 238, 7, 18, 19, 39, 40, 78, 113, + 116, 128, 164, 180, 195, 205, 220, 232, 14, 26, 27, 58, 109, 156, 157, 203, 235, 13, 28, 29, 38 + , 51, 56, 75, 85, 90, 101, 110, 112, 139, 171, 11, 37, 49, 52, 76, 83, 102, 119, 131, 150, 151 + , 167, 182, 184, 188, 197, 219, 224, 45, 55, 80, 94, 97, 133, 170, 194, 204, 221, 227, 236, 36, + 47, 73, 92, 98, 104, 118, 152, 153, 166, 202, 207, 239, 251, 22, 23, 44, 74, 91, 148, 149, 161 + , 181, 190, 233, 46, 59, 88, 137, 146, 147, 163, 196, 208, 212, 222, 250, 57, 81, 95, 106, 111, + 129, 160, 176, 199, 243, 249, 15, 53, 72, 93, 103, 115, 125, 162, 183, 185, 189, 206, 225, 255, + 186, 210, 230, 237, 242, 248, 30, 31, 62, 89, 99, 105, 114, 121, 124, 178, 209, 213, 223, 228, + 241, 254, 60, 191, 198, 247, 120, 240, 107, 127, 144, 145, 177, 211, 214, 246, 245, 123, 126, + 187, 231, 253, 63, 179, 229, 244, 61, 122, 215, 252 }; + +static int cbest_9[511] = { 1, 2, 264, 4, 132, 8, 66, 16, 33, 32, 280, 64, 140, 128, 3, 70, 265, 5, + 133, 256, 266, 6, 9, 35, 67, 134, 268, 396, 10, 17, 34, 330, 12, 18, 68, 198, 297, 20, 37, 74 + , 136, 148, 165, 281, 296, 24, 36, 41, 65, 82, 99, 164, 272, 282, 388, 40, 49, 98, 141, 194, + 284, 328, 412, 48, 97, 129, 142, 196, 346, 71, 72, 96, 130, 313, 392, 80, 206, 257, 267, 312, + 334, 7, 135, 156, 173, 192, 258, 269, 397, 404, 11, 78, 144, 161, 172, 260, 270, 299, 331, 344, + 398, 13, 19, 39, 69, 86, 103, 160, 167, 199, 202, 298, 322, 384, 14, 21, 38, 43, 75, 102, 137, + 149, 166, 204, 289, 332, 408, 462, 22, 25, 42, 51, 83, 101, 138, 150, 273, 283, 288, 301, 350, + 389, 429, 26, 50, 76, 100, 195, 274, 285, 300, 329, 363, 390, 413, 428, 28, 45, 84, 143, 197, + 200, 214, 231, 276, 286, 315, 320, 347, 362, 414, 458, 44, 53, 73, 90, 107, 131, 152, 169, 181, + 230, 314, 338, 361, 393, 400, 454, 460, 52, 57, 81, 106, 115, 168, 175, 180, 207, 229, 305, 335 + , 348, 360, 394, 421, 478, 56, 105, 114, 157, 163, 174, 193, 210, 227, 228, 259, 304, 317, 326, + 405, 420, 445, 79, 104, 113, 145, 158, 162, 212, 226, 261, 271, 316, 345, 379, 399, 406, 444, + 450, 456, 87, 88, 112, 146, 203, 225, 262, 291, 323, 336, 378, 385, 425, 452, 474, 15, 205, 222 + , 224, 239, 290, 303, 333, 367, 377, 386, 409, 424, 431, 463, 470, 476, 23, 139, 151, 189, 208, + 238, 302, 324, 351, 366, 376, 410, 430, 437, 27, 47, 77, 94, 111, 177, 188, 237, 275, 293, 342, + 365, 391, 436, 448, 29, 46, 55, 85, 110, 119, 171, 176, 183, 201, 215, 218, 235, 236, 277, 287, + 292, 321, 355, 364, 415, 417, 459, 466, 472, 30, 54, 59, 91, 109, 118, 153, 170, 182, 220, 234, + 278, 307, 339, 354, 401, 416, 423, 441, 455, 461, 468, 495, 58, 108, 117, 154, 233, 306, 319, + 349, 353, 383, 395, 402, 422, 440, 447, 479, 494, 92, 116, 211, 232, 318, 327, 340, 352, 382, + 446, 493, 61, 159, 213, 216, 247, 309, 381, 407, 427, 451, 457, 464, 491, 492, 60, 89, 123, 147 + , 185, 246, 263, 308, 337, 371, 380, 426, 433, 453, 475, 487, 490, 122, 184, 191, 223, 245, 370, + 387, 432, 439, 471, 477, 486, 489, 511, 121, 179, 190, 209, 243, 244, 295, 325, 359, 369, 411, + 438, 485, 488, 510, 95, 120, 178, 242, 294, 343, 358, 368, 419, 449, 483, 484, 509, 219, 241, + 357, 418, 443, 467, 473, 482, 507, 508, 31, 221, 240, 255, 279, 356, 442, 469, 481, 503, 506, + 155, 254, 403, 480, 502, 505, 63, 93, 127, 253, 311, 341, 375, 501, 504, 62, 126, 187, 217, 251 + , 252, 310, 374, 435, 465, 499, 500, 125, 186, 250, 373, 434, 498, 124, 249, 372, 497, 248, 496 + }; + +static int cbest_10[1023] = { 1, 2, 516, 4, 258, 8, 129, 16, 32, 580, 64, 128, 290, 145, 256, 3, 512, + 517, 5, 259, 518, 588, 6, 9, 18, 36, 72, 144, 774, 10, 17, 131, 262, 288, 524, 645, 12, 33, + 133, 266, 294, 387, 532, 576, 581, 20, 34, 65, 137, 274, 548, 582, 24, 66, 291, 838, 40, 68, + 130, 147, 161, 322, 644, 709, 806, 48, 132, 193, 257, 386, 596, 80, 136, 298, 419, 612, 661, 772 + , 96, 149, 260, 272, 306, 403, 513, 146, 153, 160, 264, 292, 385, 514, 519, 544, 584, 589, 708, + 870, 7, 19, 37, 73, 192, 354, 590, 770, 775, 11, 38, 74, 177, 263, 289, 418, 520, 525, 534, 641 + , 660, 725, 802, 836, 846, 13, 22, 76, 148, 209, 267, 295, 320, 330, 402, 526, 528, 533, 577, + 647, 717, 804, 14, 21, 26, 35, 44, 135, 152, 165, 201, 275, 304, 384, 401, 435, 549, 578, 583, + 604, 608, 782, 903, 25, 52, 67, 88, 139, 270, 296, 391, 417, 550, 620, 653, 790, 834, 839, 41, + 50, 69, 104, 141, 176, 278, 302, 323, 395, 423, 540, 598, 640, 705, 724, 807, 866, 28, 42, 49, + 70, 82, 100, 163, 208, 282, 310, 556, 592, 597, 646, 663, 677, 711, 716, 868, 878, 81, 134, 151 + , 164, 195, 200, 299, 326, 352, 362, 400, 434, 564, 613, 657, 768, 773, 902, 967, 97, 138, 155, + 169, 197, 261, 273, 307, 358, 390, 416, 433, 451, 614, 652, 733, 800, 814, 844, 854, 935, 56, 84 + , 98, 140, 181, 217, 265, 293, 328, 338, 394, 422, 515, 545, 585, 704, 788, 822, 871, 919, 162, + 179, 276, 355, 407, 427, 546, 586, 591, 616, 662, 669, 676, 710, 727, 741, 771, 780, 901, 39, 75 + , 150, 157, 194, 211, 225, 268, 280, 308, 314, 389, 411, 439, 521, 530, 535, 628, 656, 721, 803, + 832, 837, 842, 847, 966, 23, 77, 112, 154, 168, 196, 300, 321, 331, 393, 421, 432, 450, 522, 527 + , 529, 552, 606, 643, 673, 693, 713, 732, 805, 864, 874, 934, 999, 15, 27, 45, 54, 78, 90, 108, + 180, 216, 305, 483, 560, 579, 600, 605, 609, 719, 778, 783, 852, 876, 886, 899, 918, 983, 46, 53 + , 89, 167, 178, 185, 203, 213, 271, 297, 324, 334, 336, 360, 370, 406, 426, 467, 542, 551, 610, + 621, 649, 668, 726, 740, 786, 791, 810, 820, 835, 900, 917, 931, 951, 965, 975, 30, 51, 105, 156 + , 205, 210, 224, 279, 303, 356, 366, 388, 405, 410, 438, 449, 459, 536, 541, 594, 599, 622, 655, + 720, 812, 818, 862, 867, 933, 29, 43, 71, 83, 92, 101, 106, 143, 173, 283, 311, 312, 346, 392, + 409, 420, 437, 443, 557, 566, 593, 642, 659, 672, 692, 707, 712, 737, 757, 869, 879, 911, 998, + 60, 102, 241, 327, 353, 363, 399, 425, 482, 558, 565, 624, 679, 718, 735, 749, 769, 798, 898, + 963, 982, 58, 86, 166, 183, 184, 202, 212, 219, 233, 286, 359, 431, 466, 615, 636, 648, 689, 729 + , 801, 815, 840, 845, 850, 855, 884, 916, 930, 950, 964, 974, 981, 995, 1015, 57, 85, 99, 120, + 171, 199, 204, 229, 318, 329, 339, 368, 404, 448, 458, 465, 499, 654, 671, 685, 784, 789, 823, + 872, 882, 915, 932, 949, 997, 1007, 116, 142, 159, 172, 277, 408, 436, 442, 455, 481, 491, 547, + 572, 587, 617, 630, 658, 665, 706, 723, 736, 756, 776, 781, 816, 860, 894, 897, 910, 947, 991, + 114, 221, 240, 269, 281, 309, 315, 332, 342, 344, 378, 398, 424, 441, 475, 487, 531, 618, 629, + 678, 695, 734, 743, 748, 808, 833, 843, 929, 943, 962, 973, 113, 182, 189, 218, 227, 232, 301, + 364, 374, 430, 457, 523, 553, 562, 602, 607, 688, 728, 753, 796, 830, 865, 875, 927, 980, 994, + 1014, 55, 79, 91, 109, 170, 187, 198, 215, 228, 284, 415, 464, 498, 554, 561, 601, 670, 675, 684 + , 715, 745, 765, 779, 848, 853, 877, 887, 909, 914, 948, 979, 996, 1006, 1013, 47, 110, 158, 249 + , 316, 325, 335, 337, 361, 371, 397, 447, 454, 480, 490, 497, 538, 543, 611, 632, 664, 722, 787, + 811, 821, 880, 896, 913, 946, 961, 971, 990, 1011, 31, 94, 220, 245, 357, 367, 429, 440, 474, + 486, 537, 595, 623, 651, 681, 694, 701, 742, 759, 813, 819, 858, 863, 892, 928, 942, 945, 972, + 989, 993, 1003, 1023, 62, 93, 107, 188, 207, 226, 237, 243, 313, 340, 347, 376, 456, 471, 473, + 507, 567, 568, 626, 752, 890, 907, 926, 1005, 61, 103, 124, 175, 186, 214, 372, 414, 453, 463, + 489, 503, 559, 625, 638, 674, 691, 714, 731, 739, 744, 764, 794, 799, 828, 908, 925, 939, 959, + 978, 1012, 59, 87, 122, 248, 287, 350, 396, 413, 446, 485, 495, 496, 637, 751, 826, 841, 851, + 885, 912, 941, 960, 970, 977, 1010, 118, 121, 235, 244, 319, 369, 382, 428, 445, 574, 650, 667, + 680, 700, 758, 761, 785, 873, 883, 944, 988, 992, 1002, 1009, 1022, 117, 206, 223, 231, 236, 242 + , 470, 472, 506, 573, 631, 687, 777, 817, 856, 861, 895, 906, 987, 1004, 1021, 115, 174, 191, 333 + , 343, 345, 379, 452, 462, 469, 488, 502, 505, 619, 690, 697, 730, 738, 755, 809, 888, 924, 938, + 958, 969, 1019, 253, 365, 375, 412, 484, 494, 501, 563, 603, 750, 767, 792, 797, 831, 923, 940, + 957, 976, 1001, 234, 251, 285, 348, 444, 479, 555, 634, 666, 760, 824, 849, 905, 955, 1008, 111, + 222, 230, 247, 317, 380, 461, 511, 539, 633, 686, 703, 747, 881, 937, 986, 1020, 95, 190, 468, + 493, 504, 570, 696, 754, 859, 893, 968, 985, 1018, 63, 126, 252, 341, 377, 500, 569, 627, 683, + 766, 891, 922, 956, 1000, 1017, 125, 239, 250, 373, 478, 639, 795, 829, 904, 921, 954, 123, 246, + 351, 460, 477, 510, 702, 746, 763, 827, 936, 953, 119, 383, 492, 509, 575, 984, 682, 699, 857, + 1016, 238, 255, 889, 920, 476, 762, 793, 952, 349, 508, 635, 825, 381, 698, 254, 571, 127 }; + +static int cbest_11[1023] = { 1, + 2, 1026, 4, 513, 8, 16, 1282, 32, 64, 641, 128, 256, 512, 1346, 1024, 3, 673, 1027, 5, 10, 20, 40, 80, 160, 320, + 640, 6, 9, 515, 1030, 1280, 1539, 17, 517, 1034, 1283, 12, 18, 33, 521, 1042, 1362, 34, 65, 529, 1058, 1286, 1795, + 24, 36, 66, 129, 545, 643, 1090, 1290, 1667, 68, 130, 257, 577, 645, 672, 1154, 1298, 1344, 48, 72, 132, 258, 336, + 649, 681, 1314, 1347, 136, 168, 260, 514, 657, 769, 1538, 1923, 84, 96, 144, 264, 516, 1025, 1350, 1410, 1859, 42, + 272, 520, 705, 1032, 1354, 11, 21, 41, 81, 161, 192, 288, 321, 528, 675, 1028, 1537, 1699, 1794, 7, 22, 82, 162, + 322, 544, 642, 677, 897, 1031, 1046, 1066, 1106, 1186, 1281, 1366, 1378, 1666, 14, 44, 164, 324, 384, 523, 533, + 553, 576, 593, 644, 833, 1035, 1040, 1288, 1360, 1987, 13, 19, 28, 88, 328, 519, 648, 680, 689, 1043, 1056, 1284, + 1363, 1474, 1543, 1793, 1955, 26, 35, 56, 176, 656, 768, 1038, 1059, 1088, 1287, 1302, 1322, 1442, 1547, 1665, + 1922, 25, 37, 52, 67, 112, 340, 352, 525, 531, 737, 1091, 1152, 1291, 1296, 1555, 1858, 1875, 38, 69, 74, 104, 131, + 224, 547, 651, 661, 683, 704, 721, 961, 1050, 1062, 1155, 1299, 1312, 1345, 1370, 1571, 1799, 49, 70, 73, 133, 138, + 148, 170, 208, 259, 337, 448, 537, 549, 579, 647, 674, 929, 1094, 1294, 1315, 1352, 1536, 1603, 1671, 1698, 1803, + 1921, 50, 134, 137, 169, 261, 266, 276, 296, 338, 416, 581, 676, 896, 1074, 1098, 1158, 1348, 1394, 1408, 1675, + 1707, 1811, 1857, 2019, 76, 85, 97, 145, 262, 265, 522, 532, 552, 561, 585, 592, 653, 659, 685, 771, 832, 849, + 1064, 1162, 1194, 1306, 1318, 1351, 1386, 1411, 1506, 1683, 1827, 1986, 2003, 43, 86, 98, 140, 146, 172, 273, 344, + 518, 688, 773, 1033, 1110, 1122, 1170, 1355, 1490, 1542, 1697, 1792, 1927, 1954, 100, 193, 268, 274, 289, 597, 609, + 665, 697, 707, 777, 1029, 1044, 1104, 1184, 1330, 1364, 1376, 1414, 1546, 1664, 1731, 1863, 1931, 1963, 23, 46, 83, + 92, 152, 163, 184, 194, 290, 323, 368, 524, 530, 555, 693, 709, 736, 753, 785, 993, 1036, 1047, 1067, 1107, 1187, + 1218, 1320, 1358, 1367, 1379, 1418, 1450, 1545, 1554, 1867, 1874, 1939, 1985, 15, 30, 45, 60, 90, 120, 165, 180, + 196, 240, 280, 292, 325, 330, 360, 385, 480, 546, 650, 660, 679, 682, 713, 720, 745, 801, 899, 960, 977, 1041, + 1289, 1361, 1426, 1472, 1541, 1570, 1703, 1798, 1953, 29, 58, 89, 116, 166, 200, 232, 326, 329, 386, 464, 535, 536, + 548, 578, 595, 646, 835, 901, 928, 1048, 1057, 1070, 1190, 1285, 1300, 1368, 1382, 1440, 1475, 1559, 1579, 1602, + 1619, 1670, 1802, 1879, 1891, 1920, 27, 57, 177, 304, 388, 527, 557, 580, 691, 725, 837, 905, 937, 1039, 1054, + 1089, 1114, 1292, 1303, 1323, 1374, 1443, 1553, 1674, 1706, 1715, 1801, 1810, 1856, 1873, 1991, 2018, 2035, 53, + 106, 113, 178, 212, 332, 341, 353, 392, 424, 541, 560, 584, 601, 652, 658, 684, 770, 841, 848, 913, 1060, 1082, + 1096, 1153, 1202, 1297, 1402, 1478, 1522, 1569, 1673, 1682, 1705, 1797, 1826, 1959, 1995, 2002, 2027, 39, 54, 75, + 105, 114, 225, 342, 354, 400, 539, 569, 739, 772, 1051, 1063, 1078, 1092, 1138, 1160, 1192, 1304, 1313, 1326, 1371, + 1384, 1398, 1446, 1482, 1514, 1551, 1601, 1669, 1696, 1763, 1815, 1835, 1926, 71, 139, 149, 171, 209, 226, 298, + 356, 449, 565, 596, 608, 625, 663, 664, 696, 706, 723, 741, 776, 853, 865, 963, 1072, 1095, 1130, 1156, 1250, 1295, + 1310, 1353, 1392, 1687, 1730, 1747, 1809, 1862, 1930, 1962, 1971, 2007, 2017, 51, 78, 108, 135, 150, 210, 228, 267, + 277, 297, 339, 348, 417, 450, 551, 554, 587, 617, 655, 687, 692, 708, 752, 784, 931, 965, 992, 1009, 1075, 1099, + 1159, 1174, 1234, 1316, 1338, 1349, 1395, 1409, 1458, 1494, 1504, 1544, 1563, 1575, 1681, 1825, 1866, 1883, 1929, + 1938, 1961, 1984, 2001, 77, 142, 174, 263, 278, 346, 376, 418, 452, 496, 583, 669, 678, 701, 712, 729, 744, 761, + 800, 898, 933, 969, 976, 1001, 1065, 1108, 1120, 1163, 1168, 1195, 1307, 1319, 1334, 1356, 1387, 1416, 1448, 1488, + 1507, 1540, 1607, 1702, 1807, 1865, 1925, 1952, 87, 99, 141, 147, 156, 173, 188, 216, 248, 270, 300, 345, 372, 420, + 456, 488, 534, 563, 594, 667, 699, 757, 779, 789, 809, 834, 851, 900, 1102, 1111, 1123, 1171, 1328, 1412, 1491, + 1558, 1578, 1587, 1611, 1618, 1679, 1711, 1729, 1861, 1878, 1890, 1907, 1943, 2023, 94, 101, 124, 154, 186, 244, + 269, 275, 284, 526, 556, 589, 690, 724, 775, 836, 904, 936, 945, 981, 1045, 1068, 1105, 1166, 1185, 1198, 1216, + 1331, 1365, 1377, 1390, 1415, 1430, 1510, 1552, 1577, 1714, 1800, 1819, 1831, 1872, 1899, 1937, 1990, 2034, 47, 62, + 93, 102, 122, 153, 185, 195, 282, 291, 312, 362, 369, 432, 468, 540, 599, 600, 611, 715, 747, 840, 857, 912, 1037, + 1052, 1112, 1126, 1219, 1321, 1359, 1372, 1419, 1424, 1451, 1568, 1623, 1635, 1672, 1691, 1701, 1704, 1723, 1796, + 1958, 1994, 2011, 2026, 2043, 31, 61, 91, 121, 181, 197, 202, 234, 241, 281, 293, 308, 331, 361, 370, 481, 538, + 568, 613, 695, 711, 738, 755, 781, 787, 995, 1080, 1118, 1178, 1188, 1210, 1380, 1400, 1427, 1473, 1498, 1530, + 1550, 1557, 1600, 1617, 1668, 1719, 1735, 1762, 1779, 1814, 1834, 1843, 1877, 1889, 1935, 1967, 1993, 2025, 2039, + 59, 117, 167, 182, 198, 201, 233, 242, 294, 327, 387, 465, 482, 559, 564, 605, 624, 662, 722, 740, 803, 852, 864, + 881, 907, 917, 939, 962, 979, 997, 1049, 1071, 1086, 1146, 1191, 1206, 1222, 1266, 1301, 1324, 1369, 1383, 1406, + 1422, 1441, 1454, 1480, 1512, 1526, 1549, 1686, 1713, 1739, 1746, 1771, 1808, 1833, 1871, 1970, 1989, 2006, 2016, + 2033, 118, 305, 334, 364, 389, 394, 404, 426, 466, 484, 543, 550, 573, 586, 603, 616, 633, 654, 686, 717, 749, 793, + 805, 843, 873, 903, 930, 964, 1008, 1055, 1115, 1128, 1142, 1200, 1226, 1258, 1293, 1308, 1375, 1476, 1520, 1562, + 1574, 1680, 1824 }; + diff --git a/src/osd/ErasureCodePluginJerasure/cauchy.h b/src/osd/ErasureCodePluginJerasure/cauchy.h new file mode 100755 index 00000000000..67fbf3bdbde --- /dev/null +++ b/src/osd/ErasureCodePluginJerasure/cauchy.h @@ -0,0 +1,53 @@ +/* cauchy.h + * James S. Plank + +Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure Coding Techniques + +Revision 1.2A +May 24, 2011 + +James S. Plank +Department of Electrical Engineering and Computer Science +University of Tennessee +Knoxville, TN 37996 +plank@cs.utk.edu + +Copyright (c) 2011, James S. Plank +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + - Neither the name of the University of Tennessee nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +*/ + +extern int *cauchy_original_coding_matrix(int k, int m, int w); +extern int *cauchy_xy_coding_matrix(int k, int m, int w, int *x, int *y); +extern void cauchy_improve_coding_matrix(int k, int m, int w, int *matrix); +extern int *cauchy_good_general_coding_matrix(int k, int m, int w); +extern int cauchy_n_ones(int n, int w); diff --git a/src/osd/ErasureCodePluginJerasure/cauchy_best_r6.c b/src/osd/ErasureCodePluginJerasure/cauchy_best_r6.c new file mode 100755 index 00000000000..8ab7eab67fe --- /dev/null +++ b/src/osd/ErasureCodePluginJerasure/cauchy_best_r6.c @@ -0,0 +1,1985 @@ +/* cauchy_best_r6.c + * James S. Plank + +Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure Coding Techniques + +Revision 1.2A +May 24, 2011 + +James S. Plank +Department of Electrical Engineering and Computer Science +University of Tennessee +Knoxville, TN 37996 +plank@cs.utk.edu + +Copyright (c) 2011, James S. Plank +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + - Neither the name of the University of Tennessee nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "galois.h" +#include "jerasure.h" +#include "cauchy.h" + +static int PPs[33] = { -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1 }; +static int NOs[33]; +static int ONEs[33][33]; + +static int *cbest_0; +static int *cbest_1; +static int cbest_2[3]; +static int cbest_3[7]; +static int cbest_4[15]; +static int cbest_5[31]; +static int cbest_6[63]; +static int cbest_7[127]; +static int cbest_8[255]; +static int cbest_9[511]; +static int cbest_10[1023]; +static int cbest_11[1023]; +static int cbest_12[1023], cbest_13[1023], cbest_14[1023], cbest_15[1023], cbest_16[1023], cbest_17[1023], cbest_18[1023], + cbest_19[1023], cbest_20[1023], cbest_21[1023], cbest_22[1023], cbest_23[1023], cbest_24[1023], cbest_25[1023], + cbest_26[1023], cbest_27[1023], cbest_28[1023], cbest_29[1023], cbest_30[1023], cbest_31[1023]; +static unsigned int cbest_32[1023]; + +static int cbest_max_k[33] = { -1, -1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 1023, + 1023, 1023, 1023, 1023, 1023, 1023, 1023, 1023, 1023, 1023, 1023, 1023, 1023, 1023, 1023, 1023, 1023, + 1023, 1023, 1023, 1023 }; + +static int cbest_init = 0; + +static int *cbest_all[33]; + + +#define talloc(type, num) (type *) malloc(sizeof(type)*(num)) + +int cauchy_n_ones(int n, int w) +{ + int no; + int cno; + int nones; + int i, j; + int highbit; + + highbit = (1 << (w-1)); + + if (PPs[w] == -1) { + nones = 0; + PPs[w] = galois_single_multiply(highbit, 2, w); + for (i = 0; i < w; i++) { + if (PPs[w] & (1 << i)) { + ONEs[w][nones] = (1 << i); + nones++; + } + } + NOs[w] = nones; + } + + no = 0; + for (i = 0; i < w; i++) if (n & (1 << i)) no++; + cno = no; + for (i = 1; i < w; i++) { + if (n & highbit) { + n ^= highbit; + n <<= 1; + n ^= PPs[w]; + cno--; + for (j = 0; j < NOs[w]; j++) { + cno += (n & ONEs[w][j]) ? 1 : -1; + } + } else { + n <<= 1; + } + no += cno; + } + return no; +} + +int *cauchy_original_coding_matrix(int k, int m, int w) +{ + int *matrix; + int i, j, index; + + if (w < 31 && (k+m) > (1 << w)) return NULL; + matrix = talloc(int, k*m); + if (matrix == NULL) return NULL; + index = 0; + for (i = 0; i < m; i++) { + for (j = 0; j < k; j++) { + matrix[index] = galois_single_divide(1, (i ^ (m+j)), w); + index++; + } + } + return matrix; +} + +int *cauchy_xy_coding_matrix(int k, int m, int w, int *X, int *Y) +{ + int index, i, j; + int *matrix; + + matrix = talloc(int, k*m); + if (matrix == NULL) { return NULL; } + index = 0; + for (i = 0; i < m; i++) { + for (j = 0; j < k; j++) { + matrix[index] = galois_single_divide(1, (X[i] ^ Y[j]), w); + index++; + } + } + return matrix; +} + +void cauchy_improve_coding_matrix(int k, int m, int w, int *matrix) +{ + int index, i, j, x; + int tmp; + int bno, tno, bno_index; + + for (j = 0; j < k; j++) { + if (matrix[j] != 1) { + tmp = galois_single_divide(1, matrix[j], w); + index = j; + for (i = 0; i < m; i++) { + matrix[index] = galois_single_multiply(matrix[index], tmp, w); + index += k; + } + } + } + for (i = 1; i < m; i++) { + bno = 0; + index = i*k; + for (j = 0; j < k; j++) bno += cauchy_n_ones(matrix[index+j], w); + bno_index = -1; + for (j = 0; j < k; j++) { + if (matrix[index+j] != 1) { + tmp = galois_single_divide(1, matrix[index+j], w); + tno = 0; + for (x = 0; x < k; x++) { + tno += cauchy_n_ones(galois_single_multiply(matrix[index+x], tmp, w), w); + } + if (tno < bno) { + bno = tno; + bno_index = j; + } + } + } + if (bno_index != -1) { + tmp = galois_single_divide(1, matrix[index+bno_index], w); + for (j = 0; j < k; j++) { + matrix[index+j] = galois_single_multiply(matrix[index+j], tmp, w); + } + } + } +} + +int *cauchy_good_general_coding_matrix(int k, int m, int w) +{ + int *matrix, i; + + if (m == 2 && k <= cbest_max_k[w]) { + matrix = talloc(int, k*m); + if (matrix == NULL) return NULL; + if (!cbest_init) { + cbest_init = 1; + cbest_all[0] = cbest_0; cbest_all[1] = cbest_1; cbest_all[2] = cbest_2; cbest_all[3] = cbest_3; cbest_all[4] = + cbest_4; cbest_all[5] = cbest_5; cbest_all[6] = cbest_6; cbest_all[7] = cbest_7; cbest_all[8] = cbest_8; + cbest_all[9] = cbest_9; cbest_all[10] = cbest_10; cbest_all[11] = cbest_11; cbest_all[12] = cbest_12; + cbest_all[13] = cbest_13; cbest_all[14] = cbest_14; cbest_all[15] = cbest_15; cbest_all[16] = cbest_16; + cbest_all[17] = cbest_17; cbest_all[18] = cbest_18; cbest_all[19] = cbest_19; cbest_all[20] = cbest_20; + cbest_all[21] = cbest_21; cbest_all[22] = cbest_22; cbest_all[23] = cbest_23; cbest_all[24] = cbest_24; + cbest_all[25] = cbest_25; cbest_all[26] = cbest_26; cbest_all[27] = cbest_27; cbest_all[28] = cbest_28; + cbest_all[29] = cbest_29; cbest_all[30] = cbest_30; cbest_all[31] = cbest_31; cbest_all[32] = (int *) cbest_32; + } + for (i = 0; i < k; i++) { + matrix[i] = 1; + matrix[i+k] = cbest_all[w][i]; + } + return matrix; + } else { + matrix = cauchy_original_coding_matrix(k, m, w); + if (matrix == NULL) return NULL; + cauchy_improve_coding_matrix(k, m, w, matrix); + return matrix; + } +} + +static int cbest_2[3] = { 1, 2, 3 }; +static int cbest_3[7] = { 1, 2, 5, 4, 7, 3, 6 }; + +static int cbest_4[15] = { 1, 2, 9, 4, 8, 13, 3, 6, 12, 5, 11, 15, 10, 14, 7 }; + +static int cbest_5[31] = { 1, 2, 18, 4, 9, 8, 22, 16, 3, 11, 19, 5, 10, 6, 20, 27, 13, 23, 26, 12, + 17, 25, 24, 31, 30, 7, 15, 21, 29, 14, 28 }; + +static int cbest_6[63] = { 1, 2, 33, 4, 8, 49, 16, 32, 57, 3, 6, 12, 24, 48, 5, 35, 9, 37, 10, 17, + 41, 51, 56, 61, 18, 28, 53, 14, 20, 34, 7, 13, 25, 36, 59, 26, 39, 40, 45, 50, 60, 52, 63, + 11, 30, 55, 19, 22, 29, 43, 58, 15, 21, 38, 44, 47, 62, 27, 54, 42, 31, 23, 46 }; + +static int cbest_7[127] = { 1, 2, 68, 4, 34, 8, 17, 16, 76, 32, 38, 3, 64, 69, 5, 19, 35, 70, 6, 9, + 18, 102, 10, 36, 85, 12, 21, 42, 51, 72, 77, 84, 20, 25, 33, 50, 78, 98, 24, 39, 49, 100, 110 + , 48, 65, 93, 40, 66, 71, 92, 7, 46, 55, 87, 96, 103, 106, 11, 23, 37, 54, 81, 86, 108, 13, + 22, 27, 43, 53, 73, 80, 14, 26, 52, 74, 79, 99, 119, 44, 95, 101, 104, 111, 118, 29, 59, 89, + 94, 117, 28, 41, 58, 67, 88, 115, 116, 47, 57, 83, 97, 107, 114, 127, 56, 82, 109, 113, 126, + 112, 125, 15, 63, 75, 123, 124, 31, 45, 62, 91, 105, 122, 30, 61, 90, 121, 60, 120 }; + +static int cbest_8[255] = { 1, 2, 142, 4, 71, 8, 70, 173, 3, 35, 143, 16, 17, 67, 134, 140, 172, 6, 34 + , 69, 201, 216, 5, 33, 86, 12, 65, 138, 158, 159, 175, 10, 32, 43, 66, 108, 130, 193, 234, 9, + 24, 25, 50, 68, 79, 100, 132, 174, 200, 217, 20, 21, 42, 48, 87, 169, 41, 54, 64, 84, 96, 117 + , 154, 155, 165, 226, 77, 82, 135, 136, 141, 168, 192, 218, 238, 7, 18, 19, 39, 40, 78, 113, + 116, 128, 164, 180, 195, 205, 220, 232, 14, 26, 27, 58, 109, 156, 157, 203, 235, 13, 28, 29, 38 + , 51, 56, 75, 85, 90, 101, 110, 112, 139, 171, 11, 37, 49, 52, 76, 83, 102, 119, 131, 150, 151 + , 167, 182, 184, 188, 197, 219, 224, 45, 55, 80, 94, 97, 133, 170, 194, 204, 221, 227, 236, 36, + 47, 73, 92, 98, 104, 118, 152, 153, 166, 202, 207, 239, 251, 22, 23, 44, 74, 91, 148, 149, 161 + , 181, 190, 233, 46, 59, 88, 137, 146, 147, 163, 196, 208, 212, 222, 250, 57, 81, 95, 106, 111, + 129, 160, 176, 199, 243, 249, 15, 53, 72, 93, 103, 115, 125, 162, 183, 185, 189, 206, 225, 255, + 186, 210, 230, 237, 242, 248, 30, 31, 62, 89, 99, 105, 114, 121, 124, 178, 209, 213, 223, 228, + 241, 254, 60, 191, 198, 247, 120, 240, 107, 127, 144, 145, 177, 211, 214, 246, 245, 123, 126, + 187, 231, 253, 63, 179, 229, 244, 61, 122, 215, 252 }; + +static int cbest_9[511] = { 1, 2, 264, 4, 132, 8, 66, 16, 33, 32, 280, 64, 140, 128, 3, 70, 265, 5, + 133, 256, 266, 6, 9, 35, 67, 134, 268, 396, 10, 17, 34, 330, 12, 18, 68, 198, 297, 20, 37, 74 + , 136, 148, 165, 281, 296, 24, 36, 41, 65, 82, 99, 164, 272, 282, 388, 40, 49, 98, 141, 194, + 284, 328, 412, 48, 97, 129, 142, 196, 346, 71, 72, 96, 130, 313, 392, 80, 206, 257, 267, 312, + 334, 7, 135, 156, 173, 192, 258, 269, 397, 404, 11, 78, 144, 161, 172, 260, 270, 299, 331, 344, + 398, 13, 19, 39, 69, 86, 103, 160, 167, 199, 202, 298, 322, 384, 14, 21, 38, 43, 75, 102, 137, + 149, 166, 204, 289, 332, 408, 462, 22, 25, 42, 51, 83, 101, 138, 150, 273, 283, 288, 301, 350, + 389, 429, 26, 50, 76, 100, 195, 274, 285, 300, 329, 363, 390, 413, 428, 28, 45, 84, 143, 197, + 200, 214, 231, 276, 286, 315, 320, 347, 362, 414, 458, 44, 53, 73, 90, 107, 131, 152, 169, 181, + 230, 314, 338, 361, 393, 400, 454, 460, 52, 57, 81, 106, 115, 168, 175, 180, 207, 229, 305, 335 + , 348, 360, 394, 421, 478, 56, 105, 114, 157, 163, 174, 193, 210, 227, 228, 259, 304, 317, 326, + 405, 420, 445, 79, 104, 113, 145, 158, 162, 212, 226, 261, 271, 316, 345, 379, 399, 406, 444, + 450, 456, 87, 88, 112, 146, 203, 225, 262, 291, 323, 336, 378, 385, 425, 452, 474, 15, 205, 222 + , 224, 239, 290, 303, 333, 367, 377, 386, 409, 424, 431, 463, 470, 476, 23, 139, 151, 189, 208, + 238, 302, 324, 351, 366, 376, 410, 430, 437, 27, 47, 77, 94, 111, 177, 188, 237, 275, 293, 342, + 365, 391, 436, 448, 29, 46, 55, 85, 110, 119, 171, 176, 183, 201, 215, 218, 235, 236, 277, 287, + 292, 321, 355, 364, 415, 417, 459, 466, 472, 30, 54, 59, 91, 109, 118, 153, 170, 182, 220, 234, + 278, 307, 339, 354, 401, 416, 423, 441, 455, 461, 468, 495, 58, 108, 117, 154, 233, 306, 319, + 349, 353, 383, 395, 402, 422, 440, 447, 479, 494, 92, 116, 211, 232, 318, 327, 340, 352, 382, + 446, 493, 61, 159, 213, 216, 247, 309, 381, 407, 427, 451, 457, 464, 491, 492, 60, 89, 123, 147 + , 185, 246, 263, 308, 337, 371, 380, 426, 433, 453, 475, 487, 490, 122, 184, 191, 223, 245, 370, + 387, 432, 439, 471, 477, 486, 489, 511, 121, 179, 190, 209, 243, 244, 295, 325, 359, 369, 411, + 438, 485, 488, 510, 95, 120, 178, 242, 294, 343, 358, 368, 419, 449, 483, 484, 509, 219, 241, + 357, 418, 443, 467, 473, 482, 507, 508, 31, 221, 240, 255, 279, 356, 442, 469, 481, 503, 506, + 155, 254, 403, 480, 502, 505, 63, 93, 127, 253, 311, 341, 375, 501, 504, 62, 126, 187, 217, 251 + , 252, 310, 374, 435, 465, 499, 500, 125, 186, 250, 373, 434, 498, 124, 249, 372, 497, 248, 496 + }; + +static int cbest_10[1023] = { 1, 2, 516, 4, 258, 8, 129, 16, 32, 580, 64, 128, 290, 145, 256, 3, 512, + 517, 5, 259, 518, 588, 6, 9, 18, 36, 72, 144, 774, 10, 17, 131, 262, 288, 524, 645, 12, 33, + 133, 266, 294, 387, 532, 576, 581, 20, 34, 65, 137, 274, 548, 582, 24, 66, 291, 838, 40, 68, + 130, 147, 161, 322, 644, 709, 806, 48, 132, 193, 257, 386, 596, 80, 136, 298, 419, 612, 661, 772 + , 96, 149, 260, 272, 306, 403, 513, 146, 153, 160, 264, 292, 385, 514, 519, 544, 584, 589, 708, + 870, 7, 19, 37, 73, 192, 354, 590, 770, 775, 11, 38, 74, 177, 263, 289, 418, 520, 525, 534, 641 + , 660, 725, 802, 836, 846, 13, 22, 76, 148, 209, 267, 295, 320, 330, 402, 526, 528, 533, 577, + 647, 717, 804, 14, 21, 26, 35, 44, 135, 152, 165, 201, 275, 304, 384, 401, 435, 549, 578, 583, + 604, 608, 782, 903, 25, 52, 67, 88, 139, 270, 296, 391, 417, 550, 620, 653, 790, 834, 839, 41, + 50, 69, 104, 141, 176, 278, 302, 323, 395, 423, 540, 598, 640, 705, 724, 807, 866, 28, 42, 49, + 70, 82, 100, 163, 208, 282, 310, 556, 592, 597, 646, 663, 677, 711, 716, 868, 878, 81, 134, 151 + , 164, 195, 200, 299, 326, 352, 362, 400, 434, 564, 613, 657, 768, 773, 902, 967, 97, 138, 155, + 169, 197, 261, 273, 307, 358, 390, 416, 433, 451, 614, 652, 733, 800, 814, 844, 854, 935, 56, 84 + , 98, 140, 181, 217, 265, 293, 328, 338, 394, 422, 515, 545, 585, 704, 788, 822, 871, 919, 162, + 179, 276, 355, 407, 427, 546, 586, 591, 616, 662, 669, 676, 710, 727, 741, 771, 780, 901, 39, 75 + , 150, 157, 194, 211, 225, 268, 280, 308, 314, 389, 411, 439, 521, 530, 535, 628, 656, 721, 803, + 832, 837, 842, 847, 966, 23, 77, 112, 154, 168, 196, 300, 321, 331, 393, 421, 432, 450, 522, 527 + , 529, 552, 606, 643, 673, 693, 713, 732, 805, 864, 874, 934, 999, 15, 27, 45, 54, 78, 90, 108, + 180, 216, 305, 483, 560, 579, 600, 605, 609, 719, 778, 783, 852, 876, 886, 899, 918, 983, 46, 53 + , 89, 167, 178, 185, 203, 213, 271, 297, 324, 334, 336, 360, 370, 406, 426, 467, 542, 551, 610, + 621, 649, 668, 726, 740, 786, 791, 810, 820, 835, 900, 917, 931, 951, 965, 975, 30, 51, 105, 156 + , 205, 210, 224, 279, 303, 356, 366, 388, 405, 410, 438, 449, 459, 536, 541, 594, 599, 622, 655, + 720, 812, 818, 862, 867, 933, 29, 43, 71, 83, 92, 101, 106, 143, 173, 283, 311, 312, 346, 392, + 409, 420, 437, 443, 557, 566, 593, 642, 659, 672, 692, 707, 712, 737, 757, 869, 879, 911, 998, + 60, 102, 241, 327, 353, 363, 399, 425, 482, 558, 565, 624, 679, 718, 735, 749, 769, 798, 898, + 963, 982, 58, 86, 166, 183, 184, 202, 212, 219, 233, 286, 359, 431, 466, 615, 636, 648, 689, 729 + , 801, 815, 840, 845, 850, 855, 884, 916, 930, 950, 964, 974, 981, 995, 1015, 57, 85, 99, 120, + 171, 199, 204, 229, 318, 329, 339, 368, 404, 448, 458, 465, 499, 654, 671, 685, 784, 789, 823, + 872, 882, 915, 932, 949, 997, 1007, 116, 142, 159, 172, 277, 408, 436, 442, 455, 481, 491, 547, + 572, 587, 617, 630, 658, 665, 706, 723, 736, 756, 776, 781, 816, 860, 894, 897, 910, 947, 991, + 114, 221, 240, 269, 281, 309, 315, 332, 342, 344, 378, 398, 424, 441, 475, 487, 531, 618, 629, + 678, 695, 734, 743, 748, 808, 833, 843, 929, 943, 962, 973, 113, 182, 189, 218, 227, 232, 301, + 364, 374, 430, 457, 523, 553, 562, 602, 607, 688, 728, 753, 796, 830, 865, 875, 927, 980, 994, + 1014, 55, 79, 91, 109, 170, 187, 198, 215, 228, 284, 415, 464, 498, 554, 561, 601, 670, 675, 684 + , 715, 745, 765, 779, 848, 853, 877, 887, 909, 914, 948, 979, 996, 1006, 1013, 47, 110, 158, 249 + , 316, 325, 335, 337, 361, 371, 397, 447, 454, 480, 490, 497, 538, 543, 611, 632, 664, 722, 787, + 811, 821, 880, 896, 913, 946, 961, 971, 990, 1011, 31, 94, 220, 245, 357, 367, 429, 440, 474, + 486, 537, 595, 623, 651, 681, 694, 701, 742, 759, 813, 819, 858, 863, 892, 928, 942, 945, 972, + 989, 993, 1003, 1023, 62, 93, 107, 188, 207, 226, 237, 243, 313, 340, 347, 376, 456, 471, 473, + 507, 567, 568, 626, 752, 890, 907, 926, 1005, 61, 103, 124, 175, 186, 214, 372, 414, 453, 463, + 489, 503, 559, 625, 638, 674, 691, 714, 731, 739, 744, 764, 794, 799, 828, 908, 925, 939, 959, + 978, 1012, 59, 87, 122, 248, 287, 350, 396, 413, 446, 485, 495, 496, 637, 751, 826, 841, 851, + 885, 912, 941, 960, 970, 977, 1010, 118, 121, 235, 244, 319, 369, 382, 428, 445, 574, 650, 667, + 680, 700, 758, 761, 785, 873, 883, 944, 988, 992, 1002, 1009, 1022, 117, 206, 223, 231, 236, 242 + , 470, 472, 506, 573, 631, 687, 777, 817, 856, 861, 895, 906, 987, 1004, 1021, 115, 174, 191, 333 + , 343, 345, 379, 452, 462, 469, 488, 502, 505, 619, 690, 697, 730, 738, 755, 809, 888, 924, 938, + 958, 969, 1019, 253, 365, 375, 412, 484, 494, 501, 563, 603, 750, 767, 792, 797, 831, 923, 940, + 957, 976, 1001, 234, 251, 285, 348, 444, 479, 555, 634, 666, 760, 824, 849, 905, 955, 1008, 111, + 222, 230, 247, 317, 380, 461, 511, 539, 633, 686, 703, 747, 881, 937, 986, 1020, 95, 190, 468, + 493, 504, 570, 696, 754, 859, 893, 968, 985, 1018, 63, 126, 252, 341, 377, 500, 569, 627, 683, + 766, 891, 922, 956, 1000, 1017, 125, 239, 250, 373, 478, 639, 795, 829, 904, 921, 954, 123, 246, + 351, 460, 477, 510, 702, 746, 763, 827, 936, 953, 119, 383, 492, 509, 575, 984, 682, 699, 857, + 1016, 238, 255, 889, 920, 476, 762, 793, 952, 349, 508, 635, 825, 381, 698, 254, 571, 127 }; + +static int cbest_11[1023] = { 1, + 2, 1026, 4, 513, 8, 16, 1282, 32, 64, 641, 128, 256, 512, 1346, 1024, 3, 673, 1027, 5, 10, 20, 40, 80, 160, 320, + 640, 6, 9, 515, 1030, 1280, 1539, 17, 517, 1034, 1283, 12, 18, 33, 521, 1042, 1362, 34, 65, 529, 1058, 1286, 1795, + 24, 36, 66, 129, 545, 643, 1090, 1290, 1667, 68, 130, 257, 577, 645, 672, 1154, 1298, 1344, 48, 72, 132, 258, 336, + 649, 681, 1314, 1347, 136, 168, 260, 514, 657, 769, 1538, 1923, 84, 96, 144, 264, 516, 1025, 1350, 1410, 1859, 42, + 272, 520, 705, 1032, 1354, 11, 21, 41, 81, 161, 192, 288, 321, 528, 675, 1028, 1537, 1699, 1794, 7, 22, 82, 162, + 322, 544, 642, 677, 897, 1031, 1046, 1066, 1106, 1186, 1281, 1366, 1378, 1666, 14, 44, 164, 324, 384, 523, 533, + 553, 576, 593, 644, 833, 1035, 1040, 1288, 1360, 1987, 13, 19, 28, 88, 328, 519, 648, 680, 689, 1043, 1056, 1284, + 1363, 1474, 1543, 1793, 1955, 26, 35, 56, 176, 656, 768, 1038, 1059, 1088, 1287, 1302, 1322, 1442, 1547, 1665, + 1922, 25, 37, 52, 67, 112, 340, 352, 525, 531, 737, 1091, 1152, 1291, 1296, 1555, 1858, 1875, 38, 69, 74, 104, 131, + 224, 547, 651, 661, 683, 704, 721, 961, 1050, 1062, 1155, 1299, 1312, 1345, 1370, 1571, 1799, 49, 70, 73, 133, 138, + 148, 170, 208, 259, 337, 448, 537, 549, 579, 647, 674, 929, 1094, 1294, 1315, 1352, 1536, 1603, 1671, 1698, 1803, + 1921, 50, 134, 137, 169, 261, 266, 276, 296, 338, 416, 581, 676, 896, 1074, 1098, 1158, 1348, 1394, 1408, 1675, + 1707, 1811, 1857, 2019, 76, 85, 97, 145, 262, 265, 522, 532, 552, 561, 585, 592, 653, 659, 685, 771, 832, 849, + 1064, 1162, 1194, 1306, 1318, 1351, 1386, 1411, 1506, 1683, 1827, 1986, 2003, 43, 86, 98, 140, 146, 172, 273, 344, + 518, 688, 773, 1033, 1110, 1122, 1170, 1355, 1490, 1542, 1697, 1792, 1927, 1954, 100, 193, 268, 274, 289, 597, 609, + 665, 697, 707, 777, 1029, 1044, 1104, 1184, 1330, 1364, 1376, 1414, 1546, 1664, 1731, 1863, 1931, 1963, 23, 46, 83, + 92, 152, 163, 184, 194, 290, 323, 368, 524, 530, 555, 693, 709, 736, 753, 785, 993, 1036, 1047, 1067, 1107, 1187, + 1218, 1320, 1358, 1367, 1379, 1418, 1450, 1545, 1554, 1867, 1874, 1939, 1985, 15, 30, 45, 60, 90, 120, 165, 180, + 196, 240, 280, 292, 325, 330, 360, 385, 480, 546, 650, 660, 679, 682, 713, 720, 745, 801, 899, 960, 977, 1041, + 1289, 1361, 1426, 1472, 1541, 1570, 1703, 1798, 1953, 29, 58, 89, 116, 166, 200, 232, 326, 329, 386, 464, 535, 536, + 548, 578, 595, 646, 835, 901, 928, 1048, 1057, 1070, 1190, 1285, 1300, 1368, 1382, 1440, 1475, 1559, 1579, 1602, + 1619, 1670, 1802, 1879, 1891, 1920, 27, 57, 177, 304, 388, 527, 557, 580, 691, 725, 837, 905, 937, 1039, 1054, + 1089, 1114, 1292, 1303, 1323, 1374, 1443, 1553, 1674, 1706, 1715, 1801, 1810, 1856, 1873, 1991, 2018, 2035, 53, + 106, 113, 178, 212, 332, 341, 353, 392, 424, 541, 560, 584, 601, 652, 658, 684, 770, 841, 848, 913, 1060, 1082, + 1096, 1153, 1202, 1297, 1402, 1478, 1522, 1569, 1673, 1682, 1705, 1797, 1826, 1959, 1995, 2002, 2027, 39, 54, 75, + 105, 114, 225, 342, 354, 400, 539, 569, 739, 772, 1051, 1063, 1078, 1092, 1138, 1160, 1192, 1304, 1313, 1326, 1371, + 1384, 1398, 1446, 1482, 1514, 1551, 1601, 1669, 1696, 1763, 1815, 1835, 1926, 71, 139, 149, 171, 209, 226, 298, + 356, 449, 565, 596, 608, 625, 663, 664, 696, 706, 723, 741, 776, 853, 865, 963, 1072, 1095, 1130, 1156, 1250, 1295, + 1310, 1353, 1392, 1687, 1730, 1747, 1809, 1862, 1930, 1962, 1971, 2007, 2017, 51, 78, 108, 135, 150, 210, 228, 267, + 277, 297, 339, 348, 417, 450, 551, 554, 587, 617, 655, 687, 692, 708, 752, 784, 931, 965, 992, 1009, 1075, 1099, + 1159, 1174, 1234, 1316, 1338, 1349, 1395, 1409, 1458, 1494, 1504, 1544, 1563, 1575, 1681, 1825, 1866, 1883, 1929, + 1938, 1961, 1984, 2001, 77, 142, 174, 263, 278, 346, 376, 418, 452, 496, 583, 669, 678, 701, 712, 729, 744, 761, + 800, 898, 933, 969, 976, 1001, 1065, 1108, 1120, 1163, 1168, 1195, 1307, 1319, 1334, 1356, 1387, 1416, 1448, 1488, + 1507, 1540, 1607, 1702, 1807, 1865, 1925, 1952, 87, 99, 141, 147, 156, 173, 188, 216, 248, 270, 300, 345, 372, 420, + 456, 488, 534, 563, 594, 667, 699, 757, 779, 789, 809, 834, 851, 900, 1102, 1111, 1123, 1171, 1328, 1412, 1491, + 1558, 1578, 1587, 1611, 1618, 1679, 1711, 1729, 1861, 1878, 1890, 1907, 1943, 2023, 94, 101, 124, 154, 186, 244, + 269, 275, 284, 526, 556, 589, 690, 724, 775, 836, 904, 936, 945, 981, 1045, 1068, 1105, 1166, 1185, 1198, 1216, + 1331, 1365, 1377, 1390, 1415, 1430, 1510, 1552, 1577, 1714, 1800, 1819, 1831, 1872, 1899, 1937, 1990, 2034, 47, 62, + 93, 102, 122, 153, 185, 195, 282, 291, 312, 362, 369, 432, 468, 540, 599, 600, 611, 715, 747, 840, 857, 912, 1037, + 1052, 1112, 1126, 1219, 1321, 1359, 1372, 1419, 1424, 1451, 1568, 1623, 1635, 1672, 1691, 1701, 1704, 1723, 1796, + 1958, 1994, 2011, 2026, 2043, 31, 61, 91, 121, 181, 197, 202, 234, 241, 281, 293, 308, 331, 361, 370, 481, 538, + 568, 613, 695, 711, 738, 755, 781, 787, 995, 1080, 1118, 1178, 1188, 1210, 1380, 1400, 1427, 1473, 1498, 1530, + 1550, 1557, 1600, 1617, 1668, 1719, 1735, 1762, 1779, 1814, 1834, 1843, 1877, 1889, 1935, 1967, 1993, 2025, 2039, + 59, 117, 167, 182, 198, 201, 233, 242, 294, 327, 387, 465, 482, 559, 564, 605, 624, 662, 722, 740, 803, 852, 864, + 881, 907, 917, 939, 962, 979, 997, 1049, 1071, 1086, 1146, 1191, 1206, 1222, 1266, 1301, 1324, 1369, 1383, 1406, + 1422, 1441, 1454, 1480, 1512, 1526, 1549, 1686, 1713, 1739, 1746, 1771, 1808, 1833, 1871, 1970, 1989, 2006, 2016, + 2033, 118, 305, 334, 364, 389, 394, 404, 426, 466, 484, 543, 550, 573, 586, 603, 616, 633, 654, 686, 717, 749, 793, + 805, 843, 873, 903, 930, 964, 1008, 1055, 1115, 1128, 1142, 1200, 1226, 1258, 1293, 1308, 1375, 1476, 1520, 1562, + 1574, 1680, 1824 }; + +static int cbest_12[1023] = { + 1, 2, 2089, 4, 8, 3133, 16, 2088, 1044, 3, 32, 522, 261, 3639, 5, 64, 65, 2057, 2091, 6, 130, 3132, 260, + 2219, 9, 2093, 3117, 10, 128, 257, 2081, 3890, 12, 1566, 2217, 3129, 17, 514, 520, 3135, 18, 3196, 3637, 20, + 256, 1028, 1045, 3638, 24, 33, 523, 783, 2105, 3197, 3647, 34, 1040, 1945, 2056, 2090, 2595, 3125, 3891, 36, + 1046, 2348, 40, 1598, 2218, 3635, 7, 48, 66, 67, 131, 263, 269, 512, 538, 782, 1076, 1174, 1819, 2049, 2059, + 2092, 2152, 3113, 3116, 3384, 3607, 3888, 68, 69, 277, 554, 1108, 1109, 1558, 1564, 1944, 2061, 2080, 2478, + 3894, 72, 73, 134, 265, 293, 391, 526, 586, 587, 779, 2153, 2211, 2216, 2563, 3128, 4016, 11, 14, 80, 81, + 129, 132, 138, 530, 1172, 1692, 2073, 2095, 2223, 2316, 3101, 3109, 3119, 3134, 3645, 3874, 13, 96, 97, 146, + 259, 262, 268, 289, 799, 1052, 1060, 1158, 1567, 1818, 2083, 2221, 2344, 2349, 2476, 2980, 3045, 3192, 3368, + 3636, 144, 162, 276, 389, 515, 521, 578, 579, 781, 972, 1024, 1156, 2085, 2120, 2235, 2282, 2312, 3131, 3623, + 19, 22, 28, 160, 194, 195, 264, 292, 390, 778, 846, 1238, 1239, 1562, 1596, 1684, 1937, 1947, 2104, 2233, + 2593, 3193, 3385, 3633, 3643, 3646, 3889, 21, 26, 324, 518, 536, 909, 1029, 1297, 2008, 2121, 2209, 2283, + 2479, 2594, 2854, 3085, 3121, 3124, 3188, 3198, 3605, 3895, 25, 136, 258, 288, 528, 552, 798, 842, 1030, + 1140, 1141, 1490, 1556, 2470, 2603, 2626, 2721, 3892, 3898, 4017, 35, 38, 44, 56, 320, 388, 423, 486, 524, + 570, 584, 585, 618, 619, 648, 777, 780, 1041, 1594, 1817, 2097, 2107, 2317, 2561, 3105, 3164, 3189, 3199, + 3263, 3634, 3875, 37, 42, 52, 285, 309, 421, 1036, 1042, 1047, 1056, 1072, 1170, 1313, 1427, 1823, 1936, + 1946, 2048, 2053, 2058, 2065, 2109, 2280, 2345, 2477, 2579, 2627, 2981, 3041, 3044, 3112, 3127, 3369, 3388, + 3603, 3606, 3631, 3765, 3872, 41, 192, 193, 273, 399, 775, 797, 908, 968, 1104, 1105, 1168, 1236, 1237, 1281, + 1296, 1554, 1582, 1599, 1949, 2051, 2060, 2313, 2332, 2340, 2474, 3115, 3165, 3336, 3360, 3641, 4018, 49, 70, + 71, 76, 77, 88, 89, 112, 113, 387, 454, 484, 513, 516, 539, 640, 744, 745, 1004, 1077, 1175, 1234, 1235, + 1550, 1668, 1680, 1694, 1803, 2063, 2112, 2210, 2215, 2227, 2281, 2472, 2562, 2784, 3261, 3449, 3547, 3591, + 3621, 84, 85, 104, 105, 142, 154, 178, 226, 227, 242, 243, 267, 271, 385, 422, 534, 555, 771, 776, 791, 834, + 840, 844, 973, 1068, 1078, 1166, 1425, 1488, 1559, 1560, 1565, 1688, 1816, 2072, 2077, 2094, 2136, 2144, + 2154, 2222, 2298, 2336, 2350, 2468, 2543, 2611, 2624, 2658, 2729, 2850, 2855, 2982, 3081, 3097, 3100, 3108, + 3118, 3184, 3247, 3326, 3376, 3644, 3702, 4024, 82, 83, 135, 164, 170, 210, 211, 279, 284, 308, 328, 356, + 372, 417, 420, 527, 542, 576, 577, 616, 617, 656, 795, 847, 1048, 1110, 1111, 1312, 1329, 1392, 1426, 1590, + 1592, 1822, 2082, 2113, 2128, 2156, 2187, 2220, 2364, 2466, 2471, 2785, 2976, 3077, 3111, 3180, 3245, 3882, + 3893, 3899, 3902, 4081, 15, 30, 50, 60, 98, 99, 120, 121, 133, 139, 166, 272, 291, 295, 325, 340, 398, 419, + 531, 558, 582, 583, 712, 713, 774, 796, 838, 1064, 1084, 1173, 1182, 1232, 1233, 1280, 1522, 1542, 1676, + 1693, 1811, 1948, 2009, 2084, 2137, 2145, 2155, 2185, 2213, 2225, 2234, 2286, 2299, 2599, 2601, 2625, 2659, + 2745, 2870, 2988, 3047, 3130, 3185, 3194, 3327, 3352, 3386, 3545, 3601, 3619, 3622, 3629, 3703, 3733, 3773, + 3896, 3955, 140, 147, 152, 176, 186, 198, 199, 224, 225, 281, 305, 332, 386, 397, 532, 546, 590, 591, 680, + 696, 843, 901, 911, 974, 1026, 1032, 1053, 1061, 1116, 1117, 1159, 1164, 1491, 1552, 1802, 1941, 2069, 2087, + 2122, 2129, 2157, 2168, 2232, 2296, 2314, 2318, 2446, 2567, 2592, 2737, 2852, 3040, 3073, 3093, 3103, 3156, + 3181, 3389, 3627, 3632, 3642, 3858, 3873, 4020, 74, 75, 145, 163, 168, 208, 209, 266, 270, 321, 384, 395, + 452, 487, 502, 664, 770, 790, 905, 1025, 1054, 1148, 1149, 1152, 1157, 1301, 1360, 1424, 1578, 1686, 1821, + 1851, 1929, 1939, 2012, 2208, 2287, 2328, 2333, 2341, 2346, 2381, 2397, 2444, 2464, 2475, 2723, 2752, 3084, + 3120, 3195, 3337, 3361, 3370, 3417, 3514, 3579, 3604, 3615, 3780, 3878, 3984, 4019, 23, 29, 46, 92, 93, 161, + 240, 241, 278, 348, 416, 450, 562, 568, 610, 611, 650, 769, 773, 789, 794, 1154, 1222, 1223, 1328, 1376, + 1494, 1520, 1563, 1597, 1685, 1951, 2010, 2075, 2123, 2169, 2284, 2297, 2473, 2511, 2541, 2602, 2619, 2666, + 2720, 3107, 3123, 3157, 3294, 3324, 3372, 3448, 3453, 3546, 3587, 3670, 3710, 27, 196, 197, 290, 294, 418, + 482, 519, 537, 540, 574, 688, 787, 793, 969, 1062, 1142, 1143, 1435, 1548, 1588, 1662, 1663, 1801, 1810, + 1992, 2055, 2096, 2101, 2106, 2337, 2351, 2469, 2539, 2542, 2560, 2571, 2577, 2634, 2753, 2851, 2919, 2983, + 3013, 3043, 3104, 3259, 3262, 3322, 3377, 3380, 3441, 3625, 3717, 3767, 3781, 3880, 3900, 3939, 4025, 4049, + 137, 174, 280, 304, 322, 326, 344, 352, 393, 396, 455, 480, 485, 529, 553, 556, 760, 761, 832, 900, 910, 964, + 970, 1005, 1031, 1124, 1125, 1180, 1289, 1317, 1333, 1540, 1557, 1574, 1580, 1586, 1690, 1724, 1921, 1940, + 2052, 2064, 2099, 2108, 2250, 2266, 2285, 2304, 2365, 2467, 2578, 2583, 2609, 2656, 2667, 2848, 2977, 2996, + 3126, 3166, 3176, 3215, 3239, 3257, 3295, 3325, 3512, 3602, 3617, 3630, 3671, 3700, 3711, 3761, 3764, 3842, + 3883, 3903, 4080, 39, 45, 57, 148, 172, 184, 287, 297, 336, 394, 448, 525, 544, 571, 588, 589, 649, 746, 747, + 835, 841, 845, 904, 996, 1006, 1038, 1074, 1080, 1092, 1093, 1178, 1220, 1221, 1300, 1489, 1492, 1538, 1595, + 1670, 1682, 1756, 1757, 1795, 1815, 1820, 1835, 1850, 1928, 1938, 2050, 2111, 2179, 2356, 2440, 2509, 2565, + 2587, 2635, 2871, 2915, 2978, 2984, 2989, 3046, 3053, 3114, 3148, 3172, 3207, 3292, 3323, 3340, 3353, 3356, + 3364, 3387, 3401, 3445, 3544, 3589, 3640, 3729, 3757, 3769, 3796, 3897, 3954, 3959, 251, 373, 581, 1057, + 1107, 1137, 1299, 1630, 1882, 2214, 2308, 2315, 2319, 2597, 2853, 3167, 3260, 3482, 3590, 3862 }; + +static int cbest_13[1023] = { + 1, 2, 4109, 4, 6155, 8, 7176, 16, 3588, 4108, 32, 2054, 3, 1027, 1794, 6, 64, 4620, 6154, 5, 897, 4111, 6153, + 7177, 12, 2310, 3077, 4105, 128, 9, 24, 513, 1155, 10, 1026, 4101, 7689, 18, 48, 256, 896, 3589, 4365, 6159, + 7178, 17, 448, 4557, 4621, 4684, 5647, 36, 96, 224, 1025, 1792, 20, 112, 512, 769, 1538, 1795, 2052, 3076, + 3590, 6147, 6283, 7180, 33, 56, 72, 192, 2055, 2342, 3584, 4125, 6411, 7945, 28, 34, 577, 1154, 6922, 7240, + 14, 144, 384, 2050, 4110, 4493, 5134, 5903, 6152, 7168, 7304, 7, 40, 65, 1171, 3620, 4104, 4141, 5004, 288, + 768, 1153, 2308, 2567, 3652, 4097, 4107, 4397, 4685, 6157, 6171, 6379, 7688, 13, 68, 1024, 1810, 2311, 3461, + 4676, 7179, 66, 129, 576, 1826, 3079, 3844, 4100, 4173, 4622, 5646, 6347, 8073, 25, 80, 899, 905, 1031, 1091, + 1798, 2182, 2306, 2338, 2502, 3073, 3596, 4364, 4616, 5390, 5645, 6158, 6187, 7050, 7192, 11, 26, 913, 1922, + 4117, 4556, 4652, 5262, 6151, 6299, 6443, 6923, 7181, 19, 49, 130, 136, 257, 1169, 1283, 1536, 1793, 2062, + 2326, 2566, 2823, 4103, 4237, 6145, 7182, 7232, 7272, 7691, 7944, 50, 449, 515, 901, 961, 1035, 1163, 1170, + 1539, 1802, 2048, 2340, 2695, 3586, 3591, 3604, 3616, 4553, 4612, 4680, 4748, 5005, 6146, 6219, 6282, 7208, + 7241, 7288, 22, 37, 52, 97, 160, 225, 585, 1152, 1251, 1808, 2374, 2631, 3141, 3585, 4124, 4549, 4876, 5135, + 5902, 6281, 6410, 7169, 7172, 7305, 7320, 7560, 21, 38, 98, 113, 132, 226, 258, 452, 545, 898, 904, 1187, + 2053, 2070, 2278, 2438, 3205, 3460, 3525, 3636, 3972, 4127, 4137, 4157, 4367, 4677, 5132, 5839, 5901, 6163, + 6169, 6409, 6666, 7693, 57, 73, 100, 114, 193, 228, 272, 450, 456, 517, 912, 1029, 1043, 1159, 1219, 1796, + 2343, 3078, 3333, 4113, 4121, 4361, 4381, 4393, 4492, 4559, 4623, 4686, 5518, 6275, 6920, 7721, 8137, 29, 35, + 44, 74, 104, 146, 641, 1030, 1090, 1730, 1824, 2246, 2318, 2822, 3072, 3085, 3592, 3621, 3660, 3780, 4099, + 4140, 4149, 4165, 4205, 4333, 4617, 4700, 5454, 5643, 6149, 6377, 7051, 7170, 7244, 7753, 7947, 15, 30, 42, + 60, 76, 120, 145, 196, 240, 260, 292, 320, 385, 480, 514, 771, 900, 960, 1123, 1411, 1818, 1986, 2051, 2086, + 2304, 2694, 2951, 3622, 3648, 3653, 4096, 4106, 4221, 4396, 4429, 4589, 4636, 4653, 4716, 5422, 5679, 6156, + 6170, 6378, 6603, 7184, 7242, 7681, 8072, 41, 58, 194, 200, 521, 581, 584, 785, 865, 1059, 1139, 1168, 1282, + 1347, 1811, 2058, 2198, 2350, 2565, 2630, 3644, 4301, 4357, 4485, 4732, 5391, 5644, 5711, 6191, 6203, 6287, + 6297, 7193, 7296, 7306, 70, 88, 148, 208, 289, 544, 579, 801, 1034, 1089, 1099, 1162, 1315, 1570, 1827, 1830, + 1890, 2309, 2324, 2336, 2358, 2759, 3075, 3093, 3845, 4133, 4143, 4172, 4613, 4648, 4668, 4681, 4749, 4812, + 4996, 5263, 5278, 5388, 5639, 5775, 6167, 6175, 6185, 6315, 6346, 6383, 6415, 6427, 6447, 7194, 7949, 7961, + 69, 84, 116, 152, 264, 290, 386, 392, 516, 593, 773, 833, 909, 993, 1033, 1161, 1175, 1250, 1542, 1602, 1799, + 1800, 1920, 2118, 2498, 2727, 3140, 3597, 3654, 4253, 4389, 4399, 4495, 4672, 4780, 4877, 5260, 6186, 6195, + 6211, 6251, 6345, 6441, 6794, 6926, 6938, 7018, 7183, 7233, 7256, 7273, 7624, 7690, 7977, 67, 82, 400, 529, + 609, 640, 929, 1041, 1157, 1179, 1186, 1249, 1281, 1666, 1858, 1923, 2060, 2180, 2334, 2406, 2500, 2563, + 2711, 3173, 3204, 3463, 3524, 3598, 3628, 3716, 4036, 4116, 4169, 4489, 4509, 4608, 4618, 4660, 4678, 4764, + 5006, 5130, 5899, 5919, 6150, 6267, 6298, 6351, 6442, 6475, 6667, 6730, 6954, 7048, 7174, 7209, 7212, 7224, + 7289, 7308, 7400, 7432, 81, 140, 176, 232, 296, 416, 625, 770, 915, 945, 1028, 1042, 1158, 1185, 1218, 1537, + 1814, 2068, 2183, 2307, 2322, 2339, 2390, 2503, 2950, 3109, 3189, 3332, 3397, 3457, 3606, 3612, 3846, 4102, + 4236, 4525, 4687, 5000, 5020, 5582, 5641, 5871, 5935, 6144, 6339, 6403, 6539, 6921, 6986, 7173, 7188, 7196, + 7321, 7336, 7561, 7705, 7937, 8065, 8075, 27, 54, 108, 168, 216, 304, 388, 432, 520, 561, 580, 705, 777, 784, + 864, 907, 1039, 1167, 1203, 1546, 1803, 1806, 2314, 2366, 2382, 2564, 2639, 2821, 3081, 3084, 3149, 3221, + 3365, 3469, 3587, 3600, 3605, 3617, 3668, 3840, 4373, 4413, 4541, 4552, 4701, 5133, 5198, 5838, 5900, 6179, + 6218, 6303, 6395, 6914, 7042, 7216, 7234, 7274, 7322, 7368, 7692, 8169, 131, 137, 164, 464, 578, 673, 800, + 903, 1095, 1122, 1195, 1410, 1475, 1762, 1809, 1822, 1834, 2056, 2063, 2078, 2082, 2178, 2276, 2327, 2330, + 2370, 2599, 2627, 2693, 2758, 3137, 3477, 3594, 3618, 3684, 3812, 4119, 4175, 4548, 4555, 4573, 4637, 4644, + 4654, 4682, 4692, 4717, 5068, 5126, 5254, 5446, 5486, 5519, 5663, 5837, 5895, 6031, 6161, 6217, 6279, 6280, + 6291, 6435, 7210, 7236, 7276, 7290, 7695, 7720, 7725, 7737, 7745, 8136, 51, 138, 280, 352, 549, 592, 657, + 772, 832, 908, 917, 992, 1051, 1058, 1121, 1138, 1191, 1217, 1346, 1816, 1842, 2018, 2049, 2066, 2190, 2274, + 2341, 2346, 2372, 2510, 2629, 2726, 3493, 3521, 3608, 3632, 3637, 3700, 3860, 3973, 4126, 4136, 4156, 4189, + 4229, 4366, 4405, 4421, 4461, 4628, 4696, 4733, 5276, 5420, 5455, 5470, 5516, 5642, 5677, 6162, 6168, 6223, + 6273, 6363, 6408, 7171, 7245, 7264, 7752, 7946, 8077, 8089, 23, 53, 134, 161, 336, 454, 496, 528, 608, 921, + 928, 1057, 1088, 1098, 1107, 1137, 1173, 1314, 1379, 1540, 1554, 1728, 1797, 1930, 2102, 2196, 2316, 2348, + 2375, 2436, 2562, 2710, 2819, 3074, 3092, 3509, 3638, 3876, 4112, 4120, 4360, 4380, 4392, 4445, 4545, 4558, + 4565, 4605, 4614, 4649, 4669, 4740, 4813, 4997, 5386, 5406, 5418, 5423, 5452, 5675, 5678, 5695, 5703, 6274, + 6285, 6371, 6664, 7054, 7066, 7185, 7200, 7243, 7246, 7280, 7324, 7680, 7685, 7729, 7881, 39, 78, 133, 156, + 162, 227, 274, 312, 328, 453, 519, 569, 624, 914, 965, 1032, 1037, 1075, 1097, 1165, 1313, 1363, 1409, 1825, + 1828, 1938, 2074, 2084, 2214, 2242, 2279, 2439, 2534, 2691, 3143, 3593, 3650, 3656, 3661, 3781, 4123, 4148, + 4153, 4239, 4353, 4363, 4481, 4551, 4673, 4781, 5128, 5258, 5358, 5637, 5710, 5807, 5835, 5897, 5917, 6173, + 6355, 6413, 6419, 6425, 6459, 6795, 6858, 6927, 7019, 7297, 7300, 7307, 7312, 7562, 7723, 7953 }; + +static int cbest_14[1023] = { + 1, 2, 8737, 4, 8, 13105, 16, 8736, 32, 4368, 3, 2184, 17, 1092, 15289, 34, 546, 5, 64, 68, 273, 8739, 8745, + 6, 136, 13104, 272, 9, 8741, 128, 8873, 12, 6552, 13109, 10, 544, 13107, 256, 18, 24, 3276, 8753, 33, 1088, + 4369, 15288, 16381, 2185, 8705, 13089, 13173, 20, 512, 1093, 2186, 4372, 8195, 8738, 8744, 13113, 35, 66, + 547, 1094, 1638, 2188, 4370, 4376, 8752, 8865, 12832, 15291, 36, 48, 65, 69, 257, 2176, 6416, 8707, 10921, 7, + 19, 25, 70, 137, 3208, 8740, 9008, 9829, 13088, 13169, 15281, 96, 132, 138, 277, 554, 1024, 1108, 1604, 6544, + 8749, 8872, 50, 140, 275, 281, 562, 802, 819, 3272, 4504, 7644, 8747, 8801, 8805, 12833, 13073, 13108, 21, + 100, 129, 192, 276, 401, 1636, 4352, 4436, 8743, 12563, 13106, 15257, 15293, 13, 38, 40, 72, 130, 200, 264, + 274, 280, 514, 550, 818, 2216, 2252, 6553, 8193, 8875, 9009, 15259, 11, 14, 49, 384, 400, 545, 1124, 2048, + 2218, 6554, 8761, 8877, 13075, 13111, 15273, 409, 1100, 1126, 14197, 15153, 15839, 42, 528, 768, 1109, 4097, + 4432, 8937, 9144, 15016, 16377, 16380, 51, 76, 98, 102, 204, 408, 552, 563, 803, 1090, 2200, 2248, 3277, + 8203, 8704, 10785, 13172, 144, 260, 548, 560, 800, 817, 1028, 1089, 1536, 3822, 4096, 4353, 8194, 8760, + 13112, 13117, 13241, 15272, 16365, 28, 258, 265, 1606, 1634, 4380, 6556, 8864, 8869, 8993, 9136, 10913, + 12561, 12836, 15290, 26, 80, 84, 196, 385, 513, 816, 1056, 2180, 2187, 2190, 3072, 3268, 3278, 4373, 4400, + 4496, 6145, 7508, 8706, 8755, 8941, 9016, 9145, 9287, 10920, 12013, 12290, 13175, 15017, 22, 67, 152, 1095, + 1639, 2189, 3212, 3292, 4371, 4374, 4377, 4568, 4572, 6418, 6536, 6584, 8225, 8929, 9825, 9828, 12849, 13093, + 13141, 13168, 13233, 14129, 14747, 15280, 16373, 16383, 37, 529, 570, 769, 1104, 1600, 1632, 2177, 5460, + 6144, 6417, 6586, 7636, 8466, 8713, 8748, 8867, 13091, 13097, 13171, 15837, 16313, 16364, 71, 134, 168, 285, + 392, 520, 530, 770, 1026, 1058, 1096, 2056, 2178, 2201, 2284, 3209, 4233, 4360, 4402, 4508, 4914, 6424, 7640, + 8709, 8721, 8746, 8757, 8800, 8804, 8992, 9121, 9137, 10923, 11809, 12837, 13072, 13115, 13137, 14193, 15566, + 97, 133, 139, 142, 145, 261, 288, 305, 337, 555, 561, 785, 801, 1025, 1120, 1140, 1605, 1646, 2112, 2116, + 2730, 3210, 3264, 3754, 4378, 6420, 6545, 8201, 8211, 8451, 8742, 8754, 8803, 8809, 8841, 8889, 9001, 9017, + 9831, 10925, 12291, 12562, 12834, 12840, 15161, 15256, 15283, 15292, 29, 56, 141, 259, 268, 279, 284, 674, + 1110, 1142, 1538, 1570, 1911, 2280, 2457, 3273, 3820, 4232, 4505, 6281, 6546, 7628, 7645, 8192, 8715, 8751, + 8769, 8874, 9012, 9285, 9317, 12848, 12960, 13057, 13077, 13092, 14880, 15258, 15265, 15285, 15295, 27, 52, + 74, 81, 85, 101, 160, 193, 304, 336, 516, 558, 784, 1060, 1348, 1365, 1540, 1637, 2052, 2254, 2286, 3140, + 3200, 3274, 3818, 4225, 4437, 4506, 4560, 6480, 6528, 6548, 7440, 8190, 8199, 8467, 8723, 8807, 8876, 9010, + 9129, 9837, 11945, 12307, 12547, 13074, 13090, 13096, 13110, 13143, 13655, 14745, 14777, 15241, 15353, 15357, + 15831, 23, 39, 41, 44, 73, 131, 153, 194, 201, 278, 290, 307, 403, 515, 536, 551, 610, 614, 806, 823, 1116, + 1122, 1228, 1612, 1910, 2050, 2217, 2220, 2253, 2456, 2696, 3224, 3240, 3293, 3720, 4105, 4438, 6280, 6537, + 8331, 8720, 8756, 8773, 8879, 8881, 9120, 9761, 9845, 10849, 10853, 12288, 12593, 13157, 13297, 13617, 14196, + 15008, 15152, 15243, 15567, 15835, 15838, 16245, 15, 58, 112, 170, 202, 283, 341, 405, 571, 682, 810, 1040, + 1105, 1125, 1364, 1568, 1620, 1860, 1877, 2049, 2192, 2219, 2250, 3076, 4101, 4112, 4224, 4356, 4500, 5392, + 6448, 6555, 7504, 7646, 8210, 8227, 8450, 8711, 8813, 8888, 8936, 9000, 10889, 12009, 12567, 12835, 12841, + 12968, 13059, 13177, 13181, 13305, 13309, 14888, 15155, 16108, 16376, 104, 306, 386, 393, 402, 531, 568, 771, + 822, 930, 1030, 1101, 1127, 1602, 1644, 2120, 2728, 3080, 3288, 3752, 3814, 4104, 4354, 4361, 4434, 4440, + 4444, 6153, 7576, 8202, 8219, 8464, 8602, 8843, 8933, 9013, 9797, 10784, 10789, 10891, 11877, 12571, 12896, + 12900, 12961, 13056, 13237, 14097, 14181, 14474, 14881, 15249, 15264, 15275, 16357, 16369, 43, 116, 148, 224, + 266, 282, 289, 340, 388, 404, 411, 465, 522, 576, 580, 608, 672, 1072, 1102, 1141, 1220, 1876, 1909, 2208, + 2222, 2240, 2244, 2282, 3136, 4100, 4433, 4468, 4564, 4912, 5456, 6273, 6400, 6484, 7444, 7620, 7632, 8323, + 8722, 8765, 9011, 9128, 9799, 9965, 10787, 10989, 12306, 12546, 13079, 13116, 13139, 13240, 13245, 14199, + 15033, 15240, 15251, 15261, 15564, 15869, 16317, 16361, 54, 57, 77, 99, 103, 162, 205, 232, 269, 518, 553, + 556, 566, 787, 1062, 1091, 1111, 1143, 1542, 2060, 2210, 2232, 2249, 3788, 3816, 6152, 6450, 7629, 7783, + 8182, 8197, 8209, 8241, 8449, 8481, 8587, 8763, 8868, 8880, 8939, 9146, 9295, 9844, 10912, 10917, 11471, + 11813, 12005, 12289, 12560, 13153, 13156, 13301, 13653, 13685, 15009, 15018, 15157, 15242, 15562, 16379, 46, + 53, 78, 208, 320, 410, 448, 464, 549, 612, 772, 804, 821, 938, 955, 1029, 1349, 1537, 1642, 1908, 2202, 2234, + 3242, 3284, 3722, 3823, 4240, 4464, 6160, 6272, 6529, 6558, 6576, 7098, 7237, 8233, 8329, 8771, 8775, 8871, + 8905, 8940, 9020, 9283, 9286, 10793, 12012, 12294, 12844, 12857, 12969, 13058, 13174, 13587, 14099, 14521, + 14739, 14761, 14889, 15020, 15321, 15430, 15833, 16109, 16349, 82, 88, 146, 262, 291, 537, 578, 680, 776, + 786, 807, 1032, 1074, 1117, 1160, 1574, 1607, 1635, 1654, 1894, 1907, 2124, 2182, 2204, 2440, 3084, 3225, + 3308, 3810, 4099, 4120, 4301, 4381, 4488, 4643, 6147, 6557, 6578, 6616, 6620, 7782, 8188, 8207, 8218, 8224, + 8465, 8603, 8833, 8928, 9249, 9824, 11873, 12305, 12337, 12545, 12565, 12577, 12853, 12897, 12901, 12977, + 13119, 13140, 13232, 13243, 14128, 14133, 14180, 14475, 14746, 15121, 15217, 15263, 15274, 15771, 15829, + 16177, 16356, 16372, 16382, 30, 59, 113, 197, 206, 469, 811, 820, 827, 896, 928, 954, 1041, 1057, 1136, 1216, + 1344, 1361, 1572, 1621, 1861, 2080, 2144, 2181, 2193, 2321, 3073, 3204, 3214, 3269, 3279, 4113, 4357, 4384, + 4404, 4420, 4497, 4922, 7236, 7492, 7509, 8579, 8600, 8712, 8729, 8866, 9138, 9319, 9827, 9833, 10915, 10985, + 11267, 12595, 12631, 12838, 13101, 13170, 13382, 13619, 13651, 14131, 14472, 14504, 15032, 15325, 15565, + 15827, 16125, 16312 }; + +static int cbest_15[1023] = { + 1, 2, 16385, 4, 8, 24577, 16, 32, 64, 28673, 128, 256, 512, 1024, 30721, 2048, 4096, 8192, 16384, 31745, 3, + 6, 12, 24, 48, 96, 192, 384, 768, 1536, 3072, 6144, 12288, 24576, 5, 16387, 9, 16389, 10, 17, 16393, 24579, + 28672, 18, 33, 14336, 16401, 24581, 20, 34, 65, 7168, 16417, 24585, 32257, 36, 66, 129, 3584, 16449, 24593, + 28675, 40, 68, 130, 257, 1792, 16513, 24609, 28677, 30720, 72, 132, 258, 513, 896, 16641, 24641, 28681, 80, + 136, 260, 448, 514, 1025, 15360, 16897, 24705, 28689, 144, 224, 264, 516, 1026, 2049, 17409, 24833, 28705, + 30723, 112, 160, 272, 520, 1028, 2050, 4097, 7680, 18433, 25089, 28737, 30725, 56, 288, 528, 1032, 2052, + 4098, 8193, 20481, 25601, 28801, 30729, 31744, 32513, 28, 320, 544, 1040, 2056, 3840, 4100, 8194, 26625, + 28929, 30737, 14, 576, 1056, 2064, 4104, 8196, 16386, 29185, 30753, 7, 13, 25, 49, 97, 193, 385, 640, 769, + 1088, 1537, 1920, 2080, 3073, 4112, 6145, 8200, 12289, 15872, 16388, 29697, 30785, 31747, 26, 50, 98, 194, + 386, 770, 1152, 1538, 2112, 3074, 4128, 6146, 8208, 12290, 16391, 16392, 16397, 16409, 16433, 16481, 16577, + 16769, 17153, 17921, 19457, 22529, 24578, 30849, 31749, 52, 100, 196, 388, 772, 960, 1280, 1540, 2176, 3076, + 4160, 6148, 8224, 12292, 16400, 24580, 30977, 31753, 11, 104, 200, 392, 776, 1544, 2304, 3080, 4224, 6152, + 7936, 8256, 12296, 16416, 24583, 24584, 24589, 24601, 24625, 24673, 24769, 24961, 25345, 26113, 27649, 31233, + 31761, 32256, 19, 22, 208, 400, 480, 784, 1552, 2560, 3088, 4352, 6160, 8320, 12304, 14337, 16395, 16448, + 24592, 28674, 31777, 21, 35, 38, 44, 416, 800, 1568, 3104, 4608, 6176, 7169, 8448, 12320, 14338, 16403, + 16512, 24608, 28676, 31809, 32641, 37, 67, 70, 76, 88, 240, 832, 1600, 3136, 3585, 3968, 5120, 6208, 7170, + 8704, 12352, 14340, 16405, 16419, 16640, 23553, 24587, 24640, 28679, 28680, 28685, 28697, 28721, 28769, + 28865, 29057, 29441, 30209, 31873, 32259, 41, 69, 131, 134, 140, 152, 176, 1664, 1793, 3200, 3586, 6272, + 7172, 9216, 12416, 14344, 16128, 16421, 16451, 16896, 19969, 24595, 24704, 28688, 32001, 32261, 42, 73, 120, + 133, 259, 262, 268, 280, 304, 352, 897, 1794, 3328, 3588, 6400, 7176, 10240, 12544, 14352, 16425, 16453, + 16515, 17408, 18177, 24597, 24611, 24832, 28704, 30722, 32265, 74, 81, 137, 261, 449, 515, 518, 524, 536, + 560, 608, 704, 898, 1796, 1984, 3592, 6656, 7184, 12800, 14368, 15361, 16457, 16517, 16643, 17281, 18432, + 24613, 24643, 25088, 28161, 28683, 28736, 30724, 32273, 60, 82, 138, 145, 225, 265, 450, 517, 900, 1027, + 1030, 1036, 1048, 1072, 1120, 1216, 1408, 1800, 3600, 7200, 13312, 14400, 15362, 16465, 16521, 16645, 16833, + 16899, 20480, 24617, 24645, 24707, 25600, 26369, 28691, 28800, 30727, 30728, 30733, 30745, 30769, 30817, + 30913, 31105, 31489, 32289, 32512, 84, 113, 146, 161, 226, 266, 273, 452, 521, 904, 1029, 1808, 2051, 2054, + 2060, 2072, 2096, 2144, 2240, 2432, 2816, 3616, 7232, 7681, 8064, 14464, 15364, 16529, 16609, 16649, 16901, + 17411, 24649, 24709, 24835, 25473, 26624, 28693, 28707, 28928, 30736, 32321, 30, 57, 114, 148, 162, 228, 274, + 289, 456, 522, 529, 912, 992, 1033, 1824, 2053, 3648, 4099, 4102, 4108, 4120, 4144, 4192, 4288, 4480, 4864, + 5632, 7296, 7682, 14592, 15368, 16497, 16545, 16657, 16905, 17413, 18435, 24065, 24657, 24713, 24837, 25025, + 25091, 28709, 28739, 29184, 30752, 32385, 29, 58, 116, 164, 232, 276, 290, 321, 464, 530, 545, 928, 1034, + 1041, 1856, 2057, 3712, 3841, 4101, 7424, 7684, 8195, 8198, 8204, 8216, 8240, 8288, 8384, 8576, 8960, 9728, + 11264, 14848, 15376, 16441, 16673, 16913, 17417, 18437, 20483, 24721, 24801, 24841, 25093, 25603, 28713, + 28741, 28803, 29696, 30465, 30731, 30784, 31746, 32515, 15, 168, 292, 322, 532, 546, 577, 1042, 1057, 2058, + 2065, 3842, 4105, 7688, 8197, 15392, 16390, 16396, 16408, 16413, 16432, 16480, 16576, 16705, 16768, 16929, + 17152, 17425, 17920, 18441, 19456, 20225, 20485, 22528, 24689, 24737, 24849, 25097, 25605, 26627, 28745, + 28805, 28931, 29569, 30739, 30848, 31748, 32517, 32705, 296, 324, 496, 548, 578, 641, 1044, 1058, 1089, 1921, + 2066, 2081, 3844, 4032, 4106, 4113, 7696, 8201, 15424, 15873, 16256, 16399, 16961, 17441, 18449, 20489, + 24633, 24865, 25105, 25609, 26629, 28753, 28809, 28933, 29121, 29187, 30741, 30755, 30976, 31751, 31752, + 31757, 31769, 31793, 31841, 31937, 32129, 32521, 27, 51, 54, 99, 102, 108, 195, 198, 204, 216, 328, 387, 390, + 396, 408, 432, 552, 580, 642, 771, 774, 780, 792, 816, 864, 1060, 1090, 1153, 1539, 1542, 1548, 1560, 1584, + 1632, 1728, 1922, 2068, 2082, 2113, 3075, 3078, 3084, 3096, 3120, 3168, 3264, 3456, 3848, 4114, 4129, 6147, + 6150, 6156, 6168, 6192, 6240, 6336, 6528, 6912, 7712, 8202, 8209, 12291, 12294, 12300, 12312, 12336, 12384, + 12480, 12672, 13056, 13824, 15488, 15874, 17025, 17473, 18305, 18465, 20497, 24582, 24588, 24600, 24605, + 24624, 24672, 24768, 24897, 24960, 25121, 25344, 25617, 26112, 26633, 27648, 28417, 28817, 28897, 28937, + 29189, 29699, 30757, 30787, 31232, 31760, 32529, 53, 101, 197, 336, 389, 584, 644, 773, 961, 1064, 1092, + 1154, 1281, 1541, 1924, 2084, 2114, 2177, 3077, 3856, 4116, 4130, 4161, 6149, 7744, 8210, 8225, 12293, 15616, + 15876, 16394, 16411, 16435, 16483, 16579, 16771, 17155, 17537, 17923, 18497, 19459, 20513, 22531, 24591, + 25153, 25633, 26641, 28785, 28833, 28945, 29193, 29701, 30761, 30789, 30851, 31776, 32545, 105, 201, 248, + 393, 592, 648, 777, 962, 1096, 1156, 1282, 1545, 1928, 2088, 2116, 2178, 2305, 3081, 3872, 4132, 4162, 4225, + 6153, 7808, 7937, 8212, 8226, 8257, 12297, 15880, 16402, 16437, 16485, 16581, 16773, 17157, 17345, 17665, + 17925, 18561, 19461, 20545, 22533, 25217, 25665, 26497, 26657, 28729, 28961, 29201, 29705, 30793, 30853, + 30979, 31617, 31755, 31808, 32577, 32640, 23, 46, 92, 106, 184, 202, 209, 368, 394, 401, 481, 656, 736, 778, + 785, 964, 1104, 1160, 1284, 1472, 1546, 1553, 1936, 2016, 2120, 2180, 2306, 2561, 2944, 3082, 3089, 3904, + 4136, 4164, 4226, 4353, 5888, 6154, 6161, 7938, 8228, 8258, 8321, 11776, 12298, 12305, 15888, 16404, 16418, + 16489, 16585, 16777, 17161, 17929, 18689, 19465, 20609, 22537, 23552, 24321, 24586, 24603, 24627, 24675, + 24771, 24963, 25347, 25729, 26115, 26689, 27651, 28678, 28684, 28696, 28701, 28720, 28768, 28864, 28993, + 29056, 29217, 29440, 29713, 30208, 30801, 30857, 30981, 31169, 31235, 31763, 31872, 32258, 45, 78, 156, 312, + 624, 672, 801, 1554, 2562, 14384, 16593, 16785, 17937, 24677, 31237, 31765 }; + +static int cbest_16[1023] = { + 1, 2, 34821, 4, 8, 52231, 16, 60934, 34820, 32, 17410, 3, 8705, 17, 34, 5, 68, 34823, 34829, 6, 136, 30467, + 34817, 39173, 52230, 64, 272, 52229, 9, 544, 52227, 10, 1088, 12, 2176, 26115, 4352, 50311, 60935, 18, 8704, + 45956, 60932, 20, 128, 24, 34837, 47876, 17408, 30466, 33, 36, 17411, 52239, 59974, 40, 22978, 48, 256, 8707, + 17414, 34822, 34828, 43524, 60930, 65286, 19, 35, 8709, 17418, 23938, 26114, 34816, 34836, 34853, 39172, + 52247, 21, 69, 15233, 21762, 30465, 34825, 52225, 52228, 7, 25, 38, 72, 137, 13057, 19586, 34831, 34881, + 52226, 42, 65, 80, 273, 10881, 26113, 29987, 34819, 34855, 39175, 50, 66, 70, 76, 545, 9793, 11489, 45957, + 60933, 60942, 11, 14, 84, 96, 138, 1089, 11969, 34957, 37253, 50310, 52261, 13, 152, 274, 1090, 2177, 34885, + 34949, 35093, 39169, 50309, 52246, 52263, 49, 144, 257, 546, 4353, 8713, 32643, 32901, 34845, 35365, 38341, + 47877, 60950, 22, 28, 140, 512, 2180, 4354, 25155, 30471, 35909, 40261, 45716, 52291, 52295, 100, 129, 160, + 276, 2178, 8706, 17426, 26123, 39717, 49351, 52235, 59975, 44, 98, 130, 168, 548, 4360, 8708, 8721, 34833, + 51271, 52237, 60928, 81, 132, 192, 304, 514, 1092, 15232, 30464, 30475, 34839, 34861, 39181, 43525, 45958, + 47872, 50307, 52238, 52367, 53703, 60931, 60951, 60966, 65287, 51, 88, 102, 204, 280, 408, 816, 1632, 3264, + 6528, 8720, 13056, 17409, 17412, 26119, 45952, 47396, 52243, 52503, 59972, 60964, 26, 37, 85, 170, 196, 288, + 340, 552, 680, 1360, 2720, 4356, 5440, 10880, 17416, 22858, 22976, 23936, 26112, 29986, 34844, 40565, 40805, + 52245, 52775, 54663, 41, 56, 153, 162, 200, 306, 612, 1096, 1224, 2448, 4896, 7616, 8737, 9792, 11488, 17422, + 17440, 21760, 22979, 30483, 34849, 34852, 39189, 47044, 47878, 49895, 50855, 59494, 60454, 65284, 145, 320, + 336, 1028, 2184, 8711, 11968, 17415, 17442, 34824, 34889, 34893, 43520, 50583, 52224, 60940, 60994, 23, 29, + 258, 392, 560, 608, 17419, 23698, 23939, 30482, 34827, 34830, 34832, 34863, 34880, 43526, 50319, 52259, + 56583, 60943, 64806, 46, 58, 74, 176, 260, 324, 384, 1024, 1104, 3808, 8225, 8712, 14993, 17427, 17474, + 19584, 21763, 32642, 34818, 34838, 34854, 39174, 45964, 52257, 57574, 59970, 39, 73, 82, 92, 264, 290, 2192, + 5744, 5984, 8769, 11429, 19587, 25154, 26131, 29985, 30227, 30470, 34869, 35077, 38933, 39205, 41604, 43532, + 52242, 52359, 58054, 60948, 61070, 63366, 65294, 43, 52, 112, 116, 400, 2056, 4368, 8739, 12577, 16450, + 17478, 20802, 21766, 26122, 29747, 30497, 34841, 34851, 34956, 37252, 42212, 50327, 52244, 52255, 52260, + 60998, 61206, 65282, 67, 71, 77, 89, 178, 184, 576, 672, 1904, 8773, 10401, 10883, 11425, 11849, 13061, + 15235, 15241, 17546, 18626, 22850, 22982, 26121, 26130, 30499, 34884, 34948, 34965, 35092, 39168, 39188, + 42692, 45700, 45717, 45972, 47884, 50191, 50308, 52233, 52262, 59014, 60938, 15, 27, 78, 97, 139, 148, 164, + 289, 352, 356, 1120, 1216, 2992, 8717, 8841, 9313, 13059, 13065, 17538, 17682, 19594, 23522, 30469, 30474, + 32900, 34857, 34883, 35364, 37013, 37249, 38340, 39171, 44612, 45836, 52293, 59906, 60246, 57, 86, 232, 275, + 640, 784, 1091, 2872, 8715, 8977, 9797, 17424, 17434, 17954, 23942, 26118, 26147, 29953, 32641, 34897, 34901, + 34945, 34981, 35908, 37255, 38221, 39233, 40260, 49349, 51127, 52290, 52294, 59766, 59982, 60110, 60929, + 60949, 154, 224, 321, 368, 547, 648, 712, 952, 4112, 4384, 9795, 11491, 15237, 16321, 17430, 22986, 25153, + 26117, 26145, 28787, 29027, 29955, 32403, 32897, 34871, 34887, 34951, 38337, 39207, 39716, 40533, 45596, + 45959, 47873, 47892, 49350, 51007, 51269, 52234, 52241, 52271, 52289, 52303, 60946, 60967, 104, 118, 141, + 156, 259, 328, 393, 513, 516, 561, 580, 704, 768, 800, 1496, 2181, 2208, 4355, 8725, 8736, 11393, 11457, + 11971, 19590, 21106, 21346, 22306, 22918, 29507, 30473, 30535, 34868, 34917, 34959, 35009, 35013, 35076, + 40021, 45701, 45953, 45973, 47397, 49231, 50305, 50341, 51270, 52236, 52269, 52279, 52311, 52487, 59973, + 60965, 30, 59, 101, 142, 161, 177, 186, 261, 277, 325, 385, 464, 520, 578, 786, 1105, 1344, 1424, 2179, 2182, + 4362, 8710, 8723, 10885, 17420, 19170, 20130, 23946, 24675, 25635, 29991, 30603, 31683, 32647, 34835, 34840, + 34860, 34909, 34953, 34973, 35095, 35101, 35911, 39180, 39309, 39477, 40517, 40549, 40737, 50306, 50326, + 50343, 52254, 52277, 52366, 53702, 59910, 59990, 60962, 61062, 45, 54, 83, 93, 99, 114, 131, 134, 146, 169, + 172, 194, 236, 265, 278, 291, 358, 476, 528, 549, 650, 736, 1094, 1122, 1436, 1572, 2048, 2193, 2240, 2432, + 2856, 4361, 8724, 8741, 11493, 17450, 17482, 19858, 21770, 22786, 22914, 22994, 30531, 32903, 32909, 34847, + 34913, 34964, 35089, 35367, 35373, 38343, 39183, 39237, 39719, 40535, 45988, 47045, 47879, 49827, 51267, + 52327, 52365, 52502, 53583, 53701, 54661, 59426, 59495, 60455, 60958, 65285, 53, 113, 117, 133, 193, 234, + 305, 308, 372, 448, 468, 515, 550, 582, 642, 716, 748, 936, 1093, 1152, 1300, 1408, 1428, 2210, 2848, 3144, + 4369, 8224, 8722, 11153, 11459, 11841, 11973, 14992, 17446, 17448, 22798, 25163, 26127, 30481, 34877, 34983, + 35333, 35361, 35917, 39177, 39685, 40263, 40564, 40804, 43521, 43540, 45236, 45476, 45580, 45718, 47636, + 47893, 49893, 50183, 50371, 50567, 50991, 52251, 52299, 52325, 52363, 52501, 52774, 53575, 54662, 60452, + 60941, 60947, 60995, 65302, 103, 166, 179, 185, 205, 281, 296, 312, 354, 370, 409, 472, 714, 740, 817, 928, + 1164, 1432, 1480, 1568, 1633, 1872, 2244, 2600, 3265, 4358, 4364, 4386, 5712, 6288, 6529, 8768, 8833, 9729, + 10065, 10673, 10889, 11397, 11428, 11761, 17413, 17444, 22790, 22856, 23682, 23954, 26183, 29713, 29984, + 29995, 30123, 30226, 34848, 34859, 34896, 34900, 35905, 40257, 43527, 45116, 45572, 45712, 45828, 46016, + 47156, 47392, 47874, 49894, 50854, 50983, 52307, 52499, 52773, 60960, 60974, 60992, 64807, 90, 149, 171, 197, + 208, 238, 282, 341, 353, 357, 518, 553, 681, 744, 770, 1098, 1156, 1280, 1296, 1361, 1472, 1634, 2188, 2328, + 2721, 2864, 3266, 3744, 4357, 4420, 5200, 5441, 6530, 7496, 8729, 8738, 9585, 9801, 9929, 10553, 11395, + 11497, 12576, 14977, 15249, 17417, 21778, 22794, 22859, 22977, 23696, 23818, 23937, 26129, 26251, 29715, + 29746, 30055, 30479, 30480, 30496, 33989, 34870, 34888, 34892, 34905, 34915, 34919, 34989, 35017, 38213, + 39191, 39713, 40741, 45954, 45965, 45990, 46924, 47908, 49347, 50317, 50582, 50735, 50853, 52240, 52267, + 52771, 54423, 56581, 57506, 57510, 57575, 59492, 59971, 59991, 60102, 60936, 60982, 198, 374, 1097, 1225, + 7488, 15113, 17441, 17472, 26387, 29883, 30211, 34879, 35205, 50463, 50999, 52258, 61071 }; + + +static int cbest_17[1023] = { + 1, 2, 65540, 4, 32770, 8, 16385, 16, 32, 73732, 64, 128, 36866, 256, 512, 18433, 1024, 2048, 4096, 74756, + 8192, 16384, 37378, 32768, 3, 18689, 65536, 65541, 5, 32771, 65542, 6, 9, 18, 36, 72, 144, 288, 576, 1152, + 2304, 4608, 9216, 18432, 98310, 10, 17, 16387, 32774, 36864, 65548, 81925, 12, 33, 16389, 32778, 49155, + 65556, 73728, 73733, 74884, 20, 34, 65, 16393, 32786, 65572, 73734, 24, 66, 129, 16401, 32802, 36867, 65604, + 106502, 40, 68, 130, 257, 16417, 32834, 65668, 73740, 90117, 102406, 48, 132, 258, 513, 16449, 32898, 36870, + 37442, 65796, 73748, 80, 136, 260, 514, 1025, 16513, 18435, 33026, 36874, 53251, 66052, 73764, 83973, 96, + 264, 516, 1026, 2049, 16641, 18437, 33282, 36882, 51203, 66564, 73796, 160, 272, 520, 1028, 2050, 4097, + 16897, 18441, 18688, 33794, 36898, 37376, 67588, 73860, 74752, 74757, 110598, 192, 528, 1032, 2052, 4098, + 8193, 9344, 17409, 18449, 18721, 34818, 36930, 69636, 73988, 74758, 320, 544, 1040, 2056, 4100, 4672, 8194, + 18465, 36994, 74244, 92165, 107526, 384, 1056, 2064, 2336, 4104, 8196, 16386, 18497, 20481, 37122, 37379, + 40962, 74764, 81924, 91141, 640, 1088, 1168, 2080, 4112, 8200, 16388, 18561, 24577, 32769, 49154, 55299, + 74772, 75780, 102918, 584, 768, 2112, 4128, 8208, 16392, 37382, 37890, 74788, 77828, 98308, 292, 1280, 2176, + 4160, 8224, 16400, 18945, 32772, 32784, 37386, 38914, 53763, 65537, 74820, 74900, 146, 1536, 4224, 8256, + 16416, 18691, 19457, 32776, 37394, 49153, 65538, 65543, 65568, 84229, 90116, 111622, 7, 19, 37, 73, 145, 289, + 577, 1153, 2305, 2560, 4352, 4609, 8320, 9217, 16448, 18693, 37410, 45058, 51459, 75012, 98306, 98311, + 111110, 11, 38, 74, 290, 578, 1154, 2306, 3072, 4610, 8448, 9218, 16512, 18434, 18697, 22529, 32775, 32800, + 36865, 53250, 65544, 65549, 65558, 65612, 65684, 65828, 66116, 66692, 67844, 70148, 75268, 81921, 83972, + 93189, 106500, 13, 22, 76, 148, 580, 1156, 2308, 4612, 5120, 8704, 9220, 16640, 18436, 18705, 26625, 32779, + 32806, 32832, 32842, 32914, 33058, 33346, 33922, 35074, 37506, 41986, 51202, 65550, 65552, 65557, 73729, + 74880, 74885, 81927, 102404, 14, 21, 26, 35, 44, 152, 296, 1160, 2312, 4616, 6144, 9224, 16391, 16403, 16421, + 16457, 16529, 16673, 16896, 16961, 17537, 18440, 20993, 25601, 32787, 32896, 36868, 36880, 37440, 37450, + 37634, 65573, 73730, 73735, 73760, 74886, 76804, 92421, 98318, 114695, 25, 52, 67, 88, 304, 592, 2320, 4624, + 9232, 10240, 16395, 17408, 18448, 18720, 18753, 32782, 32803, 33024, 36872, 49159, 53249, 55811, 65574, + 65600, 65605, 78852, 81933, 98326, 106498, 106503, 107654, 41, 50, 69, 104, 131, 176, 608, 1184, 4640, 9248, + 12288, 16397, 18464, 18817, 32790, 32835, 33280, 38402, 49163, 51201, 55555, 65564, 65606, 65664, 65669, + 73736, 73741, 73750, 73804, 73876, 74020, 74308, 74892, 76036, 78340, 81941, 90113, 91269, 92164, 98342, + 102402, 102407, 28, 42, 49, 70, 82, 100, 133, 208, 259, 352, 1216, 2368, 9280, 9360, 16405, 16419, 18496, + 20480, 32794, 32899, 33792, 36871, 36896, 37443, 39426, 49171, 65580, 65670, 65792, 65797, 73742, 73744, + 73749, 81957, 90119, 91140, 98374, 110596, 81, 134, 137, 164, 200, 261, 416, 515, 704, 2432, 4736, 16409, + 16451, 18560, 19201, 24576, 32810, 32838, 33027, 34816, 36875, 36902, 36928, 36938, 37010, 37154, 38018, + 39170, 46082, 49187, 55298, 65588, 65798, 66048, 66053, 73765, 74916, 81989, 83969, 98438, 102982, 106510, + 112134, 122887, 97, 138, 262, 265, 274, 328, 400, 517, 832, 1027, 1408, 4680, 4864, 9472, 16425, 16453, + 16515, 18725, 19713, 32818, 32902, 33283, 36883, 36992, 37446, 45570, 49219, 65620, 65676, 66054, 66560, + 66565, 73766, 73792, 73797, 74948, 82053, 83975, 90125, 98566, 102414, 106518, 107524, 118791, 56, 84, 98, + 140, 161, 266, 273, 518, 521, 530, 548, 656, 800, 1029, 1664, 2051, 2816, 9728, 16433, 16517, 16643, 18439, + 18451, 18469, 18505, 18577, 18944, 19009, 19585, 23041, 27649, 32850, 32906, 33030, 33795, 36878, 36899, + 37120, 37377, 40960, 49283, 53255, 53762, 53827, 65636, 65804, 66566, 67584, 67589, 73756, 73798, 73856, + 73861, 74753, 82181, 90133, 98822, 102422, 106534, 110594, 110599, 111750, 116743, 162, 193, 268, 522, 529, + 1030, 1033, 1042, 1060, 1096, 1312, 1600, 2053, 2340, 3328, 4099, 5632, 9345, 16465, 16521, 16645, 16899, + 18443, 18690, 19456, 22785, 32866, 33034, 33286, 34819, 36886, 36931, 37458, 49152, 49411, 51207, 53259, + 55297, 65700, 65812, 66060, 67590, 69632, 69637, 73772, 73862, 73984, 73989, 74754, 74759, 74784, 75140, + 82437, 83981, 84228, 90149, 93445, 99334, 102438, 102916, 106566, 194, 276, 321, 524, 545, 1034, 1041, 2054, + 2057, 2066, 2084, 2120, 2192, 2624, 3200, 4101, 4673, 6656, 8195, 9346, 11264, 16481, 16649, 16901, 17411, + 18445, 18692, 18723, 26881, 32930, 33042, 33290, 33798, 36890, 36995, 37380, 37392, 37474, 37888, 49667, + 51211, 51458, 53267, 65732, 66068, 66572, 69638, 73780, 73990, 74240, 74245, 75396, 82949, 83989, 84261, + 90181, 92161, 93317, 100358, 102470, 106630, 107522, 107527, 111174, 112, 168, 196, 280, 322, 385, 532, 546, + 1036, 1057, 1170, 2058, 2065, 2337, 4102, 4105, 4114, 4132, 4168, 4240, 4384, 4674, 5248, 6400, 8197, 9348, + 13312, 16545, 16657, 16905, 17413, 18453, 18467, 18696, 22528, 32962, 33298, 33802, 34822, 36906, 36934, + 37123, 37384, 38912, 40963, 42114, 50179, 51219, 51491, 53283, 53761, 65860, 66084, 66580, 67596, 70212, + 73812, 73868, 74246, 74760, 74765, 74774, 74828, 75044, 75332, 77060, 79364, 81920, 84005, 90245, 91137, + 92167, 93188, 102534, 106758, 110606, 126983, 324, 386, 536, 641, 1044, 1058, 1089, 1169, 2060, 2081, 2338, + 4106, 4113, 4676, 8198, 8201, 8210, 8228, 8264, 8336, 8480, 8768, 9352, 10496, 12800, 16577, 16913, 17417, + 18457, 18499, 18704, 18729, 20483, 25729, 26624, 33090, 33314, 33810, 34826, 36914, 36998, 37570, 51235, + 53315, 56067, 65924, 66596, 67604, 67876, 69644, 73828, 73996, 74766, 74768, 74773, 75776, 75781, 76932, + 81926, 84037, 86021, 90373, 91143, 102662, 102914, 102919, 107014, 110614, 111620, 124935, 388, 552, 585, + 642, 769, 1048, 1090, 2068, 2082, 2113, 4108, 4129, 8202, 8209, 16390, 16402, 16420, 16456, 16528, 16672, + 16705, 16929, 16960, 17425, 17536, 18473, 18501, 18563, 18737, 20485, 20992, 21057, 24579, 25600, 32912, + 33154, 33826, 34834, 35106, 36946, 37002, 37126, 37383, 37408, 37698, 37891, 40966, 45056, 51267, 51457, + 53379, 65824, 66180, 66628, 66708, 67620, 69652, 73892, 74004, 74252, 74789, 74902, 75782, 77824, 77829, + 78980, 84101, 90629, 92173, 92420, 98304, 98309, 107534, 110630, 111108, 114694, 120839, 123911, 392, 644, + 770, 1281, 2072, 2114, 2177, 4116, 4130, 4161, 8204, 8225, 16769, 17441, 18565, 24581, 33410, 33938, 34850, + 36962, 37130, 38530, 38915, 40970, 46594, 49158, 53248, 55303, 55810, 69668, 73924, 74260, 74790, 74821, + 74896, 74901, 77830, 91149, 92181, 92453, 107542, 110662 }; + +static int cbest_18[1023] = { + 1, 2, 131136, 4, 65568, 8, 32784, 16, 16392, 32, 8196, 64, 4098, 128, 2049, 256, 512, 132160, 1024, 2048, + 66080, 4096, 33040, 8192, 16520, 8260, 16384, 3, 4130, 32768, 131137, 5, 65569, 131138, 6, 9, 2065, 32785, + 65536, 65570, 131140, 196704, 10, 17, 16393, 32786, 65572, 131144, 163920, 12, 18, 33, 8197, 16394, 32788, + 65576, 98352, 131072, 131152, 147528, 20, 34, 65, 4099, 8198, 16396, 32792, 65584, 81960, 131168, 132168, + 139332, 24, 36, 66, 129, 258, 516, 1032, 2064, 49176, 73764, 135234, 40, 68, 130, 257, 2051, 4102, 4128, + 8204, 16408, 32816, 40980, 65632, 69666, 131264, 133185, 48, 72, 132, 513, 2053, 4106, 8212, 8256, 16424, + 24588, 32848, 36882, 65696, 66084, 67617, 131392, 132161, 80, 136, 260, 514, 1025, 2057, 4114, 8228, 16456, + 16512, 20490, 32912, 34833, 65824, 131648, 132162, 96, 144, 264, 1026, 12294, 18441, 33024, 66081, 132164, + 197728, 160, 272, 520, 1028, 2050, 2081, 4162, 8324, 10245, 16648, 33042, 33296, 66048, 66082, 66592, 133184, + 164944, 197216, 192, 288, 528, 2052, 2113, 4097, 4226, 6147, 8452, 16904, 33041, 33808, 67616, 132096, + 132176, 148552, 320, 544, 1040, 2056, 2177, 4354, 8708, 17416, 34832, 66088, 98864, 132192, 135232, 140356, + 164176, 384, 576, 1056, 2305, 4100, 4610, 8193, 9220, 16521, 18440, 33044, 66096, 69664, 82472, 98608, + 136258, 640, 1088, 2080, 2561, 4104, 5122, 8194, 10244, 16522, 33048, 36880, 74276, 132288, 134209, 139268, + 139328, 147656, 768, 1152, 2112, 3073, 4112, 6146, 8261, 16385, 16524, 20488, 49432, 66144, 69634, 70178, + 73760, 82088, 132416, 1280, 2176, 8200, 8262, 12292, 16386, 33072, 34817, 40976, 41236, 49304, 66208, 68129, + 132672, 147520, 1536, 2304, 4131, 4160, 4352, 8208, 16388, 16536, 24584, 32769, 33104, 37138, 66336, 73828, + 81952, 2560, 4224, 6145, 8224, 8268, 8704, 16552, 24716, 32770, 33168, 35089, 41044, 49168, 131139, 134208, + 135266, 148544, 163904, 198240, 7, 3072, 4134, 8276, 12290, 16400, 16584, 17408, 20618, 24652, 32772, 65537, + 65571, 67104, 98336, 131141, 196705, 11, 2067, 4138, 4608, 8292, 8320, 10241, 16416, 18569, 24580, 32776, + 32787, 33552, 34816, 36914, 65538, 65573, 68128, 74272, 131142, 131145, 133201, 136256, 163921, 165200, + 196672, 196706, 13, 19, 2069, 4146, 5120, 8448, 12358, 16395, 16448, 16776, 20482, 20522, 32789, 34064, + 49160, 65540, 65574, 65577, 67633, 98353, 131073, 131146, 131153, 147529, 163922, 196708, 14, 21, 35, 2073, + 6144, 8199, 8388, 10309, 12326, 16397, 17032, 18433, 32790, 32793, 32800, 35088, 37136, 40964, 65544, 65578, + 65585, 69632, 70176, 81961, 98320, 98354, 99120, 131074, 131148, 131154, 131169, 132169, 139333, 140292, + 140352, 147530, 148680, 163924, 196712, 229488, 22, 25, 37, 67, 259, 517, 1033, 4194, 8516, 9216, 16398, + 16640, 17544, 18457, 32794, 32832, 36866, 49177, 65552, 65580, 65586, 73765, 81928, 81962, 98356, 131076, + 131156, 131170, 132170, 135235, 139334, 147532, 163928, 196640, 196720, 213096, 26, 38, 41, 69, 131, 518, + 1034, 2066, 2097, 4103, 4129, 4258, 6179, 8205, 8772, 10240, 10261, 16409, 16896, 18568, 32796, 32817, 32896, + 40981, 49178, 65588, 65633, 69667, 70146, 73732, 73766, 81964, 82600, 98360, 131080, 131160, 131172, 131265, + 131394, 131652, 132172, 133200, 139264, 163856, 180312, 197736, 204900, 28, 42, 49, 70, 73, 133, 262, 1036, + 2068, 2129, 4107, 4386, 6163, 8206, 8213, 8257, 9284, 16410, 16425, 24589, 32818, 32849, 36883, 40982, 49180, + 65592, 65600, 65634, 65697, 65826, 66085, 66600, 67632, 114744, 131088, 131176, 131266, 131393, 133187, + 135238, 135264, 139340, 147464, 147544, 163952, 164952, 172116, 200802, 44, 50, 74, 81, 134, 137, 261, 266, + 515, 524, 2055, 2072, 2193, 4115, 4132, 4642, 8214, 8229, 8258, 10308, 12288, 16412, 16426, 16457, 16513, + 18432, 20491, 20616, 24590, 32820, 32850, 32913, 33026, 33280, 33300, 33816, 35073, 41232, 49560, 65636, + 65664, 65698, 65825, 66052, 66086, 67585, 67619, 69670, 73772, 74340, 81976, 82464, 106548, 131104, 131184, + 131268, 131649, 132104, 132163, 132184, 133189, 135242, 136290, 139348, 147560, 155724, 164928, 168018, + 196832, 197220, 198753, 52, 76, 82, 97, 138, 145, 265, 274, 532, 1027, 1048, 2059, 2321, 4110, 4136, 5154, + 8230, 12295, 16428, 16458, 16514, 16650, 16908, 18456, 32824, 32852, 32914, 33025, 33792, 34835, 36886, + 36912, 40988, 65640, 65700, 65792, 67621, 69674, 73780, 73824, 90156, 98416, 102450, 131272, 131396, 131650, + 132165, 132200, 133193, 135170, 135250, 139364, 140364, 147648, 151626, 164048, 165969, 196960, 197729, 56, + 84, 98, 140, 146, 161, 273, 290, 521, 548, 1029, 1064, 2061, 2096, 2577, 4118, 4144, 4163, 6178, 8220, 8264, + 8325, 8454, 9228, 10260, 12356, 16460, 16516, 16649, 18443, 20480, 20494, 20520, 24712, 32856, 32916, 33043, + 33297, 34837, 36890, 41040, 41300, 49208, 49424, 57372, 65648, 65704, 65828, 66049, 66083, 66092, 66593, + 67625, 69682, 82024, 82080, 86058, 98480, 98848, 98868, 100401, 131200, 131280, 131400, 132166, 133121, + 134225, 136266, 143430, 149577, 164160, 164945, 197217, 197696, 197730, 88, 100, 148, 162, 193, 268, 289, + 322, 522, 529, 580, 1030, 1096, 2083, 2128, 3089, 4122, 4227, 4614, 5130, 6162, 8236, 8272, 8326, 8453, + 10247, 12324, 16440, 16905, 18445, 24604, 24648, 32920, 33028, 33298, 33809, 34841, 41012, 49240, 49296, + 53274, 65712, 65832, 66050, 66100, 66560, 66594, 77862, 82476, 84009, 98592, 131296, 131328, 131408, 131656, + 132097, 132177, 132296, 133186, 133217, 134217, 139460, 141381, 147784, 148553, 148672, 164178, 164432, + 164946, 197184, 197218, 197732, 104, 152, 164, 194, 276, 321, 386, 530, 545, 644, 1041, 1160, 2054, 2085, + 2115, 2192, 2307, 2565, 3081, 4166, 4192, 4355, 4384, 8244, 8288, 8709, 8768, 12302, 16472, 16528, 16652, + 16906, 17417, 17536, 20506, 24576, 24620, 24780, 32880, 33032, 33046, 33810, 35072, 36864, 37170, 45078, + 51225, 65840, 66089, 66596, 67584, 67618, 68145, 69730, 73892, 75813, 82216, 98610, 98865, 131424, 131584, + 131664, 132098, 132178, 132193, 132424, 133188, 135233, 135362, 137283, 139588, 140357, 148040, 148554, + 164177, 164948, 198752, 230512, 112, 168, 196, 280, 292, 385, 546, 577, 772, 1042, 1057, 1288, 2058, 2089, + 2117, 2179, 2320, 4101, 4170, 4230, 4256, 4611, 6151, 6177, 8332, 8710, 9221, 10253, 12310, 12354, 16488, + 16544, 17418, 24708, 28686, 32944, 33045, 33050, 33304, 33812, 34834, 34865, 36946, 41108, 43029, 49416, + 65760, 66056, 66090, 66097, 66148, 67620, 67681, 69665, 69794, 70144, 70182, 71715, 74020, 82473, 98609, + 98832, 98866, 99376, 131680, 132100, 132180, 132194, 132680, 133192, 133313, 135490, 136259, 139844, 140358, + 148556, 165968, 197224, 197664, 197744, 214120, 230000, 176, 200, 296, 324, 536, 641, 770, 1044, 1058, 1089, + 1544, 2060, 2121, 2181, 4105, 4178, 4234, 4358, 8460, 9222, 10305, 16576, 17420, 18442, 18473, 20554, 20610, + 32976, 33049, 33056, 34836, 34897, 38931, 41220, 65888, 66098, 66608, 67624, 67745, 68133, 74277, 74336, + 82440, 82474, 131520, 132196, 132418, 133120, 133441, 134224, 135746, 164880, 181336, 197232, 205924, 229744 }; + + +static int cbest_19[1023] = { + 1, 2, 262163, 4, 393242, 8, 196621, 16, 32, 360469, 64, 262162, 128, 131081, 3, 442393, 262161, 6, 256, + 393243, 5, 327703, 12, 196620, 512, 393240, 9, 24, 98310, 262167, 426008, 10, 458782, 48, 1024, 49155, + 196617, 17, 213004, 262171, 393234, 483359, 18, 96, 262147, 360471, 393246, 20, 2048, 16385, 196623, 286738, + 360468, 442392, 33, 192, 32770, 106502, 229391, 34, 65540, 196613, 270355, 360465, 36, 384, 4096, 131080, + 143369, 180234, 393226, 40, 65, 53251, 221196, 262195, 66, 768, 376852, 397338, 442395, 68, 8192, 24577, + 49154, 90117, 503836, 72, 129, 1536, 262160, 262227, 80, 110598, 198669, 288786, 393274, 132, 3072, 16384, + 98308, 188426, 196637, 274451, 327702, 333847, 483358, 7, 130, 136, 257, 131083, 262291, 307217, 144, 6144, + 49153, 163851, 213005, 262165, 286739, 327699, 360477, 393241, 393306, 426010, 13, 160, 55299, 131073, + 144393, 196616, 196653, 262166, 327701, 360453, 361493, 426009, 14, 26, 258, 264, 513, 12288, 53250, 94213, + 131085, 196609, 262419, 399386, 442385, 458783, 25, 52, 272, 26625, 98306, 98311, 251918, 344086, 360467, + 393370, 429080, 442397, 11, 104, 288, 24576, 32768, 90116, 106500, 196619, 196622, 196685, 229389, 262155, + 262170, 262175, 311312, 368661, 393218, 393235, 393238, 393244, 405530, 458778, 22, 28, 49, 208, 260, 320, + 514, 1025, 45058, 155656, 180235, 229390, 241679, 262146, 262169, 262675, 360470, 376853, 393232, 393247, + 442394, 458780, 44, 416, 528, 22529, 77828, 172043, 180232, 196612, 199693, 213000, 221197, 262145, 262179, + 289810, 360501, 393498, 415771, 442905, 483355, 491548, 19, 88, 97, 544, 832, 38914, 53249, 143368, 196749, + 275475, 309265, 442377, 475167, 483357, 21, 38, 50, 56, 176, 516, 576, 1026, 1664, 2049, 19457, 71684, + 114695, 131097, 135177, 202765, 214540, 263187, 270354, 288787, 334359, 360464, 426000, 76, 193, 352, 640, + 3328, 32771, 35842, 49152, 106498, 106503, 139273, 213006, 229387, 262151, 278546, 286736, 327711, 352278, + 360533, 393227, 393754, 426012, 446489, 458774, 35, 98, 152, 194, 704, 1056, 6656, 17921, 55298, 65536, + 65541, 196629, 196877, 245774, 262194, 262259, 273427, 294929, 319504, 327687, 348182, 376854, 393224, + 393258, 450585, 37, 42, 70, 112, 304, 385, 520, 1028, 1088, 1408, 2050, 4097, 13312, 16387, 49159, 94212, + 110596, 125959, 131113, 144905, 188427, 196615, 212996, 262193, 264211, 270353, 315408, 329751, 360473, + 362005, 372757, 382996, 397339, 483351, 503838, 41, 140, 608, 1152, 2816, 26624, 27649, 98318, 159752, + 188424, 198668, 221192, 262355, 271891, 331799, 360597, 394266, 399898, 442425, 499740, 503837, 67, 100, 280, + 388, 769, 1216, 1280, 5632, 47106, 65542, 98304, 99334, 107270, 157704, 176139, 196633, 196636, 197133, + 262226, 363541, 376848, 393230, 406554, 409627, 425992, 427032, 442384, 69, 74, 84, 224, 386, 560, 1032, + 2052, 2112, 2432, 4098, 8193, 11264, 16389, 49163, 49667, 79876, 90113, 122887, 131082, 131145, 143371, + 163849, 174091, 229383, 262199, 262225, 262547, 266259, 271123, 286742, 344087, 360479, 393275, 393338, + 397336, 428056, 429336, 442396, 458766, 470046, 483615, 503832, 73, 196, 1120, 1537, 2176, 4864, 22528, + 23553, 32774, 55297, 78852, 81925, 98326, 163850, 213516, 221198, 262203, 274450, 307219, 311313, 327735, + 333846, 360455, 360725, 393266, 393272, 395290, 398874, 416795, 442399, 442457, 460830, 81, 134, 770, 776, + 2240, 2304, 9728, 39938, 53248, 110594, 110599, 131072, 137225, 143361, 144392, 180226, 196649, 196652, + 196669, 197645, 198665, 214028, 251919, 262290, 262931, 270359, 286994, 303121, 307216, 327697, 356374, + 360449, 393434, 434200, 442387, 483343, 133, 148, 168, 448, 1040, 2056, 2560, 3073, 4100, 4480, 8194, 16393, + 16897, 19456, 24579, 39426, 49171, 90119, 98309, 106758, 114694, 131084, 131209, 143373, 180238, 191498, + 196608, 199949, 221188, 249870, 251916, 262164, 262211, 262231, 275987, 286746, 288784, 289811, 323600, + 327698, 333843, 360461, 360476, 361495, 393307, 398106, 415770, 426011, 442904, 443161, 448537, 466974, + 483354, 485407, 131, 137, 200, 1538, 4224, 8960, 17409, 19969, 32778, 53255, 53635, 57347, 72196, 98342, + 106510, 107014, 131077, 143497, 153608, 155657, 196639, 196717, 203277, 213020, 217100, 262235, 263699, + 270611, 274449, 286722, 325136, 327700, 327767, 333845, 350230, 360452, 360981, 361492, 393278, 393298, + 393304, 393626, 397330, 426040, 442376, 442521, 475166, 483356, 487455, 82, 138, 145, 268, 772, 1552, 4352, + 6145, 17920, 18433, 19713, 33794, 45056, 53379, 65548, 106496, 166923, 180233, 196618, 196645, 196681, + 196684, 198671, 213001, 229385, 229388, 230415, 241677, 262289, 262418, 270363, 270867, 317456, 334615, + 349206, 360485, 397342, 397722, 399387, 417819, 429082, 442369, 442389, 443929, 458814, 503828, 146, 161, + 296, 336, 392, 896, 2064, 3074, 4104, 4608, 8196, 16386, 16401, 20481, 24581, 34818, 36098, 49158, 49187, + 53507, 76804, 108550, 125958, 131087, 131089, 131337, 161800, 172041, 180746, 196813, 199437, 214668, 221452, + 241678, 262153, 262275, 262295, 265235, 270339, 271379, 273939, 278547, 286737, 290834, 311314, 327707, + 330775, 352279, 360466, 360503, 361489, 368663, 393216, 393371, 394010, 397466, 413723, 429081, 446488, + 458770, 458776, 15, 27, 54, 108, 162, 216, 259, 265, 432, 864, 1540, 1728, 3456, 5120, 6912, 12289, 13824, + 27648, 32786, 36866, 38912, 49157, 53259, 67588, 94209, 98374, 106518, 131075, 172042, 178187, 196655, + 198661, 199692, 213036, 214541, 229407, 233487, 262154, 262174, 262299, 272403, 319505, 327831, 348183, + 360497, 368660, 376855, 393219, 393239, 393245, 393290, 393310, 393362, 397594, 401434, 405531, 426014, + 426072, 442379, 442649, 450584, 458779, 483391, 503964, 514077, 53, 266, 273, 536, 3104, 6146, 8448, 18049, + 24833, 38402, 40962, 54275, 65556, 69636, 86021, 90373, 98307, 114693, 124935, 145417, 158728, 188418, + 196611, 196677, 196745, 196748, 197005, 198733, 199053, 200717, 207885, 212992, 213007, 235023, 237583, + 251914, 262168, 262173, 262417, 262674, 268307, 288790, 288850, 315409, 339991, 344082, 376860, 377364, + 382997, 393233, 393236, 394778, 397850, 399384, 426002, 442907, 458781, 458846, 483347, 483350, 483353, + 491550, 30, 105, 262, 274, 289, 400, 592, 672, 1792, 2080, 3076, 4112, 8200, 8704, 16388, 16417, 24585, + 32769, 35840, 49162, 49219, 49666, 73732, 80900, 90112, 90125, 95749, 98314, 99846, 106501, 110726, 122886, + 131096, 131593, 135176, 143385, 144395, 175115, 180250, 184330, 188430, 198797, 202764, 223244, 245775, + 262144, 262149, 262178, 262183, 262323, 262403, 272147, 274455, 286770, 288914, 309267, 333911, 344084, + 360500, 360535, 360565, 362133, 368657, 376836, 388116, 393368, 393499, 397322, 398362, 400154, 407066, + 427544, 438296, 442424, 491549, 23, 29, 46, 92, 106, 184, 209, 261, 276, 290, 321, 368, 515, 736, 1472, 1544, + 2944, 5888, 9216, 11776, 12290, 19201, 23552, 32802, 45059, 53267, 55296, 81924, 94215, 98316, 98438, 106534, + 139272, 163843, 174603, 196687, 197389, 198861, 198925, 212997, 213002, 213068, 221212, 225292, 229386, + 229423, 241675, 245772, 262427, 272019, 274579, 275474, 288978, 305169, 307221, 309264, 327959, 345622, + 360529, 362517, 363797, 376849, 378900, 393354, 393374, 393490, 396314, 405528, 409626, 426136, 436248, + 461854, 470558, 471070, 483423, 503820, 504860, 417, 529, 1072, 6148, 10240, 92165, 101382, 125957, 144425, + 147464, 196741, 204813, 262159, 262339, 262673, 263186, 286743, 288794, 307345, 327683, 425984, 426001, + 432152, 442409, 458910, 505884 }; + +static int cbest_20[1023] = { + 1, 2, 524292, 4, 262146, 8, 131073, 16, 32, 589828, 64, 128, 294914, 256, 512, 147457, 1024, 2048, 4096, + 598020, 8192, 16384, 32768, 299010, 65536, 131072, 149505, 262144, 3, 524288, 524293, 5, 262147, 524294, + 599044, 6, 9, 18, 36, 72, 144, 288, 576, 1152, 2304, 4608, 9216, 18432, 36864, 73728, 147456, 786438, 10, 17, + 131075, 262150, 294912, 524300, 655365, 12, 33, 131077, 262154, 393219, 524308, 589824, 589829, 20, 34, 65, + 131081, 262162, 299522, 524324, 589830, 24, 66, 129, 131089, 262178, 294915, 524356, 851974, 40, 68, 130, + 257, 131105, 262210, 524420, 589836, 720901, 819206, 48, 132, 258, 513, 131137, 262274, 294918, 524548, + 589844, 80, 136, 260, 514, 1025, 131201, 147459, 149761, 262402, 294922, 425987, 524804, 589860, 671749, 96, + 264, 516, 1026, 2049, 131329, 147461, 262658, 294930, 409603, 525316, 589892, 160, 272, 520, 1028, 2050, + 4097, 131585, 147465, 149504, 263170, 294946, 299008, 526340, 589956, 598016, 598021, 884742, 192, 528, 1032, + 2052, 4098, 8193, 74752, 132097, 147473, 264194, 294978, 528388, 590084, 598022, 320, 544, 1040, 2056, 4100, + 8194, 16385, 37376, 133121, 147489, 266242, 295042, 532484, 590340, 737285, 860166, 384, 1056, 2064, 4104, + 8196, 16386, 18688, 32769, 135169, 147521, 270338, 295170, 299011, 540676, 590852, 598028, 599172, 729093, + 640, 1088, 2080, 4112, 8200, 9344, 16388, 32770, 65537, 139265, 147585, 278530, 295426, 442371, 557060, + 591876, 598036, 823302, 768, 2112, 4128, 4672, 8208, 16392, 32772, 65538, 147713, 295938, 299014, 593924, + 598052, 1280, 2176, 2336, 4160, 8224, 16400, 32776, 65540, 131074, 147969, 163841, 296962, 299018, 327682, + 430083, 598084, 655364, 1168, 1536, 4224, 8256, 16416, 32784, 65544, 131076, 148481, 149507, 196609, 262145, + 299026, 393218, 598148, 606212, 673797, 892934, 584, 2560, 4352, 8320, 16448, 32800, 65552, 131080, 149509, + 299042, 299586, 303106, 411651, 598276, 622596, 786436, 888838, 292, 3072, 8448, 16512, 32832, 65568, 131088, + 149513, 151553, 262148, 262160, 299074, 311298, 524289, 598532, 745477, 146, 5120, 8704, 16640, 32896, 65600, + 131104, 149521, 155649, 262152, 299138, 393217, 524290, 524295, 524320, 599040, 599045, 720900, 7, 19, 37, + 73, 145, 289, 577, 1153, 2305, 4609, 6144, 9217, 16896, 18433, 33024, 36865, 65664, 73729, 131136, 149537, + 299266, 299520, 360450, 599046, 600068, 739333, 786434, 786439, 11, 38, 74, 290, 578, 1154, 2306, 4610, 9218, + 10240, 17408, 18434, 33280, 36866, 65792, 73730, 131200, 147458, 149569, 149760, 180225, 262151, 262176, + 294913, 425986, 446467, 524296, 524301, 524310, 524364, 524436, 524580, 524868, 525444, 526596, 528900, + 533508, 542724, 561156, 602116, 655361, 671748, 851972, 861190, 13, 22, 76, 148, 580, 1156, 2308, 4612, 9220, + 12288, 18436, 33792, 36868, 66048, 73732, 131328, 147460, 149633, 149793, 212993, 262155, 262182, 262208, + 262218, 262290, 262434, 262722, 263298, 264450, 266754, 271362, 280578, 300034, 335874, 409602, 444419, + 524302, 524304, 524309, 589825, 599052, 655367, 730117, 819204, 14, 21, 26, 35, 44, 152, 296, 1160, 2312, + 4616, 9224, 18440, 20480, 34816, 36872, 66560, 73736, 74880, 131079, 131091, 131109, 131145, 131217, 131361, + 131584, 131649, 132225, 133377, 135681, 140289, 147464, 167937, 204801, 262163, 262272, 294916, 294928, + 299523, 301058, 524325, 589826, 589831, 589856, 599060, 614404, 786446, 917511, 25, 52, 67, 88, 304, 592, + 2320, 4624, 9232, 18448, 24576, 36880, 67584, 73744, 131083, 132096, 147472, 150017, 262158, 262179, 262400, + 294920, 393223, 425985, 524326, 524352, 524357, 599076, 630788, 655373, 786454, 823814, 851970, 851975, + 897030, 41, 50, 69, 104, 131, 176, 608, 1184, 4640, 9248, 18464, 36896, 37440, 40960, 69632, 73760, 131085, + 133120, 147488, 150529, 262166, 262211, 262656, 299526, 307202, 393227, 409601, 524316, 524358, 524416, + 524421, 589832, 589837, 589846, 589900, 589972, 590116, 590404, 590980, 592132, 594436, 599108, 608260, + 626692, 655381, 720897, 737284, 786470, 819202, 819207, 28, 42, 49, 70, 82, 100, 133, 208, 259, 352, 1216, + 2368, 9280, 18496, 36928, 49152, 73792, 131093, 131107, 135168, 147520, 262170, 262275, 263168, 294919, + 294944, 299530, 315394, 393235, 430595, 524332, 524422, 524544, 524549, 589838, 589840, 589845, 655397, + 720903, 729092, 786502, 884740, 893958, 81, 134, 137, 164, 200, 261, 416, 515, 704, 2432, 4736, 18560, 18720, + 36992, 73856, 81920, 131097, 131139, 139264, 147584, 153601, 262186, 262214, 262403, 264192, 294923, 294950, + 294976, 294986, 295058, 295202, 295490, 296066, 297218, 299538, 304130, 313346, 368642, 393251, 442370, + 524340, 524550, 524800, 524805, 589861, 599300, 655429, 671745, 747525, 786566, 851982, 983047, 97, 138, 262, + 265, 274, 328, 400, 517, 832, 1027, 1408, 4864, 9472, 37120, 73984, 98304, 131113, 131141, 131203, 147712, + 149763, 157697, 262194, 262278, 262659, 266240, 294931, 295040, 299554, 364546, 393283, 524372, 524428, + 524806, 525312, 525317, 589862, 589888, 589893, 599188, 599556, 655493, 671751, 674053, 720909, 746501, + 786694, 819214, 851990, 860164, 889350, 950279, 56, 84, 98, 140, 161, 266, 273, 518, 521, 530, 548, 656, 800, + 1029, 1664, 2051, 2816, 9360, 9728, 18944, 74240, 131121, 131205, 131331, 147463, 147475, 147493, 147529, + 147601, 147745, 147968, 148033, 148609, 149765, 152065, 156673, 163840, 184321, 221185, 262226, 262282, + 262406, 263171, 270336, 294926, 294947, 295168, 299009, 393347, 411907, 425991, 430082, 524388, 524556, + 525318, 526336, 526341, 589852, 589894, 589952, 589957, 598017, 655621, 720917, 786950, 819222, 852006, + 884738, 884743, 933895, 162, 193, 268, 522, 529, 1030, 1033, 1042, 1060, 1096, 1312, 1600, 2053, 3328, 4099, + 5632, 19456, 37888, 74753, 131153, 131209, 131333, 131587, 147467, 148480, 149506, 149769, 182273, 196608, + 262242, 262410, 262662, 264195, 278528, 294934, 294979, 295424, 299650, 393475, 409607, 425995, 442369, + 448515, 524452, 524564, 524812, 526342, 528384, 528389, 589868, 589958, 590080, 590085, 598018, 598023, + 598048, 601092, 655877, 671757, 673796, 720933, 787462, 819238, 823300, 852038, 194, 276, 321, 524, 545, + 1034, 1041, 2054, 2057, 2066, 2084, 2120, 2192, 2624, 3200, 4101, 4680, 6656, 8195, 11264, 37377, 38912, + 74754, 75776, 131169, 131337, 131589, 132099, 147469, 149508, 149777, 215041, 262306, 262418, 262666, 263174, + 266243, 294938, 295043, 295936, 299012, 299024, 299778, 393731, 409611, 411650, 426003, 524484, 524820, + 525324, 528390, 532480, 532485, 589876, 590086, 590336, 590341, 603140, 656389, 671765, 720965, 737281, + 788486, 819270, 852102, 860162, 860167, 112, 168, 196, 280, 322, 385, 532, 546, 1036, 1057, 2058, 2065, 4102, + 4105, 4114, 4132, 4168, 4240, 4384, 5248, 6400, 8197, 13312, 16387, 18689, 22528, 37378, 74756, 77824, + 131233, 131345, 131593, 132101, 133123, 147477, 147491, 149512, 151552, 262338, 262674, 263178, 264198, + 270339, 294954, 294982, 295171, 296960, 299016, 327680, 336898, 394243, 409619, 426019, 430081, 446979, + 524612, 524836, 525332, 526348, 532486, 540672, 540677, 561668, 589908, 589964, 590342, 590848, 590853, + 598024, 598029, 598038, 598092, 598164, 598308, 598596, 599168, 599173, 600324, 602628, 616452, 634884, + 657413, 671781, 721029, 729089, 737287, 739589, 745476, 790534, 819334, 852230, 884750, 1015815, 386, 1089, + 4106, 8228, 8264, 8336, 10496, 131601, 147481, 147523, 149825, 155648, 262466, 263186, 295046, 300546, + 393216, 426051, 590092, 599174, 615428, 729095, 823303 }; + +static int cbest_21[1023] = { + 1, 2, 1048578, 4, 524289, 8, 16, 1310722, 32, 64, 655361, 128, 256, 512, 1376258, 1024, 2048, 4096, 688129, + 8192, 16384, 32768, 65536, 1392642, 131072, 262144, 524288, 696321, 1048576, 3, 1048579, 5, 10, 20, 40, 80, + 160, 320, 640, 1280, 2560, 5120, 10240, 20480, 40960, 81920, 163840, 327680, 655360, 6, 9, 524291, 1048582, + 1310720, 1572867, 17, 524293, 1048586, 1310723, 1396738, 12, 18, 33, 524297, 1048594, 34, 65, 524305, + 1048610, 1310726, 1835011, 24, 36, 66, 129, 524321, 655363, 1048642, 1310730, 1703939, 68, 130, 257, 524353, + 655365, 688128, 1048706, 1310738, 1376256, 48, 72, 132, 258, 513, 344064, 524417, 655369, 698369, 1048834, + 1310754, 1376259, 136, 260, 514, 1025, 172032, 524545, 655377, 1049090, 1310786, 1966083, 96, 144, 264, 516, + 1026, 2049, 86016, 524801, 655393, 1049602, 1310850, 1376262, 1900547, 272, 520, 1028, 2050, 4097, 43008, + 525313, 655425, 1050626, 1310978, 1376266, 192, 288, 528, 1032, 2052, 4098, 8193, 21504, 526337, 655489, + 688131, 1052674, 1311234, 1376274, 1736707, 544, 1040, 2056, 4100, 8194, 10752, 16385, 528385, 655617, + 688133, 1056770, 1311746, 1376290, 384, 576, 1056, 2064, 4104, 5376, 8196, 16386, 32769, 532481, 655873, + 688137, 1064962, 1312770, 1376322, 1392640, 1397762, 2031619, 1088, 2080, 2688, 4112, 8200, 16388, 32770, + 65537, 540673, 656385, 688145, 696320, 1081346, 1314818, 1376386, 1392643, 1998851, 768, 1152, 1344, 2112, + 4128, 8208, 16392, 32772, 65538, 131073, 557057, 657409, 688161, 1114114, 1318914, 1376514, 672, 2176, 4160, + 8224, 16400, 32776, 65540, 131074, 262145, 348160, 589825, 659457, 688193, 1179650, 1327106, 1376770, + 1392646, 1916931, 336, 1536, 2304, 4224, 8256, 16416, 32784, 65544, 131076, 262146, 663553, 688257, 1343490, + 1377282, 1392650, 168, 4352, 8320, 16448, 32800, 65552, 131080, 174080, 262148, 524290, 671745, 688385, + 786433, 1378306, 1392658, 1572866, 84, 3072, 4608, 8448, 16512, 32832, 65568, 131088, 262152, 524292, 688641, + 696323, 698881, 1048577, 1380354, 1392674, 1441794, 1744899, 2064387, 42, 8704, 16640, 32896, 65600, 87040, + 131104, 262160, 524296, 689153, 696325, 720897, 1048584, 1384450, 1392706, 2048003, 11, 21, 41, 81, 161, 321, + 641, 1281, 2561, 5121, 6144, 9216, 10241, 16896, 20481, 33024, 40961, 65664, 81921, 131136, 163841, 262176, + 327681, 524304, 690177, 696329, 1048580, 1392770, 1572865, 1835010, 7, 22, 82, 162, 322, 642, 1282, 2562, + 5122, 10242, 17408, 20482, 33280, 40962, 43520, 65792, 81922, 131200, 163842, 262208, 327682, 524320, 655362, + 692225, 696337, 917505, 1048583, 1048598, 1048618, 1048658, 1048738, 1048898, 1049218, 1049858, 1051138, + 1053698, 1058818, 1069058, 1089538, 1130498, 1212418, 1310721, 1392898, 1396736, 1409026, 1703938, 2007043, + 14, 44, 164, 324, 644, 1284, 2564, 5124, 10244, 12288, 18432, 20484, 33792, 40964, 66048, 81924, 131328, + 163844, 262272, 327684, 524299, 524309, 524329, 524352, 524369, 524449, 524609, 524929, 525569, 526849, + 529409, 534529, 544769, 565249, 606209, 655364, 696353, 851969, 1048587, 1048592, 1310728, 1393154, 1396739, + 13, 19, 28, 88, 328, 648, 1288, 2568, 5128, 10248, 20488, 21760, 34816, 40968, 66560, 81928, 131584, 163848, + 262400, 327688, 524295, 524416, 655368, 696385, 698368, 704513, 1048595, 1048608, 1310724, 1393666, 1507330, + 1572871, 1835009, 26, 35, 56, 176, 656, 1296, 2576, 5136, 10256, 20496, 24576, 36864, 40976, 67584, 81936, + 132096, 163856, 262656, 327696, 524544, 655376, 696449, 1048590, 1048611, 1048640, 1310727, 1310742, 1310762, + 1310802, 1310882, 1311042, 1311362, 1312002, 1313282, 1315842, 1320962, 1331202, 1351682, 1394690, 1396742, + 1474562, 1572875, 1703937, 1921027, 1966082, 25, 37, 52, 67, 112, 352, 1312, 2592, 5152, 10272, 10880, 20512, + 40992, 69632, 81952, 133120, 163872, 263168, 327712, 524301, 524307, 524800, 655392, 696577, 753665, 1048643, + 1048704, 1310731, 1310736, 1396746, 1398018, 1572883, 1900546, 2080771, 38, 69, 74, 104, 131, 224, 704, 2624, + 5184, 10304, 20544, 41024, 49152, 73728, 81984, 135168, 163904, 264192, 327744, 349184, 524323, 525312, + 655371, 655381, 655401, 655424, 655441, 655521, 655681, 656001, 656641, 657921, 660481, 665601, 675841, + 696833, 737281, 983041, 1048602, 1048614, 1048707, 1048832, 1310739, 1310752, 1376257, 1396754, 1400834, + 1572899, 1835015, 2072579, 49, 70, 73, 133, 138, 148, 208, 259, 448, 1408, 5248, 5440, 10368, 20608, 41088, + 82048, 139264, 163968, 266240, 327808, 344065, 524313, 524325, 524355, 526336, 655367, 655488, 688130, + 697345, 950273, 1048646, 1048835, 1049088, 1310734, 1310755, 1310784, 1376264, 1396770, 1572931, 1703943, + 1736706, 1835019, 1966081, 50, 134, 137, 261, 266, 276, 296, 416, 515, 896, 2816, 10496, 20736, 41216, 82176, + 98304, 147456, 164096, 172033, 270336, 327936, 344066, 524357, 524419, 528384, 655616, 688132, 698371, + 1048626, 1048650, 1048710, 1049091, 1049600, 1310787, 1310848, 1376260, 1396802, 1425410, 1572995, 1703947, + 1746947, 1835027, 1900545, 2052099, 76, 97, 145, 262, 265, 517, 522, 532, 552, 592, 832, 1027, 1792, 2720, + 5632, 20992, 41472, 82432, 86017, 164352, 172034, 174592, 278528, 328192, 344068, 524337, 524361, 524421, + 524547, 532480, 655373, 655379, 655872, 688136, 698373, 700417, 868353, 1048714, 1048838, 1049603, 1050624, + 1220610, 1310746, 1310758, 1310851, 1310976, 1376263, 1376278, 1376298, 1376338, 1376418, 1376578, 1376898, + 1377538, 1378818, 1381378, 1386498, 1396866, 1417218, 1540098, 1573123, 1703955, 1835043, 2031618, 98, 140, + 146, 273, 518, 521, 1029, 1034, 1044, 1064, 1104, 1184, 1664, 2051, 3584, 11264, 41984, 43009, 82944, 86018, + 164864, 172036, 196608, 294912, 328704, 344072, 524425, 524549, 524803, 540672, 655395, 656384, 688144, + 698377, 1048674, 1048722, 1048842, 1049094, 1050627, 1052672, 1134594, 1310790, 1310979, 1311232, 1376267, + 1376272, 1396994, 1523714, 1573379, 1703971, 1736705, 1835075, 1966087, 1998850, 100, 193, 268, 274, 289, + 529, 1030, 1033, 1360, 2053, 2058, 2068, 2088, 2128, 2208, 2368, 3328, 4099, 7168, 21505, 22528, 43010, + 83968, 86020, 165888, 172040, 329728, 344080, 524385, 524433, 524553, 524805, 525315, 557056, 610305, 655385, + 655397, 655427, 657408, 688160, 698385, 712705, 1048850, 1049098, 1049606, 1052675, 1056768, 1091586, + 1310770, 1310794, 1310854, 1311235, 1311744, 1376275, 1376288, 1397250, 1573891, 1704003, 1835139, 1900551, + 1966091, 2009091, 152, 194, 290, 524, 530, 545, 1041, 2054, 2057, 4101, 4106, 4116, 4136, 4176, 4256, 4416, + 4736, 6656, 8195, 10753, 14336, 21506, 43012, 45056, 86024, 87296, 167936, 172048, 331776, 344096, 393216, + 524561, 524809, 525317, 526339, 567297, 589824, 655429, 655491, 659456, 688139, 688149, 688169, 688192, + 688209, 688289, 688449, 688769, 689409, 690689, 693249, 698401, 699009, 708609, 770049, 1015809, 1048770, + 1048866, 1049106, 1049610, 1050630, 1056771, 1064960, 1070082, 1310858, 1310982, 1311747, 1312768, 1376270, + 1376291, 1376320, 1397760, 1482754, 1574915, 1704067, 1835267, 1900555, 1916930, 1966099, 2031617, 196, 280, + 292, 385, 546, 577, 680, 1036, 1042, 1057, 2065, 4102, 4105, 5377, 8197, 8202, 8212, 8232, 8272, 8352, 8512, + 8832, 9472, 10754, 13312, 16387, 21508, 28672, 43016, 86032, 90112, 172064, 335872, 344128, 524481, 524577, + 524817, 525321, 526341, 528387, 545793, 655409, 655433, 655493, 655619, 663552, 688135, 688256, 698433, + 761857, 999425, 1049122, 1049618, 1050634, 1052678, 1059330, 1064963, 1081344, 1310818, 1310866, 1310986, + 1311238, 1312771, 1314816, 1376323, 1376384, 1392641, 1397763, 1398786, 1576963, 1704195, 1736711, 1835523, + 1900563, 1966115, 1998849, 2088963, 200, 386, 548, 1058, 16394, 16424, 16544, 17664, 18944, 57344, 180224, + 535041, 671744, 698497, 1052682, 1056774, 1353730, 1392648, 1836035 }; + +static int cbest_22[1023] = { + 1, 2, 2097153, 4, 8, 3145729, 16, 32, 64, 3670017, 128, 256, 512, 1024, 3932161, 2048, 4096, 8192, 16384, + 32768, 4063233, 65536, 131072, 262144, 524288, 1048576, 2097152, 4128769, 3, 6, 12, 24, 48, 96, 192, 384, + 768, 1536, 3072, 6144, 12288, 24576, 49152, 98304, 196608, 393216, 786432, 1572864, 3145728, 5, 2097155, 9, + 2097157, 10, 17, 2097161, 3145731, 3670016, 18, 33, 1835008, 2097169, 3145733, 20, 34, 65, 917504, 2097185, + 3145737, 4161537, 36, 66, 129, 458752, 2097217, 3145745, 3670019, 40, 68, 130, 257, 229376, 2097281, 3145761, + 3670021, 3932160, 72, 132, 258, 513, 114688, 2097409, 3145793, 3670025, 80, 136, 260, 514, 1025, 57344, + 1966080, 2097665, 3145857, 3670033, 144, 264, 516, 1026, 2049, 28672, 2098177, 3145985, 3670049, 3932163, + 160, 272, 520, 1028, 2050, 4097, 14336, 983040, 2099201, 3146241, 3670081, 3932165, 288, 528, 1032, 2052, + 4098, 7168, 8193, 2101249, 3146753, 3670145, 3932169, 4063232, 320, 544, 1040, 2056, 3584, 4100, 8194, 16385, + 491520, 2105345, 3147777, 3670273, 3932177, 4177921, 576, 1056, 1792, 2064, 4104, 8196, 16386, 32769, + 2113537, 3149825, 3670529, 3932193, 640, 896, 1088, 2080, 4112, 8200, 16388, 32770, 65537, 245760, 2031616, + 2129921, 3153921, 3671041, 3932225, 4063235, 448, 1152, 2112, 4128, 8208, 16392, 32772, 65538, 131073, + 2162689, 3162113, 3672065, 3932289, 4063237, 224, 1280, 2176, 4160, 8224, 16400, 32776, 65540, 122880, + 131074, 262145, 2228225, 3178497, 3674113, 3932417, 4063241, 112, 2304, 4224, 8256, 16416, 32784, 65544, + 131076, 262146, 524289, 1015808, 2359297, 3211265, 3678209, 3932673, 4063249, 4128768, 56, 2560, 4352, 8320, + 16448, 32800, 61440, 65552, 131080, 262148, 524290, 1048577, 2621441, 3276801, 3686401, 3933185, 4063265, 28, + 4608, 8448, 16512, 32832, 65568, 131088, 262152, 524292, 1048578, 3407873, 3702785, 3934209, 4063297, 14, + 5120, 8704, 16640, 30720, 32896, 65600, 131104, 262160, 507904, 524296, 1048580, 2097154, 3735553, 3936257, + 4063361, 4128771, 7, 13, 25, 49, 97, 193, 385, 769, 1537, 3073, 6145, 9216, 12289, 16896, 24577, 33024, + 49153, 65664, 98305, 131136, 196609, 262176, 393217, 524304, 786433, 1048584, 1572865, 2064384, 2097156, + 3801089, 3940353, 4063489, 4128773, 4186113, 26, 50, 98, 194, 386, 770, 1538, 3074, 6146, 10240, 12290, + 15360, 17408, 24578, 33280, 49154, 65792, 98306, 131200, 196610, 262208, 393218, 524320, 786434, 1048592, + 1572866, 2097159, 2097160, 2097165, 2097177, 2097201, 2097249, 2097345, 2097537, 2097921, 2098689, 2100225, + 2103297, 2109441, 2121729, 2146305, 2195457, 2293761, 2490369, 2883585, 3145730, 3948545, 4063745, 4128777, + 52, 100, 196, 388, 772, 1540, 3076, 6148, 12292, 18432, 24580, 33792, 49156, 66048, 98308, 131328, 196612, + 253952, 262272, 393220, 524352, 786436, 1048608, 1572868, 2097168, 3145732, 3964929, 4064257, 4128785, 11, + 104, 200, 392, 776, 1544, 3080, 6152, 7680, 12296, 20480, 24584, 34816, 49160, 66560, 98312, 131584, 196616, + 262400, 393224, 524416, 786440, 1048640, 1572872, 2097184, 3145735, 3145736, 3145741, 3145753, 3145777, + 3145825, 3145921, 3146113, 3146497, 3147265, 3148801, 3151873, 3158017, 3170305, 3194881, 3244033, 3342337, + 3538945, 3997697, 4065281, 4128801, 4161536, 19, 22, 208, 400, 784, 1552, 3088, 6160, 12304, 24592, 36864, + 49168, 67584, 98320, 132096, 196624, 262656, 393232, 524544, 786448, 1032192, 1048704, 1572880, 1835009, + 2097163, 2097216, 3145744, 3670018, 4067329, 4128833, 21, 35, 38, 44, 416, 800, 1568, 3104, 3840, 6176, + 12320, 24608, 40960, 49184, 69632, 98336, 126976, 133120, 196640, 263168, 393248, 524800, 786464, 917505, + 1048832, 1572896, 1835010, 2097171, 2097280, 3145760, 3670020, 4071425, 4128897, 37, 67, 70, 76, 88, 832, + 1600, 3136, 6208, 12352, 24640, 49216, 73728, 98368, 135168, 196672, 264192, 393280, 458753, 525312, 786496, + 917506, 1049088, 1572928, 1835012, 2097173, 2097187, 2097408, 3014657, 3145739, 3145792, 3670023, 3670024, + 3670029, 3670041, 3670065, 3670113, 3670209, 3670401, 3670785, 3671553, 3673089, 3676161, 3682305, 3694593, + 3719169, 3768321, 3866625, 4079617, 4129025, 4161539, 41, 69, 131, 134, 140, 152, 176, 1664, 1920, 3200, + 6272, 12416, 24704, 49280, 81920, 98432, 139264, 196736, 229377, 266240, 393344, 458754, 526336, 786560, + 917508, 1049600, 1572992, 1835016, 2097189, 2097219, 2097664, 2555905, 3145747, 3145856, 3670032, 4096001, + 4129281, 4161541, 42, 73, 133, 259, 262, 268, 280, 304, 352, 3328, 6400, 12544, 24832, 49408, 63488, 98560, + 114689, 147456, 196864, 229378, 270336, 393472, 458756, 516096, 528384, 786688, 917512, 1050624, 1573120, + 1835024, 2080768, 2097193, 2097221, 2097283, 2098176, 2326529, 3145749, 3145763, 3145984, 3670048, 3932162, + 4129793, 4161545, 74, 81, 137, 261, 515, 518, 524, 536, 560, 608, 704, 960, 6656, 12800, 25088, 49664, 57345, + 98816, 114690, 163840, 197120, 229380, 278528, 393728, 458760, 532480, 786944, 917520, 1052672, 1573376, + 1835040, 1966081, 2097225, 2097285, 2097411, 2099200, 2211841, 3145765, 3145795, 3146240, 3604481, 3670027, + 3670080, 3932164, 4130817, 4161553, 82, 138, 145, 265, 517, 1027, 1030, 1036, 1048, 1072, 1120, 1216, 1408, + 13312, 25600, 28673, 50176, 57346, 99328, 114692, 197632, 229384, 294912, 394240, 458768, 540672, 787456, + 917536, 1056768, 1573888, 1835072, 1966082, 2097233, 2097289, 2097413, 2097667, 2101248, 2154497, 3145769, + 3145797, 3145859, 3146752, 3375105, 3670035, 3670144, 3932167, 3932168, 3932173, 3932185, 3932209, 3932257, + 3932353, 3932545, 3932929, 3933697, 3935233, 3938305, 3944449, 3956737, 3981313, 4030465, 4132865, 4161569, + 4190209, 84, 146, 161, 266, 273, 480, 521, 1029, 2051, 2054, 2060, 2072, 2096, 2144, 2240, 2432, 2816, 14337, + 26624, 28674, 31744, 51200, 57348, 100352, 114696, 198656, 229392, 327680, 395264, 458784, 557056, 788480, + 917568, 983041, 1064960, 1574912, 1835136, 1966084, 2097297, 2097417, 2097669, 2098179, 2105344, 2125825, + 3145801, 3145861, 3145987, 3147776, 3260417, 3670037, 3670051, 3670272, 3932176, 4136961, 4161601, 4177920, + 148, 162, 274, 289, 522, 529, 1033, 2053, 4099, 4102, 4108, 4120, 4144, 4192, 4288, 4480, 4864, 5632, 7169, + 14338, 28676, 53248, 57352, 102400, 114704, 200704, 229408, 258048, 397312, 458816, 589824, 790528, 917632, + 983042, 1081344, 1576960, 1835264, 1966088, 2097313, 2097425, 2097673, 2098181, 2099203, 2111489, 2113536, + 3080193, 3145809, 3145865, 3145989, 3146243, 3149824, 3203073, 3670053, 3670083, 3670528, 3932192, 4145153, + 4161665, 164, 240, 276, 290, 321, 530, 545, 1034, 1041, 2057, 3585, 4101, 7170, 8195, 8198, 8204, 8216, 8240, + 8288, 8384, 8576, 8960, 9728, 11264, 14340, 28680, 57360, 106496, 114720, 204800, 229440, 401408, 458880, + 491521, 655360, 794624, 917760, 983044, 1040384, 1114112, 1581056, 1835520, 1966096, 2097441, 2097681, + 2098185, 2099205, 2101251, 2104321, 2129920, 3145873, 3145993, 3146245, 3146755, 3153920, 3174401, 3670057, + 3670085, 3670147, 3671040, 3899393, 3932171, 3932224, 4063234, 4161793, 168, 292, 322, 532, 546, 577, 1042, + 1057, 1793, 2058, 2065, 3586, 4105, 7172, 8197, 14344, 15872, 16387, 16390, 16396, 16408, 16432, 16480, + 16576, 16768, 17152, 17920, 19456, 22528, 28688, 57376, 114752, 212992, 229504, 409600, 459008, 491522, + 802816, 918016, 983048, 1179648, 1589248, 1836032, 1966112, 2097473, 2097697, 2098193, 2099209, 2100737, + 2101253, 2105347, 2162688, 2588673, 3145889, 3146001, 3146249, 3146757, 3147779, 3160065, 3162112, 3670089, + 3670149, 3670275, 3672064, 3784705, 3932179, 3932288, 4063236, 4162049, 4177923, 120, 296, 897, 1058, 1089, + 2081, 4106, 4113, 8201, 32774, 32780, 33536, 34304, 45056, 491524, 983056, 1605632, 1966144, 2031617, + 2098209, 2098945, 2099217, 2228224, 3146257, 3146761, 3178496, 3670153, 3670277, 3670531, 3932181, 3932416, + 4063329, 4063425, 4069377, 4075521 }; + +static int cbest_23[1023] = { + 1, 2, 4194320, 4, 2097160, 8, 1048580, 16, 524290, 32, 262145, 64, 128, 4325392, 256, 512, 2162696, 1024, + 2048, 1081348, 4096, 8192, 540674, 16384, 32768, 270337, 65536, 131072, 262144, 4329488, 524288, 2164744, + 1048576, 3, 1082372, 2097152, 4194321, 5, 2097161, 4194322, 6, 9, 1048581, 2097162, 4194304, 4194324, + 6291480, 10, 17, 524291, 541186, 1048582, 2097164, 4194328, 5242900, 12, 18, 33, 66, 132, 264, 528, 1056, + 2112, 4224, 8448, 16896, 33792, 67584, 135168, 270336, 3145740, 4718610, 20, 34, 65, 262147, 524294, 540672, + 1048588, 2097176, 2621450, 4194352, 4456465, 24, 36, 129, 262149, 270593, 524298, 1048596, 1081344, 1572870, + 2097192, 2359305, 4194384, 4325393, 40, 68, 130, 257, 262153, 524306, 1048612, 1310725, 2097224, 2162688, + 4194448, 4325394, 48, 72, 258, 513, 262161, 524322, 786435, 1048644, 2097288, 2162697, 4194576, 4325376, + 4325396, 6422552, 80, 136, 260, 514, 1025, 262177, 524354, 1048708, 2097416, 2162698, 4194832, 4325400, + 5373972, 6357016, 96, 144, 516, 1026, 2049, 262209, 524418, 1048836, 1081349, 2097672, 2162700, 4195344, + 4329616, 4849682, 160, 272, 520, 1028, 2050, 4097, 262273, 524546, 1049092, 1081350, 2098184, 3211276, + 4196368, 4325424, 4587537, 5275668, 192, 288, 1032, 2052, 4098, 8193, 262401, 524802, 540675, 1049604, + 2099208, 2162712, 2686986, 3178508, 4198416, 4325456, 320, 544, 1040, 2056, 4100, 8194, 16385, 262657, + 525314, 1050628, 1081356, 2101256, 2162728, 2424841, 4202512, 4325520, 4734994, 384, 576, 2064, 4104, 8196, + 16386, 32769, 263169, 526338, 540678, 1052676, 1081364, 1605638, 2105352, 2162760, 2164808, 2637834, 4210704, + 4325648, 640, 1088, 2080, 4112, 8200, 16388, 32770, 65537, 264193, 270339, 528386, 540682, 1056772, 1081380, + 1343493, 1589254, 2113544, 2162824, 4227088, 4325904, 4464657, 6488088, 768, 1152, 4128, 8208, 16392, 32772, + 65538, 131073, 266241, 270341, 532482, 540690, 1064964, 1081412, 2129928, 2162952, 2367497, 4259856, 4326416, + 1280, 2176, 4160, 8224, 16400, 32776, 65540, 131074, 270345, 540706, 802819, 1081476, 1318917, 2163208, + 4327440, 4329489, 5406740, 1536, 2304, 8256, 16416, 32784, 65544, 131076, 262146, 270353, 278529, 540738, + 557058, 794627, 1081604, 1082404, 1114116, 2163720, 2228232, 4329490, 4456464, 2560, 4352, 8320, 16448, + 32800, 65552, 131080, 262148, 270369, 270592, 294913, 524289, 540802, 541184, 589826, 1081860, 1082368, + 1179652, 2164736, 2359304, 3244044, 4329472, 4329492, 4333584, 4866066, 6426648, 3072, 4608, 16512, 32832, + 65568, 131088, 135296, 262152, 270401, 327681, 540930, 655362, 1310724, 2164745, 2166792, 4329496, 4341776, + 4718608, 5378068, 5120, 8704, 16640, 32896, 65600, 67648, 131104, 262160, 270465, 393217, 524292, 786434, + 1048577, 1083396, 2164746, 2170888, 2621448, 2703370, 4358160, 4595729, 4853778, 6359064, 6144, 9216, 33024, + 33824, 65664, 131136, 262176, 524296, 541202, 541698, 1048578, 1085444, 1572868, 2164748, 2179080, 4329520, + 4390928, 4591633, 5242896, 10240, 16912, 17408, 33280, 65792, 131200, 262208, 270849, 524304, 524352, 542722, + 1082373, 1089540, 1622022, 2097153, 2195464, 2433033, 3145736, 3213324, 4329552, 8456, 12288, 18432, 66048, + 131328, 262272, 271361, 524320, 544770, 786433, 1048584, 1048704, 1082374, 1097732, 2097154, 2164760, + 2689034, 4194323, 4587536, 5276692, 6291472, 7, 4228, 20480, 34816, 66560, 131584, 262400, 272385, 548866, + 1048592, 1351685, 1572866, 2097156, 2097163, 2097408, 2164776, 2293768, 2426889, 3179532, 4194305, 4194325, + 4329744, 6291481, 11, 2114, 24576, 36864, 132096, 262656, 270601, 274433, 524416, 541187, 1048583, 1048608, + 1082380, 1146884, 1310721, 2097165, 2424840, 3145732, 4194306, 4194326, 4194329, 4194816, 4330000, 4849680, + 5242901, 6291482, 6492184, 13, 19, 67, 133, 265, 529, 1057, 2113, 4225, 8449, 16897, 33793, 40960, 67585, + 69632, 133120, 135169, 263168, 524544, 573442, 811011, 1048640, 1082388, 1212420, 1606662, 2097166, 2097168, + 2164872, 2621442, 3145741, 4194308, 4194330, 4330512, 4718611, 4735506, 5242902, 6291464, 6291484, 6490136, + 14, 21, 35, 134, 266, 530, 1058, 4226, 8450, 16898, 33794, 49152, 67586, 73728, 135170, 264192, 270338, + 286721, 524295, 524800, 540673, 541190, 606210, 1048589, 1343492, 1344517, 2097177, 2097184, 2165000, + 2359297, 2621451, 2638346, 2686984, 3145742, 4194312, 4194332, 4194353, 4194386, 4194452, 4194584, 4195376, + 4196432, 4198544, 4202768, 4211216, 4228112, 4261904, 4331536, 4464656, 5242884, 5373968, 5410836, 7340060, + 22, 25, 37, 70, 268, 532, 1060, 2116, 8452, 16900, 33796, 67588, 81920, 135172, 139264, 266240, 270340, + 303105, 524299, 525312, 541194, 671746, 1048590, 1048597, 1048832, 1081345, 1082436, 1572871, 1589766, + 2097178, 2097193, 2097216, 2097226, 2097292, 2097688, 2098216, 2099272, 2101384, 2105608, 2114056, 2130952, + 2165256, 2232328, 2367496, 4194354, 4194385, 4456467, 4718594, 4718614, 4734992, 5242908, 6815770, 26, 38, + 41, 69, 74, 131, 140, 536, 1064, 2120, 4232, 16904, 33800, 67592, 98304, 135176, 147456, 262151, 270344, + 270595, 335873, 524307, 526336, 540676, 802818, 1048598, 1048613, 1048646, 1048844, 1049088, 1049108, + 1049636, 1050692, 1052804, 1057028, 1065476, 1081346, 1082500, 1116164, 1183748, 1318916, 1605636, 2097180, + 2097194, 2097225, 2097280, 2162689, 2165768, 2359307, 2621454, 2637832, 3211272, 4194336, 4194356, 4194449, + 4325395, 4337680, 4456449, 4456469, 4464913, 4718618, 4870162, 5275664, 5407764, 5767190, 6291512, 6422544, + 6553625, 28, 42, 49, 73, 82, 148, 259, 280, 1072, 2128, 4240, 8464, 33808, 67600, 135184, 163840, 262155, + 270352, 270597, 278528, 401409, 524302, 524323, 524422, 524554, 524818, 525346, 526402, 528384, 528514, + 532738, 540680, 541218, 558082, 591874, 659458, 794626, 803331, 1048614, 1048645, 1049600, 1082628, 1310727, + 1589252, 2097196, 2097289, 2162690, 2359309, 2367753, 3145756, 3178504, 3246092, 3670030, 4194360, 4194368, + 4194388, 4194450, 4194577, 4325377, 4325397, 4329620, 4345872, 4456473, 5242932, 5505045, 6291544, 6357008, + 6422553, 44, 50, 81, 98, 137, 164, 261, 296, 515, 560, 2144, 4256, 8480, 16928, 67616, 135200, 196608, + 262157, 262163, 262211, 262277, 262409, 262673, 263201, 264257, 266369, 270368, 279041, 294912, 295937, + 329729, 397313, 524310, 524355, 532480, 540688, 540736, 541250, 1048604, 1048709, 1050624, 1081352, 1081472, + 1082884, 1319173, 1572878, 2097228, 2097290, 2097417, 2097664, 2162692, 2162699, 2162944, 2168840, 2621466, + 3145772, 3245068, 3407885, 4194392, 4194432, 4194578, 4194833, 4325378, 4325398, 4325401, 4325888, 4362256, + 4599825, 4718642, 4980755, 5242964, 5373973, 6291608, 6357017, 6422554, 52, 76, 97, 138, 145, 196, 262, 328, + 517, 592, 1027, 1120, 4288, 8512, 16960, 33856, 135232, 262165, 262179, 270400, 270609, 327680, 524314, + 524326, 524419, 540704, 541314, 786439, 794883, 802817, 1048620, 1048710, 1048837, 1052672, 1081360, 1310733, + 1572886, 1605634, 2097208, 2097418, 2097673, 2098176, 2162701, 2172936, 2359321, 2621482, 2705418, 2883595, + 3145804, 3211268, 4194456, 4194560, 4194580, 4194834, 4195345, 4325380, 4325402, 4329617, 4395024, 4456497, + 4718674, 4849683, 4866578, 5243028, 5373974, 6291736, 6357018, 6422536, 6422556, 56, 84, 146, 161, 194, 273, + 392, 518, 521, 656, 1029, 1184, 2051, 2240, 8576, 17024, 33920, 67712, 262169, 262181, 270464, 270625, + 393216, 524330, 524358, 524547, 541442, 557056, 786443, 794625, 1048628, 1048652, 1048838, 1049093, 1056768, + 1081351, 1081376, 1084420, 1310741, 1343489, 1572902, 1589250, 1835015, 2097240, 2097420, 2097674, 2098185, + 2099200, 2162702, 2162704, 2181128, 2359337, 2621514, 2686978, 3145868, 3178500, 3211277, 4194416, 4194836, + 4195346, 4196369, 4325384, 4325404, 4325425, 4325458, 4325524, 4325656, 4326448, 4327504, 4329618, 4333840, + 4342288, 4359184, 4392976, 4456529, 4595728, 4718738, 5243156, 5275669, 5373956, 6291992, 6357000, 6357020, + 7471132, 162, 193, 289, 1030, 1312, 2053, 4099, 34048, 262185, 262213, 524338, 524362, 524803, 786451, + 1048716, 1049605, 1064960, 1086468, 1310757, 2097256, 2097304, 2097676, 2098186, 2099209, 2101248, 2162713, + 2424833, 2621578, 2703882, 3145996, 3178509, 4194480, 4198417, 4325426, 4456593, 4587539, 4718866, 4849666, + 6292504, 6426776, 7405596 }; + +static int cbest_24[1023] = { + 1, 2, 8388675, 4, 12583010, 8, 6291505, 16, 32, 64, 11534427, 128, 256, 8388674, 4194337, 14155886, 3, 512, + 8388673, 6, 12583011, 5, 1024, 10485843, 12, 6291504, 12583008, 9, 24, 2048, 3145752, 8388679, 10, 7077943, + 14680178, 48, 1572876, 13631594, 17, 4096, 8388683, 96, 786438, 12583014, 20, 6291507, 18, 33, 192, 8192, + 393219, 7340089, 8388691, 6815797, 11534426, 12583018, 384, 131073, 6291489, 6291509, 11927640, 34, 40, 65, + 16384, 262146, 8388707, 8585282, 12582978, 66, 768, 524292, 8388611, 8454211, 11534419, 11534425, 12583026, + 1048584, 6291513, 36, 129, 1536, 32768, 2097168, 4292641, 5767213, 80, 130, 4194336, 6291473, 11796569, + 12615778, 14155882, 132, 3072, 12058719, 12582946, 68, 257, 65536, 5963820, 8388803, 11534411, 11534431, + 14155887, 258, 6144, 196609, 393218, 6307889, 7077941, 160, 260, 8388672, 72, 264, 513, 12288, 131072, + 8388931, 10534995, 11272277, 14155878, 14155884, 786436, 8486979, 10485842, 12583138, 7, 516, 24576, 4194339, + 6291569, 11927641, 136, 320, 514, 520, 1025, 393217, 5242921, 7077939, 7077942, 8388677, 8389187, 8585283, + 12583009, 14418028, 13, 528, 49152, 1572872, 2981910, 8388678, 10485841, 11542619, 12583266, 14286959, + 15466612, 14, 4194341, 6291633, 12632162, 14680179, 25, 144, 1026, 1032, 2049, 98304, 262144, 786434, + 3145753, 8389699, 11534459, 13631595, 11, 26, 640, 1040, 3145744, 6291506, 8388682, 8388687, 11534363, + 11927642, 12583012, 12583522, 12681314, 13656170, 14024809, 28, 49, 1056, 196608, 1572877, 4194345, 6291761, + 7340088, 8388681, 11010135, 12583015, 14680176, 272, 1028, 2050, 4097, 1572868, 3145754, 3538971, 6291457, + 6316081, 6815793, 6815796, 8388699, 8390723, 10485827, 10485847, 13631586, 13631592, 22, 52, 97, 2064, + 786439, 4194305, 6291488, 6291508, 7209014, 8388690, 11534403, 12584034, 14155854, 21, 50, 56, 98, 1280, + 2080, 1490955, 1572878, 3670044, 4194353, 5963821, 6291497, 6292017, 6340657, 7733306, 8388643, 8388723, + 8781888, 9175109, 9961551, 12582914, 12582994, 12583019, 12583022, 14155902, 14159982, 15728762, 19, 193, + 288, 2052, 2112, 4098, 8193, 524288, 3145736, 3145756, 3407898, 4390944, 8392771, 10485851, 11534555, + 11796571, 12583016, 13893736, 14155874, 14352495, 15204470, 15532148, 44, 104, 393216, 2195472, 6291511, + 6291512, 7077927, 7340081, 8388689, 8388706, 8388739, 12582979, 12583034, 12585058, 14680162, 14680182, 112, + 196, 385, 4128, 1097736, 1835022, 4227105, 4292640, 5767209, 5767212, 6292529, 6828085, 7077937, 8388610, + 8388695, 8454210, 11534418, 11534423, 11534424, 12583027, 13631598, 35, 41, 194, 544, 2056, 2560, 4100, 4160, + 8194, 16385, 262147, 548868, 1703949, 2146320, 4259873, 6291472, 6291517, 7340091, 8388609, 8396867, 8519746, + 11534417, 11534683, 11927632, 12976225, 13369444, 14155822, 42, 67, 88, 100, 208, 769, 4224, 274434, 524293, + 1073160, 6291493, 6815799, 6946868, 7077951, 8388705, 8585280, 8650817, 10158158, 11546715, 12058715, + 12582976, 12587106, 14155880, 14286958, 14680186, 38, 134, 224, 386, 392, 770, 131075, 137217, 536580, + 786432, 917511, 1048585, 2883606, 3604507, 4194401, 5963816, 6291491, 6291537, 6293553, 7864381, 8389059, + 8454209, 10502227, 11272279, 11796568, 12583024, 12583074, 12615779, 14155883, 15401079, 37, 70, 268, 576, + 1537, 4104, 8196, 8256, 16386, 32769, 268290, 393223, 1048576, 2097169, 3866653, 6291515, 6307888, 7077911, + 7077940, 7079991, 7340073, 7340093, 7602235, 8388715, 8405059, 10485875, 10518611, 11534429, 11534939, + 11559003, 12058718, 12582947, 12582986, 12583030, 14680146, 81, 131, 176, 416, 536, 5120, 8320, 134145, + 524294, 786446, 2981908, 3153944, 3538970, 5767205, 6684722, 6815781, 8388802, 8389443, 8847424, 8912967, + 9134150, 9240645, 10485779, 11370581, 11534410, 11534430, 11927643, 12582944, 12591202, 12714083, 13631562, + 14024808, 82, 133, 262, 388, 448, 784, 1072, 1540, 3073, 8448, 131077, 1048586, 1441803, 1572892, 1576972, + 3473434, 4194465, 6295601, 7766074, 8388627, 8388801, 9437259, 11272273, 11534443, 11796561, 11927624, + 11927644, 12615776, 12845152, 13631610, 14156014, 14418030, 69, 84, 140, 200, 524, 1088, 1538, 2144, 4112, + 8200, 16388, 32770, 65537, 262150, 393227, 788486, 1490954, 1572864, 2097170, 3145784, 4292643, 4423712, + 5767215, 5898284, 6422576, 8388711, 8390211, 8421443, 8457283, 8585286, 11535451, 12058711, 12058717, + 12582982, 12583394, 13639786, 14696562, 15466614, 15564916, 76, 259, 352, 772, 832, 1048, 4288, 6145, 16512, + 394243, 786454, 3211288, 4194338, 5963812, 6291475, 6291481, 6291568, 6815805, 8388615, 8388930, 8454215, + 10534994, 11272276, 11534409, 11802713, 12599394, 13647978, 14155879, 14155885, 14876787, 74, 161, 261, 266, + 896, 1568, 2096, 3074, 3080, 8576, 10240, 16640, 131081, 655365, 1048588, 1310730, 1572908, 1605644, 1769485, + 2211856, 2621460, 3342361, 4194593, 5079079, 5242920, 6291697, 6299697, 7077938, 7143479, 7340057, 8388807, + 8388929, 8391747, 8455747, 8486978, 8781889, 11534415, 12583139, 12583778, 13107302, 14155876, 14156142, + 14418024, 14680114, 14745714, 73, 162, 265, 280, 1152, 4192, 8208, 12289, 16392, 16896, 17152, 32772, 65538, + 132097, 262154, 393235, 802822, 1736717, 2097152, 2097172, 2981906, 3145816, 3538969, 4243489, 4292645, + 5963822, 6029359, 6291553, 6815765, 7012404, 7209015, 8585290, 8585794, 8716353, 9109574, 10485971, 11536475, + 11928664, 12320861, 12582962, 12583136, 13008993, 13631530, 14024811, 14155894, 14162030, 14286955, 14352494, + 15466608, 164, 518, 704, 776, 1664, 6146, 8384, 34304, 133121, 196611, 401411, 524300, 786437, 786470, + 1105928, 1703948, 2949142, 3145728, 4194340, 6291477, 6291632, 6291889, 6307891, 6819893, 7078007, 7209012, + 7348281, 7733307, 8388619, 8388676, 8389186, 8394819, 8454219, 8454723, 10190926, 10534993, 11534451, + 11796553, 11796573, 12189790, 12584546, 12615782, 14418029, 138, 168, 400, 517, 532, 1792, 3076, 3136, 6160, + 16768, 24577, 33024, 68608, 131089, 135169, 264194, 745477, 851974, 1490953, 1572940, 4194849, 4292897, + 4390945, 5267497, 5636138, 6553651, 6823989, 7733304, 8388835, 8388935, 8455235, 8486977, 10321997, 10485840, + 11534395, 11542618, 12058703, 12582950, 12583106, 12583267, 12617314, 14156398, 14168174, 15466613, 16089208, + 137, 152, 321, 515, 521, 560, 1544, 2176, 8224, 12290, 16400, 20480, 32776, 33280, 33536, 65540, 131074, + 137216, 262162, 266242, 393251, 425987, 552964, 917510, 1572874, 2097176, 3145880, 3407896, 3604506, 4292649, + 4567075, 6291571, 6292273, 7372857, 8388939, 8400963, 8456259, 8519747, 8585298, 8589378, 10486099, 11538523, + 11927633, 12583142, 12583264, 12586082, 12616034, 12632163, 12779619, 12976224, 14155898, 148, 522, 529, + 1036, 1408, 3328, 6148, 33792, 49153, 67072, 196613, 270338, 393222, 458755, 524308, 528388, 786502, 1572873, + 2981911, 3866652, 3883037, 4194343, 4194344, 4554787, 5242913, 6291760, 6307873, 6307893, 7077925, 7077947, + 7078071, 8388867, 8389185, 8389698, 8454227, 8585281, 8781890, 9175111, 9273413, 10158159, 10510419, + 11010131, 11534458, 11534491, 12615786, 12616290, 12616546, 12648546, 12746851, 14073961, 14418020, 14680306, + 15, 324, 1064, 3584, 6272, 12320, 24578, 131105, 134144, 163841, 276482, 393221, 532484, 1048600, 1474571, + 1573004, 1835020, 3145748, 3506202, 4195361, 4292609, 4294689, 5767181, 6291456, 6293041, 6308017, 6316080, + 6324273, 6488112, 6815792, 7077923, 7077949, 8388686, 8389191, 8413251, 8601666, 9257029, 10928208, 11403348, + 11534347, 11534362, 11542611, 11542617, 11927672, 11960408, 12582954, 12583013, 12583523, 12589154, 12616802, + 12681315, 13631722, 13656171, 13893738, 14155846, 14156910, 14286951, 14286957, 15401078, 15532150, 145, 322, + 328, 1027, 1033, 1120, 1552, 2304, 12292, 16416, 32784, 65544, 66048, 98305, 131076, 262145, 262178, 393283, + 540676, 786435, 1056776, 1441802, 2818069, 3146008, 4194304, 4194321, 4194349, 4292657, 5242923, 5767229, + 5771309, 5964332, 6291601, 6291635, 6308145, 6946869, 7209010, 7340153, 7782458, 8388680, 8388685, 8388811, + 8389195, 8470595, 8585314, 8618050, 10059855, 10485835, 10485845, 10486355, 10534979, 10534999, 10584147, + 11010134, 11272261, 11534361, 11665498, 11798617, 11927576, 11927634, 11993176, 12583270, 12615746, 12632160, + 14155790, 14155850, 14155870, 14680177, 15827066, 146, 276, 336, 641, 800, 1034, 2816, 6656, 49154, 66560, + 196617, 197121, 524324, 720901, 786442, 786566, 3158040, 3670040, 5963817, 6291637, 6292016, 6294577, + 6307897, 6308657, 6340656, 6815861, 7078199, 8389203, 8390722, 8454243, 8486983, 8585218, 8847425, 9134151, + 10485846, 10535123, 12583274, 12615794, 13631587, 14159978 }; + +static int cbest_25[1023] = { + 1, 2, 16777220, 4, 8388610, 8, 4194305, 16, 32, 18874372, 64, 128, 9437186, 256, 512, 4718593, 1024, 2048, + 4096, 19136516, 8192, 16384, 32768, 9568258, 65536, 131072, 262144, 4784129, 524288, 1048576, 2097152, + 4194304, 19169284, 8388608, 3, 9584642, 16777216, 16777221, 5, 8388611, 16777222, 6, 9, 18, 36, 72, 144, 288, + 576, 1152, 2304, 4608, 9216, 18432, 36864, 73728, 147456, 294912, 589824, 1179648, 2359296, 4718592, + 25165830, 10, 17, 4194307, 8388614, 9437184, 16777228, 20971525, 12, 33, 4194309, 4792321, 8388618, 12582915, + 16777236, 18874368, 18874373, 20, 34, 65, 4194313, 8388626, 16777252, 18874374, 24, 66, 129, 4194321, + 8388642, 9437187, 16777284, 27262982, 40, 68, 130, 257, 4194337, 8388674, 16777348, 18874380, 23068677, + 26214406, 48, 132, 258, 513, 4194369, 8388738, 9437190, 16777476, 18874388, 80, 136, 260, 514, 1025, 4194433, + 4718595, 8388866, 9437194, 13631491, 16777732, 18874404, 19173380, 21495813, 96, 264, 516, 1026, 2049, + 4194561, 4718597, 8389122, 9437202, 13107203, 16778244, 18874436, 160, 272, 520, 1028, 2050, 4097, 4194817, + 4718601, 4784128, 8389634, 9437218, 9568256, 16779268, 18874500, 19136512, 19136517, 28311558, 192, 528, + 1032, 2052, 4098, 8193, 2392064, 4195329, 4718609, 8390658, 9437250, 16781316, 18874628, 19136518, 320, 544, + 1040, 2056, 4100, 8194, 16385, 1196032, 4196353, 4718625, 8392706, 9437314, 16785412, 18874884, 23592965, + 27525126, 384, 1056, 2064, 4104, 8196, 16386, 32769, 598016, 4198401, 4718657, 8396802, 9437442, 9568259, + 9586690, 16793604, 18875396, 19136524, 23330821, 640, 1088, 2080, 4112, 8200, 16388, 32770, 65537, 299008, + 4202497, 4718721, 8404994, 9437698, 14155779, 16809988, 18876420, 19136532, 26345478, 768, 2112, 4128, 8208, + 16392, 32772, 65538, 131073, 149504, 4210689, 4718849, 8421378, 9438210, 9568262, 16842756, 18878468, + 19136548, 1280, 2176, 4160, 8224, 16400, 32776, 65540, 74752, 131074, 262145, 4227073, 4719105, 8454146, + 9439234, 9568266, 13762563, 16908292, 18882564, 19136580, 1536, 4224, 8256, 16416, 32784, 37376, 65544, + 131076, 262146, 524289, 4259841, 4719617, 4784131, 8519682, 9441282, 9568274, 17039364, 18890756, 19136644, + 21561349, 28573702, 2560, 4352, 8320, 16448, 18688, 32800, 65552, 131080, 262148, 524290, 1048577, 4325377, + 4720641, 4784133, 4793345, 8650754, 9445378, 9568290, 13172739, 17301508, 18907140, 19136772, 28442630, 3072, + 8448, 9344, 16512, 32832, 65568, 131088, 262152, 524292, 1048578, 2097153, 4456449, 4722689, 4784137, + 8912898, 9453570, 9568322, 17825796, 18939908, 19137028, 23855109, 4672, 5120, 8704, 16640, 32896, 65600, + 131104, 262160, 524296, 1048580, 2097154, 4726785, 4784145, 9469954, 9568386, 19005444, 19137540, 19169280, + 19169285, 2336, 6144, 16896, 33024, 65664, 131136, 262176, 524304, 1048584, 2097156, 4194306, 4734977, + 4784161, 5242881, 9502722, 9568514, 9584640, 10485762, 19138564, 19169286, 20971524, 23658501, 1168, 10240, + 17408, 33280, 65792, 131200, 262208, 524320, 1048592, 2097160, 4194308, 4751361, 4784193, 4792320, 6291457, + 8388609, 9568770, 12582914, 14286851, 19140612, 19398660, 27557894, 584, 12288, 33792, 66048, 131328, 262272, + 524352, 1048608, 2097168, 4194312, 4784257, 9569282, 9699330, 14221315, 19144708, 19169292, 19922948, + 23363589, 25165828, 292, 20480, 34816, 66560, 131584, 262400, 524416, 1048640, 2097184, 2396160, 4194320, + 4784385, 4849665, 8388612, 8388624, 9570306, 9584643, 9961474, 16777217, 19152900, 19169300, 19173892, 146, + 24576, 67584, 132096, 262656, 524544, 1048704, 2097216, 4194336, 4784641, 4980737, 8388616, 9572354, + 12582913, 16777218, 16777223, 16777248, 19169316, 23068676, 26361862, 28704774, 7, 19, 37, 73, 145, 289, 577, + 1153, 2305, 4609, 9217, 18433, 36865, 40960, 69632, 73729, 133120, 147457, 263168, 294913, 524800, 589825, + 1048832, 1179649, 1198080, 2097280, 2359297, 4194368, 4785153, 9576450, 9584646, 11534338, 19169348, + 19202052, 25165826, 25165831, 11, 38, 74, 290, 578, 1154, 2306, 4610, 9218, 18434, 36866, 49152, 73730, + 135168, 147458, 264192, 294914, 525312, 589826, 1049088, 1179650, 2097408, 2359298, 4194432, 4718594, + 4786177, 5767169, 8388615, 8388640, 9437185, 9584650, 13631490, 13778947, 16777224, 16777229, 16777238, + 16777292, 16777364, 16777508, 16777796, 16778372, 16779524, 16781828, 16786436, 16795652, 16814084, 16850948, + 16924676, 17072132, 17367044, 17956868, 19169412, 19267588, 20971521, 21495812, 27262980, 28606470, 13, 22, + 76, 148, 580, 1156, 2308, 4612, 9220, 18436, 36868, 73732, 81920, 139264, 147460, 266240, 294916, 526336, + 589828, 599040, 1049600, 1179652, 2097664, 2359300, 4194560, 4718596, 4788225, 6815745, 8388619, 8388646, + 8388672, 8388682, 8388754, 8388898, 8389186, 8389762, 8390914, 8393218, 8397826, 8407042, 8425474, 8462338, + 8536066, 8683522, 8978434, 9584658, 9601026, 10747906, 13107202, 16777230, 16777232, 16777237, 18874369, + 19169540, 20971527, 23920645, 26214404, 14, 21, 26, 35, 44, 152, 296, 1160, 2312, 4616, 9224, 18440, 36872, + 73736, 98304, 147464, 270336, 294920, 528384, 589832, 1050624, 1179656, 2098176, 2359304, 4194311, 4194323, + 4194341, 4194377, 4194449, 4194593, 4194816, 4194881, 4195457, 4196609, 4198913, 4203521, 4212737, 4231169, + 4268033, 4341761, 4489217, 4718600, 4792323, 5373953, 6553601, 8388627, 8388736, 9437188, 9437200, 9584674, + 9633794, 16777253, 18874370, 18874375, 18874400, 19169796, 19660804, 21569541, 23887877, 25165838, 28459014, + 29360135, 25, 52, 67, 88, 304, 592, 2320, 4624, 9232, 18448, 36880, 73744, 147472, 163840, 278528, 294928, + 299520, 532480, 589840, 1052672, 1179664, 2099200, 2359312, 4194315, 4195328, 4718608, 4792325, 4800513, + 8388622, 8388643, 8388864, 9437192, 9584706, 9586946, 12582919, 13180931, 13631489, 16777254, 16777280, + 16777285, 19170308, 20185092, 20971533, 25165846, 27262978, 27262983, 41, 50, 69, 104, 131, 176, 608, 1184, + 4640, 9248, 18464, 36896, 73760, 147488, 196608, 294944, 540672, 589856, 1056768, 1179680, 2101248, 2359328, + 4194317, 4196352, 4718624, 4792329, 4816897, 8388630, 8388675, 8389120, 9584770, 9830402, 12582923, 13107201, + 14352387, 16777244, 16777286, 16777344, 16777349, 18874376, 18874381, 18874390, 18874444, 18874516, 18874660, + 18874948, 18875524, 18876676, 18878980, 18883588, 18892804, 18911236, 18948100, 19021828, 19171332, 19464196, + 20054020, 20971541, 23068673, 23592964, 25165862, 26214402, 26214407, 28, 42, 49, 70, 82, 100, 133, 208, 259, + 352, 1216, 2368, 9280, 18496, 36928, 73792, 147520, 149760, 294976, 327680, 557056, 589888, 1064960, 1179712, + 2105344, 2359360, 4194325, 4194339, 4198400, 4718656, 4792337, 8388634, 8388739, 8389632, 9437191, 9437216, + 9584898, 10092546, 12582931, 16777260, 16777350, 16777472, 16777477, 18874382, 18874384, 18874389, 20971557, + 23068679, 23330820, 25165894, 28311556, 81, 134, 137, 164, 200, 261, 416, 515, 704, 2432, 4736, 18560, 36992, + 73856, 147584, 295040, 393216, 589952, 1081344, 1179776, 2113536, 2359424, 4194329, 4194371, 4202496, + 4718720, 4792353, 4915201, 8388650, 8388678, 8388867, 8390656, 9437195, 9437222, 9437248, 9437258, 9437330, + 9437474, 9437762, 9438338, 9439490, 9441794, 9446402, 9455618, 9474050, 9510914, 9585154, 9732098, 10027010, + 11796482, 12582947, 14155778, 14303235, 16777268, 16777478, 16777728, 16777733, 18874405, 19173376, 19173381, + 19177476, 20971589, 21495809, 23666693, 25165958, 27262990, 31457287, 97, 138, 262, 265, 274, 328, 400, 517, + 832, 1027, 1408, 4864, 9472, 37120, 73984, 74880, 147712, 295168, 590080, 655360, 1114112, 1179904, 2129920, + 2359552, 4194345, 4194373, 4194435, 4210688, 4718848, 4792385, 5046273, 8388658, 8388742, 8389123, 8392704, + 9437203, 9437312, 9585666, 11665410, 12582979, 16777300, 16777356, 16777734, 16778240, 16778245, 18874406, + 18874432, 18874437, 19173382, 19185668, 20971653, 21495815, 23068685, 25166086, 26214414, 27262998, 27525124, + 30408711, 56, 84, 98, 140, 161, 266, 518, 521, 530, 656, 800, 1029, 1664, 2051, 9728, 18944, 74240, 147968, + 295424, 590336, 786432, 1180160, 2162688, 2359808, 4194353, 4194437, 4194563, 4227072, 4718599, 4718611, + 4718629, 4718665, 4718737, 4718881, 4719104, 4719169, 4720897, 4723201, 4727809, 4737025, 4755457, 4792449, + 4866049, 5013505, 5898241, 7077889, 8388690, 8388870, 8389635, 8396800, 9437198, 9437219, 9437440, 9568257, + 12583043, 13631495, 13762562, 14229507, 16777316, 16777484, 16778246, 16779264, 16779269, 18874396, 18874438, + 18874496, 18874501, 19136513, 20971781, 23068693, 25166342, 26214422, 27263014, 27561990, 28311554, 28311559, + 28737542, 29884423 }; + +static int cbest_26[1023] = { + 1, 2, 33554467, 4, 50331698, 8, 25165849, 16, 32, 64, 46137391, 128, 256, 56623156, 512, 33554466, 16777233, + 3, 1024, 28311578, 33554465, 6, 50331699, 5, 41943083, 12, 2048, 25165848, 50331696, 9, 24, 12582924, + 33554471, 10, 4096, 14155789, 58720314, 48, 6291462, 54526006, 17, 33554475, 96, 8192, 3145731, 50331702, 20, + 25165841, 25165851, 18, 33, 192, 29360157, 33554483, 50331682, 34, 16384, 27263003, 33554435, 35127330, + 46137387, 46137390, 50331706, 384, 25165853, 40632357, 40, 65, 66, 768, 32768, 17563665, 25165833, 46137389, + 56623158, 68, 1048577, 50331666, 36, 129, 1536, 2097154, 23068695, 33554531, 46137383, 48234541, 80, 130, + 65536, 4194308, 28311579, 34078755, 47185966, 56623152, 56623157, 132, 3072, 8388616, 28311576, 136, 257, + 14155788, 16777232, 33554595, 6144, 131072, 42336299, 45088808, 50331762, 50593842, 53870641, 160, 260, + 25165881, 72, 258, 264, 513, 12288, 1572865, 3145730, 33554723, 272, 262144, 7077894, 23592983, 25296921, + 33554464, 50331826, 57671733, 24576, 22544404, 25165913, 47710254, 61866041, 320, 514, 520, 1025, 6291460, + 33554979, 34340899, 41943082, 46137407, 7, 528, 49152, 524288, 16777235, 46137359, 50331954, 544, 3145729, + 3538947, 20971541, 25165977, 33554469, 35127331, 50331697, 54722614, 56623140, 13, 144, 516, 1026, 2049, + 98304, 11272202, 12582920, 28311570, 33554470, 33555491, 41943081, 45350952, 56623164, 14, 640, 1040, + 1048576, 14155785, 16777237, 46202927, 50332210, 50724914, 58720315, 25, 1056, 196608, 6291458, 12582925, + 25166105, 27263001, 41943075, 44040233, 54526002, 54526007, 11, 26, 1028, 1088, 2050, 4097, 16777217, + 23855127, 25165825, 25165840, 25165850, 28311582, 33554474, 33554479, 33556515, 46137455, 50331700, 51118130, + 60489787, 62390329, 28, 49, 393216, 5636101, 6291463, 16777241, 29360156, 33554473, 35323938, 46137379, + 50331703, 50332722, 56623124, 58720312, 50, 1280, 2080, 12582916, 12582926, 22675476, 25165845, 25166361, + 25362457, 27263002, 28311562, 33554451, 33554491, 36700192, 39845925, 40632359, 41943087, 47185967, 50331650, + 50331690, 54526004, 22, 52, 97, 288, 1032, 2052, 2112, 4098, 8193, 786432, 14155781, 14155791, 18350096, + 25165852, 27361307, 29360153, 33554482, 33558563, 46137385, 46137519, 50331683, 56623154, 56655924, 58720306, + 60817464, 64487487, 21, 56, 2176, 2097152, 9175048, 14680078, 23068693, 25559065, 33554434, 33554499, + 46137386, 50331707, 50331710, 50333746, 62914622, 19, 100, 193, 1572864, 4587524, 13631501, 17563664, + 25165832, 25165843, 25166873, 28311577, 33554433, 40632353, 40632356, 45088809, 50331704, 53870640, 55574583, + 35, 44, 98, 104, 2056, 2560, 4100, 4160, 8194, 16385, 2293762, 8781832, 17661969, 25165855, 33554481, + 33562659, 36372513, 45482024, 46137647, 48234543, 50331680, 53477425, 56623220, 58720318, 42, 70, 112, 385, + 4224, 1146881, 4390916, 7340039, 11337738, 16777265, 23068694, 30408732, 33554487, 35127328, 46137388, + 50331686, 50335794, 56623159, 57147444, 38, 41, 140, 194, 200, 386, 4352, 2195458, 25165865, 25167897, + 28327962, 29360149, 29360159, 33554659, 41943099, 50331667, 50331730, 57671732, 58720298, 67, 88, 208, 280, + 576, 769, 2064, 4104, 8196, 16386, 32769, 1097729, 3145735, 23068691, 27262995, 28311610, 33554443, 33554530, + 33570851, 37224480, 40370213, 41943051, 46137382, 46137903, 46235695, 47710255, 48234537, 48234540, 50331664, + 54525990, 56623284, 61866040, 69, 84, 134, 224, 560, 5120, 8320, 3145728, 3538946, 4194304, 6291470, + 11534347, 16777297, 17039377, 20316178, 31457311, 33554529, 33554851, 34078754, 45088810, 46137381, 47185962, + 50339890, 53870643, 54526014, 56623153, 61341752, 37, 76, 196, 268, 400, 772, 1120, 1537, 8448, 2097155, + 7077892, 12582940, 15204366, 17301521, 18612240, 22544405, 23592981, 25165837, 25169945, 26935320, 34127907, + 34603042, 46137399, 46333999, 47235118, 64749631, 81, 131, 176, 416, 536, 770, 2240, 4112, 8200, 8704, 16388, + 32770, 65537, 1769473, 3145739, 4194309, 5668869, 14155784, 14155805, 17563667, 22741012, 25165835, 25165880, + 27263007, 28311642, 28573722, 33554439, 33554594, 33555235, 33587235, 35127334, 35651617, 46138415, 50331890, + 56098871, 56623412, 63799358, 65798204, 82, 133, 138, 168, 388, 448, 1072, 3073, 4480, 1048579, 6291478, + 8388617, 9306120, 14163981, 16777361, 24117270, 25165873, 28835866, 29360141, 33554535, 33554593, 34078753, + 34103331, 42074155, 42336298, 45350953, 47185964, 50331674, 50331763, 50348082, 50593843, 56623160, 57671735, + 58720282, 60293179, 137, 152, 800, 1538, 1544, 2144, 8960, 10240, 16640, 6291456, 12582956, 23592982, + 25165945, 25174041, 25296920, 27262987, 28311568, 30933020, 33556003, 39321638, 41943147, 42205227, 42385451, + 46137403, 48234533, 50331760, 50332082, 51740723, 53739569, 54525974, 61866043, 262, 352, 832, 1152, 4128, + 4288, 6145, 8208, 16392, 16896, 17920, 32772, 65538, 131073, 3145747, 3538945, 4194310, 4653060, 5636100, + 7602183, 10158089, 12648460, 14155821, 17563669, 25165912, 28311571, 28311580, 28311706, 30670876, 33554547, + 33554722, 33620003, 35127338, 37748775, 46139439, 47185958, 49283116, 50331746, 51380275, 56623668, 56672308, + 57409588, 74, 161, 261, 276, 336, 392, 896, 3074, 8576, 17408, 35840, 1048581, 6291494, 6324230, 7077890, + 8388608, 8388618, 11272200, 16777489, 23592979, 25166041, 33554599, 33557539, 40632365, 42336297, 44826665, + 50331670, 50331827, 50332466, 50364466, 50593840, 50618418, 52428848, 56623136, 56623166, 57671729, 60489786, + 62390328, 73, 259, 265, 304, 1540, 1600, 3088, 12289, 17152, 71680, 2097158, 2326530, 2818050, 3162115, + 12058635, 12582988, 13467660, 14155780, 14155790, 14286861, 16777234, 17563649, 21168149, 22544400, 25165883, + 25182233, 26214424, 33554563, 33554603, 34439203, 35323939, 41943211, 46137375, 47710250, 50331766, 50331824, + 54591542, 56623142, 56721460, 58851386, 164, 266, 273, 524, 704, 776, 1664, 6146, 8224, 16400, 20480, 32776, + 33280, 34304, 65540, 131074, 143360, 262145, 3145763, 5242885, 7077895, 10485770, 11370506, 11796491, + 12582912, 13107212, 14155853, 14417933, 17563673, 19660819, 20971540, 25165976, 25166233, 28311583, 28311834, + 33554721, 33554978, 33560611, 33685539, 34078759, 34340898, 35127346, 36388897, 36700193, 37355552, 42336291, + 45088800, 45350954, 46137351, 46137406, 46141487, 50333234, 50606130, 54657078, 56624180, 58720378, 61603896, + 162, 552, 672, 1792, 3076, 24577, 33792, 68608, 286720, 1048585, 1163265, 1409025, 6291526, 6553606, 8388620, + 13631500, 14155777, 15466510, 16777745, 17170449, 23068679, 23617559, 23855125, 25165897, 25165969, 28311554, + 33554727, 35127298, 35135522, 36175905, 40632373, 40636453, 46137355, 46137358, 46137423, 50331955, 50397234, + 54526070, 56623108, 56623148, 59244602, 64487486, 321, 515, 521, 608, 3200, 6176, 12290, 34816, 137216, + 573440, 1572867, 2097162, 3276803, 6291461, 6815750, 12583052, 15335438, 16777236, 20971537, 22675477, + 23068703, 24641558, 25165915, 25166617, 25198617, 28311563, 29360189, 33554468, 33554539, 33554731, 33566755, + 40632325, 41943339, 44040235, 46137357, 46137451, 47710252, 50331830, 50334770, 54722610, 54722615, 56623126, + 56623141, 148, 274, 322, 522, 529, 784, 1048, 1408, 2304, 3328, 6148, 8256, 16416, 32784, 49153, 65544, + 131076, 262146, 274432, 524289, 1146880, 3145795, 3407875, 4194316, 7340038, 14155917, 17567761, 18350097, + 18677776, 25165917, 25166104, 25296913, 25296923, 25309209, 27263000, 27263035, 27295771, 28311560, 28311574, + 28312090, 28336154, 28704794, 29425693, 32899102, 33554739, 33554977, 33555490, 33816611, 34078763, 34340897, + 37093408, 41943080, 42336303, 45088812, 46137405, 46145583, 50331794, 50331834, 50331938, 50593846, 53870645, + 55574582, 56229943, 56623165, 56625204, 58720442, 62390331, 63701054, 65929276, 518, 545, 1104, 1344, 3080, + 3584, 24578, 40960, 66560, 548864, 1048593, 3670019, 6291590, 12582922, 16777216, 16778257, 22544406, + 23855126, 25165824, 25165885, 25165905, 25166097, 25167385, 27328539, 27361305, 31195164, 33579043, 35192866, + 39583782, 40632355, 44138537, 45482025, 46202923, 46202926, 47710246, 50331952, 50332211, 50337842, 50462770, + 50724915, 51904563, 53477424, 54526000, 54526134, 60424251, 61866033, 145, 328, 517, 532, 546, 1027, 1216, + 1552, 6400, 12292, 12352, 67584, 98305, 1097728, 1572869, 2097170, 3145734, 6733830, 11272203, 12582921, + 12583180, 14155787, 14680076, 16777225, 16777239, 16777240, 25165979, 25231385, 25303065, 28360730, 29360221, + 29622301, 33554787, 33554947, 33554987, 35127329, 35258402, 36700194, 38633511, 39452710, 39845927, 40697893, + 41943074, 41943079, 41943595, 42139691, 43384874, 44040232, 46137515, 46137583, 48234557, 50331958, 50593826, + 50774066, 51216434, 51642419, 53870625, 54526003, 54722612, 54747190, 59424826, 15, 641, 6152, 6656, 49154, + 69632, 131080, 4194324, 11534346, 14156045, 17596433, 25168921, 25296925, 25362456, 27263067, 33556514, + 35389474, 41943073, 50331701, 50331770, 50332194, 50343986, 50593850, 60817466 }; + +static int cbest_27[1023] = { + 1, 2, 67108883, 4, 100663322, 8, 50331661, 16, 32, 92274709, 64, 128, 113246233, 256, 512, 67108882, + 33554441, 3, 1024, 67108881, 123731999, 6, 100663323, 5, 2048, 83886103, 12, 50331660, 100663320, 9, 24, + 4096, 25165830, 67108887, 109051928, 10, 117440542, 48, 12582915, 50331657, 17, 8192, 54525964, 67108891, + 100663314, 128974876, 18, 96, 67108867, 92274711, 100663326, 20, 50331663, 73400338, 92274708, 113246232, 33, + 192, 16384, 27262982, 58720271, 34, 50331653, 92274705, 36, 384, 36700169, 46137354, 100663306, 40, 65, + 32768, 13631491, 56623116, 67108915, 66, 768, 96469012, 113246235, 68, 23068677, 64487438, 72, 129, 1536, + 65536, 67108947, 80, 4194305, 28311558, 73924626, 100663354, 132, 3072, 8388610, 48234506, 50331677, + 85458967, 123731998, 130, 136, 257, 131072, 16777220, 67109011, 69206035, 78643217, 144, 6144, 33554440, + 92274717, 100663386, 160, 14155779, 36962313, 50331693, 92274693, 258, 264, 513, 12288, 262144, 24117253, + 67109139, 101711898, 113246225, 272, 6291457, 12582914, 32243719, 100663450, 109838360, 113246237, 288, + 24576, 50331725, 67108880, 260, 320, 514, 1025, 524288, 50855949, 61865999, 67109395, 528, 49152, 25165828, + 70254611, 74186770, 83886102, 92274741, 100663578, 106430491, 123731995, 7, 544, 33554443, 50331789, + 79167505, 113246217, 123731997, 516, 576, 1026, 2049, 98304, 1048576, 12582913, 41943051, 54525965, 54919180, + 67108885, 67109907, 73400339, 83886099, 85590039, 100663321, 109051930, 13, 640, 33554433, 50331656, + 67108886, 83886101, 92274773, 92536853, 100663834, 109051929, 14, 26, 1056, 196608, 13631490, 33554445, + 50331649, 50331917, 83230736, 102236186, 117440543, 25, 52, 520, 1028, 1088, 2050, 4097, 2097152, 6815745, + 25165826, 25165831, 37093385, 67110931, 88080406, 92274707, 98041876, 123731991, 128974878, 11, 104, 1152, + 393216, 23068676, 27262980, 50331659, 50331662, 58720269, 67108875, 67108890, 67108895, 79691792, 92274837, + 94371861, 100663298, 100663315, 100663318, 100663324, 100664346, 103809050, 113246265, 117440538, 128974877, + 22, 28, 49, 208, 1280, 11534338, 27459590, 39845896, 46137355, 50332173, 58720270, 67108866, 67108889, + 92274710, 96469013, 100663312, 100663327, 113246234, 117440540, 44, 416, 1032, 2052, 2112, 4098, 8193, + 786432, 4194304, 5767169, 19922948, 44040203, 46137352, 50331652, 51118093, 54525960, 56623117, 67108865, + 67108899, 67112979, 109903896, 113377305, 120324126, 125829148, 128974872, 19, 88, 97, 832, 2176, 9961474, + 13631489, 36700168, 70516755, 92274965, 100665370, 106692635, 113246297, 121634847, 21, 38, 50, 56, 176, + 1664, 2304, 1572864, 4980737, 18350084, 29360135, 33554457, 50332685, 51904525, 64487439, 73924627, 92274704, + 109051920, 123731983, 76, 193, 352, 1040, 2056, 2560, 3328, 4100, 8194, 16385, 9175042, 27262978, 27262983, + 41615368, 49020938, 54525966, 58720267, 64487436, 67108871, 67117075, 73400336, 83886111, 90177558, + 100663307, 109051932, 114294809, 117440534, 35, 98, 152, 194, 704, 4224, 6656, 3145728, 4587521, 13729795, + 14155778, 50331669, 62914574, 67108914, 67108979, 69992467, 81788944, 83886087, 89128982, 92275221, 96469014, + 100663304, 100663338, 100667418, 113246361, 115343385, 37, 42, 70, 112, 304, 385, 1408, 4352, 13312, + 12582919, 24117252, 28311556, 33554473, 48234507, 50331655, 50333709, 54525956, 67108913, 80740368, 85655575, + 92274713, 92667925, 95420437, 128974868, 41, 140, 608, 2064, 2816, 4104, 4608, 8196, 16386, 26624, 32769, + 6291456, 7077889, 8388608, 25165838, 32243718, 40894472, 48234504, 54951948, 56623112, 67109075, 67125267, + 69599251, 102367258, 127926300, 67, 100, 280, 388, 769, 1216, 5120, 5632, 53248, 12058626, 40370184, + 45088779, 50331673, 50331676, 67108946, 92275733, 93061141, 96469008, 100663310, 100671514, 104071194, + 109051912, 113246224, 113246489, 123732031, 131596317, 69, 74, 84, 224, 386, 560, 2432, 8448, 11264, 106496, + 12582923, 20447236, 23068673, 31457287, 33554505, 36700171, 44564491, 50335757, 58720263, 60162063, 64487434, + 67108919, 67108945, 67109267, 69402643, 73400342, 92274719, 100663355, 100663418, 113246236, 117440526, + 123797535, 73, 196, 1120, 1537, 2080, 4112, 4864, 8200, 8704, 16388, 22528, 32770, 65537, 212992, 6029313, + 14155777, 20185092, 24510469, 25165846, 56623118, 67108923, 67141651, 78643219, 83886135, 85458966, 92274695, + 99352596, 100663346, 100663352, 102105114, 113246239, 81, 134, 770, 776, 2240, 9216, 9728, 45056, 425984, + 10223618, 20807684, 28311554, 28311559, 34603017, 36700161, 36962312, 46137346, 50331689, 50331692, 50331709, + 67109010, 67109651, 69206034, 73973778, 78643216, 91226134, 92274689, 92276757, 100663514, 100679706, + 113246227, 113246745, 120455198, 123732063, 128974860, 133, 148, 168, 448, 3073, 4480, 10240, 19456, 90112, + 851968, 8388611, 10092546, 12582912, 12582931, 23068679, 32243717, 33554569, 35651593, 36700173, 46137358, + 50339853, 51183629, 56623108, 63963150, 67108931, 67108951, 70647827, 71303186, 73400346, 73924624, 74186771, + 82837520, 85458963, 92274701, 92274716, 100663387, 101908506, 106430490, 113442841, 114819097, 123731994, + 124256287, 131, 137, 200, 1538, 4128, 8208, 8960, 16392, 16896, 32772, 38912, 65538, 131073, 180224, 1703936, + 5111809, 13631495, 16121859, 16777216, 16777221, 18481156, 25165862, 27262990, 27475974, 39321608, 50331679, + 50331757, 52035597, 54525980, 67108955, 67110419, 67174419, 73400322, 75497489, 83886167, 85458965, 89653270, + 92274692, 100663358, 100663378, 100663384, 100663706, 109051960, 109936664, 113246216, 123731996, 124780575, + 82, 138, 145, 268, 772, 1552, 6145, 17408, 17920, 77824, 360448, 3407872, 4194307, 5046273, 42729483, + 50331685, 50331721, 50331724, 61865997, 64487430, 67109009, 67109138, 69206033, 81264656, 84410391, 89391126, + 92274725, 92278805, 100696090, 101711899, 101810202, 109838362, 113246209, 113246229, 113247257, 113639449, + 117440574, 123732127, 146, 161, 296, 336, 392, 896, 3074, 18432, 35840, 155648, 720896, 6815744, 9240578, + 12582947, 19660804, 33554697, 41418760, 50331853, 50348045, 50855948, 51052557, 61865998, 67108995, 67109015, + 67111955, 70123539, 84934679, 92274743, 100663451, 100664090, 109838361, 162, 259, 265, 1540, 4160, 8224, + 12289, 16400, 20480, 32776, 65540, 71680, 131074, 262145, 311296, 1441792, 13631499, 16777222, 24117249, + 25165824, 25165894, 25427974, 27262998, 45613067, 50331695, 54525996, 54919181, 58720287, 67109019, 67239955, + 83230737, 83886231, 92274737, 97189908, 100663370, 100663390, 100663442, 104857627, 109051992, 109314072, + 113246219, 128974908, 129007644, 266, 273, 536, 3104, 6146, 33792, 143360, 622592, 2883584, 4194309, 4620289, + 9830402, 12713987, 31981575, 32243715, 33554442, 36986889, 40632328, 41943049, 48234498, 49676298, 50331717, + 50331785, 50331788, 50332045, 50954253, 53215245, 67109137, 67109394, 67115027, 73924630, 79364113, 88080407, + 92282901, 96469020, 98041877, 100664858, 100728858, 101711896, 109576216, 113248281, 117440606, 123731987, + 123731990, 123731993, 123732255, 262, 274, 289, 400, 592, 672, 1792, 3076, 24577, 34816, 286720, 1245184, + 5767168, 8388614, 10403842, 12582979, 20709380, 20971525, 23068685, 33554953, 36700185, 36962315, 41943050, + 44826635, 46137370, 48234510, 50364429, 54657036, 67109043, 67109123, 69664787, 70254610, 73400370, 79167507, + 79691793, 92274740, 92274775, 92274805, 92700693, 96468996, 100663448, 100663579, 102432794, 104202266, + 113246264, 117964830, 132907037, 261, 276, 290, 321, 515, 1544, 8256, 12290, 16416, 32784, 36864, 65544, + 131076, 262146, 524289, 573440, 2490368, 4915201, 13631488, 13631507, 13737987, 24117255, 25165958, 27263014, + 33554432, 35127305, 44695563, 50331727, 50332429, 50855945, 50905101, 54526028, 54788108, 56623132, 58720303, + 61865995, 67109147, 67121171, 67371027, 69206039, 69632019, 73465874, 77594641, 78643221, 79167504, 83886097, + 83886359, 88473622, 92274769, 93126677, 100663434, 100663454, 100663570, 100666394, 109052056, 111149080, + 128974940, 129237020, 292, 322, 529, 784, 1072, 6148, 6208, 40960, 49153, 1146880, 4194313, 4980736, 6291459, + 20316164, 25165829, 27328518, 28311566, 29360134, 30932999, 33554444, 36962305, 50331648, 50331781, 50331916, + 54968332, 60227599, 67108884, 67109393, 67109906, 73924634, 75169810, 83886098, 85590038, 92274901, 92291093, + 92536855, 100794394, 109051931, 113246249, 113250329, 113377304, 117440670, 119537694, 123732511, 129499164, + 164, 324, 518, 530, 545, 1184, 1344, 3080, 3584, 24578, 67584, 2293760, 8388618, 10354690, 12583043, + 14155783, 14680067, 23068693, 27394054, 33554437, 33555465, 36700201, 36732937, 36962317, 37093384, 39845897, + 46137386, 50333197, 50397197, 55574540, 64487454, 67109171, 67109379, 67133459, 70254609, 73400402, 73924610, + 74186768, 83099664, 83230738, 83886100, 85458975, 92274772, 92536852, 92635157, 100663576, 100663835, + 100669466, 101711890, 102170650, 106430489, 106692634, 113246267, 113246296, 121634846, 123830303, 124518431, + 128974874, 517, 532, 16448, 69632, 524290, 1048577, 4587520, 27262976, 46137353, 50331658, 50331741, + 54525961, 56623148, 58720268, 58982415, 67109143, 67109203, 67633171, 78643225, 100663482, 102236187, + 106954779, 123731982 }; + +static int cbest_28[1023] = { + 1, 2, 134217732, 4, 67108866, 8, 33554433, 16, 32, 150994948, 64, 128, 75497474, 256, 512, 37748737, 1024, + 2048, 4096, 153092100, 8192, 16384, 32768, 76546050, 65536, 131072, 262144, 38273025, 524288, 1048576, + 2097152, 4194304, 153354244, 8388608, 16777216, 33554432, 76677122, 67108864, 3, 134217728, 134217733, 5, + 38338561, 67108867, 134217734, 6, 9, 18, 36, 72, 144, 288, 576, 1152, 2304, 4608, 9216, 18432, 36864, 73728, + 147456, 294912, 589824, 1179648, 2359296, 4718592, 9437184, 18874368, 37748736, 201326598, 10, 17, 33554435, + 67108870, 75497472, 134217740, 167772165, 12, 33, 33554437, 67108874, 100663299, 134217748, 150994944, + 150994949, 20, 34, 65, 33554441, 67108882, 134217764, 150994950, 24, 66, 129, 33554449, 67108898, 75497475, + 134217796, 153387012, 218103814, 40, 68, 130, 257, 33554465, 67108930, 134217860, 150994956, 184549381, + 209715206, 48, 132, 258, 513, 33554497, 67108994, 75497478, 134217988, 150994964, 80, 136, 260, 514, 1025, + 33554561, 37748739, 67109122, 75497482, 109051907, 134218244, 150994980, 171966469, 96, 264, 516, 1026, 2049, + 33554689, 37748741, 67109378, 75497490, 104857603, 134218756, 150995012, 160, 272, 520, 1028, 2050, 4097, + 33554945, 37748745, 38273024, 67109890, 75497506, 76546048, 76693506, 134219780, 150995076, 153092096, + 153092101, 226492422, 192, 528, 1032, 2052, 4098, 8193, 19136512, 33555457, 37748753, 67110914, 75497538, + 134221828, 150995204, 153092102, 320, 544, 1040, 2056, 4100, 8194, 16385, 9568256, 33556481, 37748769, + 67112962, 75497602, 134225924, 150995460, 188743685, 220200966, 384, 1056, 2064, 4104, 8196, 16386, 32769, + 4784128, 33558529, 37748801, 67117058, 75497730, 76546051, 134234116, 150995972, 153092108, 186646533, 640, + 1088, 2080, 4112, 8200, 16388, 32770, 65537, 2392064, 33562625, 37748865, 67125250, 75497986, 113246211, + 134250500, 150996996, 153092116, 210763782, 768, 2112, 4128, 8208, 16392, 32772, 65538, 131073, 1196032, + 33570817, 37748993, 38346753, 67141634, 75498498, 76546054, 134283268, 150999044, 153092132, 1280, 2176, + 4160, 8224, 16400, 32776, 65540, 131074, 262145, 598016, 33587201, 37749249, 67174402, 75499522, 76546058, + 110100483, 134348804, 151003140, 153092164, 1536, 4224, 8256, 16416, 32784, 65544, 131076, 262146, 299008, + 524289, 33619969, 37749761, 38273027, 67239938, 75501570, 76546066, 134479876, 151011332, 153092228, + 172490757, 228589574, 2560, 4352, 8320, 16448, 32800, 65552, 131080, 149504, 262148, 524290, 1048577, + 33685505, 37750785, 38273029, 67371010, 75505666, 76546082, 105381891, 134742020, 151027716, 153092356, + 227540998, 3072, 8448, 16512, 32832, 65568, 74752, 131088, 262152, 524292, 1048578, 2097153, 33816577, + 37752833, 38273033, 67633154, 75513858, 76546114, 135266308, 151060484, 153092612, 190840837, 5120, 8704, + 16640, 32896, 37376, 65600, 131104, 262160, 524296, 1048580, 2097154, 4194305, 34078721, 37756929, 38273041, + 68157442, 75530242, 76546178, 136314884, 151126020, 153093124, 153354240, 153354245, 6144, 16896, 18688, + 33024, 65664, 131136, 262176, 524304, 1048584, 2097156, 4194306, 8388609, 34603009, 37765121, 38273057, + 69206018, 75563010, 76546306, 76677120, 138412036, 151257092, 153094148, 153354246, 153391108, 189267973, + 9344, 10240, 17408, 33280, 65792, 131200, 262208, 524320, 1048592, 2097160, 4194308, 8388610, 16777217, + 35651585, 37781505, 38273089, 38338560, 71303170, 75628546, 76546562, 114294787, 142606340, 151519236, + 153096196, 220463110, 4672, 12288, 33792, 66048, 131328, 262272, 524352, 1048608, 2097168, 4194312, 8388612, + 16777218, 37814273, 38273153, 75759618, 76547074, 113770499, 152043524, 153100292, 153354252, 186908677, + 2336, 20480, 34816, 66560, 131584, 262400, 524416, 1048640, 2097184, 4194320, 8388616, 16777220, 19169280, + 33554434, 37879809, 38273281, 41943041, 76021762, 76548098, 76677123, 83886082, 153108484, 153354260, + 167772164, 1168, 24576, 67584, 132096, 262656, 524544, 1048704, 2097216, 4194336, 8388624, 16777224, + 33554436, 38010881, 38273537, 50331649, 67108865, 76550146, 100663298, 153124868, 153354276, 155189252, + 210894854, 229638150, 584, 40960, 69632, 133120, 263168, 524800, 1048832, 2097280, 4194368, 8388640, 9584640, + 16777232, 33554440, 38274049, 76554242, 76677126, 77594626, 153157636, 153354308, 159383556, 201326596, 292, + 49152, 135168, 264192, 525312, 1049088, 2097408, 4194432, 8388672, 16777248, 33554448, 38275073, 38797313, + 67108868, 67108880, 76562434, 76677130, 76695554, 79691778, 110231555, 134217729, 153223172, 153354372, + 228851718, 146, 81920, 139264, 266240, 526336, 1049600, 2097664, 4194560, 4792320, 8388736, 16777280, + 33554464, 38277121, 39845889, 67108872, 76578818, 76677138, 100663297, 134217730, 134217735, 134217760, + 153354500, 184549380, 191365125, 7, 19, 37, 73, 145, 289, 577, 1153, 2305, 4609, 9217, 18433, 36865, 73729, + 98304, 147457, 270336, 294913, 528384, 589825, 1050624, 1179649, 2098176, 2359297, 4194816, 4718593, 8388864, + 9437185, 16777344, 18874369, 33554496, 38281217, 38338563, 76611586, 76677154, 92274690, 153354756, + 153616388, 172556293, 191102981, 201326594, 201326599, 227672070, 11, 38, 74, 290, 578, 1154, 2306, 4610, + 9218, 18434, 36866, 73730, 147458, 163840, 278528, 294914, 532480, 589826, 1052672, 1179650, 2099200, + 2359298, 2396160, 4195328, 4718594, 8389120, 9437186, 16777472, 18874370, 33554560, 37748738, 38289409, + 38338565, 46137345, 67108871, 67108896, 75497473, 76677186, 105447427, 109051906, 134217736, 134217741, + 134217750, 134217804, 134217876, 134218020, 134218308, 134218884, 134220036, 134222340, 134226948, 134236164, + 134254596, 134291460, 134365188, 134512644, 134807556, 135397380, 136577028, 138936324, 143654916, 153355268, + 154140676, 167772161, 171966468, 218103812, 13, 22, 76, 148, 580, 1156, 2308, 4612, 9220, 18436, 36868, + 73732, 147460, 196608, 294916, 540672, 589828, 1056768, 1179652, 2101248, 2359300, 4196352, 4718596, 8389632, + 9437188, 16777728, 18874372, 33554688, 37748740, 38305793, 38338569, 54525953, 67108875, 67108902, 67108928, + 67108938, 67109010, 67109154, 67109442, 67110018, 67111170, 67113474, 67118082, 67127298, 67145730, 67182594, + 67256322, 67403778, 67698690, 68288514, 69468162, 71827458, 76677250, 76808194, 85983234, 104857602, + 114819075, 134217742, 134217744, 134217749, 150994945, 153356292, 167772167, 209715204, 14, 21, 26, 35, 44, + 152, 296, 1160, 2312, 4616, 9224, 18440, 36872, 73736, 147464, 294920, 327680, 557056, 589832, 1064960, + 1179656, 1198080, 2105344, 2359304, 4198400, 4718600, 8390656, 9437192, 16778240, 18874376, 33554439, + 33554451, 33554469, 33554505, 33554577, 33554721, 33554944, 33555009, 33555585, 33556737, 33559041, 33563649, + 33572865, 33591297, 33628161, 33701889, 33849345, 34144257, 34734081, 35913729, 37748744, 38338577, 42991617, + 52428801, 67108883, 67108992, 75497476, 75497488, 76677378, 77070338, 134217765, 150994946, 150994951, + 150994976, 153358340, 157286404, 201326606, 234881031, 25, 52, 67, 88, 304, 592, 2320, 4624, 9232, 18448, + 36880, 73744, 147472, 294928, 393216, 589840, 1081344, 1179664, 2113536, 2359312, 4202496, 4718608, 8392704, + 9437200, 16779264, 18874384, 33554443, 33555456, 37748752, 38338593, 38347777, 38404097, 67108878, 67108899, + 67109120, 75497480, 76677634, 100663303, 109051905, 114425859, 134217766, 134217792, 134217797, 153362436, + 153387008, 153387013, 161480708, 167772173, 189333509, 201326614, 218103810, 218103815, 41, 50, 69, 104, 131, + 176, 608, 1184, 4640, 9248, 18464, 36896, 73760, 147488, 294944, 589856, 599040, 655360, 1114112, 1179680, + 2129920, 2359328, 4210688, 4718624, 8396800, 9437216, 16781312, 18874400, 33554445, 33556480, 37748768, + 38338625, 38535169, 67108886, 67108931, 67109376, 76678146, 78643202, 100663307, 104857601, 134217756, + 134217798, 134217856, 134217861, 150994952, 150994957, 150994966, 150995020, 150995092, 150995236, 150995524, + 150996100, 150997252, 150999556, 151004164, 151013380, 151031812, 151068676, 151142404, 151289860, 151584772, + 152174596, 153370628, 153387014, 155713540, 160432132, 167772181, 184549377, 188743684, 201326630, 209715202, + 209715207, 28, 42, 49, 70, 82, 100, 133, 208, 259, 352, 1216, 2368, 9280, 18496, 36928, 73792, 147520, + 294976, 589888, 786432, 1179712, 2162688, 2359360, 4227072, 4718656, 8404992, 9437248, 16785408, 18874432, + 33554453, 33554467, 33558528, 37748800, 38338689, 67108890, 67108995, 67109888, 75497479, 75497504, 76679170, + 76693504, 80740354, 100663315, 113836035, 134217772, 134217862, 134217984, 134217989, 150994958, 150994960, + 150994965, 167772197, 184549383, 186646532, 201326662, 220495878, 226492420, 229900294, 81, 134, 137, 164, + 200, 261, 416, 515, 704, 4736, 36992, 73856, 147584, 295040, 299520, 1179776, 2228224, 2359424, 8421376, + 9437312, 18874496, 33554457, 33554499, 33562624, 37748864, 67108906, 67108934, 67109123, 67110912, 75497483, + 75497510, 75497536, 75497546, 75497762, 75498050, 75499778, 75502082, 75506690, 75515906, 75534338, 75571202, + 75644930, 75792386, 76087298, 76681218, 77856770, 80216066, 94371842, 100663331, 113246210, 134217780, + 134217990, 134218240, 150994981, 153419780, 167772229, 171966465, 201326726, 218103822, 229769222, 251658247 }; + +static int cbest_29[1023] = { + 1, 2, 268435458, 4, 134217729, 8, 16, 335544322, 32, 64, 167772161, 128, 256, 512, 352321538, 1024, 2048, + 4096, 176160769, 8192, 16384, 32768, 65536, 356515842, 131072, 262144, 524288, 1048576, 178257921, 2097152, + 4194304, 8388608, 16777216, 33554432, 357564418, 67108864, 134217728, 268435456, 3, 178782209, 268435459, 5, + 10, 20, 40, 80, 160, 320, 640, 1280, 2560, 5120, 10240, 20480, 40960, 81920, 163840, 327680, 655360, 1310720, + 2621440, 5242880, 10485760, 20971520, 41943040, 83886080, 167772160, 6, 9, 134217731, 268435462, 335544320, + 402653187, 17, 134217733, 268435466, 335544323, 12, 18, 33, 134217737, 268435474, 34, 65, 134217745, + 268435490, 335544326, 469762051, 24, 36, 66, 129, 134217761, 167772163, 268435522, 335544330, 357826562, + 436207619, 68, 130, 257, 134217793, 167772165, 176160768, 268435586, 335544338, 352321536, 48, 72, 132, 258, + 513, 88080384, 134217857, 167772169, 268435714, 335544354, 352321539, 136, 260, 514, 1025, 44040192, + 134217985, 167772177, 268435970, 335544386, 503316483, 96, 144, 264, 516, 1026, 2049, 22020096, 134218241, + 167772193, 268436482, 335544450, 352321542, 486539267, 272, 520, 1028, 2050, 4097, 11010048, 134218753, + 167772225, 268437506, 335544578, 352321546, 192, 288, 528, 1032, 2052, 4098, 8193, 5505024, 134219777, + 167772289, 176160771, 178913281, 268439554, 335544834, 352321554, 444596227, 544, 1040, 2056, 4100, 8194, + 16385, 2752512, 134221825, 167772417, 176160773, 268443650, 335545346, 352321570, 384, 576, 1056, 2064, 4104, + 8196, 16386, 32769, 1376256, 134225921, 167772673, 176160777, 268451842, 335546370, 352321602, 356515840, + 520093699, 1088, 2080, 4112, 8200, 16388, 32770, 65537, 688128, 134234113, 167773185, 176160785, 178257920, + 268468226, 335548418, 352321666, 356515843, 511705091, 768, 1152, 2112, 4128, 8208, 16392, 32772, 65538, + 131073, 344064, 134250497, 167774209, 176160801, 268500994, 335552514, 352321794, 2176, 4160, 8224, 16400, + 32776, 65540, 131074, 172032, 262145, 89128960, 134283265, 167776257, 176160833, 268566530, 335560706, + 352322050, 356515846, 490733571, 1536, 2304, 4224, 8256, 16416, 32784, 65544, 86016, 131076, 262146, 524289, + 134348801, 167780353, 176160897, 268697602, 335577090, 352322562, 356515850, 4352, 8320, 16448, 32800, 43008, + 65552, 131080, 262148, 524290, 1048577, 44564480, 134479873, 167788545, 176161025, 268959746, 335609858, + 352323586, 356515858, 357892098, 3072, 4608, 8448, 16512, 21504, 32832, 65568, 131088, 262152, 524292, + 1048578, 2097153, 134742017, 167804929, 176161281, 178257923, 269484034, 335675394, 352325634, 356515874, + 446693379, 528482307, 8704, 10752, 16640, 32896, 65600, 131104, 262160, 524296, 1048580, 2097154, 4194305, + 22282240, 135266305, 167837697, 176161793, 178257925, 270532610, 335806466, 352329730, 356515906, 524288003, + 5376, 6144, 9216, 16896, 33024, 65664, 131136, 262176, 524304, 1048584, 2097156, 4194306, 8388609, 136314881, + 167903233, 176162817, 178257929, 272629762, 336068610, 352337922, 356515970, 2688, 17408, 33280, 65792, + 131200, 262208, 524320, 1048592, 2097160, 4194308, 8388610, 11141120, 16777217, 138412033, 168034305, + 176164865, 178257937, 276824066, 336592898, 352354306, 356516098, 357564416, 513802243, 1344, 12288, 18432, + 33792, 66048, 131328, 262272, 524352, 1048608, 2097168, 4194312, 8388612, 16777218, 33554433, 142606337, + 168296449, 176168961, 178257953, 285212674, 337641474, 352387074, 356516354, 357564419, 672, 34816, 66560, + 131584, 262400, 524416, 1048640, 2097184, 4194320, 5570560, 8388616, 16777220, 33554434, 67108865, 150994945, + 168820737, 176177153, 178257985, 178782208, 301989890, 339738626, 352452610, 356516866, 336, 24576, 36864, + 67584, 132096, 262656, 524544, 1048704, 2097216, 4194336, 8388624, 16777224, 33554436, 67108866, 169869313, + 176193537, 178258049, 178946049, 343932930, 352583682, 356517890, 357564422, 491782147, 168, 69632, 133120, + 263168, 524800, 1048832, 2097280, 2785280, 4194368, 8388640, 16777232, 33554440, 67108868, 134217730, + 171966465, 176226305, 178258177, 201326593, 352845826, 356519938, 357564426, 402653186, 532676611, 84, 49152, + 73728, 135168, 264192, 525312, 1049088, 2097408, 4194432, 8388672, 16777248, 33554448, 67108872, 89391104, + 134217732, 176291841, 178258433, 268435457, 353370114, 356524034, 357564434, 369098754, 530579459, 42, + 139264, 266240, 526336, 1049600, 1392640, 2097664, 4194560, 8388736, 16777280, 33554464, 67108880, 134217736, + 176422913, 178258945, 184549377, 268435464, 354418690, 356532226, 357564450, 11, 21, 41, 81, 161, 321, 641, + 1281, 2561, 5121, 10241, 20481, 40961, 81921, 98304, 147456, 163841, 270336, 327681, 528384, 655361, 1050624, + 1310721, 2098176, 2621441, 4194816, 5242881, 8388864, 10485761, 16777344, 20971521, 33554496, 41943041, + 67108896, 83886081, 134217744, 176685057, 178259969, 178782211, 268435460, 356548610, 357564482, 402653185, + 447217667, 469762050, 525336579, 7, 22, 82, 162, 322, 642, 1282, 2562, 5122, 10242, 20482, 40962, 81922, + 163842, 278528, 327682, 532480, 655362, 696320, 1052672, 1310722, 2099200, 2621442, 4195328, 5242882, + 8389120, 10485762, 16777472, 20971522, 33554560, 41943042, 44695552, 67108928, 83886082, 134217760, + 167772162, 177209345, 178262017, 178782213, 234881025, 268435463, 268435478, 268435498, 268435538, 268435618, + 268435778, 268436098, 268436738, 268438018, 268440578, 268445698, 268455938, 268476418, 268517378, 268599298, + 268763138, 269090818, 269746178, 271056898, 273678338, 278921218, 289406978, 310378498, 335544321, 356581378, + 357564546, 360710146, 436207618, 14, 44, 164, 324, 644, 1284, 2564, 5124, 10244, 20484, 40964, 81924, 163844, + 196608, 294912, 327684, 540672, 655364, 1056768, 1310724, 2101248, 2621444, 4196352, 5242884, 8389632, + 10485764, 16777728, 20971524, 33554688, 41943044, 67108992, 83886084, 134217739, 134217749, 134217769, + 134217792, 134217809, 134217889, 134218049, 134218369, 134219009, 134220289, 134222849, 134227969, 134238209, + 134258689, 134299649, 134381569, 134545409, 134873089, 135528449, 136839169, 139460609, 144703489, 155189249, + 167772164, 178266113, 178782217, 218103809, 268435467, 268435472, 335544328, 356646914, 357564674, 13, 19, + 28, 88, 328, 648, 1288, 2568, 5128, 10248, 20488, 40968, 81928, 163848, 327688, 348160, 557056, 655368, + 1064960, 1310728, 2105344, 2621448, 4198400, 5242888, 8390656, 10485768, 16778240, 20971528, 33554944, + 41943048, 67109120, 83886088, 134217735, 134217856, 167772168, 178274305, 178782225, 180355073, 268435475, + 268435488, 335544324, 356777986, 357564930, 385875970, 402653191, 469762049, 514326531, 26, 35, 56, 176, 656, + 1296, 2576, 5136, 10256, 20496, 40976, 81936, 163856, 327696, 393216, 589824, 655376, 1081344, 1310736, + 2113536, 2621456, 4202496, 5242896, 8392704, 10485776, 16779264, 20971536, 22347776, 33555456, 41943056, + 67109376, 83886096, 134217984, 167772176, 178290689, 178782241, 268435470, 268435491, 268435520, 335544327, + 335544342, 335544362, 335544402, 335544482, 335544642, 335544962, 335545602, 335546882, 335549442, 335554562, + 335564802, 335585282, 335626242, 335708162, 335872002, 336199682, 336855042, 338165762, 340787202, 346030082, + 357040130, 357565442, 357826560, 357908482, 377487362, 402653195, 436207617, 503316482, 25, 37, 52, 67, 112, + 352, 1312, 2592, 5152, 10272, 20512, 40992, 81952, 163872, 174080, 327712, 655392, 1114112, 1310752, 2129920, + 2621472, 4210688, 5242912, 8396800, 10485792, 16781312, 20971552, 33556480, 41943072, 67109888, 83886112, + 134217741, 134217747, 134218240, 167772192, 178323457, 178782273, 192937985, 268435523, 268435584, 335544331, + 335544336, 357566466, 357826563, 402653203, 486539266, 534773763, 38, 69, 74, 104, 131, 224, 704, 2624, 5184, + 10304, 20544, 41024, 81984, 163904, 327744, 655424, 786432, 1179648, 1310784, 2162688, 2621504, 4227072, + 5242944, 8404992, 10485824, 16785408, 20971584, 33558528, 41943104, 67110912, 83886144, 134217763, 134218752, + 167772171, 167772181, 167772201, 167772224, 167772241, 167772321, 167772481, 167772801, 167773441, 167774721, + 167777281, 167782401, 167792641, 167813121, 167854081, 167936001, 168099841, 168427521, 169082881, 170393601, + 173015041, 178388993, 178782337, 188743681, 251658241, 268435482, 268435494, 268435587, 268435712, 335544339, + 335544352, 352321537, 357568514, 358612994, 402653219, 469762055, 533725187, 49, 70, 73, 133, 138, 148, 208, + 259, 448, 1408, 5248, 10368, 20608, 41088, 82048, 87040, 163968, 327808, 655488, 1310848, 2228224, 2621568, + 4259840, 5243008, 8421376, 10485888, 11173888, 16793600, 20971648, 33562624, 41943168, 67112960, 83886208, + 88080385, 134217753, 134217765, 134217795, 134219776, 167772167, 167772288, 176160770, 178520065, 178782465, + 178913280, 243269633, 268435526, 268435715, 268435968, 335544334, 335544355, 335544384, 352321544, 357572610, + 357826566, 402653251, 436207623, 444596226, 469762059, 492044291, 503316481, 50, 134, 137, 261, 266, 296, + 416, 515, 896, 2816, 20736, 41216, 82176, 164096, 327936, 655616, 1310976, 1572864, 2359296, 2621696, + 8454144, 10486016, 16809984, 20971776, 33570816, 44040193, 67117056, 83886336, 134217797, 167772416, + 178782721, 268435506, 268435530, 268435590, 268435971, 268436480, 335544387, 335544448, 352321540, 357826570, + 364904450, 402653315, 436207627, 469762067, 486539265 }; + +static int cbest_30[1023] = { + 1, 2, 541065219, 4, 811597826, 8, 405798913, 16, 32, 64, 743964675, 128, 913047554, 256, 541065218, + 270532609, 3, 512, 456523777, 541065217, 6, 811597827, 5, 676331523, 12, 129, 405798912, 258, 811597824, 9, + 24, 516, 1024, 202899456, 541065223, 541065283, 10, 1032, 946864130, 48, 2064, 101449728, 879230978, 17, + 4128, 541065227, 811597858, 96, 8256, 50724864, 811597830, 20, 16512, 405798915, 18, 33, 192, 2048, 33024, + 25362432, 405798929, 473432065, 541065235, 769327107, 66048, 439615489, 743964674, 811597834, 132096, + 12681216, 405798917, 34, 40, 65, 264192, 541065251, 528384, 6340608, 743964673, 811597842, 384, 1056768, + 405798921, 36, 66, 4096, 2113536, 3170304, 371982337, 743964683, 80, 4227072, 1585152, 8454144, 777781251, + 68, 130, 768, 16908288, 541065347, 743964679, 760872963, 913047555, 792576, 33816576, 811597890, 917340162, + 160, 67633152, 405798945, 72, 132, 257, 8192, 396288, 135266304, 913047552, 1536, 270532608, 811597954, + 913047558, 260, 198144, 405798977, 727056387, 136, 456523776, 541065475, 743964691, 193, 386, 99072, 514, + 3072, 405799041, 541065216, 929955842, 144, 320, 513, 16384, 49536, 743964707, 921501698, 997588994, 264, + 520, 772, 541065315, 676331522, 811598082, 7, 24768, 270532611, 456523779, 458670081, 913047562, 385, 6144, + 228261888, 338165761, 541065221, 541065411, 541065731, 743964739, 811597825, 13, 1028, 1544, 12384, + 270532641, 541065222, 541065282, 676331521, 14, 259, 270532613, 270532673, 405799169, 541065346, 811597874, + 904593410, 913047570, 946864131, 25, 272, 517, 770, 1025, 1040, 6192, 202899457, 541065473, 743964803, + 879230979, 11, 26, 131, 640, 1026, 1033, 3088, 12288, 405798914, 456523781, 541065226, 541065231, 541065281, + 676331539, 811597828, 811597859, 811597922, 811598338, 28, 49, 528, 2065, 3096, 101449729, 270532617, + 405798928, 464977921, 473432064, 541065225, 676331555, 710148099, 811597831, 913047586, 946864128, 161, 288, + 518, 1540, 2056, 4129, 32768, 114130944, 202899458, 202899464, 405798937, 439615488, 460750849, 498794497, + 541065243, 541065735, 541066243, 676331527, 811597955, 879230976, 22, 52, 97, 133, 1034, 1548, 6176, 8257, + 50724865, 101449732, 405798916, 456523785, 541065234, 541065299, 541066251, 769327106, 811597856, 811598080, + 21, 50, 56, 262, 322, 2066, 2080, 16513, 24576, 50724866, 101449730, 236716032, 270532625, 405798961, + 405799040, 405799425, 541065267, 541067283, 642514947, 811597835, 811597838, 879230986, 913047618, 946864146, + 1014497282, 19, 774, 2049, 3080, 4130, 33025, 25362433, 202899460, 202899520, 219807744, 541065287, + 541069347, 676331531, 743964931, 811597832, 811597866, 879230994, 980680706, 44, 98, 104, 137, 644, 1036, + 8258, 12352, 66049, 101449760, 405798919, 405798920, 456523793, 541065233, 541065250, 541073475, 591790083, + 769327105, 811597850, 811598342, 811598850, 946864134, 946864194, 112, 266, 387, 2052, 2068, 4112, 16514, + 132097, 12681217, 50724880, 118358016, 371982336, 405798933, 452296705, 541065239, 541081731, 743964672, + 811597843, 811598858, 879230982, 896139266, 913047682, 35, 41, 194, 524, 544, 1056, 1280, 2050, 4132, 6160, + 33026, 264193, 25362434, 25362440, 57065472, 109903872, 405798925, 405798931, 405800961, 439615493, + 473432067, 473432073, 541065291, 541067267, 541073411, 541098243, 545357827, 566427651, 770400259, 811597862, + 811599890, 811601922, 42, 88, 100, 145, 208, 4160, 8260, 16385, 24704, 66050, 528385, 6340609, 12681220, + 50724868, 405799171, 439615491, 439615497, 456523809, 541065249, 541131267, 743964682, 743964687, 811601954, + 946864138, 38, 224, 274, 2072, 16516, 132098, 1056769, 6340610, 12681218, 59179008, 101449736, 185991168, + 405799429, 405799937, 473432097, 507248641, 539017219, 541065410, 541197315, 743964681, 805355522, 811597840, + 811606082, 862322690, 37, 67, 196, 532, 576, 1288, 4097, 4136, 12320, 33028, 49152, 65536, 264194, 2113537, + 3170305, 25362436, 54951936, 202899472, 405798923, 405799945, 473432069, 490340353, 541065259, 541329411, + 743964677, 743964699, 743965187, 743965699, 765165571, 769327111, 777781250, 811597846, 811614338, 81, 176, + 388, 1048, 8224, 8264, 49408, 66052, 528386, 4227073, 50724872, 384663553, 402677761, 405798944, 405800977, + 456523841, 541593603, 743964678, 760872962, 782073859, 811598018, 811599874, 811630850, 815890434, 836960258, + 70, 290, 16520, 32770, 132100, 1056770, 1585153, 4227074, 8454145, 29589504, 92995584, 101449744, 270532705, + 270532737, 371982341, 405803041, 448069633, 536903683, 541065603, 542121987, 743964802, 760872961, 811597891, + 811597986, 811663874, 913047810, 917340163, 69, 84, 200, 548, 769, 4098, 4104, 4144, 8320, 24640, 33032, + 264196, 2113538, 3170306, 8454146, 16908289, 27475968, 28532736, 202899488, 371982339, 405799009, 405807169, + 541065255, 541065539, 541069315, 543178755, 544235523, 676331587, 743964929, 769327115, 771440643, 777781249, + 809517058, 810573826, 811729922, 76, 82, 416, 1064, 2112, 8272, 66056, 528388, 792577, 6340612, 8454148, + 16908290, 33816577, 404758529, 405798976, 405798993, 405815425, 456523905, 541065345, 777781255, 811862018, + 913047553, 913047556, 913048066, 946864162, 134, 448, 641, 2096, 2560, 2576, 16528, 132104, 1056772, 1585154, + 12681224, 14794752, 16908292, 46497792, 67633153, 405799105, 405831937, 407945217, 431161345, 542650371, + 549519363, 760872967, 811597888, 812126210, 879231010, 913047559, 917340160, 73, 392, 580, 776, 1088, 4100, + 8193, 16448, 33040, 65540, 98304, 264200, 396289, 2113540, 3170308, 13737984, 16908296, 25362448, 33816580, + 135266305, 371982401, 388890625, 405864961, 473432081, 541065987, 557973507, 676331651, 727056386, 742404099, + 743965191, 758824963, 769327123, 777781267, 807436290, 811598210, 812654594, 74, 140, 352, 1096, 1537, 8288, + 66064, 98816, 528392, 792578, 4227076, 6340616, 33816578, 33816584, 50724896, 380436481, 403718145, + 405286913, 405931009, 418480129, 439615505, 541065474, 541857795, 574881795, 727056385, 743964690, 743965707, + 162, 168, 261, 1538, 2128, 16544, 132112, 198145, 1056776, 1585156, 7397376, 12681232, 23248896, 33816592, + 67633154, 67633160, 101449792, 270532865, 405798947, 405798953, 406063105, 456524033, 541065379, 608698371, + 676331571, 743444483, 743964723, 743966739, 794689539, 811597906, 811597952, 879231042, 913047566, 152, 832, + 1152, 4192, 8194, 24577, 33056, 49280, 131072, 264208, 396290, 2113544, 3170312, 6868992, 14266368, 25362464, + 67633168, 135266306, 202899584, 371982345, 406327297, 541065351, 541461507, 743964715, 743968803, 756711427, + 769327139, 786235395, 813182978, 820051970, 915259394, 917876738, 929955843, 138, 896, 1160, 1282, 5152, + 8208, 66080, 99073, 131080, 528400, 792580, 4227080, 6340624, 50724928, 67633184, 135266320, 270532610, + 405799297, 439615521, 456523778, 458670080, 541066755, 541077507, 743964689, 743964706, 743964771, 743972931, + 811597894, 811598594, 828506114, 913047683, 917340166, 921501699, 923713538, 929955840, 946864258, 997588995, + 164, 268, 400, 515, 645, 784, 1290, 1552, 2192, 2580, 3073, 4224, 5160, 10320, 16576, 16640, 20640, 41280, + 49154, 82560, 132128, 165120, 198146, 330240, 660480, 1056784, 1320960, 1585160, 2641920, 3698688, 5283840, + 8454152, 10567680, 11624448, 12681248, 21135360, 42270720, 67633156, 84541440, 101449856, 135266336, + 169082880, 338165760, 405798949, 405798979, 405803009, 541065314, 541065479, 541065601, 541263363, 725008387, + 742924291, 743964695, 743981187, 777781259, 811598083, 812390402, 845414402, 879231106, 913047808, 921501696, + 991346690, 997588992, 321, 3074, 4256, 8196, 33088, 49537, 196608, 264224, 396292, 2113552, 3170320, 3434496, + 25362496, 135266308, 135266368, 270532640, 270532657, 363528193, 371982353, 410025985, 473432129, 541065355, + 541065537, 541065729, 676331779, 743964867, 743966723, 743997699, 745037827, 748257283, 760872971, 769327171, + 811603970, 912267266, 913047563, 195, 265, 280, 521, 704, 773, 3076, 5120, 8384, 66112, 99074, 197632, + 528416, 792584, 4227088, 6340640, 50724992, 270532612, 270532672, 405799168, 414253057, 439615553, 541065220, + 541065730, 541164291, 727056419, 743964705, 743964738, 744030723, 811597898, 811597958, 811606018, 811994114, + 912787458, 913047560, 917340170, 148, 209, 324, 522, 1664, 24769, 32896, 132160, 198148, 262160, 1056800, + 1585168, 1849344, 5812224, 8454160, 12681280, 270532736, 270532801, 270533121, 405798981, 405799043, + 405799681, 405801985, 422707201, 541065313, 541068291, 541089795, 676331520, 722894851, 727056391, 741916675, + 743708675, 744096771, 811599362, 913047578, 913048070, 913048578, 918396930, 929955850, 146, 225, 418, 1546, + 2176, 2564, 6145, 8200, 10304, 16386, 33152, 49538, 98560, 264256, 396296, 1717248, 2113568, 3170336, + 7133184, 16908304, 25362560, 135266312, 202899712, 228261889, 371982369, 405799057, 406591489, 456523780, + 473432193, 541065363, 541081603, 541114755, 735510531, 744228867, 760872979, 769327235, 811597875, 811598019, + 811598086, 811598208, 811795970, 879231002, 904593411, 913047571, 913047574, 913048586, 197, 336, 1029, 1545, + 1568, 1792, 12385, 49153, 66176, 98308, 99076, 792592, 4227104, 270532615, 270532616, 456524289, 464977920, + 541065409, 541065472, 541065602, 541065991, 676331547, 743964737, 811597962, 811597987, 811598336, 811610114, + 912527362, 913049618, 917340178, 946864386 }; + +static int cbest_31[1023] = { + 1, 2, 1073741828, 4, 536870914, 8, 268435457, 16, 32, 1207959556, 64, 128, 603979778, 256, 512, 301989889, + 1024, 2048, 4096, 1224736772, 8192, 16384, 32768, 612368386, 65536, 131072, 262144, 306184193, 524288, + 1048576, 2097152, 4194304, 1226833924, 8388608, 16777216, 33554432, 67108864, 613416962, 134217728, + 268435456, 306708481, 536870912, 3, 1073741824, 1073741829, 5, 536870915, 1073741830, 6, 9, 18, 36, 72, 144, + 288, 576, 1152, 2304, 4608, 9216, 18432, 36864, 73728, 147456, 294912, 589824, 1179648, 2359296, 4718592, + 9437184, 18874368, 37748736, 75497472, 150994944, 301989888, 1610612742, 10, 17, 268435459, 536870918, + 603979776, 1073741836, 1227096068, 1342177285, 12, 33, 268435461, 536870922, 805306371, 1073741844, + 1207959552, 1207959557, 20, 34, 65, 268435465, 536870930, 1073741860, 1207959558, 24, 66, 129, 268435473, + 536870946, 603979779, 1073741892, 1744830470, 40, 68, 130, 257, 268435489, 536870978, 1073741956, 1207959564, + 1476395013, 1677721606, 48, 132, 258, 513, 268435521, 536871042, 603979782, 613548034, 1073742084, + 1207959572, 80, 136, 260, 514, 1025, 268435585, 301989891, 536871170, 603979786, 872415235, 1073742340, + 1207959588, 1375731717, 96, 264, 516, 1026, 2049, 268435713, 301989893, 536871426, 603979794, 838860803, + 1073742852, 1207959620, 160, 272, 520, 1028, 2050, 4097, 268435969, 301989897, 306184192, 536871938, + 603979810, 612368384, 1073743876, 1207959684, 1224736768, 1224736773, 1811939334, 192, 528, 1032, 2052, 4098, + 8193, 153092096, 268436481, 301989905, 536872962, 603979842, 1073745924, 1207959812, 1224736774, 320, 544, + 1040, 2056, 4100, 8194, 16385, 76546048, 268437505, 301989921, 306774017, 536875010, 603979906, 1073750020, + 1207960068, 1509949445, 1761607686, 384, 1056, 2064, 4104, 8196, 16386, 32769, 38273024, 268439553, + 301989953, 536879106, 603980034, 612368387, 1073758212, 1207960580, 1224736780, 1493172229, 640, 1088, 2080, + 4112, 8200, 16388, 32770, 65537, 19136512, 268443649, 301990017, 536887298, 603980290, 905969667, 1073774596, + 1207961604, 1224736788, 1686110214, 768, 2112, 4128, 8208, 16392, 32772, 65538, 131073, 9568256, 268451841, + 301990145, 536903682, 603980802, 612368390, 1073807364, 1207963652, 1224736804, 1280, 2176, 4160, 8224, + 16400, 32776, 65540, 131074, 262145, 4784128, 268468225, 301990401, 536936450, 603981826, 612368394, + 880803843, 1073872900, 1207967748, 1224736836, 1536, 4224, 8256, 16416, 32784, 65544, 131076, 262146, 524289, + 2392064, 268500993, 301990913, 306184195, 537001986, 603983874, 612368402, 1074003972, 1207975940, + 1224736900, 1379926021, 1828716550, 2560, 4352, 8320, 16448, 32800, 65552, 131080, 262148, 524290, 1048577, + 1196032, 268566529, 301991937, 306184197, 537133058, 603987970, 612368418, 843055107, 1074266116, 1207992324, + 1224737028, 1227128836, 1820327942, 3072, 8448, 16512, 32832, 65568, 131088, 262152, 524292, 598016, 1048578, + 2097153, 268697601, 301993985, 306184201, 537395202, 603996162, 612368450, 1074790404, 1208025092, + 1224737284, 1526726661, 5120, 8704, 16640, 32896, 65600, 131104, 262160, 299008, 524296, 1048580, 2097154, + 4194305, 268959745, 301998081, 306184209, 537919490, 604012546, 612368514, 1075838980, 1208090628, + 1224737796, 1226833920, 1226833925, 6144, 16896, 33024, 65664, 131136, 149504, 262176, 524304, 1048584, + 2097156, 4194306, 8388609, 269484033, 302006273, 306184225, 538968066, 604045314, 612368642, 613416960, + 1077936132, 1208221700, 1224738820, 1226833926, 1514143749, 10240, 17408, 33280, 65792, 74752, 131200, + 262208, 524320, 1048592, 2097160, 4194308, 8388610, 16777217, 270532609, 302022657, 306184257, 306708480, + 541065218, 604110850, 612368898, 914358275, 1082130436, 1208483844, 1224740868, 1763704838, 12288, 33792, + 37376, 66048, 131328, 262272, 524352, 1048608, 2097168, 4194312, 8388612, 16777218, 33554433, 272629761, + 302055425, 306184321, 545259522, 604241922, 612369410, 910163971, 1090519044, 1209008132, 1224744964, + 1226833932, 1495269381, 18688, 20480, 34816, 66560, 131584, 262400, 524416, 1048640, 2097184, 4194320, + 8388616, 16777220, 33554434, 67108865, 153354240, 276824065, 302120961, 306184449, 553648130, 604504066, + 612370434, 613416963, 613564418, 1107296260, 1210056708, 1224753156, 1226833940, 9344, 24576, 67584, 132096, + 262656, 524544, 1048704, 2097216, 4194336, 8388624, 16777224, 33554436, 67108866, 134217729, 285212673, + 302252033, 306184705, 570425346, 605028354, 612372482, 1140850692, 1212153860, 1224769540, 1226833956, + 1687158790, 1837105158, 4672, 40960, 69632, 133120, 263168, 524800, 1048832, 2097280, 4194368, 8388640, + 16777232, 33554440, 67108868, 76677120, 134217730, 302514177, 306185217, 606076930, 612376578, 613416966, + 1216348164, 1224802308, 1226833988, 2336, 49152, 135168, 264192, 525312, 1049088, 2097408, 4194432, 8388672, + 16777248, 33554448, 67108872, 134217732, 268435458, 303038465, 306186241, 335544321, 608174082, 612384770, + 613416970, 671088642, 881852419, 1224867844, 1226834052, 1342177284, 1830813702, 1168, 81920, 139264, 266240, + 526336, 1049600, 2097664, 4194560, 8388736, 16777280, 33554464, 38338560, 67108880, 134217736, 268435460, + 304087041, 306188289, 402653185, 536870913, 612401154, 613416978, 805306370, 1224998916, 1226834180, + 1241513988, 1530920965, 584, 98304, 270336, 528384, 1050624, 2098176, 4194816, 8388864, 16777344, 33554496, + 67108896, 134217744, 268435464, 306192385, 306708483, 612433922, 613416994, 620756994, 1225261060, + 1226834436, 1275068420, 1380450309, 1528823813, 1610612740, 1821376518, 292, 163840, 278528, 532480, 1052672, + 2099200, 4195328, 8389120, 16777472, 19169280, 33554560, 67108928, 134217760, 268435472, 306200577, + 306708485, 306782209, 310378497, 536870916, 536870928, 612499458, 613417026, 637534210, 843579395, + 1073741825, 1225785348, 1226834948, 146, 196608, 540672, 1056768, 2101248, 4196352, 8389632, 16777728, + 33554688, 67108992, 134217792, 268435488, 306216961, 306708489, 318767105, 536870920, 612630530, 613417090, + 805306369, 918552579, 1073741826, 1073741831, 1073741856, 1226835972, 1476395012, 7, 19, 37, 73, 145, 289, + 577, 1153, 2305, 4609, 9217, 18433, 36865, 73729, 147457, 294913, 327680, 557056, 589825, 1064960, 1179649, + 2105344, 2359297, 4198400, 4718593, 8390656, 9437185, 9584640, 16778240, 18874369, 33554944, 37748737, + 67109120, 75497473, 134217856, 150994945, 268435520, 306249729, 306708497, 612892674, 613417218, 738197506, + 1226838020, 1228931076, 1610612738, 1610612743, 11, 38, 74, 290, 578, 1154, 2306, 4610, 9218, 18434, 36866, + 73730, 147458, 294914, 393216, 589826, 1081344, 1179650, 2113536, 2359298, 4202496, 4718594, 8392704, + 9437186, 16779264, 18874370, 33555456, 37748738, 67109376, 75497474, 134217984, 150994946, 268435584, + 301989890, 306315265, 306708513, 369098753, 536870919, 536870944, 603979777, 613417474, 872415234, 915406851, + 1073741832, 1073741837, 1073741846, 1073741900, 1073741972, 1073742116, 1073742404, 1073742980, 1073744132, + 1073746436, 1073751044, 1073760260, 1073778692, 1073815556, 1073889284, 1074036740, 1074331652, 1074921476, + 1076101124, 1078460420, 1083179012, 1092616196, 1111490564, 1149239300, 1226842116, 1227096064, 1227096069, + 1233125380, 1342177281, 1375731716, 1514668037, 1744830468, 13, 22, 76, 148, 580, 1156, 2308, 4612, 9220, + 18436, 36868, 73732, 147460, 294916, 589828, 655360, 1114112, 1179652, 2129920, 2359300, 4210688, 4718596, + 4792320, 8396800, 9437188, 16781312, 18874372, 33556480, 37748740, 67109888, 75497476, 134218240, 150994948, + 268435712, 301989892, 306446337, 306708545, 436207617, 536870923, 536870950, 536870976, 536870986, 536871058, + 536871202, 536871490, 536872066, 536873218, 536875522, 536880130, 536889346, 536907778, 536944642, 537018370, + 537165826, 537460738, 538050562, 539230210, 541589506, 546308098, 555745282, 574619650, 613417986, 614465538, + 687865858, 838860802, 1073741838, 1073741840, 1073741845, 1207959553, 1226850308, 1227096070, 1342177287, + 1677721604, 14, 21, 26, 35, 44, 152, 296, 1160, 2312, 4616, 9224, 18440, 36872, 73736, 147464, 294920, + 589832, 786432, 1179656, 2162688, 2359304, 4227072, 4718600, 8404992, 9437192, 16785408, 18874376, 33558528, + 37748744, 67110912, 75497480, 134218752, 150994952, 268435463, 268435475, 268435493, 268435529, 268435601, + 268435745, 268435968, 268436033, 268436609, 268437761, 268440065, 268444673, 268453889, 268472321, 268509185, + 268582913, 268730369, 269025281, 269615105, 270794753, 273154049, 277872641, 287309825, 301989896, 306708609, + 343932929, 419430401, 536870931, 536871040, 603979780, 603979792, 613419010, 613548032, 616562690, 910688259, + 1073741861, 1207959554, 1207959559, 1207959584, 1226866692, 1258291204, 1610612750, 1763966982, 1839202310, + 1879048199, 25, 52, 67, 88, 304, 592, 2320, 4624, 9232, 18448, 36880, 73744, 147472, 294928, 589840, 1179664, + 1310720, 2228224, 2359312, 2396160, 4259840, 4718608, 8421376, 9437200, 16793600, 18874384, 33562624, + 37748752, 67112960, 75497488, 134219776, 150994960, 268435467, 268436480, 301989904, 306708737, 307232769, + 536870926, 536870947, 536871168, 603979784, 613421058, 805306375, 872415233, 1073741862, 1073741888, + 1073741893, 1226899460, 1227096076, 1291845636, 1342177293, 1495531525, 1610612758, 1744830466, 1744830471, + 1838153734, 41, 50, 69, 104, 131, 176, 608, 18464, 73760, 147488, 589856, 1179680, 1572864, 2359328, 4325376, + 4718624, 8454144, 9437216, 16809984, 18874400, 33570816, 67117056, 75497504, 134221824, 301989920, 306708993, + 306774016, 308281345, 536870934, 536870979, 629145602, 805306379, 1073741894, 1073741952, 1207959560, + 1207959700, 1207959844, 1207960708, 1207961860, 1207964164, 1207968772, 1207977988, 1207996420, 1208033284, + 1208107012, 1208549380, 1209139204, 1210318852, 1212678148, 1217396740, 1226964996, 1283457028, 1342177301, + 1476395009, 1509949444, 1610612774, 1677721607 }; + +static unsigned int cbest_32[1023] = { + 1, 2149580803, 2, 4, 3224371202, 8, 1612185601, 16, 32, 2955673603, 64, 128, 256, 3627417602, 512, 1024, + 2149580802, 1074790401, 1813708801, 3, 2149580801, 2048, 3224371203, 6, 2686976003, 5, 12, 1612185600, + 3224371200, 2149580807, 9, 24, 806092800, 3761766402, 10, 1025, 4096, 3493068802, 48, 2050, 403046400, + 2149580811, 2149581315, 17, 4100, 3056435203, 3224371206, 96, 8200, 201523200, 20, 16400, 1612185603, + 2149580819, 3224371458, 18, 33, 192, 32800, 100761600, 1880883201, 2955673602, 3224371210, 8192, 65600, + 1746534401, 384, 131200, 50380800, 1612185605, 2149580835, 34, 40, 65, 262400, 1612185729, 2955673601, + 3224371218, 768, 524800, 25190400, 1049600, 1612185609, 2149580867, 36, 66, 129, 1536, 2099200, 12595200, + 1477836801, 3224371234, 80, 16384, 4198400, 3090022403, 6297600, 8396800, 1612185617, 2149580931, 2955673607, + 2955673667, 3022848003, 3627417603, 68, 130, 257, 16793600, 3224371266, 3677798402, 3148800, 33587200, 160, + 3072, 67174400, 1612185633, 2149581059, 2955673611, 3627417600, 72, 132, 258, 513, 1574400, 134348800, + 3224371330, 32768, 268697600, 2888499203, 787200, 537395200, 1612185665, 2955673619, 136, 260, 320, 514, + 6144, 1074790400, 1813708800, 3627417634, 393600, 3627417606, 3694592002, 2149581827, 2955673635, 3661004802, + 3963289602, 144, 264, 516, 1026, 1537, 3074, 196800, 2149580800, 3224371714, 640, 12288, 65536, 3627417610, + 98400, 1612185857, 2149581571, 2686976002, 272, 520, 1028, 2049, 6148, 906854400, 3224372226, 7, 49200, + 1074790403, 1813708803, 1838899201, 2149580805, 2149582339, 3224371201, 3593830402, 3627417618, 1280, 2052, + 3073, 24576, 1343488001, 1612186113, 1813708817, 2149580806, 2149582851, 2686976001, 2955673731, 13, 288, + 528, 1032, 12296, 24600, 3224371586, 3761766403, 14, 131072, 1074790405, 1813708805, 3493068803, 25, 4098, + 6146, 12300, 806092801, 1612186625, 1847296001, 2149580810, 2149580815, 2149581314, 2955673859, 3224371204, + 3224371970, 11, 26, 544, 1040, 4097, 24592, 49152, 453427200, 1074790657, 1612185602, 1830502401, 1981644801, + 2149580809, 2149581826, 2821324803, 3056435202, 3224371207, 3224373250, 3761766400, 28, 49, 2051, 2056, 4104, + 6150, 403046401, 1074790409, 1074790913, 1813708809, 1880883200, 2149580827, 2149582849, 2686976007, + 3493068800, 3627417666, 4101, 12292, 806092802, 1612185793, 1746534400, 2149580818, 2149581313, 2149584899, + 2686976131, 2955674115, 3224371459, 22, 52, 97, 576, 1027, 1056, 2560, 3075, 8201, 49184, 201523201, + 1612185604, 2149580851, 2552627203, 2686976259, 3056435201, 3224371211, 3224371214, 4030464002, 21, 50, 56, + 8196, 16401, 98304, 262144, 403046402, 940441600, 1074790417, 1612185728, 1612185985, 2149584903, 2686976011, + 3224371208, 3224372227, 3627417730, 3896115202, 19, 193, 4102, 24584, 32801, 100761601, 806092804, 806092864, + 873267200, 1612187649, 1796915201, 2149580817, 2149580834, 2149580899, 2149582338, 2149589003, 2351104003, + 2955674627, 3056435211, 3224371226, 3224371456, 3224373248, 3761766406, 44, 98, 104, 1029, 1088, 2064, 8193, + 8202, 8208, 65601, 98368, 201523202, 226713600, 403046432, 1612185607, 1612185608, 2149580823, 2149597203, + 2955673600, 3224371219, 3224375298, 3493068806, 3493068866, 3560243202, 3761766530, 112, 385, 2054, 8194, + 16402, 131201, 50380801, 201523216, 403046404, 470220800, 1074790433, 1477836800, 1612186624, 1813708833, + 2149580995, 2149581319, 2149613603, 2250342403, 2686976019, 3224371250, 3493068930, 3627417858, 35, 41, 194, + 1281, 4112, 32802, 49168, 196608, 262401, 100761602, 100761608, 436633600, 806092808, 806093312, 1612185613, + 1880883203, 2149580833, 2149580866, 2149581443, 2149588995, 2149646403, 2955673699, 3056435207, 3224375302, + 3761766410, 3761766914, 42, 88, 100, 208, 769, 1033, 1152, 2080, 8204, 16392, 65602, 196736, 524801, + 25190401, 50380804, 201523204, 403046656, 1074791169, 1612185616, 1746534403, 2149581187, 2149712003, + 2199961603, 3069030403, 3224371216, 3224371235, 3224371298, 3224379402, 3425894402, 3493068810, 38, 224, 386, + 2058, 2562, 5120, 16404, 131202, 524288, 1049601, 25190402, 50380802, 201523328, 235110400, 403046408, + 738918400, 1074790465, 1528217601, 1612185625, 1813708865, 2015232001, 2149580843, 2149581323, 2149843203, + 2686976035, 2955673605, 2955673795, 3073228803, 3090022402, 3224371222, 3224371462, 3224371522, 3224387602, + 3627418114, 37, 67, 196, 4108, 16416, 32804, 98336, 262402, 2099201, 12595201, 100761604, 100761664, + 218316800, 806092816, 1612185611, 1612185731, 1612189697, 1746534433, 1880883205, 1880883265, 1948057601, + 2147485699, 2149580865, 2149580930, 2150105603, 2174771203, 2955673606, 2955673666, 2955675651, 3022848002, + 3064832003, 3140403203, 3224371394, 3224404002, 3325132802, 3761766418, 81, 176, 416, 770, 1041, 2112, 5124, + 16385, 65604, 393216, 393472, 524802, 2099202, 4198401, 50380832, 113356800, 201523208, 1612185632, + 1612185649, 1612187651, 1746534405, 1746534465, 2150630403, 3223323650, 3224371232, 3224371267, 3224379394, + 3224436802, 3493068818, 3677798403, 70, 388, 448, 2066, 16408, 131204, 1049602, 4198402, 6297601, 8396801, + 25190416, 117555200, 369459200, 403046416, 1074790529, 1612185761, 1612189701, 1780121601, 1813708929, + 1880883457, 2149580839, 2149581331, 2151684099, 2162176003, 2686976067, 3090022401, 3224371466, 3224502402, + 3274752002, 3627418626, 69, 84, 131, 200, 1538, 4116, 16388, 32784, 32808, 196672, 262404, 4198404, 8396802, + 12595202, 12595208, 16793601, 109158400, 806092832, 1477836803, 1611661825, 1612185697, 1612185733, + 1612193801, 1880883209, 2149580883, 2149580929, 2149581058, 2149597187, 2153779203, 2686976387, 2955673610, + 2955673615, 2955673665, 3022848001, 3056435219, 3224371242, 3224633602, 3627417601, 3761766434, 76, 82, 352, + 772, 832, 1057, 2176, 4128, 8216, 8224, 16386, 65608, 524804, 786944, 3148801, 6297604, 8396804, 25190404, + 33587201, 1612185621, 1612185664, 1612202001, 1746534409, 2148535299, 2157977603, 2955673609, 2955674626, + 3221228546, 3222276098, 3224371264, 3224371331, 3224896002, 3249561602, 3493068834, 3677798400, 134, 161, + 392, 896, 2082, 32832, 131208, 786432, 1048576, 1049604, 3148802, 6297602, 8396808, 16793604, 50380808, + 58777600, 67174401, 184729600, 1477836833, 1612185619, 1612218401, 1712947201, 1813709057, 2149580871, + 2149581347, 2166374403, 2888499202, 2955673627, 2955675649, 3048038403, 3224371474, 3627417650, 3677798406, + 73, 133, 259, 1540, 4132, 10248, 32816, 262408, 393344, 1574401, 2099204, 12595204, 16793602, 16793608, + 54579200, 100761616, 134348801, 1477836805, 1545011201, 1611138049, 1612185737, 1612193793, 1612251201, + 1880883217, 2149581057, 2183168003, 2955411715, 2955673618, 2955673683, 2955677699, 3022848019, 3056435235, + 3090022435, 3224371238, 3224372738, 3226474498, 3236966402, 3761766466, 74, 140, 704, 776, 1089, 1664, 2304, + 3076, 4160, 8232, 10240, 32769, 65616, 524808, 1573888, 16793616, 25190408, 33587202, 33587208, 56678400, + 201523232, 268697601, 1511424001, 1610614273, 1612316801, 1662566401, 1746534417, 2149580963, 2149583875, + 2216755203, 2955673651, 3022848035, 3090022407, 3157196803, 3224371282, 3224371328, 3224372482, 3224387586, + 3228569602, 3627417604, 3627417635, 3627417698, 162, 168, 262, 400, 1792, 2114, 16432, 65568, 131216, 787201, + 1049608, 29388800, 33587216, 50380816, 67174402, 92364800, 403046464, 537395201, 1612185635, 1612185641, + 1612448001, 1813709313, 2149580875, 2149580935, 2149581379, 2149583363, 2151155203, 2283929603, 2888499201, + 2955673671, 2955677703, 3022848007, 3090022531, 3123609603, 3223848450, 3224371490, 3232768002, 3627417607, + 3627419650, 3694592003, 137, 152, 261, 321, 515, 1544, 4164, 6145, 262416, 786688, 1572864, 1574402, 2099208, + 27289600, 33587232, 67174416, 100761632, 134348802, 806092928, 1477836809, 1477837313, 1612185745, + 1612186369, 1637376001, 1880883233, 2149613571, 2418278403, 2955149827, 2955673617, 2955673634, 2955681803, + 3056435267, 3224371270, 3241164802, 3661004803, 3963289603, 138, 268, 784, 1153, 1408, 4224, 8264, 32770, + 32776, 65632, 65664, 393601, 524816, 3147776, 3148804, 4198408, 33587204, 67174432, 201523264, 268697602, + 1612185856, 1612186241, 2150368003, 2913689603, 2955673623, 2955690003, 3039641603, 3090022411, 3224371715, + 3257958402, 3493068994, 3627417632, 164, 518, 2178, 5121, 16448, 16464, 20496, 131232, 787202, 1049616, + 6297608, 14694400, 46182400, 67174404, 67174464, 134348832, 403046528, 537395202, 1074791425, 1612185637, + 1612185667, 1612187137, 1613237249, 1614284801, 1624780801, 1813709825, 2149580939, 2149581063, 2149581123, + 2149586947, 2686976515, 2888499219, 2954887939, 2955673675, 2955706403, 3022848011, 3224371362, 3224374274, + 3291545602, 3627286658, 3627417611, 3627417614, 3694592000, 145, 265, 280, 322, 517, 1552, 3080, 4228, 6152, + 32864, 196801, 262432, 1573376, 1574404, 2097152, 2099216, 12595216, 13644800, 134348804, 134348864, + 806093056, 1074790402, 1444249601, 1477836817, 1612185681, 1612201985, 1616384001, 1813708802, 1838899200, + 2149581570, 2149581825, 2149583873, 2149974403, 2955673633, 2955681795, 2955739203, 3056435331, 3224371274, + 3224371334, 3358720002, 3627417608, 3627417642, 3627418627, 3661004800, 3677798410, 3679901698, 3686195202, + 3761766658, 3963289600, 266, 336, 524, 641, 800, 3328, 4352, 5125, 8256, 8328, 10250, 12289, 20500, 32772, + 41000, 65537, 82000, 131136, 164000, 328000, 393602, 524832, 656000, 1312000, 2624000, 3148808, 4198416, + 5248000, 10496000, 20992000, 25190432, 28339200, 41984000, 83968000, 134348928, 167936000, 268697604, + 268697664, 335872000, 671744000, 1343488000, 1611924225, 1612186112, 1618483201, 1620582401, 1813708816, + 2149583361, 2955673987, 2955804803, 3006054403, 3090022419, 3224371712, 3224403970, 3493069058, 3627417626, + 3627419648, 3681996802, 3719782402, 148, 304, 2306, 3584, 12290, 16528, 98401, 131264, 787204, 1049632, + 3145728, 6297616, 8396816, 23091200, 50380864, 67174408, 268697728, 1074790785, 1612185669, 1813708825 }; diff --git a/src/osd/ErasureCodePluginJerasure/galois.c b/src/osd/ErasureCodePluginJerasure/galois.c new file mode 100755 index 00000000000..be8be59affa --- /dev/null +++ b/src/osd/ErasureCodePluginJerasure/galois.c @@ -0,0 +1,821 @@ +/* Galois.c + * James S. Plank + +Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure Coding Techniques + +Revision 1.2A +May 24, 2011 + +James S. Plank +Department of Electrical Engineering and Computer Science +University of Tennessee +Knoxville, TN 37996 +plank@cs.utk.edu + +Copyright (c) 2011, James S. Plank +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + - Neither the name of the University of Tennessee nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "galois.h" + +#define NONE (10) +#define TABLE (11) +#define SHIFT (12) +#define LOGS (13) +#define SPLITW8 (14) + +static int prim_poly[33] = +{ 0, +/* 1 */ 1, +/* 2 */ 07, +/* 3 */ 013, +/* 4 */ 023, +/* 5 */ 045, +/* 6 */ 0103, +/* 7 */ 0211, +/* 8 */ 0435, +/* 9 */ 01021, +/* 10 */ 02011, +/* 11 */ 04005, +/* 12 */ 010123, +/* 13 */ 020033, +/* 14 */ 042103, +/* 15 */ 0100003, +/* 16 */ 0210013, +/* 17 */ 0400011, +/* 18 */ 01000201, +/* 19 */ 02000047, +/* 20 */ 04000011, +/* 21 */ 010000005, +/* 22 */ 020000003, +/* 23 */ 040000041, +/* 24 */ 0100000207, +/* 25 */ 0200000011, +/* 26 */ 0400000107, +/* 27 */ 01000000047, +/* 28 */ 02000000011, +/* 29 */ 04000000005, +/* 30 */ 010040000007, +/* 31 */ 020000000011, +/* 32 */ 00020000007 }; /* Really 40020000007, but we're omitting the high order bit */ + +static int mult_type[33] = +{ NONE, +/* 1 */ TABLE, +/* 2 */ TABLE, +/* 3 */ TABLE, +/* 4 */ TABLE, +/* 5 */ TABLE, +/* 6 */ TABLE, +/* 7 */ TABLE, +/* 8 */ TABLE, +/* 9 */ TABLE, +/* 10 */ LOGS, +/* 11 */ LOGS, +/* 12 */ LOGS, +/* 13 */ LOGS, +/* 14 */ LOGS, +/* 15 */ LOGS, +/* 16 */ LOGS, +/* 17 */ LOGS, +/* 18 */ LOGS, +/* 19 */ LOGS, +/* 20 */ LOGS, +/* 21 */ LOGS, +/* 22 */ LOGS, +/* 23 */ SHIFT, +/* 24 */ SHIFT, +/* 25 */ SHIFT, +/* 26 */ SHIFT, +/* 27 */ SHIFT, +/* 28 */ SHIFT, +/* 29 */ SHIFT, +/* 30 */ SHIFT, +/* 31 */ SHIFT, +/* 32 */ SPLITW8 }; + +static int nw[33] = { 0, (1 << 1), (1 << 2), (1 << 3), (1 << 4), +(1 << 5), (1 << 6), (1 << 7), (1 << 8), (1 << 9), (1 << 10), +(1 << 11), (1 << 12), (1 << 13), (1 << 14), (1 << 15), (1 << 16), +(1 << 17), (1 << 18), (1 << 19), (1 << 20), (1 << 21), (1 << 22), +(1 << 23), (1 << 24), (1 << 25), (1 << 26), (1 << 27), (1 << 28), +(1 << 29), (1 << 30), (1 << 31), -1 }; + +static int nwm1[33] = { 0, (1 << 1)-1, (1 << 2)-1, (1 << 3)-1, (1 << 4)-1, +(1 << 5)-1, (1 << 6)-1, (1 << 7)-1, (1 << 8)-1, (1 << 9)-1, (1 << 10)-1, +(1 << 11)-1, (1 << 12)-1, (1 << 13)-1, (1 << 14)-1, (1 << 15)-1, (1 << 16)-1, +(1 << 17)-1, (1 << 18)-1, (1 << 19)-1, (1 << 20)-1, (1 << 21)-1, (1 << 22)-1, +(1 << 23)-1, (1 << 24)-1, (1 << 25)-1, (1 << 26)-1, (1 << 27)-1, (1 << 28)-1, +(1 << 29)-1, (1 << 30)-1, 0x7fffffff, 0xffffffff }; + +static int *galois_log_tables[33] = { NULL, NULL, NULL, NULL, NULL, NULL, NULL, +NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }; + +static int *galois_ilog_tables[33] = { NULL, NULL, NULL, NULL, NULL, NULL, NULL, +NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }; + +static int *galois_mult_tables[33] = { NULL, NULL, NULL, NULL, NULL, NULL, NULL, +NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }; + +static int *galois_div_tables[33] = { NULL, NULL, NULL, NULL, NULL, NULL, NULL, +NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }; + +/* Special case for w = 32 */ + +static int *galois_split_w8[7] = { NULL, NULL, NULL, NULL, NULL, NULL, NULL }; + +int galois_create_log_tables(int w) +{ + int j, b; + + if (w > 30) return -1; + if (galois_log_tables[w] != NULL) return 0; + galois_log_tables[w] = (int *) malloc(sizeof(int)*nw[w]); + if (galois_log_tables[w] == NULL) return -1; + + galois_ilog_tables[w] = (int *) malloc(sizeof(int)*nw[w]*3); + if (galois_ilog_tables[w] == NULL) { + free(galois_log_tables[w]); + galois_log_tables[w] = NULL; + return -1; + } + + for (j = 0; j < nw[w]; j++) { + galois_log_tables[w][j] = nwm1[w]; + galois_ilog_tables[w][j] = 0; + } + + b = 1; + for (j = 0; j < nwm1[w]; j++) { + if (galois_log_tables[w][b] != nwm1[w]) { + fprintf(stderr, "Galois_create_log_tables Error: j=%d, b=%d, B->J[b]=%d, J->B[j]=%d (0%o)\n", + j, b, galois_log_tables[w][b], galois_ilog_tables[w][j], (b << 1) ^ prim_poly[w]); + exit(1); + } + galois_log_tables[w][b] = j; + galois_ilog_tables[w][j] = b; + b = b << 1; + if (b & nw[w]) b = (b ^ prim_poly[w]) & nwm1[w]; + } + for (j = 0; j < nwm1[w]; j++) { + galois_ilog_tables[w][j+nwm1[w]] = galois_ilog_tables[w][j]; + galois_ilog_tables[w][j+nwm1[w]*2] = galois_ilog_tables[w][j]; + } + galois_ilog_tables[w] += nwm1[w]; + return 0; +} + +int galois_logtable_multiply(int x, int y, int w) +{ + int sum_j; + + if (x == 0 || y == 0) return 0; + + sum_j = galois_log_tables[w][x] + galois_log_tables[w][y]; + /* if (sum_j >= nwm1[w]) sum_j -= nwm1[w]; Don't need to do this, + because we replicate the ilog table twice. */ + return galois_ilog_tables[w][sum_j]; +} + +int galois_logtable_divide(int x, int y, int w) +{ + int sum_j; + int z; + + if (y == 0) return -1; + if (x == 0) return 0; + sum_j = galois_log_tables[w][x] - galois_log_tables[w][y]; + /* if (sum_j < 0) sum_j += nwm1[w]; Don't need to do this, because we replicate the ilog table twice. */ + z = galois_ilog_tables[w][sum_j]; + return z; +} + +int galois_create_mult_tables(int w) +{ + int j, x, y, logx; + + if (w >= 14) return -1; + + if (galois_mult_tables[w] != NULL) return 0; + galois_mult_tables[w] = (int *) malloc(sizeof(int) * nw[w] * nw[w]); + if (galois_mult_tables[w] == NULL) return -1; + + galois_div_tables[w] = (int *) malloc(sizeof(int) * nw[w] * nw[w]); + if (galois_div_tables[w] == NULL) { + free(galois_mult_tables[w]); + galois_mult_tables[w] = NULL; + return -1; + } + if (galois_log_tables[w] == NULL) { + if (galois_create_log_tables(w) < 0) { + free(galois_mult_tables[w]); + free(galois_div_tables[w]); + galois_mult_tables[w] = NULL; + galois_div_tables[w] = NULL; + return -1; + } + } + + /* Set mult/div tables for x = 0 */ + j = 0; + galois_mult_tables[w][j] = 0; /* y = 0 */ + galois_div_tables[w][j] = -1; + j++; + for (y = 1; y < nw[w]; y++) { /* y > 0 */ + galois_mult_tables[w][j] = 0; + galois_div_tables[w][j] = 0; + j++; + } + + for (x = 1; x < nw[w]; x++) { /* x > 0 */ + galois_mult_tables[w][j] = 0; /* y = 0 */ + galois_div_tables[w][j] = -1; + j++; + logx = galois_log_tables[w][x]; + for (y = 1; y < nw[w]; y++) { /* y > 0 */ + galois_mult_tables[w][j] = galois_ilog_tables[w][logx+galois_log_tables[w][y]]; + galois_div_tables[w][j] = galois_ilog_tables[w][logx-galois_log_tables[w][y]]; + j++; + } + } + return 0; +} + +int galois_ilog(int value, int w) +{ + if (galois_ilog_tables[w] == NULL) { + if (galois_create_log_tables(w) < 0) { + fprintf(stderr, "Error: galois_ilog - w is too big. Sorry\n"); + exit(1); + } + } + return galois_ilog_tables[w][value]; +} + +int galois_log(int value, int w) +{ + if (galois_log_tables[w] == NULL) { + if (galois_create_log_tables(w) < 0) { + fprintf(stderr, "Error: galois_log - w is too big. Sorry\n"); + exit(1); + } + } + return galois_log_tables[w][value]; +} + + +int galois_shift_multiply(int x, int y, int w) +{ + int prod; + int i, j, ind; + int k; + int scratch[33]; + + prod = 0; + for (i = 0; i < w; i++) { + scratch[i] = y; + if (y & (1 << (w-1))) { + y = y << 1; + y = (y ^ prim_poly[w]) & nwm1[w]; + } else { + y = y << 1; + } + } + for (i = 0; i < w; i++) { + ind = (1 << i); + if (ind & x) { + j = 1; + for (k = 0; k < w; k++) { + prod = prod ^ (j & scratch[i]); + j = (j << 1); + } + } + } + return prod; +} + +int galois_single_multiply(int x, int y, int w) +{ + int sum_j; + int z; + + if (x == 0 || y == 0) return 0; + + if (mult_type[w] == TABLE) { + if (galois_mult_tables[w] == NULL) { + if (galois_create_mult_tables(w) < 0) { + fprintf(stderr, "ERROR -- cannot make multiplication tables for w=%d\n", w); + exit(1); + } + } + return galois_mult_tables[w][(x<<w)|y]; + } else if (mult_type[w] == LOGS) { + if (galois_log_tables[w] == NULL) { + if (galois_create_log_tables(w) < 0) { + fprintf(stderr, "ERROR -- cannot make log tables for w=%d\n", w); + exit(1); + } + } + sum_j = galois_log_tables[w][x] + galois_log_tables[w][y]; + z = galois_ilog_tables[w][sum_j]; + return z; + } else if (mult_type[w] == SPLITW8) { + if (galois_split_w8[0] == NULL) { + if (galois_create_split_w8_tables() < 0) { + fprintf(stderr, "ERROR -- cannot make log split_w8_tables for w=%d\n", w); + exit(1); + } + } + return galois_split_w8_multiply(x, y); + } else if (mult_type[w] == SHIFT) { + return galois_shift_multiply(x, y, w); + } + fprintf(stderr, "Galois_single_multiply - no implementation for w=%d\n", w); + exit(1); +} + +int galois_multtable_multiply(int x, int y, int w) +{ + return galois_mult_tables[w][(x<<w)|y]; +} + +int galois_single_divide(int a, int b, int w) +{ + int sum_j; + + if (mult_type[w] == TABLE) { + if (galois_div_tables[w] == NULL) { + if (galois_create_mult_tables(w) < 0) { + fprintf(stderr, "ERROR -- cannot make multiplication tables for w=%d\n", w); + exit(1); + } + } + return galois_div_tables[w][(a<<w)|b]; + } else if (mult_type[w] == LOGS) { + if (b == 0) return -1; + if (a == 0) return 0; + if (galois_log_tables[w] == NULL) { + if (galois_create_log_tables(w) < 0) { + fprintf(stderr, "ERROR -- cannot make log tables for w=%d\n", w); + exit(1); + } + } + sum_j = galois_log_tables[w][a] - galois_log_tables[w][b]; + return galois_ilog_tables[w][sum_j]; + } else { + if (b == 0) return -1; + if (a == 0) return 0; + sum_j = galois_inverse(b, w); + return galois_single_multiply(a, sum_j, w); + } + fprintf(stderr, "Galois_single_divide - no implementation for w=%d\n", w); + exit(1); +} + +int galois_shift_divide(int a, int b, int w) +{ + int inverse; + + if (b == 0) return -1; + if (a == 0) return 0; + inverse = galois_shift_inverse(b, w); + return galois_shift_multiply(a, inverse, w); +} + +int galois_multtable_divide(int x, int y, int w) +{ + return galois_div_tables[w][(x<<w)|y]; +} + +void galois_w08_region_multiply(char *region, /* Region to multiply */ + int multby, /* Number to multiply by */ + int nbytes, /* Number of bytes in region */ + char *r2, /* If r2 != NULL, products go here */ + int add) +{ + unsigned char *ur1, *ur2, *cp; + unsigned char prod; + int i, srow, j; + unsigned long l, *lp2; + unsigned char *lp; + int sol; + + ur1 = (unsigned char *) region; + ur2 = (r2 == NULL) ? ur1 : (unsigned char *) r2; + +/* This is used to test its performance with respect to just calling galois_single_multiply + if (r2 == NULL || !add) { + for (i = 0; i < nbytes; i++) ur2[i] = galois_single_multiply(ur1[i], multby, 8); + } else { + for (i = 0; i < nbytes; i++) { + ur2[i] = (ur2[i]^galois_single_multiply(ur1[i], multby, 8)); + } + } + */ + + if (galois_mult_tables[8] == NULL) { + if (galois_create_mult_tables(8) < 0) { + fprintf(stderr, "galois_08_region_multiply -- couldn't make multiplication tables\n"); + exit(1); + } + } + srow = multby * nw[8]; + if (r2 == NULL || !add) { + for (i = 0; i < nbytes; i++) { + prod = galois_mult_tables[8][srow+ur1[i]]; + ur2[i] = prod; + } + } else { + sol = sizeof(long); + lp2 = &l; + lp = (unsigned char *) lp2; + for (i = 0; i < nbytes; i += sol) { + cp = ur2+i; + lp2 = (unsigned long *) cp; + for (j = 0; j < sol; j++) { + prod = galois_mult_tables[8][srow+ur1[i+j]]; + lp[j] = prod; + } + *lp2 = (*lp2) ^ l; + } + } + return; +} + +void galois_w16_region_multiply(char *region, /* Region to multiply */ + int multby, /* Number to multiply by */ + int nbytes, /* Number of bytes in region */ + char *r2, /* If r2 != NULL, products go here */ + int add) +{ + unsigned short *ur1, *ur2, *cp; + int prod; + int i, log1, j, log2; + unsigned long l, *lp2, *lptop; + unsigned short *lp; + int sol; + + ur1 = (unsigned short *) region; + ur2 = (r2 == NULL) ? ur1 : (unsigned short *) r2; + nbytes /= 2; + + +/* This is used to test its performance with respect to just calling galois_single_multiply */ +/* + if (r2 == NULL || !add) { + for (i = 0; i < nbytes; i++) ur2[i] = galois_single_multiply(ur1[i], multby, 16); + } else { + for (i = 0; i < nbytes; i++) { + ur2[i] = (ur2[i]^galois_single_multiply(ur1[i], multby, 16)); + } + } + return; + */ + + if (multby == 0) { + if (!add) { + lp2 = (unsigned long *) ur2; + ur2 += nbytes; + lptop = (unsigned long *) ur2; + while (lp2 < lptop) { *lp2 = 0; lp2++; } + } + return; + } + + if (galois_log_tables[16] == NULL) { + if (galois_create_log_tables(16) < 0) { + fprintf(stderr, "galois_16_region_multiply -- couldn't make log tables\n"); + exit(1); + } + } + log1 = galois_log_tables[16][multby]; + + if (r2 == NULL || !add) { + for (i = 0; i < nbytes; i++) { + if (ur1[i] == 0) { + ur2[i] = 0; + } else { + prod = galois_log_tables[16][ur1[i]] + log1; + ur2[i] = galois_ilog_tables[16][prod]; + } + } + } else { + sol = sizeof(long)/2; + lp2 = &l; + lp = (unsigned short *) lp2; + for (i = 0; i < nbytes; i += sol) { + cp = ur2+i; + lp2 = (unsigned long *) cp; + for (j = 0; j < sol; j++) { + if (ur1[i+j] == 0) { + lp[j] = 0; + } else { + log2 = galois_log_tables[16][ur1[i+j]]; + prod = log2 + log1; + lp[j] = galois_ilog_tables[16][prod]; + } + } + *lp2 = (*lp2) ^ l; + } + } + return; +} + +/* This will destroy mat, by the way */ + +void galois_invert_binary_matrix(int *mat, int *inv, int rows) +{ + int cols, i, j; + int tmp; + + cols = rows; + + for (i = 0; i < rows; i++) inv[i] = (1 << i); + + /* First -- convert into upper triangular */ + + for (i = 0; i < cols; i++) { + + /* Swap rows if we ave a zero i,i element. If we can't swap, then the + matrix was not invertible */ + + if ((mat[i] & (1 << i)) == 0) { + for (j = i+1; j < rows && (mat[j] & (1 << i)) == 0; j++) ; + if (j == rows) { + fprintf(stderr, "galois_invert_matrix: Matrix not invertible!!\n"); + exit(1); + } + tmp = mat[i]; mat[i] = mat[j]; mat[j] = tmp; + tmp = inv[i]; inv[i] = inv[j]; inv[j] = tmp; + } + + /* Now for each j>i, add A_ji*Ai to Aj */ + for (j = i+1; j != rows; j++) { + if ((mat[j] & (1 << i)) != 0) { + mat[j] ^= mat[i]; + inv[j] ^= inv[i]; + } + } + } + + /* Now the matrix is upper triangular. Start at the top and multiply down */ + + for (i = rows-1; i >= 0; i--) { + for (j = 0; j < i; j++) { + if (mat[j] & (1 << i)) { +/* mat[j] ^= mat[i]; */ + inv[j] ^= inv[i]; + } + } + } +} + +int galois_inverse(int y, int w) +{ + + if (y == 0) return -1; + if (mult_type[w] == SHIFT || mult_type[w] == SPLITW8) return galois_shift_inverse(y, w); + return galois_single_divide(1, y, w); +} + +int galois_shift_inverse(int y, int w) +{ + int mat2[32]; + int inv2[32]; + int i; + + for (i = 0; i < w; i++) { + mat2[i] = y; + + if (y & nw[w-1]) { + y = y << 1; + y = (y ^ prim_poly[w]) & nwm1[w]; + } else { + y = y << 1; + } + } + + galois_invert_binary_matrix(mat2, inv2, w); + + return inv2[0]; +} + +int *galois_get_mult_table(int w) +{ + if (galois_mult_tables[w] == NULL) { + if (galois_create_mult_tables(w)) { + return NULL; + } + } + return galois_mult_tables[w]; +} + +int *galois_get_div_table(int w) +{ + if (galois_mult_tables[w] == NULL) { + if (galois_create_mult_tables(w)) { + return NULL; + } + } + return galois_div_tables[w]; +} + +int *galois_get_log_table(int w) +{ + if (galois_log_tables[w] == NULL) { + if (galois_create_log_tables(w)) { + return NULL; + } + } + return galois_log_tables[w]; +} + +int *galois_get_ilog_table(int w) +{ + if (galois_ilog_tables[w] == NULL) { + if (galois_create_log_tables(w)) { + return NULL; + } + } + return galois_ilog_tables[w]; +} + +void galois_w32_region_multiply(char *region, /* Region to multiply */ + int multby, /* Number to multiply by */ + int nbytes, /* Number of bytes in region */ + char *r2, /* If r2 != NULL, products go here */ + int add) +{ + unsigned int *ur1, *ur2; + int i, j, a, b, accumulator, i8, j8, k; + int acache[4]; + + ur1 = (unsigned int *) region; + ur2 = (r2 == NULL) ? ur1 : (unsigned int *) r2; + nbytes /= sizeof(int); + + if (galois_split_w8[0]== NULL) { + if (galois_create_split_w8_tables(8) < 0) { + fprintf(stderr, "galois_32_region_multiply -- couldn't make split multiplication tables\n"); + exit(1); + } + } + + /* If we're overwriting r2, then we can't do better than just calling split_multiply. + We'll inline it here to save on the procedure call overhead */ + + i8 = 0; + for (i = 0; i < 4; i++) { + acache[i] = (((multby >> i8) & 255) << 8); + i8 += 8; + } + if (!add) { + for (k = 0; k < nbytes; k++) { + accumulator = 0; + for (i = 0; i < 4; i++) { + a = acache[i]; + j8 = 0; + for (j = 0; j < 4; j++) { + b = ((ur1[k] >> j8) & 255); + accumulator ^= galois_split_w8[i+j][a|b]; + j8 += 8; + } + } + ur2[k] = accumulator; + } + } else { + for (k = 0; k < nbytes; k++) { + accumulator = 0; + for (i = 0; i < 4; i++) { + a = acache[i]; + j8 = 0; + for (j = 0; j < 4; j++) { + b = ((ur1[k] >> j8) & 255); + accumulator ^= galois_split_w8[i+j][a|b]; + j8 += 8; + } + } + ur2[k] = (ur2[k] ^ accumulator); + } + } + return; + +} + +void galois_region_xor( char *r1, /* Region 1 */ + char *r2, /* Region 2 */ + char *r3, /* Sum region (r3 = r1 ^ r2) -- can be r1 or r2 */ + int nbytes) /* Number of bytes in region */ +{ + long *l1; + long *l2; + long *l3; + long *ltop; + char *ctop; + + ctop = r1 + nbytes; + ltop = (long *) ctop; + l1 = (long *) r1; + l2 = (long *) r2; + l3 = (long *) r3; + + while (l1 < ltop) { + *l3 = ((*l1) ^ (*l2)); + l1++; + l2++; + l3++; + } +} + +int galois_create_split_w8_tables() +{ + int p1, p2, i, j, p1elt, p2elt, index, ishift, jshift, *table; + + if (galois_split_w8[0] != NULL) return 0; + + if (galois_create_mult_tables(8) < 0) return -1; + + for (i = 0; i < 7; i++) { + galois_split_w8[i] = (int *) malloc(sizeof(int) * (1 << 16)); + if (galois_split_w8[i] == NULL) { + for (i--; i >= 0; i--) free(galois_split_w8[i]); + return -1; + } + } + + for (i = 0; i < 4; i += 3) { + ishift = i * 8; + for (j = ((i == 0) ? 0 : 1) ; j < 4; j++) { + jshift = j * 8; + table = galois_split_w8[i+j]; + index = 0; + for (p1 = 0; p1 < 256; p1++) { + p1elt = (p1 << ishift); + for (p2 = 0; p2 < 256; p2++) { + p2elt = (p2 << jshift); + table[index] = galois_shift_multiply(p1elt, p2elt, 32); + index++; + } + } + } + } + return 0; +} + +int galois_split_w8_multiply(int x, int y) +{ + int i, j, a, b, accumulator, i8, j8; + + accumulator = 0; + + i8 = 0; + for (i = 0; i < 4; i++) { + a = (((x >> i8) & 255) << 8); + j8 = 0; + for (j = 0; j < 4; j++) { + b = ((y >> j8) & 255); + accumulator ^= galois_split_w8[i+j][a|b]; + j8 += 8; + } + i8 += 8; + } + return accumulator; +} diff --git a/src/osd/ErasureCodePluginJerasure/galois.h b/src/osd/ErasureCodePluginJerasure/galois.h new file mode 100755 index 00000000000..b08fd9488cf --- /dev/null +++ b/src/osd/ErasureCodePluginJerasure/galois.h @@ -0,0 +1,111 @@ +/* Galois.h + * James S. Plank + +Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure Coding Techniques + +Revision 1.2A +May 24, 2011 + +James S. Plank +Department of Electrical Engineering and Computer Science +University of Tennessee +Knoxville, TN 37996 +plank@cs.utk.edu + +Copyright (c) 2011, James S. Plank +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + - Neither the name of the University of Tennessee nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + */ + +#ifndef _GALOIS_H +#define _GALOIS_H + +#include <stdio.h> +#include <stdlib.h> + +extern int galois_single_multiply(int a, int b, int w); +extern int galois_single_divide(int a, int b, int w); +extern int galois_log(int value, int w); +extern int galois_ilog(int value, int w); + +extern int galois_create_log_tables(int w); /* Returns 0 on success, -1 on failure */ +extern int galois_logtable_multiply(int x, int y, int w); +extern int galois_logtable_divide(int x, int y, int w); + +extern int galois_create_mult_tables(int w); /* Returns 0 on success, -1 on failure */ +extern int galois_multtable_multiply(int x, int y, int w); +extern int galois_multtable_divide(int x, int y, int w); + +extern int galois_shift_multiply(int x, int y, int w); +extern int galois_shift_divide(int x, int y, int w); + +extern int galois_create_split_w8_tables(); +extern int galois_split_w8_multiply(int x, int y); + +extern int galois_inverse(int x, int w); +extern int galois_shift_inverse(int y, int w); + +extern int *galois_get_mult_table(int w); +extern int *galois_get_div_table(int w); +extern int *galois_get_log_table(int w); +extern int *galois_get_ilog_table(int w); + +void galois_region_xor( char *r1, /* Region 1 */ + char *r2, /* Region 2 */ + char *r3, /* Sum region (r3 = r1 ^ r2) -- can be r1 or r2 */ + int nbytes); /* Number of bytes in region */ + +/* These multiply regions in w=8, w=16 and w=32. They are much faster + than calling galois_single_multiply. The regions must be long word aligned. */ + +void galois_w08_region_multiply(char *region, /* Region to multiply */ + int multby, /* Number to multiply by */ + int nbytes, /* Number of bytes in region */ + char *r2, /* If r2 != NULL, products go here. + Otherwise region is overwritten */ + int add); /* If (r2 != NULL && add) the produce is XOR'd with r2 */ + +void galois_w16_region_multiply(char *region, /* Region to multiply */ + int multby, /* Number to multiply by */ + int nbytes, /* Number of bytes in region */ + char *r2, /* If r2 != NULL, products go here. + Otherwise region is overwritten */ + int add); /* If (r2 != NULL && add) the produce is XOR'd with r2 */ + +void galois_w32_region_multiply(char *region, /* Region to multiply */ + int multby, /* Number to multiply by */ + int nbytes, /* Number of bytes in region */ + char *r2, /* If r2 != NULL, products go here. + Otherwise region is overwritten */ + int add); /* If (r2 != NULL && add) the produce is XOR'd with r2 */ + +#endif diff --git a/src/osd/ErasureCodePluginJerasure/jerasure.c b/src/osd/ErasureCodePluginJerasure/jerasure.c new file mode 100755 index 00000000000..9efae02e5fb --- /dev/null +++ b/src/osd/ErasureCodePluginJerasure/jerasure.c @@ -0,0 +1,1376 @@ +/* jerasure.c + * James S. Plank + +Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure Coding Techniques + +Revision 1.2A +May 24, 2011 + +James S. Plank +Department of Electrical Engineering and Computer Science +University of Tennessee +Knoxville, TN 37996 +plank@cs.utk.edu + +Copyright (c) 2011, James S. Plank +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + - Neither the name of the University of Tennessee nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "galois.h" +#include "jerasure.h" + +#define talloc(type, num) (type *) malloc(sizeof(type)*(num)) + +static double jerasure_total_xor_bytes = 0; +static double jerasure_total_gf_bytes = 0; +static double jerasure_total_memcpy_bytes = 0; + +void jerasure_print_matrix(int *m, int rows, int cols, int w) +{ + int i, j; + int fw; + char s[30]; + unsigned int w2; + + if (w == 32) { + fw = 10; + } else { + w2 = (1 << w); + sprintf(s, "%u", w2-1); + fw = strlen(s); + } + + for (i = 0; i < rows; i++) { + for (j = 0; j < cols; j++) { + if (j != 0) printf(" "); + printf("%*u", fw, m[i*cols+j]); + } + printf("\n"); + } +} + +void jerasure_print_bitmatrix(int *m, int rows, int cols, int w) +{ + int i, j; + + for (i = 0; i < rows; i++) { + if (i != 0 && i%w == 0) printf("\n"); + for (j = 0; j < cols; j++) { + if (j != 0 && j%w == 0) printf(" "); + printf("%d", m[i*cols+j]); + } + printf("\n"); + } +} + +int jerasure_make_decoding_matrix(int k, int m, int w, int *matrix, int *erased, int *decoding_matrix, int *dm_ids) +{ + int i, j, *tmpmat; + + j = 0; + for (i = 0; j < k; i++) { + if (erased[i] == 0) { + dm_ids[j] = i; + j++; + } + } + + tmpmat = talloc(int, k*k); + if (tmpmat == NULL) { return -1; } + for (i = 0; i < k; i++) { + if (dm_ids[i] < k) { + for (j = 0; j < k; j++) tmpmat[i*k+j] = 0; + tmpmat[i*k+dm_ids[i]] = 1; + } else { + for (j = 0; j < k; j++) { + tmpmat[i*k+j] = matrix[(dm_ids[i]-k)*k+j]; + } + } + } + + i = jerasure_invert_matrix(tmpmat, decoding_matrix, k, w); + free(tmpmat); + return i; +} + +/* Internal Routine */ +int jerasure_make_decoding_bitmatrix(int k, int m, int w, int *matrix, int *erased, int *decoding_matrix, int *dm_ids) +{ + int i, j, *tmpmat; + int index, mindex; + + j = 0; + for (i = 0; j < k; i++) { + if (erased[i] == 0) { + dm_ids[j] = i; + j++; + } + } + + tmpmat = talloc(int, k*k*w*w); + if (tmpmat == NULL) { return -1; } + for (i = 0; i < k; i++) { + if (dm_ids[i] < k) { + index = i*k*w*w; + for (j = 0; j < k*w*w; j++) tmpmat[index+j] = 0; + index = i*k*w*w+dm_ids[i]*w; + for (j = 0; j < w; j++) { + tmpmat[index] = 1; + index += (k*w+1); + } + } else { + index = i*k*w*w; + mindex = (dm_ids[i]-k)*k*w*w; + for (j = 0; j < k*w*w; j++) { + tmpmat[index+j] = matrix[mindex+j]; + } + } + } + + i = jerasure_invert_bitmatrix(tmpmat, decoding_matrix, k*w); + free(tmpmat); + return i; +} + +int jerasure_matrix_decode(int k, int m, int w, int *matrix, int row_k_ones, int *erasures, + char **data_ptrs, char **coding_ptrs, int size) +{ + int i, edd, lastdrive; + int *tmpids; + int *erased, *decoding_matrix, *dm_ids; + + if (w != 8 && w != 16 && w != 32) return -1; + + erased = jerasure_erasures_to_erased(k, m, erasures); + if (erased == NULL) return -1; + + /* Find the number of data drives failed */ + + lastdrive = k; + + edd = 0; + for (i = 0; i < k; i++) { + if (erased[i]) { + edd++; + lastdrive = i; + } + } + + /* You only need to create the decoding matrix in the following cases: + + 1. edd > 0 and row_k_ones is false. + 2. edd > 0 and row_k_ones is true and coding device 0 has been erased. + 3. edd > 1 + + We're going to use lastdrive to denote when to stop decoding data. + At this point in the code, it is equal to the last erased data device. + However, if we can't use the parity row to decode it (i.e. row_k_ones=0 + or erased[k] = 1, we're going to set it to k so that the decoding + pass will decode all data. + */ + + if (!row_k_ones || erased[k]) lastdrive = k; + + dm_ids = NULL; + decoding_matrix = NULL; + + if (edd > 1 || (edd > 0 && (!row_k_ones || erased[k]))) { + dm_ids = talloc(int, k); + if (dm_ids == NULL) { + free(erased); + return -1; + } + + decoding_matrix = talloc(int, k*k); + if (decoding_matrix == NULL) { + free(erased); + free(dm_ids); + return -1; + } + + if (jerasure_make_decoding_matrix(k, m, w, matrix, erased, decoding_matrix, dm_ids) < 0) { + free(erased); + free(dm_ids); + free(decoding_matrix); + return -1; + } + } + + /* Decode the data drives. + If row_k_ones is true and coding device 0 is intact, then only decode edd-1 drives. + This is done by stopping at lastdrive. + We test whether edd > 0 so that we can exit the loop early if we're done. + */ + + for (i = 0; edd > 0 && i < lastdrive; i++) { + if (erased[i]) { + jerasure_matrix_dotprod(k, w, decoding_matrix+(i*k), dm_ids, i, data_ptrs, coding_ptrs, size); + edd--; + } + } + + /* Then if necessary, decode drive lastdrive */ + + if (edd > 0) { + tmpids = talloc(int, k); + for (i = 0; i < k; i++) { + tmpids[i] = (i < lastdrive) ? i : i+1; + } + jerasure_matrix_dotprod(k, w, matrix, tmpids, lastdrive, data_ptrs, coding_ptrs, size); + free(tmpids); + } + + /* Finally, re-encode any erased coding devices */ + + for (i = 0; i < m; i++) { + if (erased[k+i]) { + jerasure_matrix_dotprod(k, w, matrix+(i*k), NULL, i+k, data_ptrs, coding_ptrs, size); + } + } + + free(erased); + if (dm_ids != NULL) free(dm_ids); + if (decoding_matrix != NULL) free(decoding_matrix); + + return 0; +} + + +int *jerasure_matrix_to_bitmatrix(int k, int m, int w, int *matrix) +{ + int *bitmatrix; + int rowelts, rowindex, colindex, elt, i, j, l, x; + + bitmatrix = talloc(int, k*m*w*w); + if (matrix == NULL) { return NULL; } + + rowelts = k * w; + rowindex = 0; + + for (i = 0; i < m; i++) { + colindex = rowindex; + for (j = 0; j < k; j++) { + elt = matrix[i*k+j]; + for (x = 0; x < w; x++) { + for (l = 0; l < w; l++) { + bitmatrix[colindex+x+l*rowelts] = ((elt & (1 << l)) ? 1 : 0); + } + elt = galois_single_multiply(elt, 2, w); + } + colindex += w; + } + rowindex += rowelts * w; + } + return bitmatrix; +} + +void jerasure_matrix_encode(int k, int m, int w, int *matrix, + char **data_ptrs, char **coding_ptrs, int size) +{ + int i; + + if (w != 8 && w != 16 && w != 32) { + fprintf(stderr, "ERROR: jerasure_matrix_encode() and w is not 8, 16 or 32\n"); + exit(1); + } + + for (i = 0; i < m; i++) { + jerasure_matrix_dotprod(k, w, matrix+(i*k), NULL, k+i, data_ptrs, coding_ptrs, size); + } +} + +void jerasure_bitmatrix_dotprod(int k, int w, int *bitmatrix_row, + int *src_ids, int dest_id, + char **data_ptrs, char **coding_ptrs, int size, int packetsize) +{ + int j, sindex, pstarted, index, x, y; + char *dptr, *pptr, *bdptr, *bpptr; + + if (size%(w*packetsize) != 0) { + fprintf(stderr, "jerasure_bitmatrix_dotprod - size%c(w*packetsize)) must = 0\n", '%'); + exit(1); + } + + bpptr = (dest_id < k) ? data_ptrs[dest_id] : coding_ptrs[dest_id-k]; + + for (sindex = 0; sindex < size; sindex += (packetsize*w)) { + index = 0; + for (j = 0; j < w; j++) { + pstarted = 0; + pptr = bpptr + sindex + j*packetsize; + for (x = 0; x < k; x++) { + if (src_ids == NULL) { + bdptr = data_ptrs[x]; + } else if (src_ids[x] < k) { + bdptr = data_ptrs[src_ids[x]]; + } else { + bdptr = coding_ptrs[src_ids[x]-k]; + } + for (y = 0; y < w; y++) { + if (bitmatrix_row[index]) { + dptr = bdptr + sindex + y*packetsize; + if (!pstarted) { + memcpy(pptr, dptr, packetsize); + jerasure_total_memcpy_bytes += packetsize; + pstarted = 1; + } else { + galois_region_xor(pptr, dptr, pptr, packetsize); + jerasure_total_xor_bytes += packetsize; + } + } + index++; + } + } + } + } +} + +void jerasure_do_parity(int k, char **data_ptrs, char *parity_ptr, int size) +{ + int i; + + memcpy(parity_ptr, data_ptrs[0], size); + jerasure_total_memcpy_bytes += size; + + for (i = 1; i < k; i++) { + galois_region_xor(data_ptrs[i], parity_ptr, parity_ptr, size); + jerasure_total_xor_bytes += size; + } +} + +int jerasure_invert_matrix(int *mat, int *inv, int rows, int w) +{ + int cols, i, j, k, x, rs2; + int row_start, tmp, inverse; + + cols = rows; + + k = 0; + for (i = 0; i < rows; i++) { + for (j = 0; j < cols; j++) { + inv[k] = (i == j) ? 1 : 0; + k++; + } + } + + /* First -- convert into upper triangular */ + for (i = 0; i < cols; i++) { + row_start = cols*i; + + /* Swap rows if we ave a zero i,i element. If we can't swap, then the + matrix was not invertible */ + + if (mat[row_start+i] == 0) { + for (j = i+1; j < rows && mat[cols*j+i] == 0; j++) ; + if (j == rows) return -1; + rs2 = j*cols; + for (k = 0; k < cols; k++) { + tmp = mat[row_start+k]; + mat[row_start+k] = mat[rs2+k]; + mat[rs2+k] = tmp; + tmp = inv[row_start+k]; + inv[row_start+k] = inv[rs2+k]; + inv[rs2+k] = tmp; + } + } + + /* Multiply the row by 1/element i,i */ + tmp = mat[row_start+i]; + if (tmp != 1) { + inverse = galois_single_divide(1, tmp, w); + for (j = 0; j < cols; j++) { + mat[row_start+j] = galois_single_multiply(mat[row_start+j], inverse, w); + inv[row_start+j] = galois_single_multiply(inv[row_start+j], inverse, w); + } + } + + /* Now for each j>i, add A_ji*Ai to Aj */ + k = row_start+i; + for (j = i+1; j != cols; j++) { + k += cols; + if (mat[k] != 0) { + if (mat[k] == 1) { + rs2 = cols*j; + for (x = 0; x < cols; x++) { + mat[rs2+x] ^= mat[row_start+x]; + inv[rs2+x] ^= inv[row_start+x]; + } + } else { + tmp = mat[k]; + rs2 = cols*j; + for (x = 0; x < cols; x++) { + mat[rs2+x] ^= galois_single_multiply(tmp, mat[row_start+x], w); + inv[rs2+x] ^= galois_single_multiply(tmp, inv[row_start+x], w); + } + } + } + } + } + + /* Now the matrix is upper triangular. Start at the top and multiply down */ + + for (i = rows-1; i >= 0; i--) { + row_start = i*cols; + for (j = 0; j < i; j++) { + rs2 = j*cols; + if (mat[rs2+i] != 0) { + tmp = mat[rs2+i]; + mat[rs2+i] = 0; + for (k = 0; k < cols; k++) { + inv[rs2+k] ^= galois_single_multiply(tmp, inv[row_start+k], w); + } + } + } + } + return 0; +} + +int jerasure_invertible_matrix(int *mat, int rows, int w) +{ + int cols, i, j, k, x, rs2; + int row_start, tmp, inverse; + + cols = rows; + + /* First -- convert into upper triangular */ + for (i = 0; i < cols; i++) { + row_start = cols*i; + + /* Swap rows if we ave a zero i,i element. If we can't swap, then the + matrix was not invertible */ + + if (mat[row_start+i] == 0) { + for (j = i+1; j < rows && mat[cols*j+i] == 0; j++) ; + if (j == rows) return 0; + rs2 = j*cols; + for (k = 0; k < cols; k++) { + tmp = mat[row_start+k]; + mat[row_start+k] = mat[rs2+k]; + mat[rs2+k] = tmp; + } + } + + /* Multiply the row by 1/element i,i */ + tmp = mat[row_start+i]; + if (tmp != 1) { + inverse = galois_single_divide(1, tmp, w); + for (j = 0; j < cols; j++) { + mat[row_start+j] = galois_single_multiply(mat[row_start+j], inverse, w); + } + } + + /* Now for each j>i, add A_ji*Ai to Aj */ + k = row_start+i; + for (j = i+1; j != cols; j++) { + k += cols; + if (mat[k] != 0) { + if (mat[k] == 1) { + rs2 = cols*j; + for (x = 0; x < cols; x++) { + mat[rs2+x] ^= mat[row_start+x]; + } + } else { + tmp = mat[k]; + rs2 = cols*j; + for (x = 0; x < cols; x++) { + mat[rs2+x] ^= galois_single_multiply(tmp, mat[row_start+x], w); + } + } + } + } + } + return 1; +} + +/* Converts a list-style version of the erasures into an array of k+m elements + where the element = 1 if the index has been erased, and zero otherwise */ + +int *jerasure_erasures_to_erased(int k, int m, int *erasures) +{ + int td; + int t_non_erased; + int *erased; + int i; + + td = k+m; + erased = talloc(int, td); + if (erased == NULL) return NULL; + t_non_erased = td; + + for (i = 0; i < td; i++) erased[i] = 0; + + for (i = 0; erasures[i] != -1; i++) { + if (erased[erasures[i]] == 0) { + erased[erasures[i]] = 1; + t_non_erased--; + if (t_non_erased < k) { + free(erased); + return NULL; + } + } + } + return erased; +} + +void jerasure_free_schedule(int **schedule) +{ + int i; + + for (i = 0; schedule[i][0] >= 0; i++) free(schedule[i]); + free(schedule[i]); + free(schedule); +} + +void jerasure_free_schedule_cache(int k, int m, int ***cache) +{ + int e1, e2; + + if (m != 2) { + fprintf(stderr, "jerasure_free_schedule_cache(): m must equal 2\n"); + exit(1); + } + + for (e1 = 0; e1 < k+m; e1++) { + for (e2 = 0; e2 < e1; e2++) { + jerasure_free_schedule(cache[e1*(k+m)+e2]); + } + jerasure_free_schedule(cache[e1*(k+m)+e1]); + } + free(cache); +} + +void jerasure_matrix_dotprod(int k, int w, int *matrix_row, + int *src_ids, int dest_id, + char **data_ptrs, char **coding_ptrs, int size) +{ + int init; + char *dptr, *sptr; + int i; + + if (w != 1 && w != 8 && w != 16 && w != 32) { + fprintf(stderr, "ERROR: jerasure_matrix_dotprod() called and w is not 1, 8, 16 or 32\n"); + exit(1); + } + + init = 0; + + dptr = (dest_id < k) ? data_ptrs[dest_id] : coding_ptrs[dest_id-k]; + + /* First copy or xor any data that does not need to be multiplied by a factor */ + + for (i = 0; i < k; i++) { + if (matrix_row[i] == 1) { + if (src_ids == NULL) { + sptr = data_ptrs[i]; + } else if (src_ids[i] < k) { + sptr = data_ptrs[src_ids[i]]; + } else { + sptr = coding_ptrs[src_ids[i]-k]; + } + if (init == 0) { + memcpy(dptr, sptr, size); + jerasure_total_memcpy_bytes += size; + init = 1; + } else { + galois_region_xor(sptr, dptr, dptr, size); + jerasure_total_xor_bytes += size; + } + } + } + + /* Now do the data that needs to be multiplied by a factor */ + + for (i = 0; i < k; i++) { + if (matrix_row[i] != 0 && matrix_row[i] != 1) { + if (src_ids == NULL) { + sptr = data_ptrs[i]; + } else if (src_ids[i] < k) { + sptr = data_ptrs[src_ids[i]]; + } else { + sptr = coding_ptrs[src_ids[i]-k]; + } + switch (w) { + case 8: galois_w08_region_multiply(sptr, matrix_row[i], size, dptr, init); break; + case 16: galois_w16_region_multiply(sptr, matrix_row[i], size, dptr, init); break; + case 32: galois_w32_region_multiply(sptr, matrix_row[i], size, dptr, init); break; + } + jerasure_total_gf_bytes += size; + init = 1; + } + } +} + + +int jerasure_bitmatrix_decode(int k, int m, int w, int *bitmatrix, int row_k_ones, int *erasures, + char **data_ptrs, char **coding_ptrs, int size, int packetsize) +{ + int i; + int *erased; + int *decoding_matrix; + int *dm_ids; + int edd, *tmpids, lastdrive; + + erased = jerasure_erasures_to_erased(k, m, erasures); + if (erased == NULL) return -1; + + /* See jerasure_matrix_decode for the logic of this routine. This one works just like + it, but calls the bitmatrix ops instead */ + + lastdrive = k; + + edd = 0; + for (i = 0; i < k; i++) { + if (erased[i]) { + edd++; + lastdrive = i; + } + } + + if (row_k_ones != 1 || erased[k]) lastdrive = k; + + dm_ids = NULL; + decoding_matrix = NULL; + + if (edd > 1 || (edd > 0 && (row_k_ones != 1 || erased[k]))) { + + dm_ids = talloc(int, k); + if (dm_ids == NULL) { + free(erased); + return -1; + } + + decoding_matrix = talloc(int, k*k*w*w); + if (decoding_matrix == NULL) { + free(erased); + free(dm_ids); + return -1; + } + + if (jerasure_make_decoding_bitmatrix(k, m, w, bitmatrix, erased, decoding_matrix, dm_ids) < 0) { + free(erased); + free(dm_ids); + free(decoding_matrix); + return -1; + } + } + + for (i = 0; edd > 0 && i < lastdrive; i++) { + if (erased[i]) { + jerasure_bitmatrix_dotprod(k, w, decoding_matrix+i*k*w*w, dm_ids, i, data_ptrs, coding_ptrs, size, packetsize); + edd--; + } + } + + if (edd > 0) { + tmpids = talloc(int, k); + for (i = 0; i < k; i++) { + tmpids[i] = (i < lastdrive) ? i : i+1; + } + jerasure_bitmatrix_dotprod(k, w, bitmatrix, tmpids, lastdrive, data_ptrs, coding_ptrs, size, packetsize); + free(tmpids); + } + + for (i = 0; i < m; i++) { + if (erased[k+i]) { + jerasure_bitmatrix_dotprod(k, w, bitmatrix+i*k*w*w, NULL, k+i, data_ptrs, coding_ptrs, size, packetsize); + } + } + + free(erased); + if (dm_ids != NULL) free(dm_ids); + if (decoding_matrix != NULL) free(decoding_matrix); + + return 0; +} + +static char **set_up_ptrs_for_scheduled_decoding(int k, int m, int *erasures, char **data_ptrs, char **coding_ptrs) +{ + int ddf, cdf; + int *erased; + char **ptrs; + int i, j, x; + + ddf = 0; + cdf = 0; + for (i = 0; erasures[i] != -1; i++) { + if (erasures[i] < k) ddf++; else cdf++; + } + + erased = jerasure_erasures_to_erased(k, m, erasures); + if (erased == NULL) return NULL; + + /* Set up ptrs. It will be as follows: + + - If data drive i has not failed, then ptrs[i] = data_ptrs[i]. + - If data drive i has failed, then ptrs[i] = coding_ptrs[j], where j is the + lowest unused non-failed coding drive. + - Elements k to k+ddf-1 are data_ptrs[] of the failed data drives. + - Elements k+ddf to k+ddf+cdf-1 are coding_ptrs[] of the failed data drives. + + The array row_ids contains the ids of ptrs. + The array ind_to_row_ids contains the row_id of drive i. + + However, we're going to set row_ids and ind_to_row in a different procedure. + */ + + ptrs = talloc(char *, k+m); + + j = k; + x = k; + for (i = 0; i < k; i++) { + if (erased[i] == 0) { + ptrs[i] = data_ptrs[i]; + } else { + while (erased[j]) j++; + ptrs[i] = coding_ptrs[j-k]; + j++; + ptrs[x] = data_ptrs[i]; + x++; + } + } + for (i = k; i < k+m; i++) { + if (erased[i]) { + ptrs[x] = coding_ptrs[i-k]; + x++; + } + } + free(erased); + return ptrs; +} + +static int set_up_ids_for_scheduled_decoding(int k, int m, int *erasures, int *row_ids, int *ind_to_row) +{ + int ddf, cdf; + int *erased; + int i, j, x; + + ddf = 0; + cdf = 0; + for (i = 0; erasures[i] != -1; i++) { + if (erasures[i] < k) ddf++; else cdf++; + } + + erased = jerasure_erasures_to_erased(k, m, erasures); + if (erased == NULL) return -1; + + /* See set_up_ptrs_for_scheduled_decoding for how these are set */ + + j = k; + x = k; + for (i = 0; i < k; i++) { + if (erased[i] == 0) { + row_ids[i] = i; + ind_to_row[i] = i; + } else { + while (erased[j]) j++; + row_ids[i] = j; + ind_to_row[j] = i; + j++; + row_ids[x] = i; + ind_to_row[i] = x; + x++; + } + } + for (i = k; i < k+m; i++) { + if (erased[i]) { + row_ids[x] = i; + ind_to_row[i] = x; + x++; + } + } + free(erased); + return 0; +} + +static int **jerasure_generate_decoding_schedule(int k, int m, int w, int *bitmatrix, int *erasures, int smart) +{ + int i, j, x, drive, y, index, z; + int *decoding_matrix, *inverse, *real_decoding_matrix; + int *ptr; + int *row_ids; + int *ind_to_row; + int ddf, cdf; + int **schedule; + int *b1, *b2; + + /* First, figure out the number of data drives that have failed, and the + number of coding drives that have failed: ddf and cdf */ + + ddf = 0; + cdf = 0; + for (i = 0; erasures[i] != -1; i++) { + if (erasures[i] < k) ddf++; else cdf++; + } + + row_ids = talloc(int, k+m); + ind_to_row = talloc(int, k+m); + + if (set_up_ids_for_scheduled_decoding(k, m, erasures, row_ids, ind_to_row) < 0) return NULL; + + /* Now, we're going to create one decoding matrix which is going to + decode everything with one call. The hope is that the scheduler + will do a good job. This matrix has w*e rows, where e is the + number of erasures (ddf+cdf) */ + + real_decoding_matrix = talloc(int, k*w*(cdf+ddf)*w); + + /* First, if any data drives have failed, then initialize the first + ddf*w rows of the decoding matrix from the standard decoding + matrix inversion */ + + if (ddf > 0) { + + decoding_matrix = talloc(int, k*k*w*w); + ptr = decoding_matrix; + for (i = 0; i < k; i++) { + if (row_ids[i] == i) { + bzero(ptr, k*w*w*sizeof(int)); + for (x = 0; x < w; x++) { + ptr[x+i*w+x*k*w] = 1; + } + } else { + memcpy(ptr, bitmatrix+k*w*w*(row_ids[i]-k), k*w*w*sizeof(int)); + } + ptr += (k*w*w); + } + inverse = talloc(int, k*k*w*w); + jerasure_invert_bitmatrix(decoding_matrix, inverse, k*w); + +/* printf("\nMatrix to invert\n"); + jerasure_print_bitmatrix(decoding_matrix, k*w, k*w, w); + printf("\n"); + printf("\nInverse\n"); + jerasure_print_bitmatrix(inverse, k*w, k*w, w); + printf("\n"); */ + + free(decoding_matrix); + ptr = real_decoding_matrix; + for (i = 0; i < ddf; i++) { + memcpy(ptr, inverse+k*w*w*row_ids[k+i], sizeof(int)*k*w*w); + ptr += (k*w*w); + } + free(inverse); + } + + /* Next, here comes the hard part. For each coding node that needs + to be decoded, you start by putting its rows of the distribution + matrix into the decoding matrix. If there were no failed data + nodes, then you're done. However, if there have been failed + data nodes, then you need to modify the columns that correspond + to the data nodes. You do that by first zeroing them. Then + whereever there is a one in the distribution matrix, you XOR + in the corresponding row from the failed data node's entry in + the decoding matrix. The whole process kind of makes my head + spin, but it works. + */ + + for (x = 0; x < cdf; x++) { + drive = row_ids[x+ddf+k]-k; + ptr = real_decoding_matrix + k*w*w*(ddf+x); + memcpy(ptr, bitmatrix+drive*k*w*w, sizeof(int)*k*w*w); + + for (i = 0; i < k; i++) { + if (row_ids[i] != i) { + for (j = 0; j < w; j++) { + bzero(ptr+j*k*w+i*w, sizeof(int)*w); + } + } + } + + /* There's the yucky part */ + + index = drive*k*w*w; + for (i = 0; i < k; i++) { + if (row_ids[i] != i) { + b1 = real_decoding_matrix+(ind_to_row[i]-k)*k*w*w; + for (j = 0; j < w; j++) { + b2 = ptr + j*k*w; + for (y = 0; y < w; y++) { + if (bitmatrix[index+j*k*w+i*w+y]) { + for (z = 0; z < k*w; z++) { + b2[z] = b2[z] ^ b1[z+y*k*w]; + } + } + } + } + } + } + } + +/* + printf("\n\nReal Decoding Matrix\n\n"); + jerasure_print_bitmatrix(real_decoding_matrix, (ddf+cdf)*w, k*w, w); + printf("\n"); */ + if (smart) { + schedule = jerasure_smart_bitmatrix_to_schedule(k, ddf+cdf, w, real_decoding_matrix); + } else { + schedule = jerasure_dumb_bitmatrix_to_schedule(k, ddf+cdf, w, real_decoding_matrix); + } + free(row_ids); + free(ind_to_row); + free(real_decoding_matrix); + return schedule; +} + +int jerasure_schedule_decode_lazy(int k, int m, int w, int *bitmatrix, int *erasures, + char **data_ptrs, char **coding_ptrs, int size, int packetsize, + int smart) +{ + int i, tdone; + char **ptrs; + int **schedule; + + ptrs = set_up_ptrs_for_scheduled_decoding(k, m, erasures, data_ptrs, coding_ptrs); + if (ptrs == NULL) return -1; + + schedule = jerasure_generate_decoding_schedule(k, m, w, bitmatrix, erasures, smart); + if (schedule == NULL) { + free(ptrs); + return -1; + } + + for (tdone = 0; tdone < size; tdone += packetsize*w) { + jerasure_do_scheduled_operations(ptrs, schedule, packetsize); + for (i = 0; i < k+m; i++) ptrs[i] += (packetsize*w); + } + + jerasure_free_schedule(schedule); + free(ptrs); + + return 0; +} + +int jerasure_schedule_decode_cache(int k, int m, int w, int ***scache, int *erasures, + char **data_ptrs, char **coding_ptrs, int size, int packetsize) +{ + int i, tdone; + char **ptrs; + int **schedule; + int index; + + if (erasures[1] == -1) { + index = erasures[0]*(k+m) + erasures[0]; + } else if (erasures[2] == -1) { + index = erasures[0]*(k+m) + erasures[1]; + } else { + return -1; + } + + schedule = scache[index]; + + ptrs = set_up_ptrs_for_scheduled_decoding(k, m, erasures, data_ptrs, coding_ptrs); + if (ptrs == NULL) return -1; + + + for (tdone = 0; tdone < size; tdone += packetsize*w) { + jerasure_do_scheduled_operations(ptrs, schedule, packetsize); + for (i = 0; i < k+m; i++) ptrs[i] += (packetsize*w); + } + + free(ptrs); + + return 0; +} + +/* This only works when m = 2 */ + +int ***jerasure_generate_schedule_cache(int k, int m, int w, int *bitmatrix, int smart) +{ + int ***scache; + int erasures[3]; + int e1, e2; + + /* Ok -- this is yucky, but it's how I'm doing it. You will make an index out + of erasures, which will be e1*(k+m)+(e2). If there is no e2, then e2 = e1. + Isn't that clever and confusing. Sorry. + + We're not going to worry about ordering -- in other words, the schedule for + e1,e2 will be the same as e2,e1. They will have the same pointer -- the + schedule will not be duplicated. */ + + if (m != 2) return NULL; + + scache = talloc(int **, (k+m)*(k+m+1)); + if (scache == NULL) return NULL; + + for (e1 = 0; e1 < k+m; e1++) { + erasures[0] = e1; + for (e2 = 0; e2 < e1; e2++) { + erasures[1] = e2; + erasures[2] = -1; + scache[e1*(k+m)+e2] = jerasure_generate_decoding_schedule(k, m, w, bitmatrix, erasures, smart); + scache[e2*(k+m)+e1] = scache[e1*(k+m)+e2]; + } + erasures[1] = -1; + scache[e1*(k+m)+e1] = jerasure_generate_decoding_schedule(k, m, w, bitmatrix, erasures, smart); + } + return scache; + +} + +int jerasure_invert_bitmatrix(int *mat, int *inv, int rows) +{ + int cols, i, j, k; + int tmp; + + cols = rows; + + k = 0; + for (i = 0; i < rows; i++) { + for (j = 0; j < cols; j++) { + inv[k] = (i == j) ? 1 : 0; + k++; + } + } + + /* First -- convert into upper triangular */ + + for (i = 0; i < cols; i++) { + + /* Swap rows if we have a zero i,i element. If we can't swap, then the + matrix was not invertible */ + + if ((mat[i*cols+i]) == 0) { + for (j = i+1; j < rows && (mat[j*cols+i]) == 0; j++) ; + if (j == rows) return -1; + for (k = 0; k < cols; k++) { + tmp = mat[i*cols+k]; mat[i*cols+k] = mat[j*cols+k]; mat[j*cols+k] = tmp; + tmp = inv[i*cols+k]; inv[i*cols+k] = inv[j*cols+k]; inv[j*cols+k] = tmp; + } + } + + /* Now for each j>i, add A_ji*Ai to Aj */ + for (j = i+1; j != rows; j++) { + if (mat[j*cols+i] != 0) { + for (k = 0; k < cols; k++) { + mat[j*cols+k] ^= mat[i*cols+k]; + inv[j*cols+k] ^= inv[i*cols+k]; + } + } + } + } + + /* Now the matrix is upper triangular. Start at the top and multiply down */ + + for (i = rows-1; i >= 0; i--) { + for (j = 0; j < i; j++) { + if (mat[j*cols+i]) { + for (k = 0; k < cols; k++) { + mat[j*cols+k] ^= mat[i*cols+k]; + inv[j*cols+k] ^= inv[i*cols+k]; + } + } + } + } + return 0; +} + +int jerasure_invertible_bitmatrix(int *mat, int rows) +{ + int cols, i, j, k; + int tmp; + + cols = rows; + + /* First -- convert into upper triangular */ + + for (i = 0; i < cols; i++) { + + /* Swap rows if we have a zero i,i element. If we can't swap, then the + matrix was not invertible */ + + if ((mat[i*cols+i]) == 0) { + for (j = i+1; j < rows && (mat[j*cols+i]) == 0; j++) ; + if (j == rows) return 0; + for (k = 0; k < cols; k++) { + tmp = mat[i*cols+k]; mat[i*cols+k] = mat[j*cols+k]; mat[j*cols+k] = tmp; + } + } + + /* Now for each j>i, add A_ji*Ai to Aj */ + for (j = i+1; j != rows; j++) { + if (mat[j*cols+i] != 0) { + for (k = 0; k < cols; k++) { + mat[j*cols+k] ^= mat[i*cols+k]; + } + } + } + } + return 1; +} + + +int *jerasure_matrix_multiply(int *m1, int *m2, int r1, int c1, int r2, int c2, int w) +{ + int *product, i, j, k; + + product = (int *) malloc(sizeof(int)*r1*c2); + for (i = 0; i < r1*c2; i++) product[i] = 0; + + for (i = 0; i < r1; i++) { + for (j = 0; j < c2; j++) { + for (k = 0; k < r2; k++) { + product[i*c2+j] ^= galois_single_multiply(m1[i*c1+k], m2[k*c2+j], w); + } + } + } + return product; +} + +void jerasure_get_stats(double *fill_in) +{ + fill_in[0] = jerasure_total_xor_bytes; + fill_in[1] = jerasure_total_gf_bytes; + fill_in[2] = jerasure_total_memcpy_bytes; + jerasure_total_xor_bytes = 0; + jerasure_total_gf_bytes = 0; + jerasure_total_memcpy_bytes = 0; +} + +void jerasure_do_scheduled_operations(char **ptrs, int **operations, int packetsize) +{ + char *sptr; + char *dptr; + int op; + + for (op = 0; operations[op][0] >= 0; op++) { + sptr = ptrs[operations[op][0]] + operations[op][1]*packetsize; + dptr = ptrs[operations[op][2]] + operations[op][3]*packetsize; + if (operations[op][4]) { +/* printf("%d,%d %d,%d\n", operations[op][0], + operations[op][1], + operations[op][2], + operations[op][3]); + printf("xor(0x%x, 0x%x -> 0x%x, %d)\n", sptr, dptr, dptr, packetsize); */ + galois_region_xor(sptr, dptr, dptr, packetsize); + jerasure_total_xor_bytes += packetsize; + } else { +/* printf("memcpy(0x%x <- 0x%x)\n", dptr, sptr); */ + memcpy(dptr, sptr, packetsize); + jerasure_total_memcpy_bytes += packetsize; + } + } +} + +void jerasure_schedule_encode(int k, int m, int w, int **schedule, + char **data_ptrs, char **coding_ptrs, int size, int packetsize) +{ + char **ptr_copy; + int i, tdone; + + ptr_copy = talloc(char *, (k+m)); + for (i = 0; i < k; i++) ptr_copy[i] = data_ptrs[i]; + for (i = 0; i < m; i++) ptr_copy[i+k] = coding_ptrs[i]; + for (tdone = 0; tdone < size; tdone += packetsize*w) { + jerasure_do_scheduled_operations(ptr_copy, schedule, packetsize); + for (i = 0; i < k+m; i++) ptr_copy[i] += (packetsize*w); + } + free(ptr_copy); +} + +int **jerasure_dumb_bitmatrix_to_schedule(int k, int m, int w, int *bitmatrix) +{ + int **operations; + int op; + int index, optodo, i, j; + + operations = talloc(int *, k*m*w*w+1); + op = 0; + + index = 0; + for (i = 0; i < m*w; i++) { + optodo = 0; + for (j = 0; j < k*w; j++) { + if (bitmatrix[index]) { + operations[op] = talloc(int, 5); + operations[op][4] = optodo; + operations[op][0] = j/w; + operations[op][1] = j%w; + operations[op][2] = k+i/w; + operations[op][3] = i%w; + optodo = 1; + op++; + + } + index++; + } + } + operations[op] = talloc(int, 5); + operations[op][0] = -1; + return operations; +} + +int **jerasure_smart_bitmatrix_to_schedule(int k, int m, int w, int *bitmatrix) +{ + int **operations; + int op; + int i, j; + int *diff, *from, *b1, *flink, *blink; + int *ptr, no, row; + int optodo; + int bestrow = 0, bestdiff, top; + +/* printf("Scheduling:\n\n"); + jerasure_print_bitmatrix(bitmatrix, m*w, k*w, w); */ + + operations = talloc(int *, k*m*w*w+1); + op = 0; + + diff = talloc(int, m*w); + from = talloc(int, m*w); + flink = talloc(int, m*w); + blink = talloc(int, m*w); + + ptr = bitmatrix; + + bestdiff = k*w+1; + top = 0; + for (i = 0; i < m*w; i++) { + no = 0; + for (j = 0; j < k*w; j++) { + no += *ptr; + ptr++; + } + diff[i] = no; + from[i] = -1; + flink[i] = i+1; + blink[i] = i-1; + if (no < bestdiff) { + bestdiff = no; + bestrow = i; + } + } + + flink[m*w-1] = -1; + + while (top != -1) { + row = bestrow; + /* printf("Doing row %d - %d from %d\n", row, diff[row], from[row]); */ + + if (blink[row] == -1) { + top = flink[row]; + if (top != -1) blink[top] = -1; + } else { + flink[blink[row]] = flink[row]; + if (flink[row] != -1) { + blink[flink[row]] = blink[row]; + } + } + + ptr = bitmatrix + row*k*w; + if (from[row] == -1) { + optodo = 0; + for (j = 0; j < k*w; j++) { + if (ptr[j]) { + operations[op] = talloc(int, 5); + operations[op][4] = optodo; + operations[op][0] = j/w; + operations[op][1] = j%w; + operations[op][2] = k+row/w; + operations[op][3] = row%w; + optodo = 1; + op++; + } + } + } else { + operations[op] = talloc(int, 5); + operations[op][4] = 0; + operations[op][0] = k+from[row]/w; + operations[op][1] = from[row]%w; + operations[op][2] = k+row/w; + operations[op][3] = row%w; + op++; + b1 = bitmatrix + from[row]*k*w; + for (j = 0; j < k*w; j++) { + if (ptr[j] ^ b1[j]) { + operations[op] = talloc(int, 5); + operations[op][4] = 1; + operations[op][0] = j/w; + operations[op][1] = j%w; + operations[op][2] = k+row/w; + operations[op][3] = row%w; + optodo = 1; + op++; + } + } + } + bestdiff = k*w+1; + for (i = top; i != -1; i = flink[i]) { + no = 1; + b1 = bitmatrix + i*k*w; + for (j = 0; j < k*w; j++) no += (ptr[j] ^ b1[j]); + if (no < diff[i]) { + from[i] = row; + diff[i] = no; + } + if (diff[i] < bestdiff) { + bestdiff = diff[i]; + bestrow = i; + } + } + } + + operations[op] = talloc(int, 5); + operations[op][0] = -1; + free(from); + free(diff); + free(blink); + free(flink); + + return operations; +} + +void jerasure_bitmatrix_encode(int k, int m, int w, int *bitmatrix, + char **data_ptrs, char **coding_ptrs, int size, int packetsize) +{ + int i; + + if (packetsize%sizeof(long) != 0) { + fprintf(stderr, "jerasure_bitmatrix_encode - packetsize(%d) %c sizeof(long) != 0\n", packetsize, '%'); + exit(1); + } + if (size%(packetsize*w) != 0) { + fprintf(stderr, "jerasure_bitmatrix_encode - size(%d) %c (packetsize(%d)*w(%d))) != 0\n", + size, '%', packetsize, w); + exit(1); + } + + for (i = 0; i < m; i++) { + jerasure_bitmatrix_dotprod(k, w, bitmatrix+i*k*w*w, NULL, k+i, data_ptrs, coding_ptrs, size, packetsize); + } +} + diff --git a/src/osd/ErasureCodePluginJerasure/jerasure.h b/src/osd/ErasureCodePluginJerasure/jerasure.h new file mode 100755 index 00000000000..8cc25cad839 --- /dev/null +++ b/src/osd/ErasureCodePluginJerasure/jerasure.h @@ -0,0 +1,300 @@ +/* jerasure.h - header of kernel procedures + * James S. Plank + +Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure Coding Techniques + +Revision 1.2A +May 24, 2011 + +James S. Plank +Department of Electrical Engineering and Computer Science +University of Tennessee +Knoxville, TN 37996 +plank@cs.utk.edu + +Copyright (c) 2011, James S. Plank +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + - Neither the name of the University of Tennessee nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + */ + +#ifndef _JERASURE_H +#define _JERASURE_H + +/* This uses procedures from the Galois Field arithmetic library */ + +#include "galois.h" + +/* ------------------------------------------------------------ */ +/* In all of the routines below: + + k = Number of data devices + m = Number of coding devices + w = Word size + + data_ptrs = An array of k pointers to data which is size bytes. + Size must be a multiple of sizeof(long). + Pointers must also be longword aligned. + + coding_ptrs = An array of m pointers to coding data which is size bytes. + + packetsize = The size of a coding block with bitmatrix coding. + When you code with a bitmatrix, you will use w packets + of size packetsize. + + matrix = an array of k*m integers. + It represents an m by k matrix. + Element i,j is in matrix[i*k+j]; + + bitmatrix = an array of k*m*w*w integers. + It represents an mw by kw matrix. + Element i,j is in matrix[i*k*w+j]; + + erasures = an array of id's of erased devices. + Id's are integers between 0 and k+m-1. + Id's 0 to k-1 are id's of data devices. + Id's k to k+m-1 are id's of coding devices: + Coding device id = id-k. + If there are e erasures, erasures[e] = -1. + + schedule = an array of schedule operations. + + If there are m operations, then schedule[m][0] = -1. + + operation = an array of 5 integers: + + 0 = operation: 0 for copy, 1 for xor (-1 for end) + 1 = source device (0 - k+m-1) + 2 = source packet (0 - w-1) + 3 = destination device (0 - k+m-1) + 4 = destination packet (0 - w-1) + */ + +/* --------------------------------------------------------------- */ +/* Bitmatrices / schedules ---------------------------------------- */ +/* + - jerasure_matrix_to_bitmatrix turns a m X k matrix in GF(2^w) into a + wm X wk bitmatrix (in GF(2)). This is + explained in the Cauchy Reed-Solomon coding + paper. + + - jerasure_dumb_bitmatrix_to_schedule turns a bitmatrix into a schedule + using the straightforward algorithm -- just + schedule the dot products defined by each + row of the matrix. + + - jerasure_smart_bitmatrix_to_schedule turns a bitmatrix into a schedule, + but tries to use previous dot products to + calculate new ones. This is the optimization + explained in the original Liberation code paper. + + - jerasure_generate_schedule_cache precalcalculate all the schedule for the + given distribution bitmatrix. M must equal 2. + + - jerasure_free_schedule frees a schedule that was allocated with + jerasure_XXX_bitmatrix_to_schedule. + + - jerasure_free_schedule_cache frees a schedule cache that was created with + jerasure_generate_schedule_cache. + */ + +int *jerasure_matrix_to_bitmatrix(int k, int m, int w, int *matrix); +int **jerasure_dumb_bitmatrix_to_schedule(int k, int m, int w, int *bitmatrix); +int **jerasure_smart_bitmatrix_to_schedule(int k, int m, int w, int *bitmatrix); +int ***jerasure_generate_schedule_cache(int k, int m, int w, int *bitmatrix, int smart); + +void jerasure_free_schedule(int **schedule); +void jerasure_free_schedule_cache(int k, int m, int ***cache); + + +/* ------------------------------------------------------------ */ +/* Encoding - these are all straightforward. jerasure_matrix_encode only + works with w = 8|16|32. */ + +void jerasure_do_parity(int k, char **data_ptrs, char *parity_ptr, int size); + +void jerasure_matrix_encode(int k, int m, int w, int *matrix, + char **data_ptrs, char **coding_ptrs, int size); + +void jerasure_bitmatrix_encode(int k, int m, int w, int *bitmatrix, + char **data_ptrs, char **coding_ptrs, int size, int packetsize); + +void jerasure_schedule_encode(int k, int m, int w, int **schedule, + char **data_ptrs, char **coding_ptrs, int size, int packetsize); + +/* ------------------------------------------------------------ */ +/* Decoding. -------------------------------------------------- */ + +/* These return integers, because the matrix may not be invertible. + + The parameter row_k_ones should be set to 1 if row k of the matrix + (or rows kw to (k+1)w+1) of th distribution matrix are all ones + (or all identity matrices). Then you can improve the performance + of decoding when there is more than one failure, and the parity + device didn't fail. You do it by decoding all but one of the data + devices, and then decoding the last data device from the data devices + and the parity device. + + jerasure_schedule_decode_lazy generates the schedule on the fly. + + jerasure_matrix_decode only works when w = 8|16|32. + + jerasure_make_decoding_matrix/bitmatrix make the k*k decoding matrix + (or wk*wk bitmatrix) by taking the rows corresponding to k + non-erased devices of the distribution matrix, and then + inverting that matrix. + + You should already have allocated the decoding matrix and + dm_ids, which is a vector of k integers. These will be + filled in appropriately. dm_ids[i] is the id of element + i of the survivors vector. I.e. row i of the decoding matrix + times dm_ids equals data drive i. + + Both of these routines take "erased" instead of "erasures". + Erased is a vector with k+m elements, which has 0 or 1 for + each device's id, according to whether the device is erased. + + jerasure_erasures_to_erased allocates and returns erased from erasures. + + */ + +int jerasure_matrix_decode(int k, int m, int w, + int *matrix, int row_k_ones, int *erasures, + char **data_ptrs, char **coding_ptrs, int size); + +int jerasure_bitmatrix_decode(int k, int m, int w, + int *bitmatrix, int row_k_ones, int *erasures, + char **data_ptrs, char **coding_ptrs, int size, int packetsize); + +int jerasure_schedule_decode_lazy(int k, int m, int w, int *bitmatrix, int *erasures, + char **data_ptrs, char **coding_ptrs, int size, int packetsize, + int smart); + +int jerasure_schedule_decode_cache(int k, int m, int w, int ***scache, int *erasures, + char **data_ptrs, char **coding_ptrs, int size, int packetsize); + +int jerasure_make_decoding_matrix(int k, int m, int w, int *matrix, int *erased, + int *decoding_matrix, int *dm_ids); + +int jerasure_make_decoding_bitmatrix(int k, int m, int w, int *matrix, int *erased, + int *decoding_matrix, int *dm_ids); + +int *jerasure_erasures_to_erased(int k, int m, int *erasures); + +/* ------------------------------------------------------------ */ +/* These perform dot products and schedules. -------------------*/ +/* + src_ids is a matrix of k id's (0 - k-1 for data devices, k - k+m-1 + for coding devices) that identify the source devices. Dest_id is + the id of the destination device. + + jerasure_matrix_dotprod only works when w = 8|16|32. + + jerasure_do_scheduled_operations executes the schedule on w*packetsize worth of + bytes from each device. ptrs is an array of pointers which should have as many + elements as the highest referenced device in the schedule. + + */ + +void jerasure_matrix_dotprod(int k, int w, int *matrix_row, + int *src_ids, int dest_id, + char **data_ptrs, char **coding_ptrs, int size); + +void jerasure_bitmatrix_dotprod(int k, int w, int *bitmatrix_row, + int *src_ids, int dest_id, + char **data_ptrs, char **coding_ptrs, int size, int packetsize); + +void jerasure_do_scheduled_operations(char **ptrs, int **schedule, int packetsize); + +/* ------------------------------------------------------------ */ +/* Matrix Inversion ------------------------------------------- */ +/* + The two matrix inversion functions work on rows*rows matrices of + ints. If a bitmatrix, then each int will just be zero or one. + Otherwise, they will be elements of gf(2^w). Obviously, you can + do bit matrices with crs_invert_matrix() and set w = 1, but + crs_invert_bitmatrix will be more efficient. + + The two invertible functions return whether a matrix is invertible. + They are more efficient than the inverstion functions. + + Mat will be destroyed when the matrix inversion or invertible + testing is done. Sorry. + + Inv must be allocated by the caller. + + The two invert_matrix functions return 0 on success, and -1 if the + matrix is uninvertible. + + The two invertible function simply return whether the matrix is + invertible. (0 or 1). Mat will be destroyed. + */ + +int jerasure_invert_matrix(int *mat, int *inv, int rows, int w); +int jerasure_invert_bitmatrix(int *mat, int *inv, int rows); +int jerasure_invertible_matrix(int *mat, int rows, int w); +int jerasure_invertible_bitmatrix(int *mat, int rows); + +/* ------------------------------------------------------------ */ +/* Basic matrix operations -------------------------------------*/ +/* + Each of the print_matrix routines require a w. In jerasure_print_matrix, + this is to calculate the field width. In jerasure_print_bitmatrix, it is + to put spaces between the bits. + + jerasure_matrix_multiply is a simple matrix multiplier in GF(2^w). It returns a r1*c2 + matrix, which is the product of the two input matrices. It allocates + the product. Obviously, c1 should equal r2. However, this is not + validated by the procedure. +*/ + +void jerasure_print_matrix(int *matrix, int rows, int cols, int w); +void jerasure_print_bitmatrix(int *matrix, int rows, int cols, int w); + + +int *jerasure_matrix_multiply(int *m1, int *m2, int r1, int c1, int r2, int c2, int w); + +/* ------------------------------------------------------------ */ +/* Stats ------------------------------------------------------ */ +/* + jerasure_get_stats fills in a vector of three doubles: + + fill_in[0] is the number of bytes that have been XOR'd + fill_in[1] is the number of bytes that have been copied + fill_in[2] is the number of bytes that have been multiplied + by a constant in GF(2^w) + + When jerasure_get_stats() is called, it resets its values. + */ + +void jerasure_get_stats(double *fill_in); + +#endif diff --git a/src/osd/ErasureCodePluginJerasure/liberation.c b/src/osd/ErasureCodePluginJerasure/liberation.c new file mode 100755 index 00000000000..beca10df9f4 --- /dev/null +++ b/src/osd/ErasureCodePluginJerasure/liberation.c @@ -0,0 +1,265 @@ +/* liberation.c + * James S. Plank + +Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure Coding Techniques + +Revision 1.2A +May 24, 2011 + +James S. Plank +Department of Electrical Engineering and Computer Science +University of Tennessee +Knoxville, TN 37996 +plank@cs.utk.edu + +Copyright (c) 2011, James S. Plank +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + - Neither the name of the University of Tennessee nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "galois.h" +#include "jerasure.h" +#include "liberation.h" + +#define talloc(type, num) (type *) malloc(sizeof(type)*(num)) + +int *liberation_coding_bitmatrix(int k, int w) +{ + int *matrix, i, j, index; + + if (k > w) return NULL; + matrix = talloc(int, 2*k*w*w); + if (matrix == NULL) return NULL; + bzero(matrix, sizeof(int)*2*k*w*w); + + /* Set up identity matrices */ + + for(i = 0; i < w; i++) { + index = i*k*w+i; + for (j = 0; j < k; j++) { + matrix[index] = 1; + index += w; + } + } + + /* Set up liberation matrices */ + + for (j = 0; j < k; j++) { + index = k*w*w+j*w; + for (i = 0; i < w; i++) { + matrix[index+(j+i)%w] = 1; + index += (k*w); + } + if (j > 0) { + i = (j*((w-1)/2))%w; + matrix[k*w*w+j*w+i*k*w+(i+j-1)%w] = 1; + } + } + return matrix; +} + + +int *liber8tion_coding_bitmatrix(int k) +{ + int *matrix, i, j, index; + int w; + + w = 8; + if (k > w) return NULL; + matrix = talloc(int, 2*k*w*w); + if (matrix == NULL) return NULL; + bzero(matrix, sizeof(int)*2*k*w*w); + + /* Set up identity matrices */ + + for(i = 0; i < w; i++) { + index = i*k*w+i; + for (j = 0; j < k; j++) { + matrix[index] = 1; + index += w; + } + } + + /* Set up liber8tion matrices */ + + index = k*w*w; + + if (k == 0) return matrix; + matrix[index+0*k*w+0*w+0] = 1; + matrix[index+1*k*w+0*w+1] = 1; + matrix[index+2*k*w+0*w+2] = 1; + matrix[index+3*k*w+0*w+3] = 1; + matrix[index+4*k*w+0*w+4] = 1; + matrix[index+5*k*w+0*w+5] = 1; + matrix[index+6*k*w+0*w+6] = 1; + matrix[index+7*k*w+0*w+7] = 1; + + if (k == 1) return matrix; + matrix[index+0*k*w+1*w+7] = 1; + matrix[index+1*k*w+1*w+3] = 1; + matrix[index+2*k*w+1*w+0] = 1; + matrix[index+3*k*w+1*w+2] = 1; + matrix[index+4*k*w+1*w+6] = 1; + matrix[index+5*k*w+1*w+1] = 1; + matrix[index+6*k*w+1*w+5] = 1; + matrix[index+7*k*w+1*w+4] = 1; + matrix[index+4*k*w+1*w+7] = 1; + + if (k == 2) return matrix; + matrix[index+0*k*w+2*w+6] = 1; + matrix[index+1*k*w+2*w+2] = 1; + matrix[index+2*k*w+2*w+4] = 1; + matrix[index+3*k*w+2*w+0] = 1; + matrix[index+4*k*w+2*w+7] = 1; + matrix[index+5*k*w+2*w+3] = 1; + matrix[index+6*k*w+2*w+1] = 1; + matrix[index+7*k*w+2*w+5] = 1; + matrix[index+1*k*w+2*w+3] = 1; + + if (k == 3) return matrix; + matrix[index+0*k*w+3*w+2] = 1; + matrix[index+1*k*w+3*w+5] = 1; + matrix[index+2*k*w+3*w+7] = 1; + matrix[index+3*k*w+3*w+6] = 1; + matrix[index+4*k*w+3*w+0] = 1; + matrix[index+5*k*w+3*w+3] = 1; + matrix[index+6*k*w+3*w+4] = 1; + matrix[index+7*k*w+3*w+1] = 1; + matrix[index+5*k*w+3*w+4] = 1; + + if (k == 4) return matrix; + matrix[index+0*k*w+4*w+5] = 1; + matrix[index+1*k*w+4*w+6] = 1; + matrix[index+2*k*w+4*w+1] = 1; + matrix[index+3*k*w+4*w+7] = 1; + matrix[index+4*k*w+4*w+2] = 1; + matrix[index+5*k*w+4*w+4] = 1; + matrix[index+6*k*w+4*w+3] = 1; + matrix[index+7*k*w+4*w+0] = 1; + matrix[index+2*k*w+4*w+0] = 1; + + if (k == 5) return matrix; + matrix[index+0*k*w+5*w+1] = 1; + matrix[index+1*k*w+5*w+2] = 1; + matrix[index+2*k*w+5*w+3] = 1; + matrix[index+3*k*w+5*w+4] = 1; + matrix[index+4*k*w+5*w+5] = 1; + matrix[index+5*k*w+5*w+6] = 1; + matrix[index+6*k*w+5*w+7] = 1; + matrix[index+7*k*w+5*w+0] = 1; + matrix[index+7*k*w+5*w+2] = 1; + + if (k == 6) return matrix; + matrix[index+0*k*w+6*w+3] = 1; + matrix[index+1*k*w+6*w+0] = 1; + matrix[index+2*k*w+6*w+6] = 1; + matrix[index+3*k*w+6*w+5] = 1; + matrix[index+4*k*w+6*w+1] = 1; + matrix[index+5*k*w+6*w+7] = 1; + matrix[index+6*k*w+6*w+4] = 1; + matrix[index+7*k*w+6*w+2] = 1; + matrix[index+6*k*w+6*w+5] = 1; + + if (k == 7) return matrix; + matrix[index+0*k*w+7*w+4] = 1; + matrix[index+1*k*w+7*w+7] = 1; + matrix[index+2*k*w+7*w+1] = 1; + matrix[index+3*k*w+7*w+5] = 1; + matrix[index+4*k*w+7*w+3] = 1; + matrix[index+5*k*w+7*w+2] = 1; + matrix[index+6*k*w+7*w+0] = 1; + matrix[index+7*k*w+7*w+6] = 1; + matrix[index+3*k*w+7*w+1] = 1; + + return matrix; +} + +int *blaum_roth_coding_bitmatrix(int k, int w) +{ + int *matrix, i, j, index, l, m, p; + + if (k > w) return NULL ; + + matrix = talloc(int, 2*k*w*w); + if (matrix == NULL) return NULL; + bzero(matrix, sizeof(int)*2*k*w*w); + + /* Set up identity matrices */ + + for(i = 0; i < w; i++) { + index = i*k*w+i; + for (j = 0; j < k; j++) { + matrix[index] = 1; + index += w; + } + } + + /* Set up blaum_roth matrices -- Ignore identity */ + + p = w+1; + for (j = 0; j < k; j++) { + index = k*w*w+j*w; + if (j == 0) { + for (l = 0; l < w; l++) { + matrix[index+l] = 1; + index += k*w; + } + } else { + i = j; + for (l = 1; l <= w; l++) { + if (l != p-i) { + m = l+i; + if (m >= p) m -= p; + m--; + matrix[index+m] = 1; + } else { + matrix[index+i-1] = 1; + if (i%2 == 0) { + m = i/2; + } else { + m = (p/2) + 1 + (i/2); + } + m--; + matrix[index+m] = 1; + } + index += k*w; + } + } + } + + return matrix; +} diff --git a/src/osd/ErasureCodePluginJerasure/liberation.h b/src/osd/ErasureCodePluginJerasure/liberation.h new file mode 100755 index 00000000000..ee176e877c8 --- /dev/null +++ b/src/osd/ErasureCodePluginJerasure/liberation.h @@ -0,0 +1,56 @@ +/* liberation.h + * James S. Plank + +Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure Coding Techniques + +Revision 1.2A +May 24, 2011 + +James S. Plank +Department of Electrical Engineering and Computer Science +University of Tennessee +Knoxville, TN 37996 +plank@cs.utk.edu + +Copyright (c) 2011, James S. Plank +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + - Neither the name of the University of Tennessee nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + + */ + +#ifndef _LIBERATION + +extern int *liberation_coding_bitmatrix(int k, int w); +extern int *liber8tion_coding_bitmatrix(int k); +extern int *blaum_roth_coding_bitmatrix(int k, int w); + +#endif diff --git a/src/osd/ErasureCodePluginJerasure/reed_sol.c b/src/osd/ErasureCodePluginJerasure/reed_sol.c new file mode 100755 index 00000000000..003eb419ad0 --- /dev/null +++ b/src/osd/ErasureCodePluginJerasure/reed_sol.c @@ -0,0 +1,368 @@ +/* reed_sol.c + * James S. Plank + +Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure Coding Techniques + +Revision 1.2A +May 24, 2011 + +James S. Plank +Department of Electrical Engineering and Computer Science +University of Tennessee +Knoxville, TN 37996 +plank@cs.utk.edu + +Copyright (c) 2011, James S. Plank +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + - Neither the name of the University of Tennessee nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "galois.h" +#include "jerasure.h" +#include "reed_sol.h" + +#define talloc(type, num) (type *) malloc(sizeof(type)*(num)) + +int *reed_sol_r6_coding_matrix(int k, int w) +{ + int *matrix; + int i, tmp; + + if (w != 8 && w != 16 && w != 32) return NULL; + + matrix = talloc(int, 2*k); + if (matrix == NULL) return NULL; + + for (i = 0; i < k; i++) matrix[i] = 1; + matrix[k] = 1; + tmp = 1; + for (i = 1; i < k; i++) { + tmp = galois_single_multiply(tmp, 2, w); + matrix[k+i] = tmp; + } + return matrix; +} + +int *reed_sol_vandermonde_coding_matrix(int k, int m, int w) +{ + int i, j; + int *vdm, *dist; + + vdm = reed_sol_big_vandermonde_distribution_matrix(k+m, k, w); + if (vdm == NULL) return NULL; + dist = talloc(int, m*k); + if (dist == NULL) { + free(vdm); + return NULL; + } + + i = k*k; + for (j = 0; j < m*k; j++) { + dist[j] = vdm[i]; + i++; + } + free(vdm); + return dist; +} + +static int prim32 = -1; + +#define rgw32_mask(v) ((v) & 0x80000000) + +void reed_sol_galois_w32_region_multby_2(char *region, int nbytes) +{ + int *l1; + int *ltop; + char *ctop; + + if (prim32 == -1) prim32 = galois_single_multiply((1 << 31), 2, 32); + + ctop = region + nbytes; + ltop = (int *) ctop; + l1 = (int *) region; + + while (l1 < ltop) { + *l1 = ((*l1) << 1) ^ ((*l1 & 0x80000000) ? prim32 : 0); + l1++; + } +} + +static int prim08 = -1; +static int mask08_1 = -1; +static int mask08_2 = -1; + +void reed_sol_galois_w08_region_multby_2(char *region, int nbytes) +{ + unsigned int *l1; + unsigned int *ltop; + char *ctop; + unsigned int tmp, tmp2; + + + if (prim08 == -1) { + tmp = galois_single_multiply((1 << 7), 2, 8); + prim08 = 0; + while (tmp != 0) { + prim08 |= tmp; + tmp = (tmp << 8); + } + tmp = (1 << 8) - 2; + mask08_1 = 0; + while (tmp != 0) { + mask08_1 |= tmp; + tmp = (tmp << 8); + } + tmp = (1 << 7); + mask08_2 = 0; + while (tmp != 0) { + mask08_2 |= tmp; + tmp = (tmp << 8); + } + } + + ctop = region + nbytes; + ltop = (unsigned int *) ctop; + l1 = (unsigned int *) region; + + while (l1 < ltop) { + tmp = ((*l1) << 1) & mask08_1; + tmp2 = (*l1) & mask08_2; + tmp2 = ((tmp2 << 1) - (tmp2 >> 7)); + *l1 = (tmp ^ (tmp2 & prim08)); + l1++; + } +} + +static int prim16 = -1; +static int mask16_1 = -1; +static int mask16_2 = -1; + +void reed_sol_galois_w16_region_multby_2(char *region, int nbytes) +{ + unsigned int *l1; + unsigned int *ltop; + char *ctop; + unsigned int tmp, tmp2; + + + if (prim16 == -1) { + tmp = galois_single_multiply((1 << 15), 2, 16); + prim16 = 0; + while (tmp != 0) { + prim16 |= tmp; + tmp = (tmp << 16); + } + tmp = (1 << 16) - 2; + mask16_1 = 0; + while (tmp != 0) { + mask16_1 |= tmp; + tmp = (tmp << 16); + } + tmp = (1 << 15); + mask16_2 = 0; + while (tmp != 0) { + mask16_2 |= tmp; + tmp = (tmp << 16); + } + } + + ctop = region + nbytes; + ltop = (unsigned int *) ctop; + l1 = (unsigned int *) region; + + while (l1 < ltop) { + tmp = ((*l1) << 1) & mask16_1; + tmp2 = (*l1) & mask16_2; + tmp2 = ((tmp2 << 1) - (tmp2 >> 15)); + *l1 = (tmp ^ (tmp2 & prim16)); + l1++; + } +} + +int reed_sol_r6_encode(int k, int w, char **data_ptrs, char **coding_ptrs, int size) +{ + int i; + + /* First, put the XOR into coding region 0 */ + + memcpy(coding_ptrs[0], data_ptrs[0], size); + + for (i = 1; i < k; i++) galois_region_xor(coding_ptrs[0], data_ptrs[i], coding_ptrs[0], size); + + /* Next, put the sum of (2^j)*Dj into coding region 1 */ + + memcpy(coding_ptrs[1], data_ptrs[k-1], size); + + for (i = k-2; i >= 0; i--) { + switch (w) { + case 8: reed_sol_galois_w08_region_multby_2(coding_ptrs[1], size); break; + case 16: reed_sol_galois_w16_region_multby_2(coding_ptrs[1], size); break; + case 32: reed_sol_galois_w32_region_multby_2(coding_ptrs[1], size); break; + default: return 0; + } + + galois_region_xor(coding_ptrs[1], data_ptrs[i], coding_ptrs[1], size); + } + return 1; +} + +int *reed_sol_extended_vandermonde_matrix(int rows, int cols, int w) +{ + int *vdm; + int i, j, k; + + if (w < 30 && (1 << w) < rows) return NULL; + if (w < 30 && (1 << w) < cols) return NULL; + + vdm = talloc(int, rows*cols); + if (vdm == NULL) { return NULL; } + + vdm[0] = 1; + for (j = 1; j < cols; j++) vdm[j] = 0; + if (rows == 1) return vdm; + + i=(rows-1)*cols; + for (j = 0; j < cols-1; j++) vdm[i+j] = 0; + vdm[i+j] = 1; + if (rows == 2) return vdm; + + for (i = 1; i < rows-1; i++) { + k = 1; + for (j = 0; j < cols; j++) { + vdm[i*cols+j] = k; + k = galois_single_multiply(k, i, w); + } + } + return vdm; +} + +int *reed_sol_big_vandermonde_distribution_matrix(int rows, int cols, int w) +{ + int *dist; + int i, j, k; + int sindex, srindex, siindex, tmp; + + if (cols >= rows) return NULL; + + dist = reed_sol_extended_vandermonde_matrix(rows, cols, w); + if (dist == NULL) return NULL; + + sindex = 0; + for (i = 1; i < cols; i++) { + sindex += cols; + + /* Find an appropriate row -- where i,i != 0 */ + srindex = sindex+i; + for (j = i; j < rows && dist[srindex] == 0; j++) srindex += cols; + if (j >= rows) { /* This should never happen if rows/w are correct */ + fprintf(stderr, "reed_sol_big_vandermonde_distribution_matrix(%d,%d,%d) - couldn't make matrix\n", + rows, cols, w); + exit(1); + } + + /* If necessary, swap rows */ + if (j != i) { + srindex -= i; + for (k = 0; k < cols; k++) { + tmp = dist[srindex+k]; + dist[srindex+k] = dist[sindex+k]; + dist[sindex+k] = tmp; + } + } + + /* If Element i,i is not equal to 1, multiply the column by 1/i */ + + if (dist[sindex+i] != 1) { + tmp = galois_single_divide(1, dist[sindex+i], w); + srindex = i; + for (j = 0; j < rows; j++) { + dist[srindex] = galois_single_multiply(tmp, dist[srindex], w); + srindex += cols; + } + } + + /* Now, for each element in row i that is not in column 1, you need + to make it zero. Suppose that this is column j, and the element + at i,j = e. Then you want to replace all of column j with + (col-j + col-i*e). Note, that in row i, col-i = 1 and col-j = e. + So (e + 1e) = 0, which is indeed what we want. */ + + for (j = 0; j < cols; j++) { + tmp = dist[sindex+j]; + if (j != i && tmp != 0) { + srindex = j; + siindex = i; + for (k = 0; k < rows; k++) { + dist[srindex] = dist[srindex] ^ galois_single_multiply(tmp, dist[siindex], w); + srindex += cols; + siindex += cols; + } + } + } + } + /* We desire to have row k be all ones. To do that, multiply + the entire column j by 1/dist[k,j]. Then row j by 1/dist[j,j]. */ + + sindex = cols*cols; + for (j = 0; j < cols; j++) { + tmp = dist[sindex]; + if (tmp != 1) { + tmp = galois_single_divide(1, tmp, w); + srindex = sindex; + for (i = cols; i < rows; i++) { + dist[srindex] = galois_single_multiply(tmp, dist[srindex], w); + srindex += cols; + } + } + sindex++; + } + + /* Finally, we'd like the first column of each row to be all ones. To + do that, we multiply the row by the inverse of the first element. */ + + sindex = cols*(cols+1); + for (i = cols+1; i < rows; i++) { + tmp = dist[sindex]; + if (tmp != 1) { + tmp = galois_single_divide(1, tmp, w); + for (j = 0; j < cols; j++) dist[sindex+j] = galois_single_multiply(dist[sindex+j], tmp, w); + } + sindex += cols; + } + + return dist; +} + diff --git a/src/osd/ErasureCodePluginJerasure/reed_sol.h b/src/osd/ErasureCodePluginJerasure/reed_sol.h new file mode 100755 index 00000000000..741c3177432 --- /dev/null +++ b/src/osd/ErasureCodePluginJerasure/reed_sol.h @@ -0,0 +1,59 @@ +/* reed_sol.h + * James S. Plank + +Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure Coding Techniques + +Revision 1.2A +May 24, 2011 + +James S. Plank +Department of Electrical Engineering and Computer Science +University of Tennessee +Knoxville, TN 37996 +plank@cs.utk.edu + +Copyright (c) 2011, James S. Plank +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + - Neither the name of the University of Tennessee nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + + */ + +extern int *reed_sol_vandermonde_coding_matrix(int k, int m, int w); +extern int *reed_sol_extended_vandermonde_matrix(int rows, int cols, int w); +extern int *reed_sol_big_vandermonde_distribution_matrix(int rows, int cols, int w); + +extern int reed_sol_r6_encode(int k, int w, char **data_ptrs, char **coding_ptrs, int size); +extern int *reed_sol_r6_coding_matrix(int k, int w); + +extern void reed_sol_galois_w08_region_multby_2(char *region, int nbytes); +extern void reed_sol_galois_w16_region_multby_2(char *region, int nbytes); +extern void reed_sol_galois_w32_region_multby_2(char *region, int nbytes); diff --git a/src/osd/Makefile.am b/src/osd/Makefile.am new file mode 100644 index 00000000000..ea7c036f858 --- /dev/null +++ b/src/osd/Makefile.am @@ -0,0 +1,40 @@ +## erasure code plugins +erasure_codelibdir = $(libdir)/erasure-code +erasure_codelib_LTLIBRARIES = + +include osd/ErasureCodePluginJerasure/Makefile.am + +libosd_la_SOURCES = \ + osd/ErasureCodePlugin.cc \ + osd/PG.cc \ + osd/PGLog.cc \ + osd/ReplicatedPG.cc \ + osd/Ager.cc \ + osd/OSD.cc \ + osd/OSDCap.cc \ + osd/Watch.cc \ + osd/ClassHandler.cc \ + osd/OpRequest.cc \ + osd/SnapMapper.cc \ + osd/osd_types.cc \ + objclass/class_api.cc +libosd_la_LIBADD = $(LIBOSDC) $(LIBOS) +noinst_LTLIBRARIES += libosd.la + +noinst_HEADERS += \ + osd/Ager.h \ + osd/ClassHandler.h \ + osd/ErasureCodeInterface.h \ + osd/ErasureCodePlugin.h \ + osd/OSD.h \ + osd/OSDCap.h \ + osd/OSDMap.h \ + osd/ObjectVersioner.h \ + osd/OpRequest.h \ + osd/SnapMapper.h \ + osd/PG.h \ + osd/PGLog.h \ + osd/ReplicatedPG.h \ + osd/Watch.h \ + osd/osd_types.h + diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index a9f82b51742..1cdc232b064 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -134,8 +134,6 @@ static ostream& _prefix(std::ostream* _dout, int whoami, OSDMapRef osdmap) { << " "; } -const coll_t coll_t::META_COLL("meta"); - static CompatSet get_osd_compat_set() { CompatSet::FeatureSet ceph_osd_feature_compat; CompatSet::FeatureSet ceph_osd_feature_ro_compat; @@ -156,6 +154,7 @@ static CompatSet get_osd_compat_set() { OSDService::OSDService(OSD *osd) : osd(osd), + cct(osd->cct), whoami(osd->whoami), store(osd->store), clog(osd->clog), pg_recovery_stats(osd->pg_recovery_stats), infos_oid(OSD::make_infos_oid()), @@ -186,17 +185,17 @@ OSDService::OSDService(OSD *osd) : watch_timer(osd->client_messenger->cct, watch_lock), next_notif_id(0), backfill_request_lock("OSD::backfill_request_lock"), - backfill_request_timer(g_ceph_context, backfill_request_lock, false), + backfill_request_timer(cct, backfill_request_lock, false), last_tid(0), tid_lock("OSDService::tid_lock"), - reserver_finisher(g_ceph_context), - local_reserver(&reserver_finisher, g_conf->osd_max_backfills), - remote_reserver(&reserver_finisher, g_conf->osd_max_backfills), + reserver_finisher(cct), + local_reserver(&reserver_finisher, cct->_conf->osd_max_backfills), + remote_reserver(&reserver_finisher, cct->_conf->osd_max_backfills), pg_temp_lock("OSDService::pg_temp_lock"), map_cache_lock("OSDService::map_lock"), - map_cache(g_conf->osd_map_cache_size), - map_bl_cache(g_conf->osd_map_cache_size), - map_bl_inc_cache(g_conf->osd_map_cache_size), + map_cache(cct->_conf->osd_map_cache_size), + map_bl_cache(cct->_conf->osd_map_cache_size), + map_bl_inc_cache(cct->_conf->osd_map_cache_size), in_progress_split_lock("OSDService::in_progress_split_lock"), full_status_lock("OSDService::full_status_lock"), cur_state(NONE), @@ -428,13 +427,13 @@ void OSDService::init() watch_timer.init(); } -ObjectStore *OSD::create_object_store(const std::string &dev, const std::string &jdev) +ObjectStore *OSD::create_object_store(CephContext *cct, const std::string &dev, const std::string &jdev) { struct stat st; if (::stat(dev.c_str(), &st) != 0) return 0; - if (g_conf->filestore) + if (cct->_conf->filestore) return new FileStore(dev, jdev); if (S_ISDIR(st.st_mode)) @@ -581,20 +580,20 @@ int OSD::convertfs(const std::string &dev, const std::string &jdev) return r; } -int OSD::mkfs(const std::string &dev, const std::string &jdev, uuid_d fsid, int whoami) +int OSD::mkfs(CephContext *cct, const std::string &dev, const std::string &jdev, uuid_d fsid, int whoami) { int ret; ObjectStore *store = NULL; try { - store = create_object_store(dev, jdev); + store = create_object_store(cct, dev, jdev); if (!store) { ret = -ENOENT; goto out; } // if we are fed a uuid for this osd, use it. - store->set_fsid(g_conf->osd_uuid); + store->set_fsid(cct->_conf->osd_uuid); ret = store->mkfs(); if (ret) { @@ -609,15 +608,15 @@ int OSD::mkfs(const std::string &dev, const std::string &jdev, uuid_d fsid, int } // age? - if (g_conf->osd_age_time != 0) { - if (g_conf->osd_age_time >= 0) { - dout(0) << "aging..." << dendl; - Ager ager(store); - ager.age(g_conf->osd_age_time, - g_conf->osd_age, - g_conf->osd_age - .05, - 50000, - g_conf->osd_age - .05); + if (cct->_conf->osd_age_time != 0) { + if (cct->_conf->osd_age_time >= 0) { + dout(0) << "aging..." << dendl; + Ager ager(cct, store); + ager.age(cct->_conf->osd_age_time, + cct->_conf->osd_age, + cct->_conf->osd_age - .05, + 50000, + cct->_conf->osd_age - .05); } } @@ -650,13 +649,13 @@ int OSD::mkfs(const std::string &dev, const std::string &jdev, uuid_d fsid, int sb.compat_features = get_osd_compat_set(); // benchmark? - if (g_conf->osd_auto_weight) { + if (cct->_conf->osd_auto_weight) { bufferlist bl; bufferptr bp(1048576); bp.zero(); bl.push_back(bp); dout(0) << "testing disk bandwidth..." << dendl; - utime_t start = ceph_clock_now(g_ceph_context); + utime_t start = ceph_clock_now(cct); object_t oid("disk_bw_test"); for (int i=0; i<1000; i++) { ObjectStore::Transaction *t = new ObjectStore::Transaction; @@ -664,7 +663,7 @@ int OSD::mkfs(const std::string &dev, const std::string &jdev, uuid_d fsid, int store->queue_transaction(NULL, t); } store->sync(); - utime_t end = ceph_clock_now(g_ceph_context); + utime_t end = ceph_clock_now(cct); end -= start; dout(0) << "measured " << (1000.0 / (double)end) << " mb/sec" << dendl; ObjectStore::Transaction tr; @@ -726,17 +725,17 @@ out: return ret; } -int OSD::mkjournal(const std::string &dev, const std::string &jdev) +int OSD::mkjournal(CephContext *cct, const std::string &dev, const std::string &jdev) { - ObjectStore *store = create_object_store(dev, jdev); + ObjectStore *store = create_object_store(cct, dev, jdev); if (!store) return -ENOENT; return store->mkjournal(); } -int OSD::flushjournal(const std::string &dev, const std::string &jdev) +int OSD::flushjournal(CephContext *cct, const std::string &dev, const std::string &jdev) { - ObjectStore *store = create_object_store(dev, jdev); + ObjectStore *store = create_object_store(cct, dev, jdev); if (!store) return -ENOENT; int err = store->mount(); @@ -748,9 +747,9 @@ int OSD::flushjournal(const std::string &dev, const std::string &jdev) return err; } -int OSD::dump_journal(const std::string &dev, const std::string &jdev, ostream& out) +int OSD::dump_journal(CephContext *cct, const std::string &dev, const std::string &jdev, ostream& out) { - ObjectStore *store = create_object_store(dev, jdev); + ObjectStore *store = create_object_store(cct, dev, jdev); if (!store) return -ENOENT; int err = store->dump_journal(out); @@ -907,21 +906,21 @@ int OSD::peek_journal_fsid(string path, uuid_d& fsid) // cons/des -OSD::OSD(int id, Messenger *internal_messenger, Messenger *external_messenger, +OSD::OSD(CephContext *cct_, int id, Messenger *internal_messenger, Messenger *external_messenger, Messenger *hb_clientm, Messenger *hb_front_serverm, Messenger *hb_back_serverm, Messenger *osdc_messenger, MonClient *mc, const std::string &dev, const std::string &jdev) : - Dispatcher(external_messenger->cct), + Dispatcher(cct_), osd_lock("OSD::osd_lock"), - tick_timer(external_messenger->cct, osd_lock), - authorize_handler_cluster_registry(new AuthAuthorizeHandlerRegistry(external_messenger->cct, + tick_timer(cct, osd_lock), + authorize_handler_cluster_registry(new AuthAuthorizeHandlerRegistry(cct, cct->_conf->auth_supported.length() ? cct->_conf->auth_supported : cct->_conf->auth_cluster_required)), - authorize_handler_service_registry(new AuthAuthorizeHandlerRegistry(external_messenger->cct, + authorize_handler_service_registry(new AuthAuthorizeHandlerRegistry(cct, cct->_conf->auth_supported.length() ? cct->_conf->auth_supported : cct->_conf->auth_service_required)), @@ -932,17 +931,17 @@ OSD::OSD(int id, Messenger *internal_messenger, Messenger *external_messenger, logger(NULL), recoverystate_perf(NULL), store(NULL), - clog(external_messenger->cct, client_messenger, &mc->monmap, LogClient::NO_FLAGS), + clog(cct, client_messenger, &mc->monmap, LogClient::NO_FLAGS), whoami(id), dev_path(dev), journal_path(jdev), dispatch_running(false), asok_hook(NULL), osd_compat(get_osd_compat_set()), state(STATE_INITIALIZING), boot_epoch(0), up_epoch(0), bind_epoch(0), - op_tp(external_messenger->cct, "OSD::op_tp", g_conf->osd_op_threads, "osd_op_threads"), - recovery_tp(external_messenger->cct, "OSD::recovery_tp", g_conf->osd_recovery_threads, "osd_recovery_threads"), - disk_tp(external_messenger->cct, "OSD::disk_tp", g_conf->osd_disk_threads, "osd_disk_threads"), - command_tp(external_messenger->cct, "OSD::command_tp", 1), + op_tp(cct, "OSD::op_tp", cct->_conf->osd_op_threads, "osd_op_threads"), + recovery_tp(cct, "OSD::recovery_tp", cct->_conf->osd_recovery_threads, "osd_recovery_threads"), + disk_tp(cct, "OSD::disk_tp", cct->_conf->osd_disk_threads, "osd_disk_threads"), + command_tp(cct, "OSD::command_tp", 1), paused_recovery(false), heartbeat_lock("OSD::heartbeat_lock"), heartbeat_stop(false), heartbeat_need_update(true), heartbeat_epoch(0), @@ -953,28 +952,29 @@ OSD::OSD(int id, Messenger *internal_messenger, Messenger *external_messenger, heartbeat_dispatcher(this), stat_lock("OSD::stat_lock"), finished_lock("OSD::finished_lock"), + op_tracker(cct), test_ops_hook(NULL), - op_wq(this, g_conf->osd_op_thread_timeout, &op_tp), - peering_wq(this, g_conf->osd_op_thread_timeout, &op_tp), + op_wq(this, cct->_conf->osd_op_thread_timeout, &op_tp), + peering_wq(this, cct->_conf->osd_op_thread_timeout, &op_tp), map_lock("OSD::map_lock"), peer_map_epoch_lock("OSD::peer_map_epoch_lock"), - debug_drop_pg_create_probability(g_conf->osd_debug_drop_pg_create_probability), - debug_drop_pg_create_duration(g_conf->osd_debug_drop_pg_create_duration), + debug_drop_pg_create_probability(cct->_conf->osd_debug_drop_pg_create_probability), + debug_drop_pg_create_duration(cct->_conf->osd_debug_drop_pg_create_duration), debug_drop_pg_create_left(-1), outstanding_pg_stats(false), up_thru_wanted(0), up_thru_pending(0), pg_stat_queue_lock("OSD::pg_stat_queue_lock"), osd_stat_updated(false), pg_stat_tid(0), pg_stat_tid_flushed(0), - command_wq(this, g_conf->osd_command_thread_timeout, &command_tp), + command_wq(this, cct->_conf->osd_command_thread_timeout, &command_tp), recovery_ops_active(0), - recovery_wq(this, g_conf->osd_recovery_thread_timeout, &recovery_tp), + recovery_wq(this, cct->_conf->osd_recovery_thread_timeout, &recovery_tp), replay_queue_lock("OSD::replay_queue_lock"), - snap_trim_wq(this, g_conf->osd_snap_trim_thread_timeout, &disk_tp), - scrub_wq(this, g_conf->osd_scrub_thread_timeout, &disk_tp), - scrub_finalize_wq(this, g_conf->osd_scrub_finalize_thread_timeout, &op_tp), - rep_scrub_wq(this, g_conf->osd_scrub_thread_timeout, &disk_tp), - remove_wq(store, g_conf->osd_remove_thread_timeout, &disk_tp), + snap_trim_wq(this, cct->_conf->osd_snap_trim_thread_timeout, &disk_tp), + scrub_wq(this, cct->_conf->osd_scrub_thread_timeout, &disk_tp), + scrub_finalize_wq(this, cct->_conf->osd_scrub_finalize_thread_timeout, &op_tp), + rep_scrub_wq(this, cct->_conf->osd_scrub_thread_timeout, &disk_tp), + remove_wq(store, cct->_conf->osd_remove_thread_timeout, &disk_tp), next_removal_seq(0), service(this) { @@ -986,8 +986,8 @@ OSD::~OSD() delete authorize_handler_cluster_registry; delete authorize_handler_service_registry; delete class_handler; - g_ceph_context->get_perfcounters_collection()->remove(recoverystate_perf); - g_ceph_context->get_perfcounters_collection()->remove(logger); + cct->get_perfcounters_collection()->remove(recoverystate_perf); + cct->get_perfcounters_collection()->remove(logger); delete recoverystate_perf; delete logger; delete store; @@ -1010,7 +1010,7 @@ int OSD::pre_init() return 0; assert(!store); - store = create_object_store(dev_path, journal_path); + store = create_object_store(cct, dev_path, journal_path); if (!store) { derr << "OSD::pre_init: unable to create object store" << dendl; return -ENODEV; @@ -1022,7 +1022,7 @@ int OSD::pre_init() return -EBUSY; } - g_conf->add_observer(this); + cct->_conf->add_observer(this); return 0; } @@ -1200,10 +1200,10 @@ int OSD::init() return r; } - class_handler = new ClassHandler(); + class_handler = new ClassHandler(cct); cls_initialize(class_handler); - if (g_conf->osd_open_classes_on_start) { + if (cct->_conf->osd_open_classes_on_start) { int r = class_handler->open_all_classes(); if (r) dout(1) << "warning: got an error loading one or more classes: " << cpp_strerror(r) << dendl; @@ -1262,7 +1262,7 @@ int OSD::init() heartbeat_thread.create(); // tick - tick_timer.add_event_after(g_conf->osd_heartbeat_interval, new C_Tick(this)); + tick_timer.add_event_after(cct->_conf->osd_heartbeat_interval, new C_Tick(this)); service.init(); service.publish_map(osdmap); @@ -1400,7 +1400,7 @@ void OSD::create_logger() { dout(10) << "create_logger" << dendl; - PerfCountersBuilder osd_plb(g_ceph_context, "osd", l_osd_first, l_osd_last); + PerfCountersBuilder osd_plb(cct, "osd", l_osd_first, l_osd_last); osd_plb.add_u64(l_osd_opq, "opq"); // op queue length (waiting to be processed yet) osd_plb.add_u64(l_osd_op_wip, "op_wip"); // rep ops currently being processed (primary) @@ -1460,15 +1460,19 @@ void OSD::create_logger() osd_plb.add_u64_counter(l_osd_waiting_for_map, "messages_delayed_for_map"); // dup osdmap epochs + osd_plb.add_u64(l_osd_stat_bytes, "stat_bytes"); + osd_plb.add_u64(l_osd_stat_bytes_used, "stat_bytes_used"); + osd_plb.add_u64(l_osd_stat_bytes_avail, "stat_bytes_avail"); + logger = osd_plb.create_perf_counters(); - g_ceph_context->get_perfcounters_collection()->add(logger); + cct->get_perfcounters_collection()->add(logger); } void OSD::create_recoverystate_perf() { dout(10) << "create_recoverystate_perf" << dendl; - PerfCountersBuilder rs_perf(g_ceph_context, "recoverystate_perf", rs_first, rs_last); + PerfCountersBuilder rs_perf(cct, "recoverystate_perf", rs_first, rs_last); rs_perf.add_time_avg(rs_initial_latency, "initial_latency"); rs_perf.add_time_avg(rs_started_latency, "started_latency"); @@ -1501,12 +1505,12 @@ void OSD::create_recoverystate_perf() rs_perf.add_time_avg(rs_waitupthru_latency, "waitupthru_latency"); recoverystate_perf = rs_perf.create_perf_counters(); - g_ceph_context->get_perfcounters_collection()->add(recoverystate_perf); + cct->get_perfcounters_collection()->add(recoverystate_perf); } void OSD::suicide(int exitcode) { - if (g_conf->filestore_blackhole) { + if (cct->_conf->filestore_blackhole) { derr << " filestore_blackhole=true, doing abbreviated shutdown" << dendl; _exit(exitcode); } @@ -1546,11 +1550,11 @@ int OSD::shutdown() heartbeat_lock.Unlock(); // Debugging - g_ceph_context->_conf->set_val("debug_osd", "100"); - g_ceph_context->_conf->set_val("debug_journal", "100"); - g_ceph_context->_conf->set_val("debug_filestore", "100"); - g_ceph_context->_conf->set_val("debug_ms", "100"); - g_ceph_context->_conf->apply_changes(NULL); + cct->_conf->set_val("debug_osd", "100"); + cct->_conf->set_val("debug_journal", "100"); + cct->_conf->set_val("debug_filestore", "100"); + cct->_conf->set_val("debug_ms", "100"); + cct->_conf->apply_changes(NULL); // Shutdown PGs for (hash_map<pg_t, PG*>::iterator p = pg_map.begin(); @@ -1666,7 +1670,7 @@ int OSD::shutdown() #ifdef PG_DEBUG_REFS service.dump_live_pgids(); #endif - g_conf->remove_observer(this); + cct->_conf->remove_observer(this); monc->shutdown(); osd_lock.Unlock(); @@ -2223,7 +2227,7 @@ void OSD::build_past_intervals_parallel() pg->unlock(); // don't let the transaction get too big - if (++num >= g_conf->osd_target_transaction_size) { + if (++num >= cct->_conf->osd_target_transaction_size) { store->apply_transaction(t); t = ObjectStore::Transaction(); num = 0; @@ -2516,14 +2520,14 @@ void OSD::project_pg_history(pg_t pgid, pg_history_t& h, epoch_t from, float OSDService::get_full_ratio() { - float full_ratio = g_conf->osd_failsafe_full_ratio; + float full_ratio = cct->_conf->osd_failsafe_full_ratio; if (full_ratio > 1.0) full_ratio /= 100.0; return full_ratio; } float OSDService::get_nearfull_ratio() { - float nearfull_ratio = g_conf->osd_failsafe_nearfull_ratio; + float nearfull_ratio = cct->_conf->osd_failsafe_nearfull_ratio; if (nearfull_ratio > 1.0) nearfull_ratio /= 100.0; return nearfull_ratio; } @@ -2535,7 +2539,12 @@ void OSDService::check_nearfull_warning(const osd_stat_t &osd_stat) time_t now = ceph_clock_gettime(NULL); - float ratio = ((float)osd_stat.kb_used) / ((float)osd_stat.kb); + // We base ratio on kb_avail rather than kb_used because they can + // differ significantly e.g. on btrfs volumes with a large number of + // chunks reserved for metadata, and for our purposes (avoiding + // completely filling the disk) it's far more important to know how + // much space is available to use than how much we've already used. + float ratio = ((float)(osd_stat.kb - osd_stat.kb_avail)) / ((float)osd_stat.kb); float nearfull_ratio = get_nearfull_ratio(); float full_ratio = get_full_ratio(); cur_ratio = ratio; @@ -2551,7 +2560,7 @@ void OSDService::check_nearfull_warning(const osd_stat_t &osd_stat) if (cur_state != new_state) { cur_state = new_state; - } else if (now - last_msg < g_conf->osd_op_complaint_time) { + } else if (now - last_msg < cct->_conf->osd_op_complaint_time) { return; } last_msg = now; @@ -2573,7 +2582,7 @@ bool OSDService::too_full_for_backfill(double *_ratio, double *_max_ratio) { Mutex::Locker l(full_status_lock); double max_ratio; - max_ratio = g_conf->osd_backfill_full_ratio; + max_ratio = cct->_conf->osd_backfill_full_ratio; if (_ratio) *_ratio = cur_ratio; if (_max_ratio) @@ -2588,9 +2597,17 @@ void OSD::update_osd_stat() struct statfs stbuf; store->statfs(&stbuf); - osd_stat.kb = stbuf.f_blocks * stbuf.f_bsize / 1024; - osd_stat.kb_used = (stbuf.f_blocks - stbuf.f_bfree) * stbuf.f_bsize / 1024; - osd_stat.kb_avail = stbuf.f_bavail * stbuf.f_bsize / 1024; + uint64_t bytes = stbuf.f_blocks * stbuf.f_bsize; + uint64_t used = (stbuf.f_blocks - stbuf.f_bfree) * stbuf.f_bsize; + uint64_t avail = stbuf.f_bavail * stbuf.f_bsize; + + osd_stat.kb = bytes >> 10; + osd_stat.kb_used = used >> 10; + osd_stat.kb_avail = avail >> 10; + + logger->set(l_osd_stat_bytes, bytes); + logger->set(l_osd_stat_bytes_used, used); + logger->set(l_osd_stat_bytes_avail, avail); osd_stat.hb_in.clear(); for (map<int,HeartbeatInfo>::iterator p = heartbeat_peers.begin(); p != heartbeat_peers.end(); ++p) @@ -2668,13 +2685,13 @@ void OSD::maybe_update_heartbeat_peers() assert(osd_lock.is_locked()); if (is_waiting_for_healthy()) { - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(cct); if (last_heartbeat_resample == utime_t()) { last_heartbeat_resample = now; heartbeat_need_update = true; } else if (!heartbeat_need_update) { utime_t dur = now - last_heartbeat_resample; - if (dur > g_conf->osd_heartbeat_grace) { + if (dur > cct->_conf->osd_heartbeat_grace) { dout(10) << "maybe_update_heartbeat_peers forcing update after " << dur << " seconds" << dendl; heartbeat_need_update = true; last_heartbeat_resample = now; @@ -2747,7 +2764,7 @@ void OSD::maybe_update_heartbeat_peers() // too few? int start = osdmap->get_next_up_osd_after(whoami); for (int n = start; n >= 0; ) { - if ((int)heartbeat_peers.size() >= g_conf->osd_heartbeat_min_peers) + if ((int)heartbeat_peers.size() >= cct->_conf->osd_heartbeat_min_peers) break; if (!extras.count(n) && !want.count(n) && n != whoami) { dout(10) << " adding random peer osd." << n << dendl; @@ -2761,7 +2778,7 @@ void OSD::maybe_update_heartbeat_peers() // too many? for (set<int>::iterator p = extras.begin(); - (int)heartbeat_peers.size() > g_conf->osd_heartbeat_min_peers && p != extras.end(); + (int)heartbeat_peers.size() > cct->_conf->osd_heartbeat_min_peers && p != extras.end(); ++p) { if (want.count(*p)) continue; @@ -2811,7 +2828,7 @@ void OSD::handle_osd_ping(MOSDPing *m) case MOSDPing::PING: { - if (g_conf->osd_debug_drop_ping_probability > 0) { + if (cct->_conf->osd_debug_drop_ping_probability > 0) { if (debug_heartbeat_drops_remaining.count(from)) { if (debug_heartbeat_drops_remaining[from] == 0) { debug_heartbeat_drops_remaining.erase(from); @@ -2822,10 +2839,10 @@ void OSD::handle_osd_ping(MOSDPing *m) << " remaining to drop" << dendl; break; } - } else if (g_conf->osd_debug_drop_ping_probability > + } else if (cct->_conf->osd_debug_drop_ping_probability > ((((double)(rand()%100))/100.0))) { debug_heartbeat_drops_remaining[from] = - g_conf->osd_debug_drop_ping_duration; + cct->_conf->osd_debug_drop_ping_duration; dout(5) << "Dropping heartbeat from " << from << ", " << debug_heartbeat_drops_remaining[from] << " remaining to drop" << dendl; @@ -2833,7 +2850,7 @@ void OSD::handle_osd_ping(MOSDPing *m) } } - if (!g_ceph_context->get_heartbeat_map()->is_healthy()) { + if (!cct->get_heartbeat_map()->is_healthy()) { dout(10) << "internal heartbeat not healthy, dropping ping request" << dendl; break; } @@ -2901,8 +2918,8 @@ void OSD::handle_osd_ping(MOSDPing *m) } } - utime_t cutoff = ceph_clock_now(g_ceph_context); - cutoff -= g_conf->osd_heartbeat_grace; + utime_t cutoff = ceph_clock_now(cct); + cutoff -= cct->_conf->osd_heartbeat_grace; if (i->second.is_healthy(cutoff)) { // Cancel false reports if (failure_queue.count(from)) { @@ -2938,11 +2955,11 @@ void OSD::heartbeat_entry() while (!heartbeat_stop) { heartbeat(); - double wait = .5 + ((float)(rand() % 10)/10.0) * (float)g_conf->osd_heartbeat_interval; + double wait = .5 + ((float)(rand() % 10)/10.0) * (float)cct->_conf->osd_heartbeat_interval; utime_t w; w.set_from_double(wait); dout(30) << "heartbeat_entry sleeping for " << wait << dendl; - heartbeat_cond.WaitInterval(g_ceph_context, heartbeat_lock, w); + heartbeat_cond.WaitInterval(cct, heartbeat_lock, w); if (is_stopping()) return; dout(30) << "heartbeat_entry woke up" << dendl; @@ -2952,16 +2969,16 @@ void OSD::heartbeat_entry() void OSD::heartbeat_check() { assert(heartbeat_lock.is_locked()); - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(cct); double age = hbclient_messenger->get_dispatch_queue_max_age(now); - if (age > (g_conf->osd_heartbeat_grace / 2)) { + if (age > (cct->_conf->osd_heartbeat_grace / 2)) { derr << "skipping heartbeat_check, hbqueue max age: " << age << dendl; return; // hb dispatch is too backed up for our hb status to be meaningful } // check for incoming heartbeats (move me elsewhere?) utime_t cutoff = now; - cutoff -= g_conf->osd_heartbeat_grace; + cutoff -= cct->_conf->osd_heartbeat_grace; for (map<int,HeartbeatInfo>::iterator p = heartbeat_peers.begin(); p != heartbeat_peers.end(); ++p) { @@ -3010,7 +3027,7 @@ void OSD::heartbeat() dout(5) << "heartbeat: " << osd_stat << dendl; - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(cct); // send heartbeats for (map<int,HeartbeatInfo>::iterator i = heartbeat_peers.begin(); @@ -3043,7 +3060,7 @@ void OSD::heartbeat() // hmm.. am i all alone? dout(30) << "heartbeat lonely?" << dendl; if (heartbeat_peers.empty()) { - if (now - last_mon_heartbeat > g_conf->osd_mon_heartbeat_interval && is_active()) { + if (now - last_mon_heartbeat > cct->_conf->osd_mon_heartbeat_interval && is_active()) { last_mon_heartbeat = now; dout(10) << "i have no heartbeat peers; checking mon for new map" << dendl; monc->sub_want("osdmap", osdmap->get_epoch() + 1, CEPH_SUBSCRIBE_ONETIME); @@ -3121,19 +3138,19 @@ void OSD::tick() heartbeat_lock.Unlock(); // mon report? - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(cct); if (outstanding_pg_stats && - (now - g_conf->osd_mon_ack_timeout) > last_pg_stats_ack) { + (now - cct->_conf->osd_mon_ack_timeout) > last_pg_stats_ack) { dout(1) << "mon hasn't acked PGStats in " << now - last_pg_stats_ack << " seconds, reconnecting elsewhere" << dendl; monc->reopen_session(); - last_pg_stats_ack = ceph_clock_now(g_ceph_context); // reset clock + last_pg_stats_ack = ceph_clock_now(cct); // reset clock last_pg_stats_sent = utime_t(); } - if (now - last_pg_stats_sent > g_conf->osd_mon_report_interval_max) { + if (now - last_pg_stats_sent > cct->_conf->osd_mon_report_interval_max) { osd_stat_updated = true; do_mon_report(); - } else if (now - last_mon_report > g_conf->osd_mon_report_interval_min) { + } else if (now - last_mon_report > cct->_conf->osd_mon_report_interval_min) { do_mon_report(); } @@ -3213,7 +3230,7 @@ void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store, string poolstr; - cmd_getval(g_ceph_context, cmdmap, "pool", poolstr); + cmd_getval(service->cct, cmdmap, "pool", poolstr); pool = curmap->const_lookup_pg_pool_name(poolstr.c_str()); //If we can't find it by name then maybe id specified if (pool < 0 && isdigit(poolstr[0])) @@ -3224,7 +3241,7 @@ void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store, } r = -1; string objname, nspace; - cmd_getval(g_ceph_context, cmdmap, "objname", objname); + cmd_getval(service->cct, cmdmap, "objname", objname); std::size_t found = objname.find_first_of('/'); if (found != string::npos) { nspace = objname.substr(0, found); @@ -3246,8 +3263,8 @@ void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store, map<string, bufferlist> newattrs; bufferlist val; string key, valstr; - cmd_getval(g_ceph_context, cmdmap, "key", key); - cmd_getval(g_ceph_context, cmdmap, "val", valstr); + cmd_getval(service->cct, cmdmap, "key", key); + cmd_getval(service->cct, cmdmap, "val", valstr); val.append(valstr); newattrs[key] = val; @@ -3260,7 +3277,7 @@ void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store, } else if (command == "rmomapkey") { string key; set<string> keys; - cmd_getval(g_ceph_context, cmdmap, "key", key); + cmd_getval(service->cct, cmdmap, "key", key); keys.insert(key); t.omap_rmkeys(coll_t(pgid), obj, keys); @@ -3273,7 +3290,7 @@ void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store, bufferlist newheader; string headerstr; - cmd_getval(g_ceph_context, cmdmap, "header", headerstr); + cmd_getval(service->cct, cmdmap, "header", headerstr); newheader.append(headerstr); t.omap_setheader(coll_t(pgid), obj, newheader); r = store->apply_transaction(t); @@ -3297,7 +3314,7 @@ void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store, } } else if (command == "truncobj") { int64_t trunclen; - cmd_getval(g_ceph_context, cmdmap, "len", trunclen); + cmd_getval(service->cct, cmdmap, "len", trunclen); t.truncate(coll_t(pgid), obj, trunclen); r = store->apply_transaction(t); if (r < 0) @@ -3319,6 +3336,7 @@ void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store, // ========================================= bool remove_dir( + CephContext *cct, ObjectStore *store, SnapMapper *mapper, OSDriver *osdriver, ObjectStore::Sequencer *osr, @@ -3346,7 +3364,7 @@ bool remove_dir( assert(0); } t->remove(coll, *i); - if (num >= g_conf->osd_target_transaction_size) { + if (num >= cct->_conf->osd_target_transaction_size) { C_SaferCond waiter; store->queue_transaction(osr, t, &waiter); bool cont = dstate->pause_clearing(); @@ -3386,12 +3404,12 @@ void OSD::RemoveWQ::_process(pair<PGRef, DeletingStateRef> item) if (pg->have_temp_coll()) { bool cont = remove_dir( - store, &mapper, &driver, pg->osr.get(), pg->get_temp_coll(), item.second); + pg->cct, store, &mapper, &driver, pg->osr.get(), pg->get_temp_coll(), item.second); if (!cont) return; } bool cont = remove_dir( - store, &mapper, &driver, pg->osr.get(), coll, item.second); + pg->cct, store, &mapper, &driver, pg->osr.get(), coll, item.second); if (!cont) return; @@ -3427,7 +3445,7 @@ void OSD::do_mon_report() { dout(7) << "do_mon_report" << dendl; - utime_t now(ceph_clock_now(g_ceph_context)); + utime_t now(ceph_clock_now(cct)); last_mon_report = now; // do any pending reports @@ -3450,7 +3468,7 @@ void OSD::ms_handle_connect(Connection *con) send_alive(); service.send_pg_temp(); send_failures(); - send_pg_stats(ceph_clock_now(g_ceph_context)); + send_pg_stats(ceph_clock_now(cct)); monc->sub_want("osd_pg_creates", 0, CEPH_SUBSCRIBE_ONETIME); monc->renew_subs(); @@ -3511,7 +3529,7 @@ void OSD::_maybe_boot(epoch_t oldest, epoch_t newest) // send pings sooner rather than later heartbeat_kick(); } else if (osdmap->get_epoch() >= oldest - 1 && - osdmap->get_epoch() + g_conf->osd_map_message_max > newest) { + osdmap->get_epoch() + cct->_conf->osd_map_message_max > newest) { _send_boot(); return; } @@ -3533,15 +3551,15 @@ void OSD::start_waiting_for_healthy() bool OSD::_is_healthy() { - if (!g_ceph_context->get_heartbeat_map()->is_healthy()) { + if (!cct->get_heartbeat_map()->is_healthy()) { dout(1) << "is_healthy false -- internal heartbeat failed" << dendl; return false; } if (is_waiting_for_healthy()) { Mutex::Locker l(heartbeat_lock); - utime_t cutoff = ceph_clock_now(g_ceph_context); - cutoff -= g_conf->osd_heartbeat_grace; + utime_t cutoff = ceph_clock_now(cct); + cutoff -= cct->_conf->osd_heartbeat_grace; int num = 0, up = 0; for (map<int,HeartbeatInfo>::iterator p = heartbeat_peers.begin(); p != heartbeat_peers.end(); @@ -3550,7 +3568,7 @@ bool OSD::_is_healthy() ++up; ++num; } - if ((float)up < (float)num * g_conf->osd_heartbeat_min_healthy_ratio) { + if ((float)up < (float)num * cct->_conf->osd_heartbeat_min_healthy_ratio) { dout(1) << "is_healthy false -- only " << up << "/" << num << " up peers (less than 1/3)" << dendl; return false; } @@ -3607,7 +3625,7 @@ void OSD::queue_want_up_thru(epoch_t want) up_thru_wanted = want; // expedite, a bit. WARNING this will somewhat delay other mon queries. - last_mon_report = ceph_clock_now(g_ceph_context); + last_mon_report = ceph_clock_now(cct); send_alive(); } else { dout(10) << "queue_want_up_thru want " << want << " <= queued " << up_thru_wanted @@ -3702,7 +3720,7 @@ void OSD::send_failures() heartbeat_lock.Lock(); locked = true; } - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(cct); while (!failure_queue.empty()) { int osd = failure_queue.begin()->first; int failed_for = (int)(double)(now - failure_queue.begin()->second); @@ -3771,7 +3789,7 @@ void OSD::send_pg_stats(const utime_t &now) if (!outstanding_pg_stats) { outstanding_pg_stats = true; - last_pg_stats_ack = ceph_clock_now(g_ceph_context); + last_pg_stats_ack = ceph_clock_now(cct); } monc->send_mon_message(m); } @@ -3788,7 +3806,7 @@ void OSD::handle_pg_stats_ack(MPGStatsAck *ack) return; } - last_pg_stats_ack = ceph_clock_now(g_ceph_context); + last_pg_stats_ack = ceph_clock_now(cct); pg_stat_queue_lock.Lock(); @@ -4005,7 +4023,7 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist goto out; } - cmd_getval(g_ceph_context, cmdmap, "prefix", prefix); + cmd_getval(cct, cmdmap, "prefix", prefix); if (prefix == "get_command_descriptions") { int cmdnum = 0; @@ -4027,7 +4045,7 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist goto out; } - cmd_getval(g_ceph_context, cmdmap, "format", format); + cmd_getval(cct, cmdmap, "format", format); f.reset(new_formatter(format)); if (prefix == "version") { @@ -4043,7 +4061,7 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist } else if (prefix == "injectargs") { vector<string> argsvec; - cmd_getval(g_ceph_context, cmdmap, "injected_args", argsvec); + cmd_getval(cct, cmdmap, "injected_args", argsvec); if (argsvec.empty()) { r = -EINVAL; @@ -4054,7 +4072,7 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist for (vector<string>::iterator a = ++argsvec.begin(); a != argsvec.end(); ++a) args += " " + *a; osd_lock.Unlock(); - g_conf->injectargs(args, &ss); + cct->_conf->injectargs(args, &ss); osd_lock.Lock(); } @@ -4062,14 +4080,14 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist // 'tell <pgid>' (which comes in without any of that prefix)? else if (prefix == "pg" || - (cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr) && + (cmd_getval(cct, cmdmap, "pgid", pgidstr) && (prefix == "query" || prefix == "mark_unfound_lost" || prefix == "list_missing") )) { pg_t pgid; - if (!cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr)) { + if (!cmd_getval(cct, cmdmap, "pgid", pgidstr)) { ss << "no pgid specified"; r = -EINVAL; } else if (!pgid.parse(pgidstr.c_str())) { @@ -4083,7 +4101,7 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist } else { // simulate pg <pgid> cmd= for pg->do-command if (prefix != "pg") - cmd_putval(g_ceph_context, cmdmap, "cmd", prefix); + cmd_putval(cct, cmdmap, "cmd", prefix); r = pg->do_command(cmdmap, ss, data, odata); pg->unlock(); } @@ -4094,8 +4112,8 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist int64_t count; int64_t bsize; // default count 1G, size 4MB - cmd_getval(g_ceph_context, cmdmap, "count", count, (int64_t)1 << 30); - cmd_getval(g_ceph_context, cmdmap, "bsize", bsize, (int64_t)4 << 20); + cmd_getval(cct, cmdmap, "count", count, (int64_t)1 << 30); + cmd_getval(cct, cmdmap, "bsize", bsize, (int64_t)4 << 20); bufferlist bl; bufferptr bp(bsize); @@ -4105,7 +4123,7 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist ObjectStore::Transaction *cleanupt = new ObjectStore::Transaction; store->sync_and_flush(); - utime_t start = ceph_clock_now(g_ceph_context); + utime_t start = ceph_clock_now(cct); for (int64_t pos = 0; pos < count; pos += bsize) { char nm[30]; snprintf(nm, sizeof(nm), "disk_bw_test_%lld", (long long)pos); @@ -4117,7 +4135,7 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist cleanupt->remove(coll_t::META_COLL, soid); } store->sync_and_flush(); - utime_t end = ceph_clock_now(g_ceph_context); + utime_t end = ceph_clock_now(cct); // clean up store->queue_transaction(NULL, cleanupt); @@ -4147,7 +4165,7 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist ss << "could not issue heap profiler command -- not using tcmalloc!"; } else { string heapcmd; - cmd_getval(g_ceph_context, cmdmap, "heapcmd", heapcmd); + cmd_getval(cct, cmdmap, "heapcmd", heapcmd); // XXX 1-element vector, change at callee or make vector here? vector<string> heapcmd_vec; get_str_vec(heapcmd, heapcmd_vec); @@ -4157,7 +4175,7 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist else if (prefix == "debug dump_missing") { string file_name; - cmd_getval(g_ceph_context, cmdmap, "filename", file_name); + cmd_getval(cct, cmdmap, "filename", file_name); std::ofstream fout(file_name.c_str()); if (!fout.is_open()) { ss << "failed to open file '" << file_name << "'"; @@ -4203,27 +4221,27 @@ void OSD::do_command(Connection *con, tid_t tid, vector<string>& cmd, bufferlist } else if (prefix == "debug kick_recovery_wq") { int64_t delay; - cmd_getval(g_ceph_context, cmdmap, "delay", delay); + cmd_getval(cct, cmdmap, "delay", delay); ostringstream oss; oss << delay; - r = g_conf->set_val("osd_recovery_delay_start", oss.str().c_str()); + r = cct->_conf->set_val("osd_recovery_delay_start", oss.str().c_str()); if (r != 0) { ss << "kick_recovery_wq: error setting " << "osd_recovery_delay_start to '" << delay << "': error " << r; goto out; } - g_conf->apply_changes(NULL); + cct->_conf->apply_changes(NULL); ss << "kicking recovery queue. set osd_recovery_delay_start " - << "to " << g_conf->osd_recovery_delay_start; - defer_recovery_until = ceph_clock_now(g_ceph_context); - defer_recovery_until += g_conf->osd_recovery_delay_start; + << "to " << cct->_conf->osd_recovery_delay_start; + defer_recovery_until = ceph_clock_now(cct); + defer_recovery_until += cct->_conf->osd_recovery_delay_start; recovery_wq.wake(); } else if (prefix == "cpu_profiler") { string arg; - cmd_getval(g_ceph_context, cmdmap, "arg", arg); + cmd_getval(cct, cmdmap, "arg", arg); vector<string> argvec; get_str_vec(arg, argvec); cpu_profiler_handle_command(argvec, ds); @@ -4527,7 +4545,7 @@ bool OSD::ms_verify_authorizer(Connection *con, int peer_type, uint64_t global_id; uint64_t auid = CEPH_AUTH_UID_DEFAULT; - isvalid = authorize_handler->verify_authorizer(g_ceph_context, monc->rotating_secrets, + isvalid = authorize_handler->verify_authorizer(cct, monc->rotating_secrets, authorizer_data, authorizer_reply, name, global_id, caps_info, session_key, &auid); if (isvalid) { @@ -4801,17 +4819,17 @@ bool OSD::scrub_should_schedule() return false; } - if (loadavgs[0] >= g_conf->osd_scrub_load_threshold) { + if (loadavgs[0] >= cct->_conf->osd_scrub_load_threshold) { dout(20) << "scrub_should_schedule loadavg " << loadavgs[0] - << " >= max " << g_conf->osd_scrub_load_threshold + << " >= max " << cct->_conf->osd_scrub_load_threshold << " = no, load too high" << dendl; return false; } dout(20) << "scrub_should_schedule loadavg " << loadavgs[0] - << " < max " << g_conf->osd_scrub_load_threshold + << " < max " << cct->_conf->osd_scrub_load_threshold << " = yes" << dendl; - return loadavgs[0] < g_conf->osd_scrub_load_threshold; + return loadavgs[0] < cct->_conf->osd_scrub_load_threshold; } void OSD::sched_scrub() @@ -4822,7 +4840,7 @@ void OSD::sched_scrub() dout(20) << "sched_scrub load_is_low=" << (int)load_is_low << dendl; - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(cct); //dout(20) << " " << last_scrub_pg << dendl; @@ -4834,15 +4852,15 @@ void OSD::sched_scrub() dout(30) << "sched_scrub examine " << pgid << " at " << t << dendl; utime_t diff = now - t; - if ((double)diff < g_conf->osd_scrub_min_interval) { + if ((double)diff < cct->_conf->osd_scrub_min_interval) { dout(10) << "sched_scrub " << pgid << " at " << t - << ": " << (double)diff << " < min (" << g_conf->osd_scrub_min_interval << " seconds)" << dendl; + << ": " << (double)diff << " < min (" << cct->_conf->osd_scrub_min_interval << " seconds)" << dendl; break; } - if ((double)diff < g_conf->osd_scrub_max_interval && !load_is_low) { + if ((double)diff < cct->_conf->osd_scrub_max_interval && !load_is_low) { // save ourselves some effort dout(10) << "sched_scrub " << pgid << " high load at " << t - << ": " << (double)diff << " < max (" << g_conf->osd_scrub_max_interval << " seconds)" << dendl; + << ": " << (double)diff << " < max (" << cct->_conf->osd_scrub_max_interval << " seconds)" << dendl; break; } @@ -4850,11 +4868,11 @@ void OSD::sched_scrub() if (pg) { if (pg->is_active() && (load_is_low || - (double)diff >= g_conf->osd_scrub_max_interval || + (double)diff >= cct->_conf->osd_scrub_max_interval || pg->scrubber.must_scrub)) { dout(10) << "sched_scrub scrubbing " << pgid << " at " << t << (pg->scrubber.must_scrub ? ", explicitly requested" : - ( (double)diff >= g_conf->osd_scrub_max_interval ? ", diff >= max" : "")) + ( (double)diff >= cct->_conf->osd_scrub_max_interval ? ", diff >= max" : "")) << dendl; if (pg->sched_scrub()) { pg->unlock(); @@ -4873,13 +4891,13 @@ bool OSDService::inc_scrubs_pending() bool result = false; sched_scrub_lock.Lock(); - if (scrubs_pending + scrubs_active < g_conf->osd_max_scrubs) { + if (scrubs_pending + scrubs_active < cct->_conf->osd_max_scrubs) { dout(20) << "inc_scrubs_pending " << scrubs_pending << " -> " << (scrubs_pending+1) - << " (max " << g_conf->osd_max_scrubs << ", active " << scrubs_active << ")" << dendl; + << " (max " << cct->_conf->osd_max_scrubs << ", active " << scrubs_active << ")" << dendl; result = true; ++scrubs_pending; } else { - dout(20) << "inc_scrubs_pending " << scrubs_pending << " + " << scrubs_active << " active >= max " << g_conf->osd_max_scrubs << dendl; + dout(20) << "inc_scrubs_pending " << scrubs_pending << " + " << scrubs_active << " active >= max " << cct->_conf->osd_max_scrubs << dendl; } sched_scrub_lock.Unlock(); @@ -4890,7 +4908,7 @@ void OSDService::dec_scrubs_pending() { sched_scrub_lock.Lock(); dout(20) << "dec_scrubs_pending " << scrubs_pending << " -> " << (scrubs_pending-1) - << " (max " << g_conf->osd_max_scrubs << ", active " << scrubs_active << ")" << dendl; + << " (max " << cct->_conf->osd_max_scrubs << ", active " << scrubs_active << ")" << dendl; --scrubs_pending; assert(scrubs_pending >= 0); sched_scrub_lock.Unlock(); @@ -4903,12 +4921,12 @@ void OSDService::inc_scrubs_active(bool reserved) if (reserved) { --(scrubs_pending); dout(20) << "inc_scrubs_active " << (scrubs_active-1) << " -> " << scrubs_active - << " (max " << g_conf->osd_max_scrubs + << " (max " << cct->_conf->osd_max_scrubs << ", pending " << (scrubs_pending+1) << " -> " << scrubs_pending << ")" << dendl; assert(scrubs_pending >= 0); } else { dout(20) << "inc_scrubs_active " << (scrubs_active-1) << " -> " << scrubs_active - << " (max " << g_conf->osd_max_scrubs + << " (max " << cct->_conf->osd_max_scrubs << ", pending " << scrubs_pending << ")" << dendl; } sched_scrub_lock.Unlock(); @@ -4918,7 +4936,7 @@ void OSDService::dec_scrubs_active() { sched_scrub_lock.Lock(); dout(20) << "dec_scrubs_active " << scrubs_active << " -> " << (scrubs_active-1) - << " (max " << g_conf->osd_max_scrubs << ", pending " << scrubs_pending << ")" << dendl; + << " (max " << cct->_conf->osd_max_scrubs << ", pending " << scrubs_pending << ")" << dendl; --scrubs_active; sched_scrub_lock.Unlock(); } @@ -4937,10 +4955,10 @@ bool OSDService::prepare_to_stop() osdmap->get_epoch(), false )); - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(cct); utime_t timeout; - timeout.set_from_double(now + g_conf->osd_mon_shutdown_timeout); - while ((ceph_clock_now(g_ceph_context) < timeout) && + timeout.set_from_double(now + cct->_conf->osd_mon_shutdown_timeout); + while ((ceph_clock_now(cct) < timeout) && (state != STOPPING)) { is_stopping_cond.WaitUntil(is_stopping_lock, timeout); } @@ -5159,7 +5177,7 @@ void OSD::handle_osd_map(MOSDMap *m) t.remove(coll_t::META_COLL, get_inc_osdmap_pobject_name(e)); superblock.oldest_map = e+1; num++; - if (num >= g_conf->osd_target_transaction_size && + if (num >= cct->_conf->osd_target_transaction_size && (uint64_t)num > (last - first)) // make sure we at least keep pace with incoming maps break; } @@ -5172,7 +5190,7 @@ void OSD::handle_osd_map(MOSDMap *m) map_lock.get_write(); - C_Contexts *fin = new C_Contexts(g_ceph_context); + C_Contexts *fin = new C_Contexts(cct); // advance through the new maps for (epoch_t cur = start; cur <= superblock.newest_map; cur++) { @@ -5199,7 +5217,7 @@ void OSD::handle_osd_map(MOSDMap *m) superblock.current_epoch = cur; advance_map(t, fin); - had_map_since = ceph_clock_now(g_ceph_context); + had_map_since = ceph_clock_now(cct); } if (osdmap->is_up(whoami) && @@ -5633,15 +5651,15 @@ void OSD::send_incremental_map(epoch_t since, Connection *con) return; } - if (to > since && (int64_t)(to - since) > g_conf->osd_map_share_max_epochs) { - dout(10) << " " << (to - since) << " > max " << g_conf->osd_map_share_max_epochs + if (to > since && (int64_t)(to - since) > cct->_conf->osd_map_share_max_epochs) { + dout(10) << " " << (to - since) << " > max " << cct->_conf->osd_map_share_max_epochs << ", only sending most recent" << dendl; - since = to - g_conf->osd_map_share_max_epochs; + since = to - cct->_conf->osd_map_share_max_epochs; } while (since < to) { - if (to - since > (epoch_t)g_conf->osd_map_message_max) - to = since + g_conf->osd_map_message_max; + if (to - since > (epoch_t)cct->_conf->osd_map_message_max) + to = since + cct->_conf->osd_map_message_max; MOSDMap *m = build_incremental_map_msg(since, to); send_map(m, con); since = to; @@ -5708,7 +5726,7 @@ OSDMapRef OSDService::_add_map(OSDMap *o) { epoch_t e = o->get_epoch(); - if (g_conf->osd_map_dedup) { + if (cct->_conf->osd_map_dedup) { // Dedup against an existing map at a nearby epoch OSDMapRef for_dedup = map_cache.lower_bound(e); if (for_dedup) { @@ -5900,7 +5918,7 @@ void OSD::handle_pg_create(OpRequestRef op) // drop the next N pg_creates in a row? if (debug_drop_pg_create_left < 0 && - g_conf->osd_debug_drop_pg_create_probability > + cct->_conf->osd_debug_drop_pg_create_probability > ((((double)(rand()%100))/100.0))) { debug_drop_pg_create_left = debug_drop_pg_create_duration; } @@ -6024,8 +6042,8 @@ void OSD::handle_pg_create(OpRequestRef op) PG::RecoveryCtx OSD::create_context() { ObjectStore::Transaction *t = new ObjectStore::Transaction; - C_Contexts *on_applied = new C_Contexts(g_ceph_context); - C_Contexts *on_safe = new C_Contexts(g_ceph_context); + C_Contexts *on_applied = new C_Contexts(cct); + C_Contexts *on_safe = new C_Contexts(cct); map< int, map<pg_t,pg_query_t> > *query_map = new map<int, map<pg_t, pg_query_t> >; map<int,vector<pair<pg_notify_t, pg_interval_map_t> > > *notify_list = @@ -6046,8 +6064,8 @@ void OSD::dispatch_context_transaction(PG::RecoveryCtx &ctx, PG *pg) ctx.transaction, ctx.on_applied, ctx.on_safe); assert(tr == 0); ctx.transaction = new ObjectStore::Transaction; - ctx.on_applied = new C_Contexts(g_ceph_context); - ctx.on_safe = new C_Contexts(g_ceph_context); + ctx.on_applied = new C_Contexts(cct); + ctx.on_safe = new C_Contexts(cct); } } @@ -6701,7 +6719,7 @@ void OSD::check_replay_queue() { assert(osd_lock.is_locked()); - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(cct); list< pair<pg_t,utime_t> > pgids; replay_queue_lock.Lock(); while (!replay_queue.empty() && @@ -6745,12 +6763,12 @@ bool OSDService::queue_for_recovery(PG *pg) bool OSD::_recover_now() { - if (recovery_ops_active >= g_conf->osd_recovery_max_active) { + if (recovery_ops_active >= cct->_conf->osd_recovery_max_active) { dout(15) << "_recover_now active " << recovery_ops_active - << " >= max " << g_conf->osd_recovery_max_active << dendl; + << " >= max " << cct->_conf->osd_recovery_max_active << dendl; return false; } - if (ceph_clock_now(g_ceph_context) < defer_recovery_until) { + if (ceph_clock_now(cct) < defer_recovery_until) { dout(15) << "_recover_now defer until " << defer_recovery_until << dendl; return false; } @@ -6762,14 +6780,14 @@ void OSD::do_recovery(PG *pg, ThreadPool::TPHandle &handle) { // see how many we should try to start. note that this is a bit racy. recovery_wq.lock(); - int max = MAX(g_conf->osd_recovery_max_active - recovery_ops_active, - g_conf->osd_recovery_max_single_start); + int max = MIN(cct->_conf->osd_recovery_max_active - recovery_ops_active, + cct->_conf->osd_recovery_max_single_start); if (max > 0) { - dout(10) << "do_recovery can start " << max << " (" << recovery_ops_active << "/" << g_conf->osd_recovery_max_active + dout(10) << "do_recovery can start " << max << " (" << recovery_ops_active << "/" << cct->_conf->osd_recovery_max_active << " rops)" << dendl; recovery_ops_active += max; // take them now, return them if we don't use them. } else { - dout(10) << "do_recovery can start 0 (" << recovery_ops_active << "/" << g_conf->osd_recovery_max_active + dout(10) << "do_recovery can start 0 (" << recovery_ops_active << "/" << cct->_conf->osd_recovery_max_active << " rops)" << dendl; } recovery_wq.unlock(); @@ -6830,7 +6848,7 @@ void OSD::start_recovery_op(PG *pg, const hobject_t& soid) { recovery_wq.lock(); dout(10) << "start_recovery_op " << *pg << " " << soid - << " (" << recovery_ops_active << "/" << g_conf->osd_recovery_max_active << " rops)" + << " (" << recovery_ops_active << "/" << cct->_conf->osd_recovery_max_active << " rops)" << dendl; assert(recovery_ops_active >= 0); recovery_ops_active++; @@ -6849,7 +6867,7 @@ void OSD::finish_recovery_op(PG *pg, const hobject_t& soid, bool dequeue) recovery_wq.lock(); dout(10) << "finish_recovery_op " << *pg << " " << soid << " dequeue=" << dequeue - << " (" << recovery_ops_active << "/" << g_conf->osd_recovery_max_active << " rops)" + << " (" << recovery_ops_active << "/" << cct->_conf->osd_recovery_max_active << " rops)" << dendl; // adjust count @@ -6954,9 +6972,9 @@ void OSD::handle_op(OpRequestRef op) } } - if (g_conf->osd_debug_drop_op_probability > 0 && + if (cct->_conf->osd_debug_drop_op_probability > 0 && !m->get_source().is_mds()) { - if ((double)rand() / (double)RAND_MAX < g_conf->osd_debug_drop_op_probability) { + if ((double)rand() / (double)RAND_MAX < cct->_conf->osd_debug_drop_op_probability) { dout(0) << "handle_op DEBUG artificially dropping op " << *m << dendl; return; } @@ -6978,11 +6996,11 @@ void OSD::handle_op(OpRequestRef op) } // too big? - if (g_conf->osd_max_write_size && - m->get_data_len() > g_conf->osd_max_write_size << 20) { + if (cct->_conf->osd_max_write_size && + m->get_data_len() > cct->_conf->osd_max_write_size << 20) { // journal can't hold commit! derr << "handle_op msg data len " << m->get_data_len() - << " > osd_max_write_size " << (g_conf->osd_max_write_size << 20) + << " > osd_max_write_size " << (cct->_conf->osd_max_write_size << 20) << " on " << *m << dendl; service.reply_op_error(op, -OSD_WRITETOOBIG); return; @@ -7103,7 +7121,7 @@ bool OSD::op_is_discardable(MOSDOp *op) */ void OSD::enqueue_op(PG *pg, OpRequestRef op) { - utime_t latency = ceph_clock_now(g_ceph_context) - op->request->get_recv_stamp(); + utime_t latency = ceph_clock_now(cct) - op->request->get_recv_stamp(); dout(15) << "enqueue_op " << op << " prio " << op->request->get_priority() << " cost " << op->request->get_cost() << " latency " << latency @@ -7194,7 +7212,7 @@ void OSD::dequeue_op( PGRef pg, OpRequestRef op, ThreadPool::TPHandle &handle) { - utime_t latency = ceph_clock_now(g_ceph_context) - op->request->get_recv_stamp(); + utime_t latency = ceph_clock_now(cct) - op->request->get_recv_stamp(); dout(10) << "dequeue_op " << op << " prio " << op->request->get_priority() << " cost " << op->request->get_cost() << " latency " << latency @@ -7308,8 +7326,8 @@ void OSD::handle_conf_change(const struct md_config_t *conf, const std::set <std::string> &changed) { if (changed.count("osd_max_backfills")) { - service.local_reserver.set_max(g_conf->osd_max_backfills); - service.remote_reserver.set_max(g_conf->osd_max_backfills); + service.local_reserver.set_max(cct->_conf->osd_max_backfills); + service.remote_reserver.set_max(cct->_conf->osd_max_backfills); } } @@ -7386,3 +7404,34 @@ int OSD::init_op_flags(OpRequestRef op) return 0; } + +bool OSD::RecoveryWQ::_enqueue(PG *pg) { + if (!pg->recovery_item.is_on_list()) { + pg->get("RecoveryWQ"); + osd->recovery_queue.push_back(&pg->recovery_item); + + if (osd->cct->_conf->osd_recovery_delay_start > 0) { + osd->defer_recovery_until = ceph_clock_now(osd->cct); + osd->defer_recovery_until += osd->cct->_conf->osd_recovery_delay_start; + } + return true; + } + return false; +} + +void OSD::PeeringWQ::_dequeue(list<PG*> *out) { + set<PG*> got; + for (list<PG*>::iterator i = peering_queue.begin(); + i != peering_queue.end() && + out->size() < osd->cct->_conf->osd_peering_wq_batch_size; + ) { + if (in_use.count(*i)) { + ++i; + } else { + out->push_back(*i); + got.insert(*i); + peering_queue.erase(i++); + } + } + in_use.insert(got.begin(), got.end()); +} diff --git a/src/osd/OSD.h b/src/osd/OSD.h index bd5e3d0bbbd..c2f45196870 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -27,6 +27,7 @@ #include "common/WorkQueue.h" #include "common/LogClient.h" #include "common/AsyncReserver.h" +#include "common/ceph_context.h" #include "os/ObjectStore.h" #include "OSDCap.h" @@ -114,6 +115,10 @@ enum { l_osd_waiting_for_map, + l_osd_stat_bytes, + l_osd_stat_bytes_used, + l_osd_stat_bytes_avail, + l_osd_last, }; @@ -174,8 +179,6 @@ class HistoricOpsSocketHook; class TestOpsSocketHook; struct C_CompleteSplits; -extern const coll_t meta_coll; - typedef std::tr1::shared_ptr<ObjectStore::Sequencer> SequencerRef; class DeletingState { @@ -282,6 +285,7 @@ class OSD; class OSDService { public: OSD *osd; + CephContext *cct; SharedPtrRegistry<pg_t, ObjectStore::Sequencer> osr_registry; SharedPtrRegistry<pg_t, DeletingState> deleting_pgs; const int whoami; @@ -433,7 +437,7 @@ public: bool ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, bool force_new); - ObjecterDispatcher(OSDService *o) : Dispatcher(g_ceph_context), osd(o) {} + ObjecterDispatcher(OSDService *o) : Dispatcher(cct), osd(o) {} } objecter_dispatcher; friend class ObjecterDispatcher; @@ -856,7 +860,7 @@ public: struct HeartbeatDispatcher : public Dispatcher { OSD *osd; - HeartbeatDispatcher(OSD *o) : Dispatcher(g_ceph_context), osd(o) {} + HeartbeatDispatcher(OSD *o) : Dispatcher(cct), osd(o) {} bool ms_dispatch(Message *m) { return osd->heartbeat_dispatch(m); }; @@ -1003,22 +1007,7 @@ private: bool _empty() { return peering_queue.empty(); } - void _dequeue(list<PG*> *out) { - set<PG*> got; - for (list<PG*>::iterator i = peering_queue.begin(); - i != peering_queue.end() && - out->size() < g_conf->osd_peering_wq_batch_size; - ) { - if (in_use.count(*i)) { - ++i; - } else { - out->push_back(*i); - got.insert(*i); - peering_queue.erase(i++); - } - } - in_use.insert(got.begin(), got.end()); - } + void _dequeue(list<PG*> *out); void _process( const list<PG *> &pgs, ThreadPool::TPHandle &handle) { @@ -1394,19 +1383,7 @@ protected: bool _empty() { return osd->recovery_queue.empty(); } - bool _enqueue(PG *pg) { - if (!pg->recovery_item.is_on_list()) { - pg->get("RecoveryWQ"); - osd->recovery_queue.push_back(&pg->recovery_item); - - if (g_conf->osd_recovery_delay_start > 0) { - osd->defer_recovery_until = ceph_clock_now(g_ceph_context); - osd->defer_recovery_until += g_conf->osd_recovery_delay_start; - } - return true; - } - return false; - } + bool _enqueue(PG *pg); void _dequeue(PG *pg) { if (pg->recovery_item.remove_myself()) pg->put("RecoveryWQ"); @@ -1693,7 +1670,8 @@ protected: public: /* internal and external can point to the same messenger, they will still * be cleaned up properly*/ - OSD(int id, + OSD(CephContext *cct_, + int id, Messenger *internal, Messenger *external, Messenger *hb_client, @@ -1705,15 +1683,15 @@ protected: // static bits static int find_osd_dev(char *result, int whoami); - static ObjectStore *create_object_store(const std::string &dev, const std::string &jdev); + static ObjectStore *create_object_store(CephContext *cct, const std::string &dev, const std::string &jdev); static int convertfs(const std::string &dev, const std::string &jdev); static int do_convertfs(ObjectStore *store); static int convert_collection(ObjectStore *store, coll_t cid); - static int mkfs(const std::string &dev, const std::string &jdev, + static int mkfs(CephContext *cct, const std::string &dev, const std::string &jdev, uuid_d fsid, int whoami); - static int mkjournal(const std::string &dev, const std::string &jdev); - static int flushjournal(const std::string &dev, const std::string &jdev); - static int dump_journal(const std::string &dev, const std::string &jdev, ostream& out); + static int mkjournal(CephContext *cct, const std::string &dev, const std::string &jdev); + static int flushjournal(CephContext *cct, const std::string &dev, const std::string &jdev); + static int dump_journal(CephContext *cct, const std::string &dev, const std::string &jdev, ostream& out); /* remove any non-user xattrs from a map of them */ void filter_xattrs(map<string, bufferptr>& attrs) { for (map<string, bufferptr>::iterator iter = attrs.begin(); diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc index 4b35b0c48ea..8007d613b8c 100644 --- a/src/osd/OSDMap.cc +++ b/src/osd/OSDMap.cc @@ -1845,7 +1845,9 @@ void OSDMap::build_simple_crush_map(CephContext *cct, CrushWrapper& crush, crush.set_type_name(6, "root"); // root - int rootid = crush.add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_DEFAULT, 6 /* pool */, 0, NULL, NULL); + int rootid; + int r = crush.add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_DEFAULT, 6 /* pool */, 0, NULL, NULL, &rootid); + assert(r == 0); crush.set_item_name(rootid, "default"); for (int o=0; o<nosd; o++) { @@ -1975,7 +1977,9 @@ void OSDMap::build_simple_crush_map_from_conf(CephContext *cct, CrushWrapper& cr set<string> hosts, racks; // root - int rootid = crush.add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_DEFAULT, 6 /* pool */, 0, NULL, NULL); + int rootid; + int r = crush.add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_DEFAULT, 6 /* pool */, 0, NULL, NULL, &rootid); + assert(r == 0); crush.set_item_name(rootid, "default"); // add osds diff --git a/src/osd/OpRequest.cc b/src/osd/OpRequest.cc index c694362a8a5..1ffe3073051 100644 --- a/src/osd/OpRequest.cc +++ b/src/osd/OpRequest.cc @@ -30,9 +30,9 @@ OpRequest::OpRequest(Message *req, OpTracker *tracker) : seq(0) { received_time = request->get_recv_stamp(); tracker->register_inflight_op(&xitem); - if (req->get_priority() < g_conf->osd_client_op_priority) { + if (req->get_priority() < tracker->cct->_conf->osd_client_op_priority) { // don't warn as quickly for low priority ops - warn_interval_multiplier = g_conf->osd_recovery_op_warn_multiple; + warn_interval_multiplier = tracker->cct->_conf->osd_recovery_op_warn_multiple; } } @@ -56,14 +56,14 @@ void OpHistory::cleanup(utime_t now) { while (arrived.size() && (now - arrived.begin()->first > - (double)(g_conf->osd_op_history_duration))) { + (double)(tracker->cct->_conf->osd_op_history_duration))) { duration.erase(make_pair( arrived.begin()->second->get_duration(), arrived.begin()->second)); arrived.erase(arrived.begin()); } - while (duration.size() > g_conf->osd_op_history_size) { + while (duration.size() > tracker->cct->_conf->osd_op_history_size) { arrived.erase(make_pair( duration.begin()->second->get_arrived(), duration.begin()->second)); @@ -75,8 +75,8 @@ void OpHistory::dump_ops(utime_t now, Formatter *f) { cleanup(now); f->open_object_section("OpHistory"); - f->dump_int("num to keep", g_conf->osd_op_history_size); - f->dump_int("duration to keep", g_conf->osd_op_history_duration); + f->dump_int("num to keep", tracker->cct->_conf->osd_op_history_size); + f->dump_int("duration to keep", tracker->cct->_conf->osd_op_history_duration); { f->open_array_section("Ops"); for (set<pair<utime_t, OpRequestRef> >::const_iterator i = @@ -95,7 +95,7 @@ void OpHistory::dump_ops(utime_t now, Formatter *f) void OpTracker::dump_historic_ops(Formatter *f) { Mutex::Locker locker(ops_in_flight_lock); - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(cct); history.dump_ops(now, f); } @@ -105,7 +105,7 @@ void OpTracker::dump_ops_in_flight(Formatter *f) f->open_object_section("ops_in_flight"); // overall dump f->dump_int("num_ops", ops_in_flight.size()); f->open_array_section("ops"); // list of OpRequests - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(cct); for (xlist<OpRequest*>::iterator p = ops_in_flight.begin(); !p.end(); ++p) { f->open_object_section("op"); (*p)->dump(now, f); @@ -126,7 +126,7 @@ void OpTracker::unregister_inflight_op(OpRequest *i) { Mutex::Locker locker(ops_in_flight_lock); assert(i->xitem.get_list() == &ops_in_flight); - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(cct); i->xitem.remove_myself(); i->request->clear_data(); history.insert(now, OpRequestRef(i)); @@ -138,9 +138,9 @@ bool OpTracker::check_ops_in_flight(std::vector<string> &warning_vector) if (!ops_in_flight.size()) return false; - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(cct); utime_t too_old = now; - too_old -= g_conf->osd_op_complaint_time; + too_old -= cct->_conf->osd_op_complaint_time; utime_t oldest_secs = now - ops_in_flight.front()->received_time; @@ -148,11 +148,11 @@ bool OpTracker::check_ops_in_flight(std::vector<string> &warning_vector) << "; oldest is " << oldest_secs << " seconds old" << dendl; - if (oldest_secs < g_conf->osd_op_complaint_time) + if (oldest_secs < cct->_conf->osd_op_complaint_time) return false; xlist<OpRequest*>::iterator i = ops_in_flight.begin(); - warning_vector.reserve(g_conf->osd_op_log_threshold + 1); + warning_vector.reserve(cct->_conf->osd_op_log_threshold + 1); int slow = 0; // total slow int warned = 0; // total logged @@ -161,13 +161,13 @@ bool OpTracker::check_ops_in_flight(std::vector<string> &warning_vector) // exponential backoff of warning intervals if (((*i)->received_time + - (g_conf->osd_op_complaint_time * + (cct->_conf->osd_op_complaint_time * (*i)->warn_interval_multiplier)) < now) { // will warn if (warning_vector.empty()) warning_vector.push_back(""); warned++; - if (warned > g_conf->osd_op_log_threshold) + if (warned > cct->_conf->osd_op_log_threshold) break; utime_t age = now - (*i)->received_time; @@ -259,7 +259,7 @@ void OpRequest::dump(utime_t now, Formatter *f) const void OpTracker::mark_event(OpRequest *op, const string &dest) { - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(cct); return _mark_event(op, dest, now); } @@ -297,7 +297,7 @@ OpRequestRef OpTracker::create_request(Message *ref) void OpRequest::mark_event(const string &event) { - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(tracker->cct); { Mutex::Locker l(lock); events.push_back(make_pair(now, event)); diff --git a/src/osd/OpRequest.h b/src/osd/OpRequest.h index fc8a8ab82c9..50ade71a1b9 100644 --- a/src/osd/OpRequest.h +++ b/src/osd/OpRequest.h @@ -26,15 +26,17 @@ #include "osd/osd_types.h" struct OpRequest; +class OpTracker; typedef std::tr1::shared_ptr<OpRequest> OpRequestRef; class OpHistory { set<pair<utime_t, OpRequestRef> > arrived; set<pair<double, OpRequestRef> > duration; void cleanup(utime_t now); bool shutdown; + OpTracker *tracker; public: - OpHistory() : shutdown(false) {} + OpHistory(OpTracker *tracker_) : shutdown(false), tracker(tracker_) {} ~OpHistory() { assert(arrived.empty()); assert(duration.empty()); @@ -52,13 +54,18 @@ class OpTracker { void operator()(OpRequest *op); }; friend class RemoveOnDelete; + friend class OpRequest; + friend class OpHistory; uint64_t seq; Mutex ops_in_flight_lock; xlist<OpRequest *> ops_in_flight; OpHistory history; +protected: + CephContext *cct; + public: - OpTracker() : seq(0), ops_in_flight_lock("OpTracker mutex") {} + OpTracker(CephContext *cct_) : seq(0), ops_in_flight_lock("OpTracker mutex"), history(this), cct(cct_) {} void dump_ops_in_flight(Formatter *f); void dump_historic_ops(Formatter *f); void register_inflight_op(xlist<OpRequest*>::item *i); diff --git a/src/osd/PG.cc b/src/osd/PG.cc index 2b81856b202..f319d160a39 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -1,3 +1,4 @@ + // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab /* @@ -142,6 +143,7 @@ PG::PG(OSDService *o, OSDMapRef curmap, const PGPool &_pool, pg_t p, const hobject_t& loid, const hobject_t& ioid) : osd(o), + cct(o->cct), osdriver(osd->store, coll_t(), OSD::make_snapmapper_oid()), snap_mapper( &osdriver, @@ -158,7 +160,7 @@ PG::PG(OSDService *o, OSDMapRef curmap, deleting(false), dirty_info(false), dirty_big_info(false), info(p), info_struct_v(0), - coll(p), pg_log(g_ceph_context), log_oid(loid), biginfo_oid(ioid), + coll(p), pg_log(cct), log_oid(loid), biginfo_oid(ioid), recovery_item(this), scrub_item(this), scrub_finalize_item(this), snap_trim_item(this), stat_queue_item(this), recovery_ops_active(0), waiting_on_backfill(0), @@ -1094,7 +1096,7 @@ void PG::activate(ObjectStore::Transaction& t, if (is_primary() && pool.info.crash_replay_interval > 0 && may_need_replay(get_osdmap())) { - replay_until = ceph_clock_now(g_ceph_context); + replay_until = ceph_clock_now(cct); replay_until += pool.info.crash_replay_interval; dout(10) << "activate starting replay interval for " << pool.info.crash_replay_interval << " until " << replay_until << dendl; @@ -1223,7 +1225,7 @@ void PG::activate(ObjectStore::Transaction& t, m = new MOSDPGLog(get_osdmap()->get_epoch(), pi); // send some recent log, so that op dup detection works well. - m->log.copy_up_to(pg_log.get_log(), g_conf->osd_min_pg_log_entries); + m->log.copy_up_to(pg_log.get_log(), cct->_conf->osd_min_pg_log_entries); m->info.log_tail = m->log.tail; pi.log_tail = m->log.tail; // sigh... @@ -1918,7 +1920,7 @@ void PG::publish_stats_to_osd() else state_clear(PG_STATE_INCONSISTENT); - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(cct); info.stats.last_fresh = now; if (info.stats.state != state) { info.stats.state = state; @@ -2625,8 +2627,8 @@ bool PG::sched_scrub() return false; } - bool time_for_deep = (ceph_clock_now(g_ceph_context) > - info.history.last_deep_scrub_stamp + g_conf->osd_deep_scrub_interval); + bool time_for_deep = (ceph_clock_now(cct) > + info.history.last_deep_scrub_stamp + cct->_conf->osd_deep_scrub_interval); //NODEEP_SCRUB so ignore time initiated deep-scrub if (osd->osd->get_osdmap()->test_flag(CEPH_OSDMAP_NODEEP_SCRUB)) @@ -2778,7 +2780,7 @@ void PG::_scan_list( int r; __u64 pos = 0; while ( (r = osd->store->read(coll, poid, pos, - g_conf->osd_deep_scrub_stride, bl, + cct->_conf->osd_deep_scrub_stride, bl, true)) > 0) { handle.reset_tp_timeout(); h << bl; @@ -2812,8 +2814,8 @@ void PG::_scan_list( assert(iter); uint64_t keys_scanned = 0; for (iter->seek_to_first(); iter->valid() ; iter->next()) { - if (g_conf->osd_scan_list_ping_tp_interval && - (keys_scanned % g_conf->osd_scan_list_ping_tp_interval == 0)) { + if (cct->_conf->osd_scan_list_ping_tp_interval && + (keys_scanned % cct->_conf->osd_scan_list_ping_tp_interval == 0)) { handle.reset_tp_timeout(); } ++keys_scanned; @@ -2969,7 +2971,7 @@ void PG::schedule_backfill_full_retry() { Mutex::Locker lock(osd->backfill_request_lock); osd->backfill_request_timer.add_event_after( - g_conf->osd_backfill_retry_interval, + cct->_conf->osd_backfill_retry_interval, new QueuePeeringEvt<RequestBackfill>( this, get_osdmap()->get_epoch(), RequestBackfill())); @@ -3629,8 +3631,8 @@ void PG::chunky_scrub(ThreadPool::TPHandle &handle) while (!boundary_found) { vector<hobject_t> objects; ret = osd->store->collection_list_partial(coll, start, - g_conf->osd_scrub_chunk_min, - g_conf->osd_scrub_chunk_max, + cct->_conf->osd_scrub_chunk_min, + cct->_conf->osd_scrub_chunk_max, 0, &objects, &scrubber.end); assert(ret >= 0); @@ -4213,7 +4215,7 @@ void PG::scrub_finish() // finish up unreg_next_scrub(); - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(cct); info.history.last_scrub = info.last_update; info.history.last_scrub_stamp = now; if (scrubber.deep) { @@ -4687,7 +4689,7 @@ void PG::proc_primary_info(ObjectStore::Transaction &t, const pg_info_t &oinfo) if (last_complete_ondisk.epoch >= info.history.last_epoch_started) { // DEBUG: verify that the snaps are empty in snap_mapper - if (g_conf->osd_debug_verify_snaps_on_info) { + if (cct->_conf->osd_debug_verify_snaps_on_info) { interval_set<snapid_t> p; p.union_of(oinfo.purged_snaps, info.purged_snaps); p.subtract(info.purged_snaps); @@ -5062,7 +5064,7 @@ void PG::handle_activate_map(RecoveryCtx *rctx) ActMap evt; recovery_state.handle_event(evt, rctx); if (osdmap_ref->get_epoch() - last_persisted_osdmap_ref->get_epoch() > - g_conf->osd_pg_epoch_persisted_max_stale) { + cct->_conf->osd_pg_epoch_persisted_max_stale) { dout(20) << __func__ << ": Dirtying info: last_persisted is " << last_persisted_osdmap_ref->get_epoch() << " while current is " << osdmap_ref->get_epoch() << dendl; @@ -5116,9 +5118,9 @@ std::ostream& operator<<(std::ostream& oss, /*------Crashed-------*/ PG::RecoveryState::Crashed::Crashed(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Crashed") { - state_name = "Crashed"; context< RecoveryMachine >().log_enter(state_name); assert(0 == "we got a bad state machine event"); } @@ -5126,9 +5128,9 @@ PG::RecoveryState::Crashed::Crashed(my_context ctx) /*------Initial-------*/ PG::RecoveryState::Initial::Initial(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Initial") { - state_name = "Initial"; context< RecoveryMachine >().log_enter(state_name); } @@ -5171,15 +5173,15 @@ void PG::RecoveryState::Initial::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_initial_latency, dur); } /*------Started-------*/ PG::RecoveryState::Started::Started(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started") { - state_name = "Started"; context< RecoveryMachine >().log_enter(state_name); } @@ -5220,15 +5222,15 @@ void PG::RecoveryState::Started::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_started_latency, dur); } /*--------Reset---------*/ PG::RecoveryState::Reset::Reset(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Reset") { - state_name = "Reset"; context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; pg->flushed = false; @@ -5295,15 +5297,15 @@ void PG::RecoveryState::Reset::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_reset_latency, dur); } /*-------Start---------*/ PG::RecoveryState::Start::Start(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Start") { - state_name = "Start"; context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; @@ -5320,15 +5322,15 @@ void PG::RecoveryState::Start::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_start_latency, dur); } /*---------Primary--------*/ PG::RecoveryState::Primary::Primary(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary") { - state_name = "Started/Primary"; context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; assert(pg->want_acting.empty()); @@ -5369,15 +5371,16 @@ void PG::RecoveryState::Primary::exit() context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; pg->want_acting.clear(); - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_primary_latency, dur); } /*---------Peering--------*/ PG::RecoveryState::Peering::Peering(my_context ctx) - : my_base(ctx), flushed(false) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering"), + flushed(false) { - state_name = "Started/Primary/Peering"; context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; @@ -5457,16 +5460,16 @@ void PG::RecoveryState::Peering::exit() pg->state_clear(PG_STATE_PEERING); pg->clear_probe_targets(); - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_peering_latency, dur); } /*------Backfilling-------*/ PG::RecoveryState::Backfilling::Backfilling(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/Backfilling") { - state_name = "Started/Primary/Active/Backfilling"; context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; pg->backfill_reserved = true; @@ -5495,16 +5498,16 @@ void PG::RecoveryState::Backfilling::exit() pg->backfill_reserved = false; pg->backfill_reserving = false; pg->state_clear(PG_STATE_BACKFILL); - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_backfilling_latency, dur); } /*--WaitRemoteBackfillReserved--*/ PG::RecoveryState::WaitRemoteBackfillReserved::WaitRemoteBackfillReserved(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/WaitRemoteBackfillReserved") { - state_name = "Started/Primary/Active/WaitRemoteBackfillReserved"; context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; pg->state_set(PG_STATE_BACKFILL_WAIT); @@ -5530,7 +5533,7 @@ void PG::RecoveryState::WaitRemoteBackfillReserved::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_waitremotebackfillreserved_latency, dur); } @@ -5557,9 +5560,9 @@ PG::RecoveryState::WaitRemoteBackfillReserved::react(const RemoteReservationReje /*--WaitLocalBackfillReserved--*/ PG::RecoveryState::WaitLocalBackfillReserved::WaitLocalBackfillReserved(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/WaitLocalBackfillReserved") { - state_name = "Started/Primary/Active/WaitLocalBackfillReserved"; context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; pg->state_set(PG_STATE_BACKFILL_WAIT); @@ -5575,15 +5578,15 @@ void PG::RecoveryState::WaitLocalBackfillReserved::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_waitlocalbackfillreserved_latency, dur); } /*----NotBackfilling------*/ PG::RecoveryState::NotBackfilling::NotBackfilling(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/NotBackfilling") { - state_name = "Started/Primary/Active/NotBackfilling"; context< RecoveryMachine >().log_enter(state_name); } @@ -5591,15 +5594,15 @@ void PG::RecoveryState::NotBackfilling::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_notbackfilling_latency, dur); } /*---RepNotRecovering----*/ PG::RecoveryState::RepNotRecovering::RepNotRecovering(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/ReplicaActive/RepNotRecovering") { - state_name = "Started/ReplicaActive/RepNotRecovering"; context< RecoveryMachine >().log_enter(state_name); } @@ -5607,15 +5610,15 @@ void PG::RecoveryState::RepNotRecovering::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_repnotrecovering_latency, dur); } /*---RepWaitRecoveryReserved--*/ PG::RecoveryState::RepWaitRecoveryReserved::RepWaitRecoveryReserved(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/ReplicaActive/RepWaitRecoveryReserved") { - state_name = "Started/ReplicaActive/RepWaitRecoveryReserved"; context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; @@ -5644,15 +5647,15 @@ void PG::RecoveryState::RepWaitRecoveryReserved::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_repwaitrecoveryreserved_latency, dur); } /*-RepWaitBackfillReserved*/ PG::RecoveryState::RepWaitBackfillReserved::RepWaitBackfillReserved(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/ReplicaActive/RepWaitBackfillReserved") { - state_name = "Started/ReplicaActive/RepWaitBackfillReserved"; context< RecoveryMachine >().log_enter(state_name); } @@ -5663,7 +5666,7 @@ PG::RecoveryState::RepNotRecovering::react(const RequestBackfillPrio &evt) double ratio, max_ratio; if (pg->osd->too_full_for_backfill(&ratio, &max_ratio) && - !g_conf->osd_debug_skip_full_check_in_backfill_reservation) { + !pg->cct->_conf->osd_debug_skip_full_check_in_backfill_reservation) { dout(10) << "backfill reservation rejected: full ratio is " << ratio << ", which is greater than max allowed ratio " << max_ratio << dendl; @@ -5682,7 +5685,7 @@ void PG::RecoveryState::RepWaitBackfillReserved::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_repwaitbackfillreserved_latency, dur); } @@ -5710,9 +5713,9 @@ PG::RecoveryState::RepWaitBackfillReserved::react(const RemoteReservationRejecte /*---RepRecovering-------*/ PG::RecoveryState::RepRecovering::RepRecovering(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/ReplicaActive/RepRecovering") { - state_name = "Started/ReplicaActive/RepRecovering"; context< RecoveryMachine >().log_enter(state_name); } @@ -5729,15 +5732,15 @@ void PG::RecoveryState::RepRecovering::exit() context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; pg->osd->remote_reserver.cancel_reservation(pg->info.pgid); - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_RepRecovering_latency, dur); } /*------Activating--------*/ PG::RecoveryState::Activating::Activating(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/Activating") { - state_name = "Started/Primary/Active/Activating"; context< RecoveryMachine >().log_enter(state_name); } @@ -5745,14 +5748,14 @@ void PG::RecoveryState::Activating::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_activating_latency, dur); } PG::RecoveryState::WaitLocalRecoveryReserved::WaitLocalRecoveryReserved(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/WaitLocalRecoveryReserved") { - state_name = "Started/Primary/Active/WaitLocalRecoveryReserved"; context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; pg->state_set(PG_STATE_RECOVERY_WAIT); @@ -5767,15 +5770,15 @@ void PG::RecoveryState::WaitLocalRecoveryReserved::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_waitlocalrecoveryreserved_latency, dur); } PG::RecoveryState::WaitRemoteRecoveryReserved::WaitRemoteRecoveryReserved(my_context ctx) : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/WaitRemoteRecoveryReserved"), acting_osd_it(context< Active >().sorted_acting_set.begin()) { - state_name = "Started/Primary/Active/WaitRemoteRecoveryReserved"; context< RecoveryMachine >().log_enter(state_name); post_event(RemoteRecoveryReserved()); } @@ -5814,14 +5817,14 @@ void PG::RecoveryState::WaitRemoteRecoveryReserved::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_waitremoterecoveryreserved_latency, dur); } PG::RecoveryState::Recovering::Recovering(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/Recovering") { - state_name = "Started/Primary/Active/Recovering"; context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; @@ -5877,16 +5880,16 @@ void PG::RecoveryState::Recovering::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_recovering_latency, dur); } PG::RecoveryState::Recovered::Recovered(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/Recovered") { int newest_update_osd; - state_name = "Started/Primary/Active/Recovered"; context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; @@ -5911,14 +5914,14 @@ void PG::RecoveryState::Recovered::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_recovered_latency, dur); } PG::RecoveryState::Clean::Clean(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/Clean") { - state_name = "Started/Primary/Active/Clean"; context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; @@ -5939,18 +5942,18 @@ void PG::RecoveryState::Clean::exit() context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; pg->state_clear(PG_STATE_CLEAN); - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_clean_latency, dur); } /*---------Active---------*/ PG::RecoveryState::Active::Active(my_context ctx) : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active"), sorted_acting_set(context< RecoveryMachine >().pg->acting.begin(), context< RecoveryMachine >().pg->acting.end()), all_replicas_activated(false) { - state_name = "Started/Primary/Active"; context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; @@ -6004,7 +6007,7 @@ boost::statechart::result PG::RecoveryState::Active::react(const AdvMap& advmap) } // if we haven't reported our PG stats in a long time, do so now. - if (pg->info.stats.reported_epoch + g_conf->osd_pg_stat_report_interval_max < advmap.osdmap->get_epoch()) { + if (pg->info.stats.reported_epoch + pg->cct->_conf->osd_pg_stat_report_interval_max < advmap.osdmap->get_epoch()) { dout(20) << "reporting stats to osd after " << (advmap.osdmap->get_epoch() - pg->info.stats.reported_epoch) << " epochs" << dendl; pg->publish_stats_to_osd(); @@ -6025,13 +6028,13 @@ boost::statechart::result PG::RecoveryState::Active::react(const ActMap&) pg->discover_all_missing(*context< RecoveryMachine >().get_query_map()); } - if (g_conf->osd_check_for_log_corruption) + if (pg->cct->_conf->osd_check_for_log_corruption) pg->check_log_for_corruption(pg->osd->store); int unfound = pg->pg_log.get_missing().num_missing() - pg->missing_loc.size(); if (unfound > 0 && pg->all_unfound_are_queried_or_lost(pg->get_osdmap())) { - if (g_conf->osd_auto_mark_unfound_lost) { + if (pg->cct->_conf->osd_auto_mark_unfound_lost) { pg->osd->clog.error() << pg->info.pgid << " has " << unfound << " objects unfound and apparently lost, would automatically marking lost but NOT IMPLEMENTED\n"; //pg->mark_all_unfound_lost(*context< RecoveryMachine >().get_cur_transaction()); @@ -6190,16 +6193,15 @@ void PG::RecoveryState::Active::exit() pg->state_clear(PG_STATE_BACKFILL_WAIT); pg->state_clear(PG_STATE_RECOVERY_WAIT); pg->state_clear(PG_STATE_REPLAY); - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_active_latency, dur); } /*------ReplicaActive-----*/ PG::RecoveryState::ReplicaActive::ReplicaActive(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/ReplicaActive") { - state_name = "Started/ReplicaActive"; - context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; @@ -6283,14 +6285,15 @@ void PG::RecoveryState::ReplicaActive::exit() context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; pg->osd->remote_reserver.cancel_reservation(pg->info.pgid); - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_replicaactive_latency, dur); } /*-------Stray---*/ PG::RecoveryState::Stray::Stray(my_context ctx) - : my_base(ctx) { - state_name = "Started/Stray"; + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Stray") +{ context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; @@ -6385,15 +6388,15 @@ void PG::RecoveryState::Stray::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_stray_latency, dur); } /*--------GetInfo---------*/ PG::RecoveryState::GetInfo::GetInfo(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/GetInfo") { - state_name = "Started/Primary/Peering/GetInfo"; context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; @@ -6560,15 +6563,16 @@ void PG::RecoveryState::GetInfo::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_getinfo_latency, dur); } /*------GetLog------------*/ -PG::RecoveryState::GetLog::GetLog(my_context ctx) : - my_base(ctx), newest_update_osd(-1), msg(0) +PG::RecoveryState::GetLog::GetLog(my_context ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/GetLog"), + newest_update_osd(-1), msg(0) { - state_name = "Started/Primary/Peering/GetLog"; context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; @@ -6675,15 +6679,15 @@ void PG::RecoveryState::GetLog::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_getlog_latency, dur); } /*------WaitActingChange--------*/ PG::RecoveryState::WaitActingChange::WaitActingChange(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/WaitActingChange") { - state_name = "Started/Primary/Peering/WaitActingChange"; context< RecoveryMachine >().log_enter(state_name); } @@ -6735,15 +6739,15 @@ void PG::RecoveryState::WaitActingChange::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_waitactingchange_latency, dur); } /*------Incomplete--------*/ PG::RecoveryState::Incomplete::Incomplete(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/Incomplete") { - state_name = "Started/Primary/Peering/Incomplete"; context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; @@ -6772,15 +6776,15 @@ void PG::RecoveryState::Incomplete::exit() PG *pg = context< RecoveryMachine >().pg; pg->state_clear(PG_STATE_INCOMPLETE); - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_incomplete_latency, dur); } /*------GetMissing--------*/ PG::RecoveryState::GetMissing::GetMissing(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/GetMissing") { - state_name = "Started/Primary/Peering/GetMissing"; context< RecoveryMachine >().log_enter(state_name); PG *pg = context< RecoveryMachine >().pg; @@ -6897,15 +6901,15 @@ void PG::RecoveryState::GetMissing::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_getmissing_latency, dur); } /*---WaitFlushedPeering---*/ PG::RecoveryState::WaitFlushedPeering::WaitFlushedPeering(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/WaitFlushedPeering") { - state_name = "Started/Primary/Peering/WaitFlushedPeering"; PG *pg = context< RecoveryMachine >().pg; context< RecoveryMachine >().log_enter(state_name); if (context< RecoveryMachine >().pg->flushed) @@ -6933,9 +6937,9 @@ PG::RecoveryState::WaitFlushedPeering::react(const QueryState &q) /*------WaitUpThru--------*/ PG::RecoveryState::WaitUpThru::WaitUpThru(my_context ctx) - : my_base(ctx) + : my_base(ctx), + NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/WaitUpThru") { - state_name = "Started/Primary/Peering/WaitUpThru"; context< RecoveryMachine >().log_enter(state_name); } @@ -6978,7 +6982,7 @@ void PG::RecoveryState::WaitUpThru::exit() { context< RecoveryMachine >().log_exit(state_name, enter_time); PG *pg = context< RecoveryMachine >().pg; - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; pg->osd->recoverystate_perf->tinc(rs_waitupthru_latency, dur); } @@ -6994,9 +6998,9 @@ void PG::RecoveryState::RecoveryMachine::log_enter(const char *state_name) void PG::RecoveryState::RecoveryMachine::log_exit(const char *state_name, utime_t enter_time) { - utime_t dur = ceph_clock_now(g_ceph_context) - enter_time; + utime_t dur = ceph_clock_now(pg->cct) - enter_time; dout(5) << "exit " << state_name << " " << dur << " " << event_count << " " << event_time << dendl; - pg->osd->pg_recovery_stats.log_exit(state_name, ceph_clock_now(g_ceph_context) - enter_time, + pg->osd->pg_recovery_stats.log_exit(state_name, ceph_clock_now(pg->cct) - enter_time, event_count, event_time); event_count = 0; event_time = utime_t(); @@ -7193,6 +7197,22 @@ bool PG::PriorSet::affected_by_map(const OSDMapRef osdmap, const PG *debug_pg) c return false; } +void PG::RecoveryState::start_handle(RecoveryCtx *new_ctx) { + assert(!rctx); + rctx = new_ctx; + if (rctx) + rctx->start_time = ceph_clock_now(pg->cct); +} + +void PG::RecoveryState::end_handle() { + if (rctx) { + utime_t dur = ceph_clock_now(pg->cct) - rctx->start_time; + machine.event_time += dur; + } + machine.event_count++; + rctx = 0; +} + void intrusive_ptr_add_ref(PG *pg) { pg->get("intptr"); } void intrusive_ptr_release(PG *pg) { pg->put("intptr"); } diff --git a/src/osd/PG.h b/src/osd/PG.h index 720ce67bca3..cdbe827a4a9 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -46,6 +46,7 @@ #include "common/cmdparse.h" #include "common/tracked_int_ptr.hpp" #include "common/WorkQueue.h" +#include "common/ceph_context.h" #include "include/str_list.h" #include <list> @@ -189,6 +190,7 @@ public: /*** PG ****/ protected: OSDService *osd; + CephContext *cct; OSDriver osdriver; SnapMapper snap_mapper; public: @@ -383,7 +385,9 @@ public: const char *state_name; utime_t enter_time; const char *get_state_name() { return state_name; } - NamedState() : state_name(0), enter_time(ceph_clock_now(g_ceph_context)) {} + NamedState(CephContext *cct_, const char *state_name_) + : state_name(state_name_), + enter_time(ceph_clock_now(cct_)) {}; virtual ~NamedState() {} }; @@ -519,7 +523,8 @@ protected: list<OpRequestRef> waiting_for_active; list<OpRequestRef> waiting_for_all_missing; map<hobject_t, list<OpRequestRef> > waiting_for_missing_object, - waiting_for_degraded_object; + waiting_for_degraded_object, + waiting_for_blocked_object; // Callbacks should assume pg (and nothing else) is locked map<hobject_t, list<Context*> > callbacks_for_degraded_object; map<eversion_t,list<OpRequestRef> > waiting_for_ack, waiting_for_ondisk; @@ -1060,21 +1065,8 @@ public: /* Encapsulates PG recovery process */ class RecoveryState { - void start_handle(RecoveryCtx *new_ctx) { - assert(!rctx); - rctx = new_ctx; - if (rctx) - rctx->start_time = ceph_clock_now(g_ceph_context); - } - - void end_handle() { - if (rctx) { - utime_t dur = ceph_clock_now(g_ceph_context) - rctx->start_time; - machine.event_time += dur; - } - machine.event_count++; - rctx = 0; - } + void start_handle(RecoveryCtx *new_ctx); + void end_handle(); /* States */ struct Initial; diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index d8aa8b1d02b..a92403ae370 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -50,6 +50,7 @@ #include "include/compat.h" #include "common/cmdparse.h" +#include "mon/MonClient.h" #include "osdc/Objecter.h" #include "json_spirit/json_spirit_value.h" @@ -126,8 +127,8 @@ void ReplicatedPG::wait_for_missing_object(const hobject_t& soid, OpRequestRef o else { dout(7) << "missing " << soid << " v " << v << ", pulling." << dendl; map<int, vector<PullOp> > pulls; - prepare_pull(soid, v, g_conf->osd_client_op_priority, &pulls); - send_pulls(g_conf->osd_client_op_priority, pulls); + prepare_pull(soid, v, cct->_conf->osd_client_op_priority, &pulls); + send_pulls(cct->_conf->osd_client_op_priority, pulls); } waiting_for_missing_object[soid].push_back(op); op->mark_delayed("waiting for missing object"); @@ -184,13 +185,20 @@ void ReplicatedPG::wait_for_degraded_object(const hobject_t& soid, OpRequestRef } } map<int, vector<PushOp> > pushes; - prep_object_replica_pushes(soid, v, g_conf->osd_client_op_priority, &pushes); - send_pushes(g_conf->osd_client_op_priority, pushes); + prep_object_replica_pushes(soid, v, cct->_conf->osd_client_op_priority, &pushes); + send_pushes(cct->_conf->osd_client_op_priority, pushes); } waiting_for_degraded_object[soid].push_back(op); op->mark_delayed("waiting for degraded object"); } +void ReplicatedPG::wait_for_blocked_object(const hobject_t& soid, OpRequestRef op) +{ + dout(10) << __func__ << " " << soid << " " << op << dendl; + waiting_for_blocked_object[soid].push_back(op); + op->mark_delayed("waiting for blocked object"); +} + void ReplicatedPG::wait_for_backfill_pos(OpRequestRef op) { waiting_for_backfill_pos.push_back(op); @@ -280,14 +288,14 @@ int ReplicatedPG::do_command(cmdmap_t cmdmap, ostream& ss, string prefix; string format; - cmd_getval(g_ceph_context, cmdmap, "format", format); + cmd_getval(cct, cmdmap, "format", format); boost::scoped_ptr<Formatter> f(new_formatter(format)); // demand that we have a formatter if (!f) f.reset(new_formatter("json")); string command; - cmd_getval(g_ceph_context, cmdmap, "cmd", command); + cmd_getval(cct, cmdmap, "cmd", command); if (command == "query") { f->open_object_section("pg"); f->dump_string("state", pg_state_string(get_state())); @@ -314,7 +322,7 @@ int ReplicatedPG::do_command(cmdmap_t cmdmap, ostream& ss, } else if (command == "mark_unfound_lost") { string mulcmd; - cmd_getval(g_ceph_context, cmdmap, "mulcmd", mulcmd); + cmd_getval(cct, cmdmap, "mulcmd", mulcmd); if (mulcmd != "revert") { ss << "mode must be 'revert'; mark and delete not yet implemented"; return -EINVAL; @@ -345,7 +353,7 @@ int ReplicatedPG::do_command(cmdmap_t cmdmap, ostream& ss, else if (command == "list_missing") { hobject_t offset; string offset_json; - if (cmd_getval(g_ceph_context, cmdmap, "offset", offset_json)) { + if (cmd_getval(cct, cmdmap, "offset", offset_json)) { json_spirit::Value v; try { if (!json_spirit::read(offset_json, v)) @@ -369,7 +377,7 @@ int ReplicatedPG::do_command(cmdmap_t cmdmap, ostream& ss, f->open_array_section("objects"); int32_t num = 0; bufferlist bl; - while (p != missing.missing.end() && num < g_conf->osd_command_max_records) { + while (p != missing.missing.end() && num < cct->_conf->osd_command_max_records) { f->open_object_section("object"); { f->open_object_section("oid"); @@ -459,7 +467,7 @@ void ReplicatedPG::do_pg_op(OpRequestRef op) dout(10) << " pgls pg=" << m->get_pg() << " != " << info.pgid << dendl; result = 0; // hmm? } else { - unsigned list_size = MIN(g_conf->osd_max_pgls, p->op.pgls.count); + unsigned list_size = MIN(cct->_conf->osd_max_pgls, p->op.pgls.count); dout(10) << " pgls pg=" << m->get_pg() << " count " << list_size << dendl; // read into a buffer @@ -584,14 +592,14 @@ void ReplicatedPG::calc_trim_to() return; } - size_t target = g_conf->osd_min_pg_log_entries; + size_t target = cct->_conf->osd_min_pg_log_entries; if (is_degraded() || state_test(PG_STATE_RECOVERING | PG_STATE_RECOVERY_WAIT | PG_STATE_BACKFILL | PG_STATE_BACKFILL_WAIT | PG_STATE_BACKFILL_TOOFULL)) { - target = g_conf->osd_max_pg_log_entries; + target = cct->_conf->osd_max_pg_log_entries; } if (min_last_complete_ondisk != eversion_t() && @@ -622,7 +630,9 @@ ReplicatedPG::ReplicatedPG(OSDService *o, OSDMapRef curmap, PG(o, curmap, _pool, p, oid, ioid), snapset_contexts_lock("ReplicatedPG::snapset_contexts"), temp_created(false), - temp_coll(coll_t::make_temp_coll(p)), snap_trimmer_machine(this) + temp_coll(coll_t::make_temp_coll(p)), + temp_seq(0), + snap_trimmer_machine(this) { snap_trimmer_machine.initiate(); } @@ -712,27 +722,32 @@ void ReplicatedPG::do_op(OpRequestRef op) m->get_object_locator().get_pool(), m->get_object_locator().nspace), &obc, can_create, &snapid); - if (r) { - if (r == -EAGAIN) { - // If we're not the primary of this OSD, and we have - // CEPH_OSD_FLAG_LOCALIZE_READS set, we just return -EAGAIN. Otherwise, - // we have to wait for the object. - if (is_primary() || - (!(m->get_flags() & CEPH_OSD_FLAG_BALANCE_READS) && - !(m->get_flags() & CEPH_OSD_FLAG_LOCALIZE_READS))) { - // missing the specific snap we need; requeue and wait. - assert(!can_create); // only happens on a read - hobject_t soid(m->get_oid(), m->get_object_locator().key, - snapid, m->get_pg().ps(), - info.pgid.pool(), m->get_object_locator().nspace); - wait_for_missing_object(soid, op); - return; - } + + if (r == -EAGAIN) { + // If we're not the primary of this OSD, and we have + // CEPH_OSD_FLAG_LOCALIZE_READS set, we just return -EAGAIN. Otherwise, + // we have to wait for the object. + if (is_primary() || + (!(m->get_flags() & CEPH_OSD_FLAG_BALANCE_READS) && + !(m->get_flags() & CEPH_OSD_FLAG_LOCALIZE_READS))) { + // missing the specific snap we need; requeue and wait. + assert(!can_create); // only happens on a read + hobject_t soid(m->get_oid(), m->get_object_locator().key, + snapid, m->get_pg().ps(), + info.pgid.pool(), m->get_object_locator().nspace); + wait_for_missing_object(soid, op); + return; } + } + + if (maybe_handle_cache(op, obc, r)) + return; + + if (r) { osd->reply_op_error(op, r); return; } - + // make sure locator is consistent object_locator_t oloc(obc->obs.oi.soid); if (m->get_object_locator() != oloc) { @@ -743,6 +758,12 @@ void ReplicatedPG::do_op(OpRequestRef op) << " op " << *m << "\n"; } + // io blocked on obc? + if (obc->is_blocked()) { + wait_for_blocked_object(obc->obs.oi.soid, op); + return; + } + if ((op->may_read()) && (obc->obs.oi.lost)) { // This object is lost. Reading from it returns an error. dout(20) << __func__ << ": object " << obc->obs.oi.soid @@ -887,6 +908,53 @@ void ReplicatedPG::do_op(OpRequestRef op) execute_ctx(ctx); } +bool ReplicatedPG::maybe_handle_cache(OpRequestRef op, ObjectContextRef obc, + int r) +{ + switch(pool.info.cache_mode) { + case pg_pool_t::CACHEMODE_NONE: + return false; + break; + case pg_pool_t::CACHEMODE_WRITEBACK: + if (obc.get()) { + return false; + } else { + do_cache_redirect(op, obc); + return true; + } + break; + case pg_pool_t::CACHEMODE_INVALIDATE_FORWARD: + do_cache_redirect(op, obc); + return true; + break; + case pg_pool_t::CACHEMODE_READONLY: + if (obc.get() && !r) { + return false; + } else { + do_cache_redirect(op, obc); + return true; + } + break; + default: + assert(0); + } + return false; +} + +void ReplicatedPG::do_cache_redirect(OpRequestRef op, ObjectContextRef obc) +{ + MOSDOp *m = static_cast<MOSDOp*>(op->request); + int flags = m->get_flags() & (CEPH_OSD_FLAG_ACK|CEPH_OSD_FLAG_ONDISK); + MOSDOpReply *reply = new MOSDOpReply(m, -ENOENT, + get_osdmap()->get_epoch(), flags); + request_redirect_t redir(m->get_object_locator(), pool.info.tier_of); + reply->set_redirect(redir); + dout(10) << "sending redirect to pool " << pool.info.tier_of << " for op " + << op << dendl; + m->get_connection()->get_messenger()->send_message(reply, m->get_connection()); + return; +} + void ReplicatedPG::execute_ctx(OpContext *ctx) { dout(10) << __func__ << " " << ctx << dendl; @@ -974,13 +1042,7 @@ void ReplicatedPG::execute_ctx(OpContext *ctx) ctx->user_at_version = obc->obs.oi.user_version; // note my stats - utime_t now = ceph_clock_now(g_ceph_context); - - // note some basic context for op replication that prepare_transaction may clobber - eversion_t old_last_update = pg_log.get_head(); - bool old_exists = obc->obs.exists; - uint64_t old_size = obc->obs.oi.size; - eversion_t old_version = obc->obs.oi.version; + utime_t now = ceph_clock_now(cct); if (op->may_read()) { dout(10) << " taking ondisk_read_lock" << dendl; @@ -1075,7 +1137,7 @@ void ReplicatedPG::execute_ctx(OpContext *ctx) append_log(ctx->log, pg_trim_to, ctx->local_t); // verify that we are doing this in order? - if (g_conf->osd_debug_op_order && m->get_source().is_client()) { + if (cct->_conf->osd_debug_op_order && m->get_source().is_client()) { map<client_t,tid_t>& cm = debug_op_order[obc->obs.oi.soid]; tid_t t = m->get_tid(); client_t n = m->get_source().num(); @@ -1100,7 +1162,7 @@ void ReplicatedPG::execute_ctx(OpContext *ctx) repop->src_obc.swap(src_obc); // and src_obc. - issue_repop(repop, now, old_last_update, old_exists, old_size, old_version); + issue_repop(repop, now); eval_repop(repop); repop->put(); @@ -1123,7 +1185,7 @@ void ReplicatedPG::log_op_stats(OpContext *ctx) OpRequestRef op = ctx->op; MOSDOp *m = static_cast<MOSDOp*>(op->request); - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(cct); utime_t latency = now; latency -= ctx->op->request->get_recv_stamp(); @@ -1169,7 +1231,7 @@ void ReplicatedPG::log_op_stats(OpContext *ctx) void ReplicatedPG::log_subop_stats(OpRequestRef op, int tag_inb, int tag_lat) { - utime_t now = ceph_clock_now(g_ceph_context); + utime_t now = ceph_clock_now(cct); utime_t latency = now; latency -= op->request->get_recv_stamp(); @@ -1286,8 +1348,8 @@ void ReplicatedPG::do_scan( BackfillInterval bi; osr->flush(); scan_range( - m->begin, g_conf->osd_backfill_scan_min, - g_conf->osd_backfill_scan_max, &bi, handle); + m->begin, cct->_conf->osd_backfill_scan_min, + cct->_conf->osd_backfill_scan_max, &bi, handle); MOSDPGScan *reply = new MOSDPGScan(MOSDPGScan::OP_SCAN_DIGEST, get_osdmap()->get_epoch(), m->query_epoch, info.pgid, bi.begin, bi.end); @@ -1353,7 +1415,7 @@ void ReplicatedPG::_do_push(OpRequestRef op) reply->pgid = info.pgid; reply->map_epoch = m->map_epoch; reply->replies.swap(replies); - reply->compute_cost(g_ceph_context); + reply->compute_cost(cct); t->register_on_complete(new C_OSD_SendMessageOnConn( osd, reply, m->get_connection())); @@ -1384,7 +1446,7 @@ void ReplicatedPG::_do_pull_response(OpRequestRef op) reply->pgid = info.pgid; reply->map_epoch = m->map_epoch; reply->pulls.swap(replies); - reply->compute_cost(g_ceph_context); + reply->compute_cost(cct); t->register_on_complete(new C_OSD_SendMessageOnConn( osd, reply, m->get_connection())); @@ -1442,12 +1504,12 @@ void ReplicatedPG::do_backfill(OpRequestRef op) case MOSDPGBackfill::OP_BACKFILL_FINISH: { assert(is_replica()); - assert(g_conf->osd_kill_backfill_at != 1); + assert(cct->_conf->osd_kill_backfill_at != 1); MOSDPGBackfill *reply = new MOSDPGBackfill(MOSDPGBackfill::OP_BACKFILL_FINISH_ACK, get_osdmap()->get_epoch(), m->query_epoch, info.pgid); - reply->set_priority(g_conf->osd_recovery_op_priority); + reply->set_priority(cct->_conf->osd_recovery_op_priority); osd->send_message_osd_cluster(reply, m->get_connection()); queue_peering_event( CephPeeringEvtRef( @@ -1461,7 +1523,7 @@ void ReplicatedPG::do_backfill(OpRequestRef op) case MOSDPGBackfill::OP_BACKFILL_PROGRESS: { assert(is_replica()); - assert(g_conf->osd_kill_backfill_at != 2); + assert(cct->_conf->osd_kill_backfill_at != 2); info.last_backfill = m->last_backfill; if (m->compat_stat_sum) { @@ -1481,7 +1543,7 @@ void ReplicatedPG::do_backfill(OpRequestRef op) case MOSDPGBackfill::OP_BACKFILL_FINISH_ACK: { assert(is_primary()); - assert(g_conf->osd_kill_backfill_at != 3); + assert(cct->_conf->osd_kill_backfill_at != 3); finish_recovery_op(hobject_t::get_max()); } break; @@ -1530,7 +1592,7 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid) &obc->obs, obc->ssc, this); - ctx->mtime = ceph_clock_now(g_ceph_context); + ctx->mtime = ceph_clock_now(cct); ctx->at_version.epoch = get_osdmap()->get_epoch(); ctx->at_version.version = pg_log.get_head().version + 1; @@ -2052,11 +2114,11 @@ int ReplicatedPG::do_tmapup(OpContext *ctx, bufferlist::iterator& bp, OSDOp& osd return result; } -static int check_offset_and_length(uint64_t offset, uint64_t length) +static int check_offset_and_length(uint64_t offset, uint64_t length, uint64_t max) { - if (offset >= g_conf->osd_max_object_size || - length > g_conf->osd_max_object_size || - offset + length > g_conf->osd_max_object_size) + if (offset >= max || + length > max || + offset + length > max) return -EFBIG; return 0; @@ -2115,9 +2177,9 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) // munge ZERO -> TRUNCATE? (don't munge to DELETE or we risk hosing attributes) if (op.op == CEPH_OSD_OP_ZERO && obs.exists && - op.extent.offset < g_conf->osd_max_object_size && + op.extent.offset < cct->_conf->osd_max_object_size && op.extent.length >= 1 && - op.extent.length <= g_conf->osd_max_object_size && + op.extent.length <= cct->_conf->osd_max_object_size && op.extent.offset + op.extent.length >= oi.size) { if (op.extent.offset >= oi.size) { // no-op @@ -2217,7 +2279,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) uint64_t last = op.extent.offset; for (miter = m.begin(); miter != m.end(); ++miter) { // verify hole? - if (g_conf->osd_verify_sparse_read_holes && + if (cct->_conf->osd_verify_sparse_read_holes && last < miter->first) { bufferlist t; uint64_t len = miter->first - last; @@ -2242,7 +2304,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) } // verify trailing hole? - if (g_conf->osd_verify_sparse_read_holes) { + if (cct->_conf->osd_verify_sparse_read_holes) { uint64_t end = MIN(op.extent.offset + op.extent.length, oi.size); if (last < end) { bufferlist t; @@ -2596,7 +2658,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) timeout = 0; } if (!timeout) - timeout = g_conf->osd_default_notify_timeout; + timeout = cct->_conf->osd_default_notify_timeout; notify_info_t n; n.timeout = timeout; @@ -2666,7 +2728,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) oi.truncate_size = op.extent.truncate_size; } } - result = check_offset_and_length(op.extent.offset, op.extent.length); + result = check_offset_and_length(op.extent.offset, op.extent.length, cct->_conf->osd_max_object_size); if (result < 0) break; t.write(coll, soid, op.extent.offset, op.extent.length, osd_op.indata); @@ -2686,7 +2748,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) result = -EINVAL; break; } - result = check_offset_and_length(op.extent.offset, op.extent.length); + result = check_offset_and_length(op.extent.offset, op.extent.length, cct->_conf->osd_max_object_size); if (result < 0) break; if (obs.exists) { @@ -2718,7 +2780,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) case CEPH_OSD_OP_ZERO: ++ctx->num_write; { // zero - result = check_offset_and_length(op.extent.offset, op.extent.length); + result = check_offset_and_length(op.extent.offset, op.extent.length, cct->_conf->osd_max_object_size); if (result < 0) break; assert(op.extent.length); @@ -2781,7 +2843,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) break; } - if (op.extent.offset > g_conf->osd_max_object_size) { + if (op.extent.offset > cct->_conf->osd_max_object_size) { result = -EFBIG; break; } @@ -2865,7 +2927,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) dout(10) << "watch: peer_addr=" << ctx->op->request->get_connection()->get_peer_addr() << dendl; - watch_info_t w(cookie, g_conf->osd_client_watch_timeout, + watch_info_t w(cookie, cct->_conf->osd_client_watch_timeout, ctx->op->request->get_connection()->get_peer_addr()); if (do_watch) { if (oi.watchers.count(make_pair(cookie, entity))) { @@ -2898,7 +2960,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) case CEPH_OSD_OP_SETXATTR: ++ctx->num_write; { - if (op.xattr.value_len > g_conf->osd_max_attr_size) { + if (op.xattr.value_len > cct->_conf->osd_max_attr_size) { result = -EFBIG; break; } @@ -2996,8 +3058,8 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) } } - if (g_conf->osd_tmapput_sets_uses_tmap) { - assert(g_conf->osd_auto_upgrade_tmap); + if (cct->_conf->osd_tmapput_sets_uses_tmap) { + assert(cct->_conf->osd_auto_upgrade_tmap); oi.uses_tmap = true; } @@ -3046,7 +3108,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) } set<string> out_set; - if (oi.uses_tmap && g_conf->osd_auto_upgrade_tmap) { + if (oi.uses_tmap && cct->_conf->osd_auto_upgrade_tmap) { dout(20) << "CEPH_OSD_OP_OMAPGETKEYS: " << " Reading " << oi.soid << " omap from tmap" << dendl; map<string, bufferlist> vals; @@ -3104,7 +3166,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) } map<string, bufferlist> out_set; - if (oi.uses_tmap && g_conf->osd_auto_upgrade_tmap) { + if (oi.uses_tmap && cct->_conf->osd_auto_upgrade_tmap) { dout(20) << "CEPH_OSD_OP_OMAPGETVALS: " << " Reading " << oi.soid << " omap from tmap" << dendl; map<string, bufferlist> vals; @@ -3155,7 +3217,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) case CEPH_OSD_OP_OMAPGETHEADER: ++ctx->num_read; { - if (oi.uses_tmap && g_conf->osd_auto_upgrade_tmap) { + if (oi.uses_tmap && cct->_conf->osd_auto_upgrade_tmap) { dout(20) << "CEPH_OSD_OP_OMAPGETHEADER: " << " Reading " << oi.soid << " omap from tmap" << dendl; map<string, bufferlist> vals; @@ -3186,7 +3248,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) goto fail; } map<string, bufferlist> out; - if (oi.uses_tmap && g_conf->osd_auto_upgrade_tmap) { + if (oi.uses_tmap && cct->_conf->osd_auto_upgrade_tmap) { dout(20) << "CEPH_OSD_OP_OMAPGET: " << " Reading " << oi.soid << " omap from tmap" << dendl; map<string, bufferlist> vals; @@ -3285,7 +3347,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) case CEPH_OSD_OP_OMAPSETVALS: ++ctx->num_write; { - if (oi.uses_tmap && g_conf->osd_auto_upgrade_tmap) { + if (oi.uses_tmap && cct->_conf->osd_auto_upgrade_tmap) { _copy_up_tmap(ctx); } if (!obs.exists) { @@ -3315,7 +3377,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) case CEPH_OSD_OP_OMAPSETHEADER: ++ctx->num_write; { - if (oi.uses_tmap && g_conf->osd_auto_upgrade_tmap) { + if (oi.uses_tmap && cct->_conf->osd_auto_upgrade_tmap) { _copy_up_tmap(ctx); } if (!obs.exists) { @@ -3335,7 +3397,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) result = -ENOENT; break; } - if (oi.uses_tmap && g_conf->osd_auto_upgrade_tmap) { + if (oi.uses_tmap && cct->_conf->osd_auto_upgrade_tmap) { _copy_up_tmap(ctx); } t.touch(coll, soid); @@ -3351,7 +3413,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) result = -ENOENT; break; } - if (oi.uses_tmap && g_conf->osd_auto_upgrade_tmap) { + if (oi.uses_tmap && cct->_conf->osd_auto_upgrade_tmap) { _copy_up_tmap(ctx); } t.touch(coll, soid); @@ -3438,7 +3500,11 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) } ::encode(out_omap, osd_op.outdata); - dout(20) << " cursor.is_complete=" << cursor.is_complete() << dendl; + dout(20) << " cursor.is_complete=" << cursor.is_complete() + << " " << out_attrs.size() << " attrs" + << " " << bl.length() << " bytes" + << " " << out_omap.size() << " keys" + << dendl; ::encode(cursor, osd_op.outdata); result = 0; } @@ -3459,44 +3525,20 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops) result = -EINVAL; goto fail; } - pg_t raw_pg; - get_osdmap()->object_locator_to_pg(src_name, src_oloc, raw_pg); - hobject_t src(src_name, src_oloc.key, src_snapid, - raw_pg.ps(), raw_pg.pool(), - src_oloc.nspace); if (!ctx->copy_op) { // start + pg_t raw_pg; + get_osdmap()->object_locator_to_pg(src_name, src_oloc, raw_pg); + hobject_t src(src_name, src_oloc.key, src_snapid, + raw_pg.ps(), raw_pg.pool(), + src_oloc.nspace); result = start_copy(ctx, src, src_oloc, src_version, &ctx->copy_op); if (result < 0) goto fail; result = -EINPROGRESS; } else { // finish - CopyOpRef cop = ctx->copy_op; - - if (!obs.exists) { - ctx->delta_stats.num_objects++; - obs.exists = true; - } else { - t.remove(coll, soid); - } - t.write(coll, soid, 0, cop->data.length(), cop->data); - for (map<string,bufferlist>::iterator p = cop->attrs.begin(); p != cop->attrs.end(); ++p) - t.setattr(coll, soid, string("_") + p->first, p->second); - t.omap_setkeys(coll, soid, cop->omap); - - interval_set<uint64_t> ch; - if (oi.size > 0) - ch.insert(0, oi.size); - ctx->modified_ranges.union_of(ch); - - if (cop->data.length() != oi.size) { - ctx->delta_stats.num_bytes -= oi.size; - oi.size = cop->data.length(); - ctx->delta_stats.num_bytes += oi.size; - } - ctx->delta_stats.num_wr++; - ctx->delta_stats.num_wr_kb += SHIFT_ROUND_UP(cop->data.length(), 10); + result = finish_copy(ctx); } } break; @@ -3774,7 +3816,7 @@ void ReplicatedPG::make_writeable(OpContext *ctx) ctx->log.push_back(pg_log_entry_t(pg_log_entry_t::CLONE, coid, ctx->at_version, ctx->obs->oi.version, ctx->obs->oi.user_version, - ctx->reqid, ctx->new_obs.oi.mtime)); + osd_reqid_t(), ctx->new_obs.oi.mtime)); ::encode(snaps, ctx->log.back().snaps); ctx->at_version.version++; @@ -3935,6 +3977,15 @@ coll_t ReplicatedPG::get_temp_coll(ObjectStore::Transaction *t) return temp_coll; } +hobject_t ReplicatedPG::generate_temp_object() +{ + ostringstream ss; + ss << "temp_" << info.pgid << "_" << get_role() << "_" << osd->monc->get_global_id() << "_" << (++temp_seq); + hobject_t hoid(object_t(ss.str()), "", CEPH_NOSNAP, 0, -1, ""); + dout(20) << __func__ << " " << hoid << dendl; + return hoid; +} + int ReplicatedPG::prepare_transaction(OpContext *ctx) { assert(!ctx->ops.empty()); @@ -4133,8 +4184,14 @@ void ReplicatedPG::_copy_some(OpContext *ctx, CopyOpRef cop) { dout(10) << __func__ << " " << ctx << " " << cop << dendl; ObjectOperation op; - op.assert_version(cop->version); - op.copy_get(&cop->cursor, g_conf->osd_copyfrom_max_chunk, + if (cop->version) { + op.assert_version(cop->version); + } else { + // we should learn the version after the first chunk, if we didn't know + // it already! + assert(cop->cursor.is_initial()); + } + op.copy_get(&cop->cursor, cct->_conf->osd_copyfrom_max_chunk, &cop->size, &cop->mtime, &cop->attrs, &cop->data, &cop->omap, &cop->rval); @@ -4143,10 +4200,11 @@ void ReplicatedPG::_copy_some(OpContext *ctx, CopyOpRef cop) get_last_peering_reset()); osd->objecter_lock.Lock(); tid_t tid = osd->objecter->read(cop->src.oid, cop->oloc, op, - cop->src.snap, NULL, 0, - new C_OnFinisher(fin, - &osd->objecter_finisher), - NULL); + cop->src.snap, NULL, 0, + new C_OnFinisher(fin, + &osd->objecter_finisher), + // discover the object version if we don't know it yet + cop->version ? NULL : &cop->version); fin->tid = tid; cop->objecter_tid = tid; osd->objecter_lock.Unlock(); @@ -4171,14 +4229,33 @@ void ReplicatedPG::process_copy_chunk(hobject_t oid, tid_t tid, int r) if (r < 0) { copy_ops.erase(ctx->obc->obs.oi.soid); --ctx->obc->copyfrom_readside; + kick_object_context_blocked(ctx->obc); reply_ctx(ctx, r); return; } assert(cop->rval >= 0); - // FIXME: this is accumulating the entire object in memory. - if (!cop->cursor.is_complete()) { + // write out what we have so far + vector<OSDOp> ops; + tid_t rep_tid = osd->get_tid(); + osd_reqid_t reqid(osd->get_cluster_msgr_name(), 0, rep_tid); + OpContext *tctx = new OpContext(OpRequestRef(), reqid, ops, &ctx->obc->obs, ctx->obc->ssc, this); + tctx->mtime = ceph_clock_now(g_ceph_context); + RepGather *repop = new_repop(tctx, ctx->obc, rep_tid); + + if (cop->temp_cursor.is_initial()) { + cop->temp_coll = get_temp_coll(&tctx->local_t); + cop->temp_oid = generate_temp_object(); + temp_contents.insert(cop->temp_oid); + repop->ctx->new_temp_oid = cop->temp_oid; + } + + _write_copy_chunk(cop, &tctx->op_t); + + issue_repop(repop, repop->ctx->mtime); + eval_repop(repop); + dout(10) << __func__ << " fetching more" << dendl; _copy_some(ctx, cop); return; @@ -4190,6 +4267,73 @@ void ReplicatedPG::process_copy_chunk(hobject_t oid, tid_t tid, int r) copy_ops.erase(ctx->obc->obs.oi.soid); --ctx->obc->copyfrom_readside; ctx->copy_op.reset(); + kick_object_context_blocked(ctx->obc); +} + +void ReplicatedPG::_write_copy_chunk(CopyOpRef cop, ObjectStore::Transaction *t) +{ + dout(20) << __func__ << " " << cop + << " " << cop->attrs.size() << " attrs" + << " " << cop->data.length() << " bytes" + << " " << cop->omap.size() << " keys" + << dendl; + if (!cop->temp_cursor.attr_complete) { + t->touch(cop->temp_coll, cop->temp_oid); + for (map<string,bufferlist>::iterator p = cop->attrs.begin(); p != cop->attrs.end(); ++p) + t->setattr(cop->temp_coll, cop->temp_oid, string("_") + p->first, p->second); + cop->attrs.clear(); + } + if (!cop->temp_cursor.data_complete) { + t->write(cop->temp_coll, cop->temp_oid, cop->temp_cursor.data_offset, cop->data.length(), cop->data); + cop->data.clear(); + } + if (!cop->temp_cursor.omap_complete) { + t->omap_setkeys(cop->temp_coll, cop->temp_oid, cop->omap); + cop->omap.clear(); + } + cop->temp_cursor = cop->cursor; +} + +int ReplicatedPG::finish_copy(OpContext *ctx) +{ + CopyOpRef cop = ctx->copy_op; + ObjectState& obs = ctx->new_obs; + ObjectStore::Transaction& t = ctx->op_t; + + if (!obs.exists) { + ctx->delta_stats.num_objects++; + obs.exists = true; + } else { + t.remove(coll, obs.oi.soid); + } + + if (cop->temp_cursor.is_initial()) { + // write directly to final object + cop->temp_coll = coll; + cop->temp_oid = obs.oi.soid; + _write_copy_chunk(cop, &t); + } else { + // finish writing to temp object, then move into place + _write_copy_chunk(cop, &t); + t.collection_move_rename(cop->temp_coll, cop->temp_oid, coll, obs.oi.soid); + temp_contents.erase(cop->temp_oid); + ctx->discard_temp_oid = cop->temp_oid; + } + + interval_set<uint64_t> ch; + if (obs.oi.size > 0) + ch.insert(0, obs.oi.size); + ctx->modified_ranges.union_of(ch); + + if (cop->cursor.data_offset != obs.oi.size) { + ctx->delta_stats.num_bytes -= obs.oi.size; + ctx->delta_stats.num_bytes += obs.oi.size; + obs.oi.size = cop->cursor.data_offset; + } + ctx->delta_stats.num_wr++; + ctx->delta_stats.num_wr_kb += SHIFT_ROUND_UP(obs.oi.size, 10); + + return 0; } void ReplicatedPG::cancel_copy(CopyOpRef cop) @@ -4209,6 +4353,8 @@ void ReplicatedPG::cancel_copy(CopyOpRef cop) --ctx->obc->copyfrom_readside; ctx->copy_op.reset(); + kick_object_context_blocked(ctx->obc); + delete ctx; } @@ -4314,9 +4460,11 @@ void ReplicatedPG::op_applied(RepGather *repop) repop->waitfor_disk.count(whoami) == 0); // commit before ondisk repop->waitfor_ack.erase(whoami); - assert(info.last_update >= repop->v); - assert(last_update_applied < repop->v); - last_update_applied = repop->v; + if (repop->v != eversion_t()) { + assert(info.last_update >= repop->v); + assert(last_update_applied < repop->v); + last_update_applied = repop->v; + } // chunky scrub if (scrubber.active && scrubber.is_chunky) { @@ -4363,9 +4511,10 @@ void ReplicatedPG::op_commit(RepGather *repop) // is no separate reply sent. repop->waitfor_ack.erase(whoami); - last_update_ondisk = repop->v; - - last_complete_ondisk = repop->pg_local_last_complete; + if (repop->v != eversion_t()) { + last_update_ondisk = repop->v; + last_complete_ondisk = repop->pg_local_last_complete; + } eval_repop(repop); } @@ -4487,7 +4636,7 @@ void ReplicatedPG::eval_repop(RepGather *repop) // _prior_ to being committed; it will not get set with // writeahead journaling, for instance. if (repop->ctx->readable_stamp == utime_t()) - repop->ctx->readable_stamp = ceph_clock_now(g_ceph_context); + repop->ctx->readable_stamp = ceph_clock_now(cct); } } @@ -4516,8 +4665,7 @@ void ReplicatedPG::eval_repop(RepGather *repop) } } -void ReplicatedPG::issue_repop(RepGather *repop, utime_t now, - eversion_t old_last_update, bool old_exists, uint64_t old_size, eversion_t old_version) +void ReplicatedPG::issue_repop(RepGather *repop, utime_t now) { OpContext *ctx = repop->ctx; const hobject_t& soid = ctx->obs->oi.soid; @@ -4555,34 +4703,31 @@ void ReplicatedPG::issue_repop(RepGather *repop, utime_t now, ((static_cast<MOSDOp *>(ctx->op->request))->get_flags() & CEPH_OSD_FLAG_PARALLELEXEC)) { // replicate original op for parallel execution on replica assert(0 == "broken implementation, do not use"); - wr->oloc = object_locator_t(repop->ctx->obs->oi.soid); - wr->ops = repop->ctx->ops; - wr->mtime = repop->ctx->mtime; - wr->old_exists = old_exists; - wr->old_size = old_size; - wr->old_version = old_version; - wr->snapset = repop->obc->ssc->snapset; - wr->snapc = repop->ctx->snapc; - wr->set_data(repop->ctx->op->request->get_data()); // _copy_ bufferlist - } else { - // ship resulting transaction, log entries, and pg_stats - if (peer == backfill_target && soid >= backfill_pos) { - dout(10) << "issue_repop shipping empty opt to osd." << peer << ", object beyond backfill_pos " - << backfill_pos << ", last_backfill is " << pinfo.last_backfill << dendl; - ObjectStore::Transaction t; - ::encode(t, wr->get_data()); - } else { - ::encode(repop->ctx->op_t, wr->get_data()); - } - ::encode(repop->ctx->log, wr->logbl); + } - if (backfill_target >= 0 && backfill_target == peer) - wr->pg_stats = pinfo.stats; // reflects backfill progress - else - wr->pg_stats = info.stats; + // ship resulting transaction, log entries, and pg_stats + if (peer == backfill_target && soid >= backfill_pos && + soid.pool == (int64_t)info.pgid.pool()) { // only skip normal (not temp pool=-1) objects + dout(10) << "issue_repop shipping empty opt to osd." << peer << ", object beyond backfill_pos " + << backfill_pos << ", last_backfill is " << pinfo.last_backfill << dendl; + ObjectStore::Transaction t; + ::encode(t, wr->get_data()); + } else { + ::encode(repop->ctx->op_t, wr->get_data()); } + + ::encode(repop->ctx->log, wr->logbl); + + if (backfill_target >= 0 && backfill_target == peer) + wr->pg_stats = pinfo.stats; // reflects backfill progress + else + wr->pg_stats = info.stats; wr->pg_trim_to = pg_trim_to; + + wr->new_temp_oid = repop->ctx->new_temp_oid; + wr->discard_temp_oid = repop->ctx->discard_temp_oid; + osd->send_message_osd_cluster(peer, wr, get_osdmap()->get_epoch()); // keep peer_info up to date @@ -4602,7 +4747,7 @@ ReplicatedPG::RepGather *ReplicatedPG::new_repop(OpContext *ctx, ObjectContextRe RepGather *repop = new RepGather(ctx, obc, rep_tid, info.last_complete); - repop->start = ceph_clock_now(g_ceph_context); + repop->start = ceph_clock_now(cct); repop_queue.push_back(&repop->queue_item); repop_map[repop->rep_tid] = repop; @@ -4801,7 +4946,7 @@ void ReplicatedPG::handle_watch_timeout(WatchRef watch) osd_reqid_t reqid(osd->get_cluster_msgr_name(), 0, rep_tid); OpContext *ctx = new OpContext(OpRequestRef(), reqid, ops, &obc->obs, obc->ssc, this); - ctx->mtime = ceph_clock_now(g_ceph_context); + ctx->mtime = ceph_clock_now(cct); ctx->at_version.epoch = get_osdmap()->get_epoch(); ctx->at_version.version = pg_log.get_head().version + 1; @@ -4818,12 +4963,7 @@ void ReplicatedPG::handle_watch_timeout(WatchRef watch) 0, osd_reqid_t(), ctx->mtime)); - eversion_t old_last_update = pg_log.get_head(); - bool old_exists = repop->obc->obs.exists; - uint64_t old_size = repop->obc->obs.oi.size; - eversion_t old_version = repop->obc->obs.oi.version; - - obc->obs.oi.prior_version = old_version; + obc->obs.oi.prior_version = repop->obc->obs.oi.version; obc->obs.oi.version = ctx->at_version; bufferlist bl; ::encode(obc->obs.oi, bl); @@ -4832,8 +4972,7 @@ void ReplicatedPG::handle_watch_timeout(WatchRef watch) append_log(repop->ctx->log, eversion_t(), repop->ctx->local_t); // obc ref swallowed by repop! - issue_repop(repop, repop->ctx->mtime, old_last_update, old_exists, - old_size, old_version); + issue_repop(repop, repop->ctx->mtime); eval_repop(repop); } @@ -5077,6 +5216,24 @@ void ReplicatedPG::add_object_context_to_pg_stat(ObjectContextRef obc, pg_stat_t pgstat->stats.cat_sum[oi.category].add(stat); } +void ReplicatedPG::kick_object_context_blocked(ObjectContextRef obc) +{ + const hobject_t& soid = obc->obs.oi.soid; + map<hobject_t, list<OpRequestRef> >::iterator p = waiting_for_blocked_object.find(soid); + if (p == waiting_for_blocked_object.end()) + return; + + if (obc->is_blocked()) { + dout(10) << __func__ << " " << soid << " still blocked" << dendl; + return; + } + + list<OpRequestRef>& ls = waiting_for_blocked_object[soid]; + dout(10) << __func__ << " " << soid << " requeuing " << ls.size() << " requests" << dendl; + requeue_ops(ls); + waiting_for_blocked_object.erase(soid); +} + SnapSetContext *ReplicatedPG::create_snapset_context(const object_t& oid) { Mutex::Locker l(snapset_contexts_lock); @@ -5186,6 +5343,16 @@ void ReplicatedPG::sub_op_modify(OpRequestRef op) bufferlist::iterator p = m->get_data().begin(); + if (m->new_temp_oid != hobject_t()) { + dout(20) << __func__ << " start tracking temp " << m->new_temp_oid << dendl; + temp_contents.insert(m->new_temp_oid); + get_temp_coll(&rm->localt); + } + if (m->discard_temp_oid != hobject_t()) { + dout(20) << __func__ << " stop tracking temp " << m->discard_temp_oid << dendl; + temp_contents.erase(m->discard_temp_oid); + } + ::decode(rm->opt, p); if (!(m->get_connection()->get_features() & CEPH_FEATURE_OSD_SNAPMAPPER)) rm->opt.set_tolerate_collection_add_enoent(); @@ -5257,9 +5424,11 @@ void ReplicatedPG::sub_op_modify_applied(RepModify *rm) osd->send_message_osd_cluster(rm->ackerosd, ack, get_osdmap()->get_epoch()); } - assert(info.last_update >= m->version); - assert(last_update_applied < m->version); - last_update_applied = m->version; + if (m->version != eversion_t()) { + assert(info.last_update >= m->version); + assert(last_update_applied < m->version); + last_update_applied = m->version; + } if (scrubber.active_rep_scrub) { if (last_update_applied == scrubber.active_rep_scrub->scrub_to) { osd->rep_scrub_wq.queue(scrubber.active_rep_scrub); @@ -5360,7 +5529,7 @@ void ReplicatedPG::calc_head_subsets(ObjectContextRef obc, SnapSet& snapset, con if (size) data_subset.insert(0, size); - if (!g_conf->osd_recover_clone_overlap) { + if (!cct->_conf->osd_recover_clone_overlap) { dout(10) << "calc_head_subsets " << head << " -- osd_recover_clone_overlap disabled" << dendl; return; } @@ -5387,7 +5556,7 @@ void ReplicatedPG::calc_head_subsets(ObjectContextRef obc, SnapSet& snapset, con } - if (cloning.num_intervals() > g_conf->osd_recover_clone_overlap_limit) { + if (cloning.num_intervals() > cct->_conf->osd_recover_clone_overlap_limit) { dout(10) << "skipping clone, too many holes" << dendl; clone_subsets.clear(); cloning.clear(); @@ -5414,7 +5583,7 @@ void ReplicatedPG::calc_clone_subsets(SnapSet& snapset, const hobject_t& soid, if (size) data_subset.insert(0, size); - if (!g_conf->osd_recover_clone_overlap) { + if (!cct->_conf->osd_recover_clone_overlap) { dout(10) << "calc_clone_subsets " << soid << " -- osd_recover_clone_overlap disabled" << dendl; return; } @@ -5463,7 +5632,7 @@ void ReplicatedPG::calc_clone_subsets(SnapSet& snapset, const hobject_t& soid, << " overlap " << next << dendl; } - if (cloning.num_intervals() > g_conf->osd_recover_clone_overlap_limit) { + if (cloning.num_intervals() > cct->_conf->osd_recover_clone_overlap_limit) { dout(10) << "skipping clone, too many holes" << dendl; clone_subsets.clear(); cloning.clear(); @@ -5754,7 +5923,7 @@ int ReplicatedPG::send_pull_legacy(int prio, int peer, subop->set_priority(prio); subop->ops = vector<OSDOp>(1); subop->ops[0].op.op = CEPH_OSD_OP_PULL; - subop->ops[0].op.extent.length = g_conf->osd_recovery_max_chunk; + subop->ops[0].op.extent.length = cct->_conf->osd_recovery_max_chunk; subop->recovery_info = recovery_info; subop->recovery_progress = progress; @@ -6111,16 +6280,16 @@ void ReplicatedPG::send_pushes(int prio, map<int, vector<PushOp> > &pushes) msg->set_priority(prio); for (; (j != i->second.end() && - cost < g_conf->osd_max_push_cost && - pushes < g_conf->osd_max_push_objects) ; + cost < cct->_conf->osd_max_push_cost && + pushes < cct->_conf->osd_max_push_objects) ; ++j) { dout(20) << __func__ << ": sending push " << *j << " to osd." << i->first << dendl; - cost += j->cost(g_ceph_context); + cost += j->cost(cct); pushes += 1; msg->pushes.push_back(*j); } - msg->compute_cost(g_ceph_context); + msg->compute_cost(cct); osd->send_message_osd_cluster(msg, con); } } @@ -6157,7 +6326,7 @@ void ReplicatedPG::send_pulls(int prio, map<int, vector<PullOp> > &pulls) msg->pgid = info.pgid; msg->map_epoch = get_osdmap()->get_epoch(); msg->pulls.swap(i->second); - msg->compute_cost(g_ceph_context); + msg->compute_cost(cct); osd->send_message_osd_cluster(msg, con); } } @@ -6212,7 +6381,7 @@ int ReplicatedPG::build_push_op(const ObjectRecoveryInfo &recovery_info, new_progress.first = false; } - uint64_t available = g_conf->osd_recovery_max_chunk; + uint64_t available = cct->_conf->osd_recovery_max_chunk; if (!progress.omap_complete) { ObjectMap::ObjectMapIterator iter = osd->store->get_omap_iterator(coll, @@ -6766,7 +6935,7 @@ void ReplicatedPG::mark_all_unfound_lost(int what) ObjectStore::Transaction *t = new ObjectStore::Transaction; C_PG_MarkUnfoundLost *c = new C_PG_MarkUnfoundLost(this); - utime_t mtime = ceph_clock_now(g_ceph_context); + utime_t mtime = ceph_clock_now(cct); info.last_update.epoch = get_osdmap()->get_epoch(); const pg_missing_t &missing = pg_log.get_missing(); map<hobject_t, pg_missing_t::item>::const_iterator m = missing.missing.begin(); @@ -7020,6 +7189,14 @@ void ReplicatedPG::on_change(ObjectStore::Transaction *t) p->second.clear(); finish_degraded_object(p->first); } + for (map<hobject_t,list<OpRequestRef> >::iterator p = waiting_for_blocked_object.begin(); + p != waiting_for_blocked_object.end(); + waiting_for_blocked_object.erase(p++)) { + if (is_primary()) + requeue_ops(p->second); + else + p->second.clear(); + } if (is_primary()) requeue_ops(waiting_for_all_missing); @@ -7241,6 +7418,13 @@ int ReplicatedPG::start_recovery_ops( return started; } + if (needs_recovery()) { + // this shouldn't happen! + // We already checked num_missing() so we must have missing replicas + osd->clog.error() << info.pgid << " recovery ending with missing replicas\n"; + return started; + } + if (state_test(PG_STATE_RECOVERING)) { state_clear(PG_STATE_RECOVERING); if (needs_backfill()) { @@ -7407,7 +7591,7 @@ int ReplicatedPG::recover_primary(int max, ThreadPool::TPHandle &handle) ++skipped; } else { int r = prepare_pull( - soid, need, g_conf->osd_recovery_op_priority, &pulls); + soid, need, cct->_conf->osd_recovery_op_priority, &pulls); switch (r) { case PULL_YES: ++started; @@ -7430,7 +7614,7 @@ int ReplicatedPG::recover_primary(int max, ThreadPool::TPHandle &handle) pg_log.set_last_requested(v); } - send_pulls(g_conf->osd_recovery_op_priority, pulls); + send_pulls(cct->_conf->osd_recovery_op_priority, pulls); return started; } @@ -7530,12 +7714,12 @@ int ReplicatedPG::recover_replicas(int max, ThreadPool::TPHandle &handle) dout(10) << __func__ << ": recover_object_replicas(" << soid << ")" << dendl; map<hobject_t,pg_missing_t::item>::const_iterator r = m.missing.find(soid); started += prep_object_replica_pushes(soid, r->second.need, - g_conf->osd_recovery_op_priority, + cct->_conf->osd_recovery_op_priority, &pushes); } } - send_pushes(g_conf->osd_recovery_op_priority, pushes); + send_pushes(cct->_conf->osd_recovery_op_priority, pushes); return started; } @@ -7707,7 +7891,7 @@ int ReplicatedPG::recover_backfill( prep_backfill_object_push( i->first, i->second.first, i->second.second, backfill_target, &pushes); } - send_pushes(g_conf->osd_recovery_op_priority, pushes); + send_pushes(cct->_conf->osd_recovery_op_priority, pushes); release_waiting_for_backfill_pos(); dout(5) << "backfill_pos is " << backfill_pos << " and pinfo.last_backfill is " @@ -7764,7 +7948,7 @@ void ReplicatedPG::prep_backfill_object_push( ObjectContextRef obc = get_object_context(oid, false); obc->ondisk_read_lock(); (*pushes)[peer].push_back(PushOp()); - prep_push_to_replica(obc, oid, peer, g_conf->osd_recovery_op_priority, + prep_push_to_replica(obc, oid, peer, cct->_conf->osd_recovery_op_priority, &((*pushes)[peer].back())); obc->ondisk_read_unlock(); } @@ -7816,7 +8000,7 @@ void ReplicatedPG::check_local() assert(info.last_update >= pg_log.get_tail()); // otherwise we need some help! - if (!g_conf->osd_debug_verify_stray_on_activate) + if (!cct->_conf->osd_debug_verify_stray_on_activate) return; // just scan the log. @@ -8066,9 +8250,10 @@ void ReplicatedPG::SnapTrimmer::log_exit(const char *state_name, utime_t enter_t << "SnapTrimmer state<" << get_state_name() << ">: ") /* NotTrimming */ -ReplicatedPG::NotTrimming::NotTrimming(my_context ctx) : my_base(ctx) +ReplicatedPG::NotTrimming::NotTrimming(my_context ctx) + : my_base(ctx), + NamedState(context< SnapTrimmer >().pg->cct, "NotTrimming") { - state_name = "NotTrimming"; context< SnapTrimmer >().requeue = false; context< SnapTrimmer >().log_enter(state_name); } @@ -8107,9 +8292,10 @@ boost::statechart::result ReplicatedPG::NotTrimming::react(const SnapTrim&) } /* TrimmingObjects */ -ReplicatedPG::TrimmingObjects::TrimmingObjects(my_context ctx) : my_base(ctx) +ReplicatedPG::TrimmingObjects::TrimmingObjects(my_context ctx) + : my_base(ctx), + NamedState(context< SnapTrimmer >().pg->cct, "Trimming/TrimmingObjects") { - state_name = "Trimming/TrimmingObjects"; context< SnapTrimmer >().log_enter(state_name); } @@ -8142,24 +8328,20 @@ boost::statechart::result ReplicatedPG::TrimmingObjects::react(const SnapTrim&) dout(10) << "TrimmingObjects react trimming " << pos << dendl; RepGather *repop = pg->trim_object(pos); assert(repop); - repop->queue_snap_trimmer = true; - eversion_t old_last_update = pg->pg_log.get_head(); - bool old_exists = repop->obc->obs.exists; - uint64_t old_size = repop->obc->obs.oi.size; - eversion_t old_version = repop->obc->obs.oi.version; pg->append_log(repop->ctx->log, eversion_t(), repop->ctx->local_t); - pg->issue_repop(repop, repop->ctx->mtime, old_last_update, old_exists, old_size, old_version); + pg->issue_repop(repop, repop->ctx->mtime); pg->eval_repop(repop); repops.insert(repop); return discard_event(); } /* WaitingOnReplicasObjects */ -ReplicatedPG::WaitingOnReplicas::WaitingOnReplicas(my_context ctx) : my_base(ctx) +ReplicatedPG::WaitingOnReplicas::WaitingOnReplicas(my_context ctx) + : my_base(ctx), + NamedState(context< SnapTrimmer >().pg->cct, "Trimming/WaitingOnReplicas") { - state_name = "Trimming/WaitingOnReplicas"; context< SnapTrimmer >().log_enter(state_name); context< SnapTrimmer >().requeue = false; } diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h index 254b5842ffc..e880bdecade 100644 --- a/src/osd/ReplicatedPG.h +++ b/src/osd/ReplicatedPG.h @@ -109,6 +109,10 @@ public: map<string,bufferlist> omap; int rval; + coll_t temp_coll; + hobject_t temp_oid; + object_copy_cursor_t temp_cursor; + CopyOp(OpContext *c, hobject_t s, object_locator_t l, version_t v) : ctx(c), src(s), oloc(l), version(v), objecter_tid(0), @@ -180,6 +184,8 @@ public: CopyOpRef copy_op; + hobject_t new_temp_oid, discard_temp_oid; ///< temp objects we should start/stop tracking + OpContext(const OpContext& other); const OpContext& operator=(const OpContext& other); @@ -279,8 +285,7 @@ protected: void op_applied(RepGather *repop); void op_commit(RepGather *repop); void eval_repop(RepGather*); - void issue_repop(RepGather *repop, utime_t now, - eversion_t old_last_update, bool old_exists, uint64_t old_size, eversion_t old_version); + void issue_repop(RepGather *repop, utime_t now); RepGather *new_repop(OpContext *ctx, ObjectContextRef obc, tid_t rep_tid); void remove_repop(RepGather *repop); void repop_ack(RepGather *repop, @@ -419,9 +424,6 @@ protected: }; map<hobject_t, PullInfo> pulling; - // Track contents of temp collection, clear on reset - set<hobject_t> temp_contents; - ObjectRecoveryInfo recalc_subsets(const ObjectRecoveryInfo& recovery_info); static void trim_pushed_data(const interval_set<uint64_t> ©_subset, const interval_set<uint64_t> &intervals_received, @@ -619,7 +621,10 @@ protected: void write_update_size_and_usage(object_stat_sum_t& stats, object_info_t& oi, SnapSet& ss, interval_set<uint64_t>& modified, uint64_t offset, uint64_t length, bool count_bytes); - void add_interval_usage(interval_set<uint64_t>& s, object_stat_sum_t& st); + void add_interval_usage(interval_set<uint64_t>& s, object_stat_sum_t& st); + + inline bool maybe_handle_cache(OpRequestRef op, ObjectContextRef obc, int r); + void do_cache_redirect(OpRequestRef op, ObjectContextRef obc); int prepare_transaction(OpContext *ctx); @@ -790,7 +795,9 @@ protected: int start_copy(OpContext *ctx, hobject_t src, object_locator_t oloc, version_t version, CopyOpRef *pcop); void process_copy_chunk(hobject_t oid, tid_t tid, int r); + void _write_copy_chunk(CopyOpRef cop, ObjectStore::Transaction *t); void _copy_some(OpContext *ctx, CopyOpRef cop); + int finish_copy(OpContext *ctx); void cancel_copy(CopyOpRef cop); void requeue_cancel_copy_ops(bool requeue=true); @@ -852,7 +859,10 @@ public: private: bool temp_created; coll_t temp_coll; + set<hobject_t> temp_contents; ///< contents of temp collection, clear on reset + uint64_t temp_seq; ///< last id for naming temp objects coll_t get_temp_coll(ObjectStore::Transaction *t); + hobject_t generate_temp_object(); ///< generate a new temp object name public: bool have_temp_coll(); coll_t get_temp_coll() { @@ -929,6 +939,9 @@ public: bool is_degraded_object(const hobject_t& oid); void wait_for_degraded_object(const hobject_t& oid, OpRequestRef op); + void wait_for_blocked_object(const hobject_t& soid, OpRequestRef op); + void kick_object_context_blocked(ObjectContextRef obc); + void mark_all_unfound_lost(int what); eversion_t pick_newest_available(const hobject_t& oid); ObjectContextRef mark_object_lost(ObjectStore::Transaction *t, diff --git a/src/osd/SnapMapper.h b/src/osd/SnapMapper.h index 560cc43497f..f0d0baa2190 100644 --- a/src/osd/SnapMapper.h +++ b/src/osd/SnapMapper.h @@ -21,7 +21,7 @@ #include <string.h> #include "common/map_cacher.hpp" -#include "os/hobject.h" +#include "common/hobject.h" #include "include/buffer.h" #include "include/encoding.h" #include "include/object.h" diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc index 3451d520ff2..aa20dc592fa 100644 --- a/src/osd/osd_types.cc +++ b/src/osd/osd_types.cc @@ -101,6 +101,41 @@ void object_locator_t::generate_test_instances(list<object_locator_t*>& o) o.push_back(new object_locator_t(12, "n1", "key2")); } +// -- request_redirect_t -- +void request_redirect_t::encode(bufferlist& bl) const +{ + ENCODE_START(1, 1, bl); + ::encode(redirect_locator, bl); + ::encode(redirect_object, bl); + ::encode(osd_instructions, bl); + ENCODE_FINISH(bl); +} + +void request_redirect_t::decode(bufferlist::iterator& bl) +{ + DECODE_START(1, bl); + ::decode(redirect_locator, bl); + ::decode(redirect_object, bl); + ::decode(osd_instructions, bl); + DECODE_FINISH(bl); +} + +void request_redirect_t::dump(Formatter *f) const +{ + f->dump_string("object", redirect_object); + f->open_object_section("locator"); + redirect_locator.dump(f); + f->close_section(); // locator +} + +void request_redirect_t::generate_test_instances(list<request_redirect_t*>& o) +{ + object_locator_t loc(1, "redir_obj"); + o.push_back(new request_redirect_t()); + o.push_back(new request_redirect_t(loc, 0)); + o.push_back(new request_redirect_t(loc, "redir_obj")); + o.push_back(new request_redirect_t(loc)); +} // -- pow2_hist_t -- void pow2_hist_t::dump(Formatter *f) const @@ -358,6 +393,8 @@ ostream& operator<<(ostream& out, const pg_t &pg) // -- coll_t -- +const coll_t coll_t::META_COLL("meta"); + bool coll_t::is_temp(pg_t& pgid) const { const char *cstr(str.c_str()); @@ -649,6 +686,14 @@ void pg_pool_t::dump(Formatter *f) const f->dump_int("read_tier", read_tier); f->dump_int("write_tier", write_tier); f->dump_string("cache_mode", get_cache_mode_name()); + f->open_array_section("properties"); + for (map<string,string>::const_iterator i = properties.begin(); + i != properties.end(); + ++i) { + string name = i->first; + f->dump_string(name.c_str(), i->second); + } + f->close_section(); } @@ -853,7 +898,7 @@ void pg_pool_t::encode(bufferlist& bl, uint64_t features) const return; } - ENCODE_START(9, 5, bl); + ENCODE_START(10, 5, bl); ::encode(type, bl); ::encode(size, bl); ::encode(crush_ruleset, bl); @@ -880,6 +925,7 @@ void pg_pool_t::encode(bufferlist& bl, uint64_t features) const ::encode(c, bl); ::encode(read_tier, bl); ::encode(write_tier, bl); + ::encode(properties, bl); ENCODE_FINISH(bl); } @@ -947,6 +993,9 @@ void pg_pool_t::decode(bufferlist::iterator& bl) ::decode(read_tier, bl); ::decode(write_tier, bl); } + if (struct_v >= 10) { + ::decode(properties, bl); + } DECODE_FINISH(bl); calc_pg_masks(); } @@ -988,6 +1037,8 @@ void pg_pool_t::generate_test_instances(list<pg_pool_t*>& o) a.cache_mode = CACHEMODE_WRITEBACK; a.read_tier = 1; a.write_tier = 1; + a.properties["p-1"] = "v-1"; + a.properties["empty"] = string(); o.push_back(new pg_pool_t(a)); } diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index 312eb81e3fd..091b2b95e8f 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -26,7 +26,7 @@ #include "include/interval_set.h" #include "common/snap_types.h" #include "common/Formatter.h" -#include "os/hobject.h" +#include "common/hobject.h" #include "Watch.h" #define CEPH_OSD_ONDISK_MAGIC "ceph osd volume v026" @@ -129,6 +129,10 @@ struct object_locator_t { nspace = ""; } + bool empty() const { + return pool == -1; + } + void encode(bufferlist& bl) const; void decode(bufferlist::iterator& p); void dump(Formatter *f) const; @@ -153,6 +157,47 @@ inline ostream& operator<<(ostream& out, const object_locator_t& loc) return out; } +struct request_redirect_t { +private: + object_locator_t redirect_locator; ///< this is authoritative + string redirect_object; ///< If non-empty, the request goes to this object name + bufferlist osd_instructions; ///< a bufferlist for the OSDs, passed but not interpreted by clients + + friend ostream& operator<<(ostream& out, const request_redirect_t& redir); +public: + + request_redirect_t() {} + explicit request_redirect_t(const object_locator_t& orig, int64_t rpool) : + redirect_locator(orig) { redirect_locator.pool = rpool; } + explicit request_redirect_t(const object_locator_t& rloc) : + redirect_locator(rloc) {} + explicit request_redirect_t(const object_locator_t& orig, + const string& robj) : + redirect_locator(orig), redirect_object(robj) {} + + void set_instructions(const bufferlist& bl) { osd_instructions = bl; } + const bufferlist& get_instructions() { return osd_instructions; } + + bool empty() const { return redirect_locator.empty() && + redirect_object.empty(); } + + void combine_with_locator(object_locator_t& orig, string& obj) const { + orig = redirect_locator; + if (!redirect_object.empty()) + obj = redirect_object; + } + + void encode(bufferlist& bl) const; + void decode(bufferlist::iterator& bl); + void dump(Formatter *f) const; + static void generate_test_instances(list<request_redirect_t*>& o); +}; +WRITE_CLASS_ENCODER(request_redirect_t) + +inline ostream& operator<<(ostream& out, const request_redirect_t& redir) { + out << "object " << redir.redirect_object << ", locator{" << redir.redirect_locator << "}"; + return out; +} // Internal OSD op flags - set by the OSD based on the op types enum { @@ -778,6 +823,7 @@ private: public: + map<string,string> properties; /// interpreted according to the pool type epoch_t last_change; /// most recent epoch changed, exclusing snapshot changes snapid_t snap_seq; /// seq for per-pool snapshot epoch_t snap_epoch; /// osdmap epoch of last snap @@ -2151,6 +2197,11 @@ public: if (destructor_callback) destructor_callback->complete(0); } + + bool is_blocked() const { + return copyfrom_readside > 0; + } + // do simple synchronous mutual exclusion, for now. now waitqueues or anything fancy. void ondisk_write_lock() { lock.Lock(); diff --git a/src/osdc/Makefile.am b/src/osdc/Makefile.am new file mode 100644 index 00000000000..3a8a2165aaa --- /dev/null +++ b/src/osdc/Makefile.am @@ -0,0 +1,17 @@ +libosdc_la_SOURCES = \ + osdc/Objecter.cc \ + osdc/ObjectCacher.cc \ + osdc/Filer.cc \ + osdc/Striper.cc \ + osdc/Journaler.cc +noinst_LTLIBRARIES += libosdc.la + +noinst_HEADERS += \ + osdc/Blinker.h \ + osdc/Filer.h \ + osdc/Journaler.h \ + osdc/ObjectCacher.h \ + osdc/Objecter.h \ + osdc/Striper.h \ + osdc/WritebackHandler.h + diff --git a/src/osdc/ObjectCacher.cc b/src/osdc/ObjectCacher.cc index 01eeccc03be..590b5d473a8 100644 --- a/src/osdc/ObjectCacher.cc +++ b/src/osdc/ObjectCacher.cc @@ -899,11 +899,10 @@ void ObjectCacher::bh_write_commit(int64_t poolid, sobject_t oid, loff_t start, ob->last_commit_tid = tid; // waiters? + list<Context*> ls; if (ob->waitfor_commit.count(tid)) { - list<Context*> ls; ls.splice(ls.begin(), ob->waitfor_commit[tid]); ob->waitfor_commit.erase(tid); - finish_contexts(cct, ls, r); } // is the entire object set now clean and fully committed? @@ -915,6 +914,9 @@ void ObjectCacher::bh_write_commit(int64_t poolid, sobject_t oid, loff_t start, oset->dirty_or_tx == 0) { // nothing dirty/tx flush_set_callback(flush_set_callback_arg, oset); } + + if (!ls.empty()) + finish_contexts(cct, ls, r); } } diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc index 188c28d5e56..6c0486ce801 100644 --- a/src/osdc/Objecter.cc +++ b/src/osdc/Objecter.cc @@ -297,7 +297,7 @@ void Objecter::send_linger(LingerOp *info) if (ops.count(info->register_tid)) { Op *o = ops[info->register_tid]; op_cancel_map_check(o); - cancel_op(o); + cancel_linger_op(o); } info->register_tid = _op_submit(o); } else { @@ -659,7 +659,7 @@ void Objecter::handle_osd_map(MOSDMap *m) send_op(op); } } else { - cancel_op(op); + cancel_linger_op(op); } } for (list<LingerOp*>::iterator p = need_resend_linger.begin(); p != need_resend_linger.end(); ++p) { @@ -1003,7 +1003,7 @@ void Objecter::kick_requests(OSDSession *session) if (op->should_resend) { resend[op->tid] = op; } else { - cancel_op(op); + cancel_linger_op(op); } } while (!resend.empty()) { @@ -1243,7 +1243,7 @@ tid_t Objecter::_op_submit(Op *op) } // send? - ldout(cct, 10) << "op_submit oid " << op->oid + ldout(cct, 10) << "op_submit oid " << op->base_oid << " " << op->base_oloc << " " << op->target_oloc << " " << op->ops << " tid " << op->tid << " osd." << (op->session ? op->session->osd : -1) @@ -1328,22 +1328,33 @@ int Objecter::recalc_op_target(Op *op) bool is_read = op->flags & CEPH_OSD_FLAG_READ; bool is_write = op->flags & CEPH_OSD_FLAG_WRITE; - op->target_oloc = op->base_oloc; - const pg_pool_t *pi = osdmap->get_pg_pool(op->base_oloc.pool); - if (pi) { - if (is_read && pi->has_read_tier()) - op->target_oloc.pool = pi->read_tier; - if (is_write && pi->has_write_tier()) - op->target_oloc.pool = pi->write_tier; + bool need_check_tiering = false; + if (op->target_oid.name.empty()) { + op->target_oid = op->base_oid; + need_check_tiering = true; + } + if (op->target_oloc.empty()) { + op->target_oloc = op->base_oloc; + need_check_tiering = true; + } + + if (need_check_tiering) { + const pg_pool_t *pi = osdmap->get_pg_pool(op->base_oloc.pool); + if (pi) { + if (is_read && pi->has_read_tier()) + op->target_oloc.pool = pi->read_tier; + if (is_write && pi->has_write_tier()) + op->target_oloc.pool = pi->write_tier; + } } if (op->precalc_pgid) { - assert(op->oid.name.empty()); // make sure this is a listing op + assert(op->base_oid.name.empty()); // make sure this is a listing op ldout(cct, 10) << "recalc_op_target have " << pgid << " pool " << osdmap->have_pg_pool(pgid.pool()) << dendl; if (!osdmap->have_pg_pool(pgid.pool())) return RECALC_OP_TARGET_POOL_DNE; } else { - int ret = osdmap->object_locator_to_pg(op->oid, op->target_oloc, pgid); + int ret = osdmap->object_locator_to_pg(op->target_oid, op->target_oloc, pgid); if (ret == -ENOENT) return RECALC_OP_TARGET_POOL_DNE; } @@ -1429,12 +1440,10 @@ bool Objecter::recalc_linger_op_target(LingerOp *linger_op) return RECALC_OP_TARGET_NO_ACTION; } -void Objecter::cancel_op(Op *op) +void Objecter::cancel_linger_op(Op *op) { ldout(cct, 15) << "cancel_op " << op->tid << dendl; - // currently this only works for linger registrations, since we just - // throw out the callbacks. assert(!op->should_resend); delete op->onack; delete op->oncommit; @@ -1485,7 +1494,8 @@ void Objecter::send_op(Op *op) op->stamp = ceph_clock_now(cct); MOSDOp *m = new MOSDOp(client_inc, op->tid, - op->oid, op->target_oloc, op->pgid, osdmap->get_epoch(), + op->target_oid, op->target_oloc, op->pgid, + osdmap->get_epoch(), flags); m->set_snapid(op->snapid); @@ -1546,6 +1556,15 @@ void Objecter::throttle_op(Op *op, int op_budget) } } +void Objecter::unregister_op(Op *op) +{ + if (op->onack) + num_unacked--; + if (op->oncommit) + num_uncommitted--; + ops.erase(op->tid); +} + /* This function DOES put the passed message before returning */ void Objecter::handle_osd_op_reply(MOSDOpReply *m) { @@ -1592,12 +1611,18 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m) int rc = m->get_result(); + if (m->is_redirect_reply()) { + ldout(cct, 5) << " got redirect reply; redirecting" << dendl; + unregister_op(op); + m->get_redirect().combine_with_locator(op->target_oloc, op->target_oid.name); + op_submit(op); + m->put(); + return; + } + if (rc == -EAGAIN) { ldout(cct, 7) << " got -EAGAIN, resubmitting" << dendl; - if (op->onack) - num_unacked--; - if (op->oncommit) - num_uncommitted--; + unregister_op(op); op_submit(op); m->put(); return; @@ -2219,7 +2244,7 @@ void Objecter::dump_active() for (map<tid_t,Op*>::iterator p = ops.begin(); p != ops.end(); ++p) { Op *op = p->second; ldout(cct, 20) << op->tid << "\t" << op->pgid << "\tosd." << (op->session ? op->session->osd : -1) - << "\t" << op->oid << "\t" << op->ops << dendl; + << "\t" << op->base_oid << "\t" << op->ops << dendl; } } @@ -2250,7 +2275,7 @@ void Objecter::dump_ops(Formatter *fmt) const fmt->dump_int("osd", op->session ? op->session->osd : -1); fmt->dump_stream("last_sent") << op->stamp; fmt->dump_int("attempts", op->attempts); - fmt->dump_stream("object_id") << op->oid; + fmt->dump_stream("object_id") << op->base_oid; fmt->dump_stream("object_locator") << op->base_oloc; fmt->dump_stream("target_object_locator") << op->target_oloc; fmt->dump_stream("snapid") << op->snapid; diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h index 154ee410fde..880023ab37b 100644 --- a/src/osdc/Objecter.h +++ b/src/osdc/Objecter.h @@ -828,8 +828,9 @@ public: xlist<Op*>::item session_item; int incarnation; - object_t oid; + object_t base_oid; object_locator_t base_oloc; + object_t target_oid; object_locator_t target_oloc; pg_t pgid; @@ -874,7 +875,7 @@ public: Op(const object_t& o, const object_locator_t& ol, vector<OSDOp>& op, int f, Context *ac, Context *co, version_t *ov) : session(NULL), session_item(this), incarnation(0), - oid(o), base_oloc(ol), + base_oid(o), base_oloc(ol), used_replica(false), con(NULL), snapid(CEPH_NOSNAP), outbl(NULL), @@ -1186,7 +1187,7 @@ public: map<epoch_t,list< pair<Context*, int> > > waiting_for_map; void send_op(Op *op); - void cancel_op(Op *op); + void cancel_linger_op(Op *op); void finish_op(Op *op); bool is_pg_changed(vector<int>& a, vector<int>& b, bool any_change=false); enum recalc_op_target_result { @@ -1308,6 +1309,7 @@ private: // low-level tid_t op_submit(Op *op); tid_t _op_submit(Op *op); + inline void unregister_op(Op *op); // public interface public: diff --git a/src/perfglue/Makefile.am b/src/perfglue/Makefile.am new file mode 100644 index 00000000000..f2b8d5030e6 --- /dev/null +++ b/src/perfglue/Makefile.am @@ -0,0 +1,23 @@ +libperfglue_la_SOURCES = + +if WITH_TCMALLOC +libperfglue_la_SOURCES += perfglue/heap_profiler.cc +libperfglue_la_LIBADD = -ltcmalloc +AM_CFLAGS += -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free +AM_CXXFLAGS += -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free +else +libperfglue_la_SOURCES += perfglue/disabled_heap_profiler.cc +endif # WITH_TCMALLOC + +if WITH_PROFILER +libperfglue_la_SOURCES += perfglue/cpu_profiler.cc +else +libperfglue_la_SOURCES += perfglue/disabled_stubs.cc +endif # WITH_PROFILER + +noinst_LTLIBRARIES += libperfglue.la + +noinst_HEADERS += \ + perfglue/cpu_profiler.h \ + perfglue/heap_profiler.h + diff --git a/src/rbd.cc b/src/rbd.cc index 7d5d46611ec..eea9733c4b9 100644 --- a/src/rbd.cc +++ b/src/rbd.cc @@ -8,6 +8,7 @@ * LGPL2. See file COPYING. * */ +#include "include/int_types.h" #include "mon/MonClient.h" #include "mon/MonMap.h" @@ -33,7 +34,6 @@ #include <boost/scoped_ptr.hpp> #include <dirent.h> #include <errno.h> -#include <inttypes.h> #include <iostream> #include <memory> #include <sstream> @@ -2484,7 +2484,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \ if (r < 0) { cerr << "rbd: error opening pool " << poolname << ": " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } } @@ -2511,7 +2511,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \ if (r < 0) { cerr << "rbd: error opening image " << imgname << ": " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } } @@ -2526,7 +2526,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \ if (r < 0) { cerr << "rbd: error setting snapshot context: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } } @@ -2535,14 +2535,14 @@ if (!set_conf_param(v, p1, p2, p3)) { \ if (r < 0) { cerr << "rbd: error opening pool " << dest_poolname << ": " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } } if (opt_cmd == OPT_CREATE || opt_cmd == OPT_RESIZE) { if (!size_set) { cerr << "rbd: must specify --size <MB>" << std::endl; - return EXIT_FAILURE; + return EINVAL; } } @@ -2558,7 +2558,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \ default: cerr << "rbd: list: " << cpp_strerror(-r) << std::endl; } - return EXIT_FAILURE; + return -r; } break; @@ -2566,19 +2566,19 @@ if (!set_conf_param(v, p1, p2, p3)) { \ if (order && (order < 12 || order > 25)) { cerr << "rbd: order must be between 12 (4 KB) and 25 (32 MB)" << std::endl; - return EXIT_FAILURE; + return EINVAL; } if ((stripe_unit && !stripe_count) || (!stripe_unit && stripe_count)) { cerr << "must specify both (or neither) of stripe-unit and stripe-count" << std::endl; usage(); - return EXIT_FAILURE; + return EINVAL; } r = do_create(rbd, io_ctx, imgname, size, &order, format, features, stripe_unit, stripe_count); if (r < 0) { cerr << "rbd: create error: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; @@ -2586,14 +2586,14 @@ if (!set_conf_param(v, p1, p2, p3)) { \ if (order && (order < 12 || order > 25)) { cerr << "rbd: order must be between 12 (4 KB) and 25 (32 MB)" << std::endl; - return EXIT_FAILURE; + return EINVAL; } r = do_clone(rbd, io_ctx, imgname, snapname, dest_io_ctx, destname, features, &order); if (r < 0) { cerr << "rbd: clone error: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; @@ -2601,7 +2601,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \ r = do_flatten(image); if (r < 0) { cerr << "rbd: flatten error: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; @@ -2609,7 +2609,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \ r = do_rename(rbd, io_ctx, imgname, destname); if (r < 0) { cerr << "rbd: rename error: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; @@ -2617,7 +2617,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \ r = do_show_info(imgname, image, snapname, formatter.get()); if (r < 0) { cerr << "rbd: info: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; @@ -2647,18 +2647,18 @@ if (!set_conf_param(v, p1, p2, p3)) { \ r = image.stat(info, sizeof(info)); if (r < 0) { cerr << "rbd: resize error: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } if (info.size > size && !resize_allow_shrink) { cerr << "rbd: shrinking an image is only allowed with the --allow-shrink flag" << std::endl; - return EXIT_FAILURE; + return EINVAL; } r = do_resize(image, size); if (r < 0) { cerr << "rbd: resize error: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; @@ -2671,87 +2671,87 @@ if (!set_conf_param(v, p1, p2, p3)) { \ if (r < 0) { cerr << "rbd: failed to list snapshots: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; case OPT_SNAP_CREATE: if (!imgname || !snapname) { cerr << "rbd: snap create requires image and snapname" << std::endl; - return EXIT_FAILURE; + return EINVAL; } r = do_add_snap(image, snapname); if (r < 0) { cerr << "rbd: failed to create snapshot: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; case OPT_SNAP_ROLLBACK: if (!imgname) { cerr << "rbd: snap rollback requires image name" << std::endl; - return EXIT_FAILURE; + return EINVAL; } r = do_rollback_snap(image, snapname); if (r < 0) { cerr << "rbd: rollback failed: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; case OPT_SNAP_REMOVE: if (!imgname) { cerr << "rbd: snap remove requires image name" << std::endl; - return EXIT_FAILURE; + return EINVAL; } r = do_remove_snap(image, snapname); - if (r == -EBUSY) { - cerr << "rbd: snapshot '" << snapname << "' is protected from removal." - << std::endl; - return EXIT_FAILURE; - } if (r < 0) { - cerr << "rbd: failed to remove snapshot: " << cpp_strerror(-r) - << std::endl; - return EXIT_FAILURE; + if (r == -EBUSY) { + cerr << "rbd: snapshot '" << snapname << "' is protected from removal." + << std::endl; + } else { + cerr << "rbd: failed to remove snapshot: " << cpp_strerror(-r) + << std::endl; + } + return -r; } break; case OPT_SNAP_PURGE: if (!imgname) { cerr << "rbd: snap purge requires image name" << std::endl; - return EXIT_FAILURE; + return EINVAL; } r = do_purge_snaps(image); if (r < 0) { cerr << "rbd: removing snaps failed: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; case OPT_SNAP_PROTECT: if (!imgname) { cerr << "rbd: snap protect requires image name" << std::endl; - return EXIT_FAILURE; + return EINVAL; } r = do_protect_snap(image, snapname); if (r < 0) { cerr << "rbd: protecting snap failed: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; case OPT_SNAP_UNPROTECT: if (!imgname) { cerr << "rbd: snap unprotect requires image name" << std::endl; - return EXIT_FAILURE; + return EINVAL; } r = do_unprotect_snap(image, snapname); if (r < 0) { cerr << "rbd: unprotecting snap failed: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; @@ -2759,19 +2759,19 @@ if (!set_conf_param(v, p1, p2, p3)) { \ r = do_list_children(image, formatter.get()); if (r < 0) { cerr << "rbd: listing children failed: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; case OPT_EXPORT: if (!path) { cerr << "rbd: export requires pathname" << std::endl; - return EXIT_FAILURE; + return EINVAL; } r = do_export(image, path); if (r < 0) { cerr << "rbd: export error: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; @@ -2779,32 +2779,32 @@ if (!set_conf_param(v, p1, p2, p3)) { \ r = do_diff(image, fromsnapname, formatter.get()); if (r < 0) { cerr << "rbd: diff error: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; case OPT_EXPORT_DIFF: if (!path) { cerr << "rbd: export-diff requires pathname" << std::endl; - return EXIT_FAILURE; + return EINVAL; } r = do_export_diff(image, fromsnapname, snapname, path); if (r < 0) { cerr << "rbd: export-diff error: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; case OPT_IMPORT: if (!path) { cerr << "rbd: import requires pathname" << std::endl; - return EXIT_FAILURE; + return EINVAL; } r = do_import(rbd, dest_io_ctx, destname, &order, path, format, features, size); if (r < 0) { cerr << "rbd: import failed: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; @@ -2813,7 +2813,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \ r = do_import_diff(image, path); if (r < 0) { cerr << "rbd: import-diff failed: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; @@ -2821,7 +2821,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \ r = do_copy(image, dest_io_ctx, destname); if (r < 0) { cerr << "rbd: copy failed: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; @@ -2829,7 +2829,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \ r = do_watch(io_ctx, imgname); if (r < 0) { cerr << "rbd: watch failed: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; @@ -2837,7 +2837,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \ r = do_kernel_add(poolname, imgname, snapname); if (r < 0) { cerr << "rbd: add failed: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; @@ -2845,7 +2845,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \ r = do_kernel_rm(devpath); if (r < 0) { cerr << "rbd: remove failed: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; @@ -2853,7 +2853,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \ r = do_kernel_showmapped(formatter.get()); if (r < 0) { cerr << "rbd: showmapped failed: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; @@ -2861,7 +2861,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \ r = do_lock_list(image, formatter.get()); if (r < 0) { cerr << "rbd: listing locks failed: " << cpp_strerror(r) << std::endl; - return EXIT_FAILURE; + return -r; } break; @@ -2878,7 +2878,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \ } else { cerr << "rbd: taking lock failed: " << cpp_strerror(r) << std::endl; } - return EXIT_FAILURE; + return -r; } break; @@ -2886,7 +2886,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \ r = do_lock_remove(image, lock_cookie, lock_client); if (r < 0) { cerr << "rbd: releasing lock failed: " << cpp_strerror(r) << std::endl; - return EXIT_FAILURE; + return -r; } break; @@ -2894,7 +2894,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \ r = do_bench_write(image, bench_io_size, bench_io_threads, bench_bytes, bench_pattern); if (r < 0) { cerr << "bench-write failed: " << cpp_strerror(-r) << std::endl; - return EXIT_FAILURE; + return -r; } break; } diff --git a/src/rbd_fuse/rbd-fuse.c b/src/rbd_fuse/rbd-fuse.c index 5a4bfe2702c..eea6edb9eb8 100644 --- a/src/rbd_fuse/rbd-fuse.c +++ b/src/rbd_fuse/rbd-fuse.c @@ -3,6 +3,8 @@ */ #define FUSE_USE_VERSION 26 +#include "include/int_types.h" + #include <stdio.h> #include <stdlib.h> #include <stddef.h> @@ -15,7 +17,6 @@ #include <sys/types.h> #include <unistd.h> #include <getopt.h> -#include <inttypes.h> #include "include/rbd/librbd.h" diff --git a/src/rgw/Makefile.am b/src/rgw/Makefile.am new file mode 100644 index 00000000000..24060b52e25 --- /dev/null +++ b/src/rgw/Makefile.am @@ -0,0 +1,150 @@ +if WITH_RADOSGW +librgw_la_SOURCES = \ + rgw/librgw.cc \ + rgw/rgw_acl.cc \ + rgw/rgw_acl_s3.cc \ + rgw/rgw_acl_swift.cc \ + rgw/rgw_client_io.cc \ + rgw/rgw_fcgi.cc \ + rgw/rgw_xml.cc \ + rgw/rgw_usage.cc \ + rgw/rgw_json_enc.cc \ + rgw/rgw_user.cc \ + rgw/rgw_bucket.cc\ + rgw/rgw_tools.cc \ + rgw/rgw_rados.cc \ + rgw/rgw_http_client.cc \ + rgw/rgw_rest_client.cc \ + rgw/rgw_rest_conn.cc \ + rgw/rgw_op.cc \ + rgw/rgw_common.cc \ + rgw/rgw_cache.cc \ + rgw/rgw_formats.cc \ + rgw/rgw_log.cc \ + rgw/rgw_multi.cc \ + rgw/rgw_policy_s3.cc \ + rgw/rgw_gc.cc \ + rgw/rgw_multi_del.cc \ + rgw/rgw_env.cc \ + rgw/rgw_cors.cc \ + rgw/rgw_cors_s3.cc \ + rgw/rgw_auth_s3.cc \ + rgw/rgw_metadata.cc \ + rgw/rgw_replica_log.cc \ + rgw/rgw_keystone.cc +librgw_la_CXXFLAGS = -Woverloaded-virtual ${AM_CXXFLAGS} +noinst_LTLIBRARIES += librgw.la + +LIBRGW_DEPS += \ + $(LIBRADOS) \ + libcls_rgw_client.la \ + libcls_log_client.a \ + libcls_statelog_client.a \ + libcls_replica_log_client.a \ + libcls_lock_client.la \ + libcls_refcount_client.la \ + libcls_version_client.a \ + -lcurl \ + -lexpat \ + -lm \ + -lfcgi + +radosgw_SOURCES = \ + rgw/rgw_resolve.cc \ + rgw/rgw_rest.cc \ + rgw/rgw_rest_swift.cc \ + rgw/rgw_rest_s3.cc \ + rgw/rgw_rest_usage.cc \ + rgw/rgw_rest_user.cc \ + rgw/rgw_rest_bucket.cc \ + rgw/rgw_rest_metadata.cc \ + rgw/rgw_replica_log.cc \ + rgw/rgw_rest_log.cc \ + rgw/rgw_rest_opstate.cc \ + rgw/rgw_rest_replica_log.cc \ + rgw/rgw_rest_config.cc \ + rgw/rgw_http_client.cc \ + rgw/rgw_swift.cc \ + rgw/rgw_swift_auth.cc \ + rgw/rgw_main.cc +radosgw_LDADD = $(LIBRGW) $(LIBRGW_DEPS) -lresolv $(CEPH_GLOBAL) +bin_PROGRAMS += radosgw + +radosgw_admin_SOURCES = rgw/rgw_admin.cc +radosgw_admin_LDADD = $(LIBRGW) $(LIBRGW_DEPS) $(CEPH_GLOBAL) +bin_PROGRAMS += radosgw-admin + +ceph_rgw_multiparser_SOURCES = rgw/rgw_multiparser.cc +ceph_rgw_multiparser_LDADD = $(LIBRGW) $(LIBRGW_DEPS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_rgw_multiparser + +ceph_rgw_jsonparser_SOURCES = \ + rgw/rgw_jsonparser.cc \ + rgw/rgw_common.cc \ + rgw/rgw_env.cc \ + rgw/rgw_json_enc.cc +ceph_rgw_jsonparser_LDADD = $(LIBRGW) $(LIBRGW_DEPS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_rgw_jsonparser + +# inject rgw stuff in the decoder testcase +DENCODER_SOURCES += \ + rgw/rgw_dencoder.cc \ + rgw/rgw_acl.cc \ + rgw/rgw_common.cc \ + rgw/rgw_env.cc \ + rgw/rgw_json_enc.cc + + +endif # WITH_RADOSGW + + +noinst_HEADERS += \ + rgw/logrotate.conf \ + rgw/rgw_acl.h \ + rgw/rgw_acl_s3.h \ + rgw/rgw_acl_swift.h \ + rgw/rgw_client_io.h \ + rgw/rgw_fcgi.h \ + rgw/rgw_xml.h \ + rgw/rgw_cache.h \ + rgw/rgw_common.h \ + rgw/rgw_cors.h \ + rgw/rgw_cors_s3.h \ + rgw/rgw_cors_swift.h \ + rgw/rgw_string.h \ + rgw/rgw_formats.h \ + rgw/rgw_http_errors.h \ + rgw/rgw_log.h \ + rgw/rgw_multi.h \ + rgw/rgw_policy_s3.h \ + rgw/rgw_gc.h \ + rgw/rgw_metadata.h \ + rgw/rgw_multi_del.h \ + rgw/rgw_op.h \ + rgw/rgw_http_client.h \ + rgw/rgw_swift.h \ + rgw/rgw_swift_auth.h \ + rgw/rgw_rados.h \ + rgw/rgw_replica_log.h \ + rgw/rgw_resolve.h \ + rgw/rgw_rest.h \ + rgw/rgw_rest_swift.h \ + rgw/rgw_rest_s3.h \ + rgw/rgw_auth_s3.h \ + rgw/rgw_rest_admin.h \ + rgw/rgw_rest_usage.h \ + rgw/rgw_rest_user.h \ + rgw/rgw_rest_bucket.h \ + rgw/rgw_rest_client.h \ + rgw/rgw_rest_conn.h \ + rgw/rgw_tools.h \ + rgw/rgw_rest_metadata.h \ + rgw/rgw_rest_log.h \ + rgw/rgw_rest_opstate.h \ + rgw/rgw_rest_replica_log.h \ + rgw/rgw_rest_config.h \ + rgw/rgw_usage.h \ + rgw/rgw_user.h \ + rgw/rgw_bucket.h \ + rgw/rgw_keystone.h + diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 644a1760aaf..81abb231b6f 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -127,6 +127,7 @@ void _usage() cerr << " mdlog trim\n"; cerr << " replica mdlog get/delete\n"; cerr << " replica datalog get/delete\n"; + cerr << " --metadata-key=<key> key to retrieve metadata from with metadata get\n"; cerr << " --rgw-region=<region> region in which radosgw is running\n"; cerr << " --rgw-zone=<zone> zone in which radosgw is running\n"; cerr << " --fix besides checking bucket index, will also fix it\n"; diff --git a/src/test/Makefile.am b/src/test/Makefile.am new file mode 100644 index 00000000000..9ce4a246673 --- /dev/null +++ b/src/test/Makefile.am @@ -0,0 +1,853 @@ +## Unknown/other tests + +ceph_test_timers_SOURCES = test/TestTimers.cc +ceph_test_timers_LDADD = $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_test_timers + +ceph_test_signal_handlers_SOURCES = test/TestSignalHandlers.cc +ceph_test_signal_handlers_LDADD = $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_test_signal_handlers + +ceph_test_rados_SOURCES = \ + test/osd/TestRados.cc \ + test/osd/TestOpStat.cc \ + test/osd/Object.cc \ + test/osd/RadosModel.cc +ceph_test_rados_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_test_rados + +ceph_test_mutate_SOURCES = test/test_mutate.cc +ceph_test_mutate_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_test_mutate + +ceph_test_rewrite_latency_SOURCES = test/test_rewrite_latency.cc +ceph_test_rewrite_latency_LDADD = $(LIBCOMMON) $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS) +bin_DEBUGPROGRAMS += ceph_test_rewrite_latency + +ceph_test_msgr_SOURCES = test/testmsgr.cc +ceph_test_msgr_LDADD = $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_test_msgr + +ceph_streamtest_SOURCES = test/streamtest.cc +ceph_streamtest_LDADD = $(LIBOS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_streamtest + +ceph_test_trans_SOURCES = test/test_trans.cc +ceph_test_trans_LDADD = $(LIBOS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_test_trans + +ceph_test_crypto_SOURCES = test/testcrypto.cc +ceph_test_crypto_LDADD = $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_test_crypto + +ceph_test_keys_SOURCES = test/testkeys.cc +ceph_test_keys_LDADD = $(LIBMON) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_test_keys + + +## Dencoder test + +ceph_dencoder_SOURCES = \ + test/encoding/ceph_dencoder.cc \ + $(DENCODER_SOURCES) +ceph_dencoder_LDADD = \ + $(LIBOSD) $(LIBMDS) $(LIBMON) \ + $(DENCODER_DEPS) $(CEPH_GLOBAL) + +# These should always use explicit _CFLAGS/_CXXFLAGS so avoid basename conflicts +ceph_dencoder_CFLAGS = ${AM_CFLAGS} +ceph_dencoder_CXXFLAGS = ${AM_CXXFLAGS} + +if COMPILER_HAS_VTA +ceph_dencoder_CFLAGS += -fno-var-tracking-assignments +ceph_dencoder_CXXFLAGS += -fno-var-tracking-assignments +endif + +bin_PROGRAMS += ceph-dencoder + + +## Build tests +# These should all use explicit _CXXFLAGS so avoid basename conflicts + +if WITH_BUILD_TESTS +test_build_libcommon_SOURCES = \ + test/buildtest_skeleton.cc \ + $(libcommon_la_SOURCES) +test_build_libcommon_LDADD = \ + $(LIBCOMMON_DEPS) \ + $(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS) +test_build_libcommon_LDFLAGS = -static-libtool-libs +test_build_libcommon_CFLAGS = $(AM_CFLAGS) +test_build_libcommon_CXXFLAGS = $(AM_CXXFLAGS) +bin_DEBUGPROGRAMS += test_build_libcommon + +test_build_librados_SOURCES = \ + test/buildtest_skeleton.cc \ + $(librados_la_SOURCES) +test_build_librados_LDADD = \ + $(LIBRADOS_DEPS) \ + $(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS) +test_build_librados_LDFLAGS = -static-libtool-libs +test_build_librados_CFLAGS = $(AM_CFLAGS) +test_build_librados_CXXFLAGS = $(AM_CXXFLAGS) +bin_DEBUGPROGRAMS += test_build_librados + +test_build_librgw_SOURCES = \ + test/buildtest_skeleton.cc \ + $(librgw_la_SOURCES) +test_build_librgw_LDADD = \ + $(LIBRGW_DEPS) \ + $(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS) \ + $(CEPH_GLOBAL) +test_build_librgw_LDFLAGS = -static-libtool-libs +test_build_librgw_CFLAGS = $(AM_CFLAGS) +test_build_librgw_CXXFLAGS = $(AM_CXXFLAGS) +bin_DEBUGPROGRAMS += test_build_librgw + +# I dont get this one... testing the osdc build but link in libcephfs? +test_build_libcephfs_SOURCES = \ + test/buildtest_skeleton.cc \ + $(libosdc_la_SOURCES) +test_build_libcephfs_LDADD = \ + $(LIBCEPHFS) -lexpat \ + $(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS) +test_build_libcephfs_LDFLAGS = -static-libtool-libs +test_build_libcephfs_CFLAGS = $(AM_CFLAGS) +test_build_libcephfs_CXXFLAGS = $(AM_CXXFLAGS) +bin_DEBUGPROGRAMS += test_build_libcephfs + +endif # WITH_BUILD_TESTS + + +## Benchmarks + +ceph_smalliobench_SOURCES = \ + test/bench/small_io_bench.cc \ + test/bench/rados_backend.cc \ + test/bench/detailed_stat_collector.cc \ + test/bench/bencher.cc +ceph_smalliobench_LDADD = $(LIBRADOS) -lboost_program_options $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_smalliobench + +ceph_smalliobenchfs_SOURCES = \ + test/bench/small_io_bench_fs.cc \ + test/bench/testfilestore_backend.cc \ + test/bench/detailed_stat_collector.cc \ + test/bench/bencher.cc +ceph_smalliobenchfs_LDADD = $(LIBRADOS) -lboost_program_options $(LIBOS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_smalliobenchfs + +ceph_smalliobenchdumb_SOURCES = \ + test/bench/small_io_bench_dumb.cc \ + test/bench/dumb_backend.cc \ + test/bench/detailed_stat_collector.cc \ + test/bench/bencher.cc +ceph_smalliobenchdumb_LDADD = $(LIBRADOS) -lboost_program_options $(LIBOS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_smalliobenchdumb + +ceph_smalliobenchrbd_SOURCES = \ + test/bench/small_io_bench_rbd.cc \ + test/bench/rbd_backend.cc \ + test/bench/detailed_stat_collector.cc \ + test/bench/bencher.cc +ceph_smalliobenchrbd_LDADD = $(LIBRBD) $(LIBRADOS) -lboost_program_options $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_smalliobenchrbd + +ceph_tpbench_SOURCES = \ + test/bench/tp_bench.cc \ + test/bench/detailed_stat_collector.cc +ceph_tpbench_LDADD = $(LIBRADOS) -lboost_program_options $(LIBOS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_tpbench + +ceph_omapbench_SOURCES = test/omap_bench.cc +ceph_omapbench_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_omapbench + +ceph_kvstorebench_SOURCES = \ + test/kv_store_bench.cc \ + key_value_store/kv_flat_btree_async.cc +ceph_kvstorebench_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_kvstorebench + +ceph_multi_stress_watch_SOURCES = \ + test/multi_stress_watch.cc \ + test/librados/test.cc +ceph_multi_stress_watch_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_multi_stress_watch + + + + + + +## System tests + +libsystest_la_SOURCES = \ + test/system/cross_process_sem.cc \ + test/system/systest_runnable.cc \ + test/system/systest_settings.cc +libsystest_la_LIBADD = $(CEPH_GLOBAL) +noinst_LTLIBRARIES += libsystest.la + +ceph_test_rados_list_parallel_SOURCES = \ + test/system/rados_list_parallel.cc \ + test/system/st_rados_create_pool.cc \ + test/system/st_rados_list_objects.cc +ceph_test_rados_list_parallel_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS) +bin_DEBUGPROGRAMS += ceph_test_rados_list_parallel + +ceph_test_rados_open_pools_parallel_SOURCES = \ + test/system/rados_open_pools_parallel.cc \ + test/system/st_rados_create_pool.cc +ceph_test_rados_open_pools_parallel_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS) +bin_DEBUGPROGRAMS += ceph_test_rados_open_pools_parallel + +ceph_test_rados_delete_pools_parallel_SOURCES = \ + test/system/rados_delete_pools_parallel.cc \ + test/system/st_rados_create_pool.cc \ + test/system/st_rados_delete_pool.cc \ + test/system/st_rados_list_objects.cc +ceph_test_rados_delete_pools_parallel_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS) +bin_DEBUGPROGRAMS += ceph_test_rados_delete_pools_parallel + +ceph_test_rados_watch_notify_SOURCES = \ + test/system/rados_watch_notify.cc \ + test/system/st_rados_create_pool.cc \ + test/system/st_rados_delete_pool.cc \ + test/system/st_rados_delete_objs.cc \ + test/system/st_rados_watch.cc \ + test/system/st_rados_notify.cc +ceph_test_rados_watch_notify_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS) +bin_DEBUGPROGRAMS += ceph_test_rados_watch_notify + +ceph_bench_log_SOURCES = test/bench_log.cc +ceph_bench_log_LDADD = $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_bench_log + + + +## Unit tests + +# target to build but not run the unit tests +unittests:: $(check_PROGRAMS) + +UNITTEST_CXXFLAGS = \ + $(AM_CXXFLAGS) \ + -I$(top_srcdir)/src/gtest/include \ + -I$(top_builddir)/src/gtest/include +UNITTEST_LDADD = \ + $(top_builddir)/src/gtest/lib/libgtest.a \ + $(top_builddir)/src/gtest/lib/libgtest_main.a \ + $(PTHREAD_LIBS) + +unittest_encoding_SOURCES = test/encoding.cc +unittest_encoding_LDADD = $(LIBCEPHFS) $(LIBRADOS) -lm $(UNITTEST_LDADD) +unittest_encoding_CXXFLAGS = $(UNITTEST_CXXFLAGS) -fno-strict-aliasing +check_PROGRAMS += unittest_encoding + +unittest_addrs_SOURCES = test/test_addrs.cc +unittest_addrs_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_addrs_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +check_PROGRAMS += unittest_addrs + +unittest_sharedptr_registry_SOURCES = test/common/test_sharedptr_registry.cc +unittest_sharedptr_registry_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_sharedptr_registry_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +check_PROGRAMS += unittest_sharedptr_registry + +unittest_util_SOURCES = test/common/test_util.cc +unittest_util_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_util_LDADD = $(LIBCOMMON) -lm $(UNITTEST_LDADD) $(CRYPTO_LIBS) $(EXTRALIBS) +check_PROGRAMS += unittest_util + +unittest_workqueue_SOURCES = test/test_workqueue.cc +unittest_workqueue_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_workqueue_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +check_PROGRAMS += unittest_workqueue + +unittest_striper_SOURCES = test/test_striper.cc +unittest_striper_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_striper_LDADD = $(LIBOSDC) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +check_PROGRAMS += unittest_striper + +unittest_prebufferedstreambuf_SOURCES = test/test_prebufferedstreambuf.cc +unittest_prebufferedstreambuf_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_prebufferedstreambuf_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD) $(EXTRALIBS) +check_PROGRAMS += unittest_prebufferedstreambuf + +unittest_str_list_SOURCES = test/test_str_list.cc +unittest_str_list_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_str_list_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +check_PROGRAMS += unittest_str_list + +unittest_log_SOURCES = log/test.cc +unittest_log_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD) +unittest_log_CXXFLAGS = $(UNITTEST_CXXFLAGS) -O2 +check_PROGRAMS += unittest_log + +unittest_throttle_SOURCES = test/common/Throttle.cc +unittest_throttle_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_throttle_CXXFLAGS = $(UNITTEST_CXXFLAGS) -O2 +check_PROGRAMS += unittest_throttle + +unittest_base64_SOURCES = test/base64.cc +unittest_base64_LDADD = $(LIBCEPHFS) -lm $(UNITTEST_LDADD) +unittest_base64_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_base64 + +unittest_ceph_argparse_SOURCES = test/ceph_argparse.cc +unittest_ceph_argparse_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_ceph_argparse_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_ceph_argparse + +libec_example_la_SOURCES = test/osd/ErasureCodePluginExample.cc +libec_example_la_CFLAGS = ${AM_CFLAGS} +libec_example_la_CXXFLAGS= ${AM_CXXFLAGS} +libec_example_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libec_example_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' +erasure_codelib_LTLIBRARIES += libec_example.la + +unittest_erasure_code_plugin_SOURCES = test/osd/TestErasureCodePluginExample.cc +unittest_erasure_code_plugin_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_erasure_code_plugin_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +if LINUX +unittest_erasure_code_plugin_LDADD += -ldl +endif +check_PROGRAMS += unittest_erasure_code_plugin + +unittest_erasure_code_jerasure_SOURCES = \ + test/osd/TestErasureCodeJerasure.cc \ + $(libec_jerasure_la_SOURCES) +unittest_erasure_code_jerasure_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_erasure_code_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +if LINUX +unittest_erasure_code_jerasure_LDADD += -ldl +endif +check_PROGRAMS += unittest_erasure_code_jerasure + +unittest_erasure_code_plugin_jerasure_SOURCES = \ + test/osd/TestErasureCodePluginJerasure.cc +unittest_erasure_code_plugin_jerasure_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} +unittest_erasure_code_plugin_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +if LINUX +unittest_erasure_code_plugin_jerasure_LDADD += -ldl +endif +check_PROGRAMS += unittest_erasure_code_plugin_jerasure + +unittest_erasure_code_example_SOURCES = test/osd/TestErasureCodeExample.cc +noinst_HEADERS += test/osd/ErasureCodeExample.h +unittest_erasure_code_example_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_erasure_code_example_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +check_PROGRAMS += unittest_erasure_code_example + +unittest_osd_types_SOURCES = test/test_osd_types.cc +unittest_osd_types_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_osd_types_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +check_PROGRAMS += unittest_osd_types + +unittest_pglog_SOURCES = test/osd/TestPGLog.cc +unittest_pglog_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_pglog_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +check_PROGRAMS += unittest_pglog + +if LINUX +unittest_pglog_LDADD += -ldl +endif # LINUX + +unittest_gather_SOURCES = test/gather.cc +unittest_gather_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_gather_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_gather + +unittest_run_cmd_SOURCES = test/run_cmd.cc +unittest_run_cmd_LDADD = $(LIBCEPHFS) $(UNITTEST_LDADD) +unittest_run_cmd_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_run_cmd + +unittest_signals_SOURCES = test/signals.cc +unittest_signals_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_signals_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_signals + +unittest_simple_spin_SOURCES = test/simple_spin.cc +unittest_simple_spin_LDADD = $(LIBCEPHFS) $(UNITTEST_LDADD) +unittest_simple_spin_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_simple_spin + +unittest_librados_SOURCES = test/librados/librados.cc +unittest_librados_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) +unittest_librados_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_librados + +unittest_bufferlist_SOURCES = test/bufferlist.cc +unittest_bufferlist_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_bufferlist_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_bufferlist + +unittest_crc32c_SOURCES = test/common/test_crc32c.cc +unittest_crc32c_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_crc32c_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_crc32c + +unittest_arch_SOURCES = test/test_arch.c +unittest_arch_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_arch_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_arch + +unittest_crypto_SOURCES = test/crypto.cc +unittest_crypto_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_crypto_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_crypto + +unittest_perf_counters_SOURCES = test/perf_counters.cc +unittest_perf_counters_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_perf_counters_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_perf_counters + +unittest_admin_socket_SOURCES = test/admin_socket.cc +unittest_admin_socket_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_admin_socket_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_admin_socket + +unittest_ceph_crypto_SOURCES = test/ceph_crypto.cc +unittest_ceph_crypto_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_ceph_crypto_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_ceph_crypto + +unittest_utf8_SOURCES = test/utf8.cc +unittest_utf8_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_utf8_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_utf8 + +unittest_mime_SOURCES = test/mime.cc +unittest_mime_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_mime_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_mime + +unittest_escape_SOURCES = test/escape.cc +unittest_escape_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_escape_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_escape + +unittest_chain_xattr_SOURCES = test/filestore/chain_xattr.cc +unittest_chain_xattr_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_chain_xattr_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_chain_xattr + +unittest_flatindex_SOURCES = test/os/TestFlatIndex.cc +unittest_flatindex_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_flatindex_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_flatindex + +unittest_strtol_SOURCES = test/strtol.cc +unittest_strtol_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_strtol_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_strtol + +unittest_confutils_SOURCES = test/confutils.cc +unittest_confutils_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_confutils_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_confutils + +unittest_heartbeatmap_SOURCES = test/heartbeat_map.cc +unittest_heartbeatmap_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_heartbeatmap_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_heartbeatmap + +# why does this include rgw/rgw_formats.cc...? +unittest_formatter_SOURCES = \ + test/formatter.cc \ + rgw/rgw_formats.cc +unittest_formatter_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_formatter_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_formatter + +unittest_libcephfs_config_SOURCES = test/libcephfs_config.cc +unittest_libcephfs_config_LDADD = $(LIBCEPHFS) $(UNITTEST_LDADD) +unittest_libcephfs_config_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_libcephfs_config + +unittest_lfnindex_SOURCES = test/os/TestLFNIndex.cc +unittest_lfnindex_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_lfnindex_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_lfnindex + +unittest_librados_config_SOURCES = test/librados/librados_config.cc +unittest_librados_config_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) +unittest_librados_config_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_librados_config + +#unittest_librgw_link_SOURCES = test/librgw_link.cc +#unittest_librgw_link_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} +#unittest_librgw_link_LDADD = $(LIBRGW) ${UNITTEST_LDADD} +#unittest_librgw_link_CXXFLAGS = ${CRYPTO_CFLAGS} ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} +#check_PROGRAMS += unittest_librgw_link + +unittest_daemon_config_SOURCES = test/daemon_config.cc +unittest_daemon_config_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_daemon_config_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_daemon_config + +unittest_osd_osdcap_SOURCES = test/osd/osdcap.cc +unittest_osd_osdcap_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_osd_osdcap_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_osd_osdcap + +unittest_mon_moncap_SOURCES = test/mon/moncap.cc +unittest_mon_moncap_LDADD = $(LIBMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_mon_moncap_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_mon_moncap + +#if WITH_RADOSGW +#unittest_librgw_SOURCES = test/librgw.cc +#unittest_librgw_LDFLAGS = -lrt $(PTHREAD_CFLAGS) -lcurl ${AM_LDFLAGS} +#unittest_librgw_LDADD = librgw.la $(LIBRADOS) ${UNITTEST_LDADD} -lexpat $(CEPH_GLOBAL) +#unittest_librgw_CXXFLAGS = ${CRYPTO_CFLAGS} ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} +#check_PROGRAMS += unittest_librgw +#endif # WITH_RADOSGW + +unittest_ipaddr_SOURCES = test/test_ipaddr.cc +unittest_ipaddr_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_ipaddr_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_ipaddr + +unittest_texttable_SOURCES = test/test_texttable.cc +unittest_texttable_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD) +unittest_texttable_CXXFLAGS = $(UNITTEST_CXXFLAGS) +check_PROGRAMS += unittest_texttable + +if WITH_RADOSGW +ceph_test_cors_SOURCES = test/test_cors.cc +ceph_test_cors_LDADD = \ + $(LIBRADOS) $(LIBRGW) $(CEPH_GLOBAL) \ + $(UNITTEST_LDADD) \ + -lcurl -luuid -lexpat +ceph_test_cors_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_cors + +ceph_test_cls_rgw_meta_SOURCES = test/test_rgw_admin_meta.cc +ceph_test_cls_rgw_meta_LDADD = \ + $(LIBRADOS) $(LIBRGW) $(CEPH_GLOBAL) \ + $(UNITTEST_LDADD) $(CRYPTO_LIBS) \ + -lcurl -luuid -lexpat \ + libcls_version_client.a libcls_log_client.a \ + libcls_statelog_client.a libcls_refcount_client.la \ + libcls_rgw_client.la libcls_lock_client.la +ceph_test_cls_rgw_meta_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_cls_rgw_meta + +ceph_test_cls_rgw_log_SOURCES = test/test_rgw_admin_log.cc +ceph_test_cls_rgw_log_LDADD = \ + $(LIBRADOS) $(LIBRGW) $(CEPH_GLOBAL) \ + $(UNITTEST_LDADD) $(CRYPTO_LIBS) \ + -lcurl -luuid -lexpat \ + libcls_version_client.a libcls_log_client.a \ + libcls_statelog_client.a libcls_refcount_client.la \ + libcls_rgw_client.la libcls_lock_client.la +ceph_test_cls_rgw_log_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_cls_rgw_log + +ceph_test_cls_rgw_opstate_SOURCES = test/test_rgw_admin_opstate.cc +ceph_test_cls_rgw_opstate_LDADD = \ + $(LIBRADOS) $(LIBRGW) $(CEPH_GLOBAL) \ + $(UNITTEST_LDADD) $(CRYPTO_LIBS) \ + -lcurl -luuid -lexpat \ + libcls_version_client.a libcls_log_client.a \ + libcls_statelog_client.a libcls_refcount_client.la \ + libcls_rgw_client.la libcls_lock_client.la +ceph_test_cls_rgw_opstate_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_cls_rgw_opstate +endif # WITH_RADOSGW + +ceph_test_librbd_SOURCES = \ + test/librbd/test_librbd.cc \ + test/librados/test.cc +ceph_test_librbd_LDADD = $(LIBRBD) $(LIBRADOS) $(UNITTEST_LDADD) +ceph_test_librbd_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_librbd + +ceph_test_librbd_fsx_SOURCES = test/librbd/fsx.c +ceph_test_librbd_fsx_LDADD = $(LIBRBD) $(LIBRADOS) -lm +ceph_test_librbd_fsx_CFLAGS = ${AM_CFLAGS} -Wno-format +bin_DEBUGPROGRAMS += ceph_test_librbd_fsx + +ceph_test_cls_rbd_SOURCES = \ + test/cls_rbd/test_cls_rbd.cc \ + test/librados/test.cc +ceph_test_cls_rbd_LDADD = $(LIBRADOS) libcls_rbd_client.la libcls_lock_client.la $(UNITTEST_LDADD) +ceph_test_cls_rbd_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_cls_rbd + +ceph_test_cls_refcount_SOURCES = \ + test/cls_refcount/test_cls_refcount.cc \ + test/librados/test.cc +ceph_test_cls_refcount_LDADD = $(LIBRADOS) libcls_refcount_client.la $(UNITTEST_LDADD) +ceph_test_cls_refcount_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_cls_refcount + +ceph_test_cls_version_SOURCES = \ + test/cls_version/test_cls_version.cc \ + test/librados/test.cc +ceph_test_cls_version_LDADD = $(LIBRADOS) libcls_version_client.a $(UNITTEST_LDADD) +ceph_test_cls_version_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_cls_version + +ceph_test_cls_log_SOURCES = \ + test/cls_log/test_cls_log.cc \ + test/librados/test.cc +ceph_test_cls_log_LDADD = $(LIBRADOS) libcls_log_client.a $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_test_cls_log_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_cls_log + +ceph_test_cls_statelog_SOURCES = \ + test/cls_statelog/test_cls_statelog.cc \ + test/librados/test.cc +ceph_test_cls_statelog_LDADD = $(LIBRADOS) libcls_statelog_client.a $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_test_cls_statelog_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_cls_statelog + +ceph_test_cls_replica_log_SOURCES = \ + test/cls_replica_log/test_cls_replica_log.cc \ + test/librados/test.cc +ceph_test_cls_replica_log_LDADD = \ + $(LIBRADOS) libcls_replica_log_client.a \ + $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_test_cls_replica_log_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_cls_replica_log + +ceph_test_cls_lock_SOURCES = \ + test/cls_lock/test_cls_lock.cc \ + test/librados/test.cc +ceph_test_cls_lock_LDADD = $(LIBRADOS) libcls_lock_client.la $(UNITTEST_LDADD) +ceph_test_cls_lock_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_cls_lock + +ceph_test_cls_hello_SOURCES = \ + test/cls_hello/test_cls_hello.cc \ + test/librados/test.cc +ceph_test_cls_hello_LDADD = \ + $(LIBRADOS) $(CRYPTO_LIBS) \ + $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_test_cls_hello_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_cls_hello + +if WITH_RADOSGW +ceph_test_cls_rgw_SOURCES = \ + test/cls_rgw/test_cls_rgw.cc \ + test/librados/test.cc +ceph_test_cls_rgw_LDADD = $(LIBRADOS) libcls_rgw_client.la $(UNITTEST_LDADD) +ceph_test_cls_rgw_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_cls_rgw +endif # WITH_RADOSGW + +ceph_test_mon_workloadgen_SOURCES = test/mon/test_mon_workloadgen.cc +ceph_test_mon_workloadgen_LDADD = $(LIBOS) $(LIBOSDC) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_test_mon_workloadgen + +ceph_test_rados_api_cmd_SOURCES = \ + test/librados/cmd.cc \ + test/librados/test.cc +ceph_test_rados_api_cmd_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) +ceph_test_rados_api_cmd_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_rados_api_cmd + +ceph_test_rados_api_io_SOURCES = \ + test/librados/io.cc \ + test/librados/test.cc +ceph_test_rados_api_io_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) +ceph_test_rados_api_io_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_rados_api_io + +ceph_test_rados_api_aio_SOURCES = \ + test/librados/aio.cc \ + test/librados/test.cc +ceph_test_rados_api_aio_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) +ceph_test_rados_api_aio_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_rados_api_aio + +ceph_test_rados_api_list_SOURCES = \ + test/librados/list.cc \ + test/librados/test.cc +ceph_test_rados_api_list_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) +ceph_test_rados_api_list_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_rados_api_list + +ceph_test_rados_api_pool_SOURCES = \ + test/librados/pool.cc \ + test/librados/test.cc +ceph_test_rados_api_pool_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) +ceph_test_rados_api_pool_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_rados_api_pool + +ceph_test_rados_api_stat_SOURCES = \ + test/librados/stat.cc \ + test/librados/test.cc +ceph_test_rados_api_stat_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) +ceph_test_rados_api_stat_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_rados_api_stat + +ceph_test_rados_api_watch_notify_SOURCES = \ + test/librados/watch_notify.cc \ + test/librados/test.cc +ceph_test_rados_api_watch_notify_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) +ceph_test_rados_api_watch_notify_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_rados_api_watch_notify + +ceph_test_rados_api_snapshots_SOURCES = \ + test/librados/snapshots.cc \ + test/librados/test.cc +ceph_test_rados_api_snapshots_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) +ceph_test_rados_api_snapshots_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_rados_api_snapshots + +ceph_test_rados_api_cls_SOURCES = \ + test/librados/cls.cc \ + test/librados/test.cc +ceph_test_rados_api_cls_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) +ceph_test_rados_api_cls_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_rados_api_cls + +ceph_test_rados_api_misc_SOURCES = \ + test/librados/misc.cc \ + test/librados/test.cc +ceph_test_rados_api_misc_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_test_rados_api_misc_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_rados_api_misc + +ceph_test_rados_api_lock_SOURCES = \ + test/librados/lock.cc \ + test/librados/test.cc +ceph_test_rados_api_lock_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) +ceph_test_rados_api_lock_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_rados_api_lock + +ceph_test_libcephfs_SOURCES = \ + test/libcephfs/test.cc \ + test/libcephfs/readdir_r_cb.cc \ + test/libcephfs/caps.cc \ + test/libcephfs/multiclient.cc +ceph_test_libcephfs_LDADD = $(LIBCEPHFS) $(UNITTEST_LDADD) +ceph_test_libcephfs_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_libcephfs + +ceph_test_filestore_SOURCES = test/filestore/store_test.cc +ceph_test_filestore_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_test_filestore_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_filestore + +ceph_test_filestore_workloadgen_SOURCES = \ + test/filestore/workload_generator.cc \ + test/filestore/TestFileStoreState.cc +ceph_test_filestore_workloadgen_LDADD = $(LIBOS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_test_filestore_workloadgen + +ceph_test_filestore_idempotent_SOURCES = \ + test/filestore/test_idempotent.cc \ + test/filestore/FileStoreTracker.cc \ + test/common/ObjectContents.cc +ceph_test_filestore_idempotent_LDADD = $(LIBOS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_test_filestore_idempotent + +ceph_test_filestore_idempotent_sequence_SOURCES = \ + test/filestore/test_idempotent_sequence.cc \ + test/filestore/DeterministicOpSequence.cc \ + test/filestore/TestFileStoreState.cc \ + test/filestore/FileStoreDiff.cc +ceph_test_filestore_idempotent_sequence_LDADD = $(LIBOS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_test_filestore_idempotent_sequence + +ceph_xattr_bench_SOURCES = test/xattr_bench.cc +ceph_xattr_bench_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_xattr_bench_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_xattr_bench + +ceph_test_filejournal_SOURCES = test/test_filejournal.cc +ceph_test_filejournal_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_test_filejournal_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_filejournal + +ceph_test_stress_watch_SOURCES = \ + test/test_stress_watch.cc \ + test/librados/test.cc +ceph_test_stress_watch_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) +ceph_test_stress_watch_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_stress_watch + +ceph_test_objectcacher_stress_SOURCES = \ + test/osdc/object_cacher_stress.cc \ + test/osdc/FakeWriteback.cc +ceph_test_objectcacher_stress_LDADD = $(LIBOSDC) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_test_objectcacher_stress + +ceph_test_snap_mapper_SOURCES = test/test_snap_mapper.cc +ceph_test_snap_mapper_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_test_snap_mapper_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_snap_mapper + +ceph_test_object_map_SOURCES = \ + test/ObjectMap/test_object_map.cc \ + test/ObjectMap/KeyValueDBMemory.cc +ceph_test_object_map_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_test_object_map_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_object_map + +ceph_test_keyvaluedb_atomicity_SOURCES = test/ObjectMap/test_keyvaluedb_atomicity.cc +ceph_test_keyvaluedb_atomicity_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_test_keyvaluedb_atomicity_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_keyvaluedb_atomicity + +ceph_test_keyvaluedb_iterators_SOURCES = \ + test/ObjectMap/test_keyvaluedb_iterators.cc \ + test/ObjectMap/KeyValueDBMemory.cc +ceph_test_keyvaluedb_iterators_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_test_keyvaluedb_iterators_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_keyvaluedb_iterators + +ceph_test_store_tool_SOURCES = test/ObjectMap/test_store_tool/test_store_tool.cc +ceph_test_store_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) +ceph_test_store_tool_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph_test_store_tool + +ceph_test_cfuse_cache_invalidate_SOURCES = test/test_cfuse_cache_invalidate.cc +bin_DEBUGPROGRAMS += ceph_test_cfuse_cache_invalidate + + +noinst_HEADERS += \ + test/osd/RadosModel.h \ + test/osd/Object.h \ + test/osd/TestOpStat.h \ + test/bench/distribution.h \ + test/bench/rados_backend.h \ + test/bench/rbd_backend.h \ + test/bench/bencher.h \ + test/bench/backend.h \ + test/bench/dumb_backend.h \ + test/bench/stat_collector.h \ + test/bench/detailed_stat_collector.h \ + test/bench/testfilestore_backend.h \ + test/common/ObjectContents.h \ + test/encoding/types.h \ + test/filestore/DeterministicOpSequence.h \ + test/filestore/FileStoreTracker.h \ + test/filestore/FileStoreDiff.h \ + test/filestore/TestFileStoreState.h \ + test/filestore/workload_generator.h \ + test/kv_store_bench.h \ + test/librados/test.h \ + test/ObjectMap/KeyValueDBMemory.h \ + test/omap_bench.h \ + test/osd/Object.h \ + test/osd/RadosModel.h \ + test/osd/TestOpStat.h \ + test/osdc/FakeWriteback.h \ + test/system/cross_process_sem.h \ + test/system/st_rados_create_pool.h \ + test/system/st_rados_list_objects.h \ + test/system/st_rados_delete_objs.h \ + test/system/st_rados_delete_pool.h \ + test/system/st_rados_notify.h \ + test/system/st_rados_watch.h \ + test/system/systest_runnable.h \ + test/system/systest_settings.h \ + test/unit.h + diff --git a/src/barclass.cc b/src/test/barclass.cc index f5354f1e0f3..f5354f1e0f3 100644 --- a/src/barclass.cc +++ b/src/test/barclass.cc diff --git a/src/test/test_libcommon_build.cc b/src/test/buildtest_skeleton.cc index 8215a05dbda..8215a05dbda 100644 --- a/src/test/test_libcommon_build.cc +++ b/src/test/buildtest_skeleton.cc diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t index 90f6beca133..524d0edd8ba 100644 --- a/src/test/cli/radosgw-admin/help.t +++ b/src/test/cli/radosgw-admin/help.t @@ -88,6 +88,7 @@ mdlog trim replica mdlog get/delete replica datalog get/delete + --metadata-key=<key> key to retrieve metadata from with metadata get --rgw-region=<region> region in which radosgw is running --rgw-zone=<zone> zone in which radosgw is running --fix besides checking bucket index, will also fix it diff --git a/src/test/common/test_crc32c.cc b/src/test/common/test_crc32c.cc new file mode 100644 index 00000000000..5cf88de0a80 --- /dev/null +++ b/src/test/common/test_crc32c.cc @@ -0,0 +1,84 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include <iostream> +#include <string.h> + +#include "include/types.h" +#include "include/crc32c.h" +#include "include/utime.h" +#include "common/Clock.h" + +#include "gtest/gtest.h" + +#include "common/sctp_crc32.h" +#include "common/crc32c_intel_baseline.h" + +TEST(Crc32c, Small) { + const char *a = "foo bar baz"; + const char *b = "whiz bang boom"; + ASSERT_EQ(4119623852u, ceph_crc32c(0, (unsigned char *)a, strlen(a))); + ASSERT_EQ(881700046u, ceph_crc32c(1234, (unsigned char *)a, strlen(a))); + ASSERT_EQ(2360230088u, ceph_crc32c(0, (unsigned char *)b, strlen(b))); + ASSERT_EQ(3743019208u, ceph_crc32c(5678, (unsigned char *)b, strlen(b))); +} + +TEST(Crc32c, PartialWord) { + const char *a = (const char *)malloc(5); + const char *b = (const char *)malloc(35); + memset((void *)a, 1, 5); + memset((void *)b, 1, 35); + ASSERT_EQ(2715569182u, ceph_crc32c(0, (unsigned char *)a, 5)); + ASSERT_EQ(440531800u, ceph_crc32c(0, (unsigned char *)b, 35)); +} + +TEST(Crc32c, Big) { + int len = 4096000; + char *a = (char *)malloc(len); + memset(a, 1, len); + ASSERT_EQ(31583199u, ceph_crc32c(0, (unsigned char *)a, len)); + ASSERT_EQ(1400919119u, ceph_crc32c(1234, (unsigned char *)a, len)); +} + +TEST(Crc32c, Performance) { + int len = 1000 * 1024 * 1024; + char *a = (char *)malloc(len); + std::cout << "populating large buffer" << std::endl; + for (int i=0; i<len; i++) + a[i] = i & 0xff; + std::cout << "calculating crc" << std::endl; + + { + utime_t start = ceph_clock_now(NULL); + unsigned val = ceph_crc32c(0, (unsigned char *)a, len); + utime_t end = ceph_clock_now(NULL); + float rate = (float)len / (float)(1024*1024) / (float)(end - start); + std::cout << "best choice = " << rate << " MB/sec" << std::endl; + ASSERT_EQ(261108528u, val); + } + { + utime_t start = ceph_clock_now(NULL); + unsigned val = ceph_crc32c(0xffffffff, (unsigned char *)a, len); + utime_t end = ceph_clock_now(NULL); + float rate = (float)len / (float)(1024*1024) / (float)(end - start); + std::cout << "best choice 0xffffffff = " << rate << " MB/sec" << std::endl; + ASSERT_EQ(3895876243u, val); + } + { + utime_t start = ceph_clock_now(NULL); + unsigned val = ceph_crc32c_sctp(0, (unsigned char *)a, len); + utime_t end = ceph_clock_now(NULL); + float rate = (float)len / (float)(1024*1024) / (float)(end - start); + std::cout << "sctp = " << rate << " MB/sec" << std::endl; + ASSERT_EQ(261108528u, val); + } + { + utime_t start = ceph_clock_now(NULL); + unsigned val = ceph_crc32c_intel_baseline(0, (unsigned char *)a, len); + utime_t end = ceph_clock_now(NULL); + float rate = (float)len / (float)(1024*1024) / (float)(end - start); + std::cout << "intel baseline = " << rate << " MB/sec" << std::endl; + ASSERT_EQ(261108528u, val); + } + +} diff --git a/src/test/encoding/types.h b/src/test/encoding/types.h index a6f7cfb7883..fe17f077d8e 100644 --- a/src/test/encoding/types.h +++ b/src/test/encoding/types.h @@ -32,6 +32,7 @@ TYPE(CrushWrapper) #include "osd/osd_types.h" TYPE(osd_reqid_t) TYPE(object_locator_t) +TYPE(request_redirect_t) TYPE(pg_t) TYPE(coll_t) TYPE(pow2_hist_t) @@ -75,7 +76,7 @@ TYPE(ObjectStore::Transaction) #include "os/SequencerPosition.h" TYPE(SequencerPosition) -#include "os/hobject.h" +#include "common/hobject.h" TYPE(hobject_t) #include "mon/AuthMonitor.h" diff --git a/src/test/filestore/store_test.cc b/src/test/filestore/store_test.cc index 80c775052ec..92104960127 100644 --- a/src/test/filestore/store_test.cc +++ b/src/test/filestore/store_test.cc @@ -898,6 +898,65 @@ TEST_F(StoreTest, TwoHash) { ASSERT_EQ(r, 0); } +TEST_F(StoreTest, MoveRename) { + coll_t temp_cid("mytemp"); + hobject_t temp_oid("tmp_oid", "", CEPH_NOSNAP, 0, 0, ""); + coll_t cid("dest"); + hobject_t oid("dest_oid", "", CEPH_NOSNAP, 0, 0, ""); + int r; + { + ObjectStore::Transaction t; + t.create_collection(cid); + t.touch(cid, oid); + r = store->apply_transaction(t); + ASSERT_EQ(r, 0); + } + ASSERT_TRUE(store->exists(cid, oid)); + bufferlist data, attr; + map<string, bufferlist> omap; + data.append("data payload"); + attr.append("attr value"); + omap["omap_key"].append("omap value"); + { + ObjectStore::Transaction t; + t.create_collection(temp_cid); + t.touch(temp_cid, temp_oid); + t.write(temp_cid, temp_oid, 0, data.length(), data); + t.setattr(temp_cid, temp_oid, "attr", attr); + t.omap_setkeys(temp_cid, temp_oid, omap); + r = store->apply_transaction(t); + ASSERT_EQ(r, 0); + } + ASSERT_TRUE(store->exists(temp_cid, temp_oid)); + { + ObjectStore::Transaction t; + t.remove(cid, oid); + t.collection_move_rename(temp_cid, temp_oid, cid, oid); + r = store->apply_transaction(t); + ASSERT_EQ(r, 0); + } + ASSERT_TRUE(store->exists(cid, oid)); + ASSERT_FALSE(store->exists(temp_cid, temp_oid)); + { + bufferlist newdata; + r = store->read(cid, oid, 0, 1000, newdata); + ASSERT_GE(r, 0); + ASSERT_TRUE(newdata.contents_equal(data)); + bufferlist newattr; + r = store->getattr(cid, oid, "attr", newattr); + ASSERT_GE(r, 0); + ASSERT_TRUE(newattr.contents_equal(attr)); + set<string> keys; + keys.insert("omap_key"); + map<string, bufferlist> newomap; + r = store->omap_get_values(cid, oid, keys, &newomap); + ASSERT_GE(r, 0); + ASSERT_EQ(1u, newomap.size()); + ASSERT_TRUE(newomap.count("omap_key")); + ASSERT_TRUE(newomap["omap_key"].contents_equal(omap["omap_key"])); + } +} + // // support tests for qa/workunits/filestore/filestore.sh // diff --git a/src/fooclass.cc b/src/test/fooclass.cc index 2db2d815bb0..2db2d815bb0 100644 --- a/src/fooclass.cc +++ b/src/test/fooclass.cc diff --git a/src/test/libcephfs/caps.cc b/src/test/libcephfs/caps.cc index 96f1a90024d..9fa92cf29dc 100644 --- a/src/test/libcephfs/caps.cc +++ b/src/test/libcephfs/caps.cc @@ -11,11 +11,11 @@ * Foundation. See file COPYING. * */ +#include "include/int_types.h" #include "gtest/gtest.h" #include "include/cephfs/libcephfs.h" #include <linux/types.h> -#include <inttypes.h> #include "include/ceph_fs.h" #include <errno.h> #include <sys/fcntl.h> diff --git a/src/test/librados/misc.cc b/src/test/librados/misc.cc index af17847aeab..803c8b1cc77 100644 --- a/src/test/librados/misc.cc +++ b/src/test/librados/misc.cc @@ -581,18 +581,33 @@ TEST(LibRadosMisc, CopyPP) { ASSERT_EQ(0, ioctx.write_full("foo", blc)); ASSERT_EQ(0, ioctx.setxattr("foo", "myattr", xc)); - ObjectWriteOperation op; - op.copy_from("foo", ioctx, ioctx.get_last_version()); - ASSERT_EQ(0, ioctx.operate("foo.copy", &op)); + { + ObjectWriteOperation op; + op.copy_from("foo", ioctx, ioctx.get_last_version()); + ASSERT_EQ(0, ioctx.operate("foo.copy", &op)); + + bufferlist bl2, x2; + ASSERT_EQ((int)bl.length(), ioctx.read("foo.copy", bl2, 10000, 0)); + ASSERT_TRUE(bl.contents_equal(bl2)); + ASSERT_EQ((int)x.length(), ioctx.getxattr("foo.copy", "myattr", x2)); + ASSERT_TRUE(x.contents_equal(x2)); + } - bufferlist bl2, x2; - ASSERT_EQ((int)bl.length(), ioctx.read("foo.copy", bl2, 10000, 0)); - ASSERT_TRUE(bl.contents_equal(bl2)); - ASSERT_EQ((int)x.length(), ioctx.getxattr("foo.copy", "myattr", x2)); - ASSERT_TRUE(x.contents_equal(x2)); + // small object without a version + { + ObjectWriteOperation op; + op.copy_from("foo", ioctx, 0); + ASSERT_EQ(0, ioctx.operate("foo.copy2", &op)); + + bufferlist bl2, x2; + ASSERT_EQ((int)bl.length(), ioctx.read("foo.copy2", bl2, 10000, 0)); + ASSERT_TRUE(bl.contents_equal(bl2)); + ASSERT_EQ((int)x.length(), ioctx.getxattr("foo.copy2", "myattr", x2)); + ASSERT_TRUE(x.contents_equal(x2)); + } // do a big object - bl.append(buffer::create(8000000)); + bl.append(buffer::create(g_conf->osd_copyfrom_max_chunk * 3)); bl.zero(); bl.append("tail"); blc = bl; @@ -600,15 +615,29 @@ TEST(LibRadosMisc, CopyPP) { ASSERT_EQ(0, ioctx.write_full("big", blc)); ASSERT_EQ(0, ioctx.setxattr("big", "myattr", xc)); - ObjectWriteOperation op2; - op.copy_from("big", ioctx, ioctx.get_last_version()); - ASSERT_EQ(0, ioctx.operate("big.copy", &op)); + { + ObjectWriteOperation op; + op.copy_from("big", ioctx, ioctx.get_last_version()); + ASSERT_EQ(0, ioctx.operate("big.copy", &op)); + + bufferlist bl2, x2; + ASSERT_EQ((int)bl.length(), ioctx.read("big.copy", bl2, bl.length(), 0)); + ASSERT_TRUE(bl.contents_equal(bl2)); + ASSERT_EQ((int)x.length(), ioctx.getxattr("foo.copy", "myattr", x2)); + ASSERT_TRUE(x.contents_equal(x2)); + } - bl2.clear(); - ASSERT_EQ((int)bl.length(), ioctx.read("big.copy", bl2, bl.length(), 0)); - ASSERT_TRUE(bl.contents_equal(bl2)); - ASSERT_EQ((int)x.length(), ioctx.getxattr("foo.copy", "myattr", x2)); - ASSERT_TRUE(x.contents_equal(x2)); + { + ObjectWriteOperation op; + op.copy_from("big", ioctx, 0); + ASSERT_EQ(0, ioctx.operate("big.copy2", &op)); + + bufferlist bl2, x2; + ASSERT_EQ((int)bl.length(), ioctx.read("big.copy2", bl2, bl.length(), 0)); + ASSERT_TRUE(bl.contents_equal(bl2)); + ASSERT_EQ((int)x.length(), ioctx.getxattr("foo.copy2", "myattr", x2)); + ASSERT_TRUE(x.contents_equal(x2)); + } ioctx.close(); ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster)); diff --git a/src/test/librbd/test_librbd.cc b/src/test/librbd/test_librbd.cc index 562be6a6bcf..84bf3477aff 100644 --- a/src/test/librbd/test_librbd.cc +++ b/src/test/librbd/test_librbd.cc @@ -12,6 +12,7 @@ * */ +#include "include/int_types.h" #include "include/rados/librados.h" #include "include/rbd_types.h" #include "include/rbd/librbd.h" @@ -20,7 +21,6 @@ #include "gtest/gtest.h" #include <errno.h> -#include <inttypes.h> #include <stdarg.h> #include <stdio.h> #include <stdlib.h> diff --git a/src/test/osd/ErasureCodeExample.h b/src/test/osd/ErasureCodeExample.h new file mode 100644 index 00000000000..896e614c6b5 --- /dev/null +++ b/src/test/osd/ErasureCodeExample.h @@ -0,0 +1,115 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com> + * + * Author: Loic Dachary <loic@dachary.org> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#ifndef CEPH_ERASURE_CODE_EXAMPLE_H +#define CEPH_ERASURE_CODE_EXAMPLE_H + +#include <unistd.h> +#include <errno.h> +#include <sstream> +#include "osd/ErasureCodeInterface.h" + +#define DATA_CHUNKS 2u +#define CODING_CHUNKS 1u + +class ErasureCodeExample : public ErasureCodeInterface { +public: + useconds_t delay; + ErasureCodeExample(const map<std::string,std::string> ¶meters) : + delay(0) + { + if (parameters.find("usleep") != parameters.end()) { + std::istringstream ss(parameters.find("usleep")->second); + ss >> delay; + usleep(delay); + } + } + + virtual ~ErasureCodeExample() {} + + virtual int minimum_to_decode(const set<int> &want_to_read, + const set<int> &available_chunks, + set<int> *minimum) { + if (available_chunks.size() < DATA_CHUNKS) + return -EIO; + set<int>::iterator i; + unsigned j; + for (i = available_chunks.begin(), j = 0; j < DATA_CHUNKS; i++, j++) + minimum->insert(*i); + return 0; + } + + virtual int minimum_to_decode_with_cost(const set<int> &want_to_read, + const map<int, int> &available, + set<int> *minimum) { + set <int> available_chunks; + for (map<int, int>::const_iterator i = available.begin(); + i != available.end(); + i++) + available_chunks.insert(i->first); + return minimum_to_decode(want_to_read, available_chunks, minimum); + } + + virtual int encode(const set<int> &want_to_encode, + const bufferlist &in, + map<int, bufferlist> *encoded) { + unsigned chunk_length = ( in.length() / DATA_CHUNKS ) + 1; + unsigned length = chunk_length * ( DATA_CHUNKS + CODING_CHUNKS ); + bufferlist out(in); + bufferptr pad(length - in.length()); + pad.zero(0, DATA_CHUNKS); + out.push_back(pad); + char *p = out.c_str(); + for (unsigned i = 0; i < chunk_length * DATA_CHUNKS; i++) + p[i + 2 * chunk_length] = + p[i + 0 * chunk_length] ^ p[i + 1 * chunk_length]; + const bufferptr ptr = out.buffers().front(); + for (set<int>::iterator j = want_to_encode.begin(); + j != want_to_encode.end(); + j++) { + bufferptr chunk(ptr, (*j) * chunk_length, chunk_length); + (*encoded)[*j].push_front(chunk); + } + return 0; + } + + virtual int decode(const set<int> &want_to_read, + const map<int, bufferlist> &chunks, + map<int, bufferlist> *decoded) { + + unsigned chunk_length = (*chunks.begin()).second.length(); + for (set<int>::iterator i = want_to_read.begin(); + i != want_to_read.end(); + i++) { + if (chunks.find(*i) != chunks.end()) + (*decoded)[*i] = chunks.find(*i)->second; + else { + bufferptr chunk(chunk_length); + map<int, bufferlist>::const_iterator k = chunks.begin(); + const char *a = k->second.buffers().front().c_str(); + k++; + const char *b = k->second.buffers().front().c_str(); + for (unsigned j = 0; j < chunk_length; j++) { + chunk[j] = a[j] ^ b[j]; + } + (*decoded)[*i].push_front(chunk); + } + } + return 0; + } +}; + +#endif diff --git a/src/test/osd/ErasureCodePluginExample.cc b/src/test/osd/ErasureCodePluginExample.cc new file mode 100644 index 00000000000..1543b1cdaed --- /dev/null +++ b/src/test/osd/ErasureCodePluginExample.cc @@ -0,0 +1,34 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com> + * + * Author: Loic Dachary <loic@dachary.org> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include "osd/ErasureCodePlugin.h" +#include "ErasureCodeExample.h" + +class ErasureCodePluginExample : public ErasureCodePlugin { +public: + virtual int factory(const map<std::string,std::string> ¶meters, + ErasureCodeInterfaceRef *erasure_code) + { + *erasure_code = ErasureCodeInterfaceRef(new ErasureCodeExample(parameters)); + return 0; + } +}; + +int __erasure_code_init(char *plugin_name) +{ + ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); + return instance.add(plugin_name, new ErasureCodePluginExample()); +} diff --git a/src/test/osd/RadosModel.h b/src/test/osd/RadosModel.h index b9967d7af50..3a73ac33faf 100644 --- a/src/test/osd/RadosModel.h +++ b/src/test/osd/RadosModel.h @@ -1,4 +1,6 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +#include "include/int_types.h" + #include "common/Mutex.h" #include "common/Cond.h" #include "include/rados/librados.hpp" @@ -15,7 +17,6 @@ #include <time.h> #include "Object.h" #include "TestOpStat.h" -#include "inttypes.h" #include "test/librados/test.h" #ifndef RADOSMODEL_H @@ -82,14 +83,16 @@ public: class TestOp { public: + int num; RadosTestContext *context; TestOpStat *stat; bool done; - TestOp(RadosTestContext *context, - TestOpStat *stat = 0) : - context(context), - stat(stat), - done(0) + TestOp(int n, RadosTestContext *context, + TestOpStat *stat = 0) + : num(n), + context(context), + stat(stat), + done(0) {} virtual ~TestOp() {}; @@ -229,6 +232,7 @@ public: for (list<TestOp*>::iterator i = inflight.begin(); i != inflight.end();) { if ((*i)->finished()) { + cout << (*i)->num << ": done (" << (inflight.size()-1) << " left)" << std::endl; delete *i; inflight.erase(i++); } else { @@ -237,7 +241,7 @@ public: } if (inflight.size() >= (unsigned) max_in_flight || (!next && !inflight.empty())) { - cout << "Waiting on " << inflight.size() << std::endl; + cout << " waiting on " << inflight.size() << std::endl; wait(); } else { break; @@ -487,11 +491,11 @@ public: librados::ObjectWriteOperation op; librados::AioCompletion *comp; bool done; - RemoveAttrsOp(RadosTestContext *context, + RemoveAttrsOp(int n, RadosTestContext *context, const string &oid, - TestOpStat *stat) : - TestOp(context, stat), oid(oid), comp(NULL), done(false) - {} + TestOpStat *stat) + : TestOp(n, context, stat), oid(oid), comp(NULL), done(false) + {} void _begin() { @@ -576,11 +580,12 @@ public: librados::ObjectWriteOperation op; librados::AioCompletion *comp; bool done; - TmapPutOp(RadosTestContext *context, - const string &oid, - TestOpStat *stat) : - TestOp(context, stat), oid(oid), comp(NULL), done(false) - {} + TmapPutOp(int n, + RadosTestContext *context, + const string &oid, + TestOpStat *stat) + : TestOp(n, context, stat), oid(oid), comp(NULL), done(false) + {} void _begin() { @@ -646,7 +651,7 @@ public: assert(0); } done = true; - context->update_object_version(oid, comp->get_version()); + context->update_object_version(oid, comp->get_version64()); context->oid_in_use.erase(oid); context->oid_not_in_use.insert(oid); context->kick(); @@ -669,11 +674,13 @@ public: librados::ObjectWriteOperation op; librados::AioCompletion *comp; bool done; - SetAttrsOp(RadosTestContext *context, - const string &oid, - TestOpStat *stat) : - TestOp(context, stat), oid(oid), comp(NULL), done(false) - {} + SetAttrsOp(int n, + RadosTestContext *context, + const string &oid, + TestOpStat *stat) + : TestOp(n, context, stat), + oid(oid), comp(NULL), done(false) + {} void _begin() { @@ -737,7 +744,7 @@ public: assert(0); } done = true; - context->update_object_version(oid, comp->get_version()); + context->update_object_version(oid, comp->get_version64()); context->oid_in_use.erase(oid); context->oid_not_in_use.insert(oid); context->kick(); @@ -762,11 +769,12 @@ public: uint64_t waiting_on; uint64_t last_acked_tid; - WriteOp(RadosTestContext *context, + WriteOp(int n, + RadosTestContext *context, const string &oid, - TestOpStat *stat = 0) : - TestOp(context, stat), - oid(oid), waiting_on(0), last_acked_tid(0) + TestOpStat *stat = 0) + : TestOp(n, context, stat), + oid(oid), waiting_on(0), last_acked_tid(0) {} void _begin() @@ -795,16 +803,17 @@ public: } interval_set<uint64_t> ranges; context->cont_gen.get_ranges(cont, ranges); + std::cout << num << ": seq_num " << context->seq_num << " ranges " << ranges << std::endl; context->state_lock.Unlock(); int r = context->io_ctx.selfmanaged_snap_set_write_ctx(context->seq, snapset); if (r) { - cerr << "r is " << r << " snapset is " << snapset << " seq is " << context->seq << std::endl; + cerr << " r is " << r << " snapset is " << snapset << " seq is " << context->seq << std::endl; assert(0); } waiting_on = ranges.num_intervals(); - cout << "waiting_on = " << waiting_on << std::endl; + //cout << " waiting_on = " << waiting_on << std::endl; ContentsGenerator::iterator gen_pos = context->cont_gen.get_iterator(cont); uint64_t tid = 1; for (interval_set<uint64_t>::iterator i = ranges.begin(); @@ -817,9 +826,8 @@ public: } assert(to_write.length() == i.get_len()); assert(to_write.length() > 0); - std::cout << "Writing " << context->prefix+oid << " from " << i.get_start() - << " to " << i.get_len() + i.get_start() << " tid " << tid - << " ranges are " << ranges << std::endl; + std::cout << num << ": writing " << context->prefix+oid << " from " << i.get_start() + << " to " << i.get_len() + i.get_start() << " tid " << tid << std::endl; pair<TestOp*, TestOp::CallbackInfo*> *cb_arg = new pair<TestOp*, TestOp::CallbackInfo*>(this, new TestOp::CallbackInfo(tid)); @@ -837,7 +845,7 @@ public: context->state_lock.Lock(); uint64_t tid = info->id; - cout << "finishing write tid " << tid << " to " << context->prefix + oid << std::endl; + cout << num << ": finishing write tid " << tid << " to " << context->prefix + oid << std::endl; if (tid <= last_acked_tid) { cerr << "Error: finished tid " << tid @@ -888,10 +896,11 @@ class DeleteOp : public TestOp { public: string oid; - DeleteOp(RadosTestContext *context, + DeleteOp(int n, + RadosTestContext *context, const string &oid, - TestOpStat *stat = 0) : - TestOp(context, stat), oid(oid) + TestOpStat *stat = 0) + : TestOp(n, context, stat), oid(oid) {} void _begin() @@ -969,16 +978,17 @@ public: bufferlist header; map<string, bufferlist> xattrs; - ReadOp(RadosTestContext *context, + ReadOp(int n, + RadosTestContext *context, const string &oid, - TestOpStat *stat = 0) : - TestOp(context, stat), - completion(NULL), - oid(oid), - old_value(&context->cont_gen), - snap(0), - retval(0), - attrretval(0) + TestOpStat *stat = 0) + : TestOp(n, context, stat), + completion(NULL), + oid(oid), + old_value(&context->cont_gen), + snap(0), + retval(0), + attrretval(0) {} void _begin() @@ -1001,9 +1011,9 @@ public: if (ctx) { assert(old_value.exists); TestAlarm alarm; - std::cerr << "about to start" << std::endl; + std::cerr << num << ": about to start" << std::endl; ctx->start(); - std::cerr << "started" << std::endl; + std::cerr << num << ": started" << std::endl; bufferlist bl; context->io_ctx.set_notify_timeout(600); int r = context->io_ctx.notify(context->prefix+oid, 0, bl); @@ -1011,7 +1021,7 @@ public: std::cerr << "r is " << r << std::endl; assert(0); } - std::cerr << "notified, waiting" << std::endl; + std::cerr << num << ": notified, waiting" << std::endl; ctx->wait(); } if (snap >= 0) { @@ -1058,7 +1068,7 @@ public: uint64_t version = completion->get_version64(); if (int err = completion->get_return_value()) { if (!(err == -ENOENT && old_value.deleted())) { - cerr << "Error: oid " << oid << " read returned error code " + cerr << num << ": Error: oid " << oid << " read returned error code " << err << std::endl; } } else { @@ -1067,16 +1077,16 @@ public: ContDesc to_check; bufferlist::iterator p = result.begin(); if (!context->cont_gen.read_header(p, to_check)) { - cerr << "Unable to decode oid " << oid << " at snap " << context->current_snap << std::endl; + cerr << num << ": Unable to decode oid " << oid << " at snap " << context->current_snap << std::endl; context->errors++; } if (to_check != old_value.most_recent()) { - cerr << "Found incorrect object contents " << to_check + cerr << num << ": Found incorrect object contents " << to_check << ", expected " << old_value.most_recent() << " oid " << oid << std::endl; context->errors++; } if (!old_value.check(result)) { - cerr << "Object " << oid << " contents " << to_check << " corrupt" << std::endl; + cerr << num << ": oid " << oid << " contents " << to_check << " corrupt" << std::endl; context->errors++; } if (context->errors) assert(0); @@ -1084,28 +1094,28 @@ public: // Attributes if (!(old_value.header == header)) { - cerr << "oid: " << oid << " header does not match, old size: " + cerr << num << ": oid " << oid << " header does not match, old size: " << old_value.header.length() << " new size " << header.length() << std::endl; assert(old_value.header == header); } if (omap.size() != old_value.attrs.size()) { - cerr << "oid: " << oid << " tmap.size() is " << omap.size() + cerr << num << ": oid " << oid << " tmap.size() is " << omap.size() << " and old is " << old_value.attrs.size() << std::endl; assert(omap.size() == old_value.attrs.size()); } if (omap_keys.size() != old_value.attrs.size()) { - cerr << "oid: " << oid << " tmap.size() is " << omap_keys.size() + cerr << num << ": oid " << oid << " tmap.size() is " << omap_keys.size() << " and old is " << old_value.attrs.size() << std::endl; assert(omap_keys.size() == old_value.attrs.size()); } if (xattrs.size() != old_value.attrs.size()) { - cerr << "oid: " << oid << " xattrs.size() is " << xattrs.size() + cerr << num << ": oid " << oid << " xattrs.size() is " << xattrs.size() << " and old is " << old_value.attrs.size() << std::endl; assert(xattrs.size() == old_value.attrs.size()); } if (version != old_value.version) { - cerr << "oid: " << oid << " version is " << version + cerr << num << ": oid " << oid << " version is " << version << " and expected " << old_value.version << std::endl; assert(version == old_value.version); } @@ -1162,9 +1172,10 @@ public: class SnapCreateOp : public TestOp { public: - SnapCreateOp(RadosTestContext *context, - TestOpStat *stat = 0) : - TestOp(context, stat) + SnapCreateOp(int n, + RadosTestContext *context, + TestOpStat *stat = 0) + : TestOp(n, context, stat) {} void _begin() @@ -1200,11 +1211,11 @@ public: class SnapRemoveOp : public TestOp { public: int to_remove; - SnapRemoveOp(RadosTestContext *context, + SnapRemoveOp(int n, RadosTestContext *context, int snap, - TestOpStat *stat = 0) : - TestOp(context, stat), - to_remove(snap) + TestOpStat *stat = 0) + : TestOp(n, context, stat), + to_remove(snap) {} void _begin() @@ -1240,11 +1251,12 @@ public: class WatchOp : public TestOp { string oid; public: - WatchOp(RadosTestContext *context, - const string &_oid, - TestOpStat *stat = 0) : - TestOp(context, stat), - oid(_oid) + WatchOp(int n, + RadosTestContext *context, + const string &_oid, + TestOpStat *stat = 0) + : TestOp(n, context, stat), + oid(_oid) {} void _begin() @@ -1317,13 +1329,14 @@ public: librados::ObjectWriteOperation op; librados::AioCompletion *comp; - RollbackOp(RadosTestContext *context, + RollbackOp(int n, + RadosTestContext *context, const string &_oid, int snap, - TestOpStat *stat = 0) : - TestOp(context, stat), - oid(_oid), - roll_back_to(snap), done(false) + TestOpStat *stat = 0) + : TestOp(n, context, stat), + oid(_oid), + roll_back_to(snap), done(false) {} void _begin() @@ -1368,7 +1381,7 @@ public: assert(0); } done = true; - context->update_object_version(oid, comp->get_version()); + context->update_object_version(oid, comp->get_version64()); context->oid_in_use.erase(oid); context->oid_not_in_use.insert(oid); context->kick(); @@ -1391,16 +1404,20 @@ public: ObjectDesc src_value; librados::ObjectWriteOperation op; librados::AioCompletion *comp; + librados::AioCompletion *comp_racing_read; int snap; - bool done; - tid_t tid; - CopyFromOp(RadosTestContext *context, + int done; + uint64_t version; + int r; + CopyFromOp(int n, + RadosTestContext *context, const string &oid, const string &oid_src, TestOpStat *stat) - : TestOp(context, stat), oid(oid), oid_src(oid_src), + : TestOp(n, context, stat), + oid(oid), oid_src(oid_src), src_value(&context->cont_gen), - comp(NULL), done(false), tid(0) + comp(NULL), done(0), version(0), r(0) {} void _begin() @@ -1432,35 +1449,67 @@ public: new TestOp::CallbackInfo(0)); comp = context->rados.aio_create_completion((void*) cb_arg, &write_callback, NULL); - tid = context->io_ctx.aio_operate(context->prefix+oid, comp, &op); + context->io_ctx.aio_operate(context->prefix+oid, comp, &op); + + // queue up a racing read, too. + pair<TestOp*, TestOp::CallbackInfo*> *read_cb_arg = + new pair<TestOp*, TestOp::CallbackInfo*>(this, + new TestOp::CallbackInfo(1)); + comp_racing_read = context->rados.aio_create_completion((void*) read_cb_arg, &write_callback, + NULL); + context->io_ctx.aio_stat(context->prefix+oid, comp_racing_read, NULL, NULL); } void _finish(CallbackInfo *info) { Mutex::Locker l(context->state_lock); - done = true; - int r; - assert(comp->is_complete()); - cout << "finishing copy_from tid " << tid << " to " << context->prefix + oid << std::endl; - if ((r = comp->get_return_value())) { - if (!(r == -ENOENT && src_value.deleted())) { - cerr << "Error: oid " << oid << " copy_from " << oid_src << " returned error code " - << r << std::endl; + + // note that the read can (and atm will) come back before the + // write reply, but will reflect the update and the versions will + // match. + + if (info->id == 0) { + // copy_from + assert(comp->is_complete()); + cout << num << ": finishing copy_from to " << context->prefix + oid << std::endl; + if ((r = comp->get_return_value())) { + if (!(r == -ENOENT && src_value.deleted())) { + cerr << "Error: oid " << oid << " copy_from " << oid_src << " returned error code " + << r << std::endl; + } + } else { + assert(!version || comp->get_version64() == version); + version = comp->get_version64(); + context->update_object_full(oid, src_value); + context->update_object_version(oid, comp->get_version64()); } - } else { - context->update_object_full(oid, src_value); - context->update_object_version(oid, comp->get_version()); + context->oid_in_use.erase(oid_src); + context->oid_not_in_use.insert(oid_src); + context->kick(); + } else if (info->id == 1) { + // racing read + assert(comp_racing_read->is_complete()); + cout << num << ": finishing copy_from racing read to " << context->prefix + oid << std::endl; + if ((r = comp_racing_read->get_return_value())) { + if (!(r == -ENOENT && src_value.deleted())) { + cerr << "Error: oid " << oid << " copy_from " << oid_src << " returned error code " + << r << std::endl; + } + } else { + assert(comp_racing_read->get_return_value() == 0); + assert(!version || comp_racing_read->get_version64() == version); + version = comp_racing_read->get_version64(); + } + context->oid_in_use.erase(oid); + context->oid_not_in_use.insert(oid); + context->kick(); } - context->oid_in_use.erase(oid); - context->oid_not_in_use.insert(oid); - context->oid_in_use.erase(oid_src); - context->oid_not_in_use.insert(oid_src); - context->kick(); + ++done; } bool finished() { - return done; + return done == 2; } string getType() diff --git a/src/test/osd/TestErasureCodeExample.cc b/src/test/osd/TestErasureCodeExample.cc new file mode 100644 index 00000000000..66f521d7863 --- /dev/null +++ b/src/test/osd/TestErasureCodeExample.cc @@ -0,0 +1,146 @@ +// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com> + * + * Author: Loic Dachary <loic@dachary.org> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include "global/global_init.h" +#include "ErasureCodeExample.h" +#include "common/ceph_argparse.h" +#include "global/global_context.h" +#include "gtest/gtest.h" + +TEST(ErasureCodeExample, constructor) +{ + map<std::string,std::string> parameters; + { + ErasureCodeExample example(parameters); + EXPECT_EQ(0u, example.delay); + } + parameters["usleep"] = "10"; + { + ErasureCodeExample example(parameters); + EXPECT_EQ(10u, example.delay); + } +} + +TEST(ErasureCodeExample, minimum_to_decode) +{ + map<std::string,std::string> parameters; + ErasureCodeExample example(parameters); + set<int> available_chunks; + set<int> want_to_read; + want_to_read.insert(1); + { + set<int> minimum; + EXPECT_EQ(-EIO, example.minimum_to_decode(want_to_read, + available_chunks, + &minimum)); + } + available_chunks.insert(0); + available_chunks.insert(2); + { + set<int> minimum; + EXPECT_EQ(0, example.minimum_to_decode(want_to_read, + available_chunks, + &minimum)); + EXPECT_EQ(available_chunks, minimum); + EXPECT_EQ(2u, minimum.size()); + EXPECT_EQ(1u, minimum.count(0)); + EXPECT_EQ(1u, minimum.count(2)); + } + { + set<int> minimum; + available_chunks.insert(1); + EXPECT_EQ(0, example.minimum_to_decode(want_to_read, + available_chunks, + &minimum)); + EXPECT_EQ(2u, minimum.size()); + EXPECT_EQ(1u, minimum.count(0)); + EXPECT_EQ(1u, minimum.count(1)); + } +} + +TEST(ErasureCodeExample, encode_decode) +{ + map<std::string,std::string> parameters; + ErasureCodeExample example(parameters); + + bufferlist in; + in.append("ABCDE"); + int want_to_encode[] = { 0, 1, 2 }; + map<int, bufferlist> encoded; + EXPECT_EQ(0, example.encode(set<int>(want_to_encode, want_to_encode+3), + in, + &encoded)); + EXPECT_EQ(3u, encoded.size()); + EXPECT_EQ(3u, encoded[0].length()); + EXPECT_EQ('A', encoded[0][0]); + EXPECT_EQ('B', encoded[0][1]); + EXPECT_EQ('C', encoded[0][2]); + EXPECT_EQ('D', encoded[1][0]); + EXPECT_EQ('E', encoded[1][1]); + EXPECT_EQ('A'^'D', encoded[2][0]); + EXPECT_EQ('B'^'E', encoded[2][1]); + EXPECT_EQ('C'^0, encoded[2][2]); + + // all chunks are available + { + int want_to_decode[] = { 0, 1 }; + map<int, bufferlist> decoded; + EXPECT_EQ(0, example.decode(set<int>(want_to_decode, want_to_decode+2), + encoded, + &decoded)); + EXPECT_EQ(2u, decoded.size()); + EXPECT_EQ(3u, decoded[0].length()); + EXPECT_EQ('A', decoded[0][0]); + EXPECT_EQ('B', decoded[0][1]); + EXPECT_EQ('C', decoded[0][2]); + EXPECT_EQ('D', decoded[1][0]); + EXPECT_EQ('E', decoded[1][1]); + } + + // one chunk is missing + { + map<int, bufferlist> degraded = encoded; + degraded.erase(0); + EXPECT_EQ(2u, degraded.size()); + int want_to_decode[] = { 0, 1 }; + map<int, bufferlist> decoded; + EXPECT_EQ(0, example.decode(set<int>(want_to_decode, want_to_decode+2), + degraded, + &decoded)); + EXPECT_EQ(2u, decoded.size()); + EXPECT_EQ(3u, decoded[0].length()); + EXPECT_EQ('A', decoded[0][0]); + EXPECT_EQ('B', decoded[0][1]); + EXPECT_EQ('C', decoded[0][2]); + EXPECT_EQ('D', decoded[1][0]); + EXPECT_EQ('E', decoded[1][1]); + } +} + +int main(int argc, char **argv) { + vector<const char*> args; + argv_to_vec(argc, (const char **)argv, args); + + global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); + common_init_finish(g_ceph_context); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +// Local Variables: +// compile-command: "cd ../.. ; make -j4 && make unittest_erasure_code_example && ./unittest_erasure_code_example --gtest_filter=*.* --log-to-stderr=true --debug-osd=20" +// End: diff --git a/src/test/osd/TestErasureCodeJerasure.cc b/src/test/osd/TestErasureCodeJerasure.cc new file mode 100644 index 00000000000..266b1735659 --- /dev/null +++ b/src/test/osd/TestErasureCodeJerasure.cc @@ -0,0 +1,108 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com> + * + * Author: Loic Dachary <loic@dachary.org> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include "global/global_init.h" +#include "osd/ErasureCodePluginJerasure/ErasureCodeJerasure.h" +#include "common/ceph_argparse.h" +#include "global/global_context.h" +#include "gtest/gtest.h" + +template <typename T> +class ErasureCodeTest : public ::testing::Test { + public: +}; + +typedef ::testing::Types< + ErasureCodeJerasureReedSolomonVandermonde, + ErasureCodeJerasureReedSolomonRAID6, + ErasureCodeJerasureCauchyOrig, + ErasureCodeJerasureCauchyGood, + ErasureCodeJerasureLiberation, + ErasureCodeJerasureBlaumRoth, + ErasureCodeJerasureLiber8tion +> JerasureTypes; +TYPED_TEST_CASE(ErasureCodeTest, JerasureTypes); + +TYPED_TEST(ErasureCodeTest, encode_decode) { + TypeParam jerasure; + map<std::string,std::string> parameters; + parameters["erasure-code-k"] = "2"; + parameters["erasure-code-m"] = "2"; + parameters["erasure-code-w"] = "7"; + parameters["erasure-code-packetsize"] = "8"; + jerasure.init(parameters); + + bufferlist in; + for (int i = 0; i < 5; i++) + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"); + int want_to_encode[] = { 0, 1, 2, 3 }; + map<int, bufferlist> encoded; + EXPECT_EQ(0, jerasure.encode(set<int>(want_to_encode, want_to_encode+4), + in, + &encoded)); + EXPECT_EQ(4u, encoded.size()); + unsigned length = encoded[0].length(); + EXPECT_EQ(0, strncmp(encoded[0].c_str(), in.c_str(), length)); + EXPECT_EQ(0, strncmp(encoded[1].c_str(), in.c_str() + length, in.length() - length)); + + + // all chunks are available + { + int want_to_decode[] = { 0, 1 }; + map<int, bufferlist> decoded; + EXPECT_EQ(0, jerasure.decode(set<int>(want_to_decode, want_to_decode+2), + encoded, + &decoded)); + // always decode all, regardless of want_to_decode + EXPECT_EQ(4u, decoded.size()); + EXPECT_EQ(length, decoded[0].length()); + EXPECT_EQ(0, strncmp(decoded[0].c_str(), in.c_str(), length)); + EXPECT_EQ(0, strncmp(decoded[1].c_str(), in.c_str() + length, in.length() - length)); + } + + // two chunks are missing + { + map<int, bufferlist> degraded = encoded; + degraded.erase(0); + degraded.erase(1); + EXPECT_EQ(2u, degraded.size()); + int want_to_decode[] = { 0, 1 }; + map<int, bufferlist> decoded; + EXPECT_EQ(0, jerasure.decode(set<int>(want_to_decode, want_to_decode+2), + degraded, + &decoded)); + // always decode all, regardless of want_to_decode + EXPECT_EQ(4u, decoded.size()); + EXPECT_EQ(length, decoded[0].length()); + EXPECT_EQ(0, strncmp(decoded[0].c_str(), in.c_str(), length)); + EXPECT_EQ(0, strncmp(decoded[1].c_str(), in.c_str() + length, in.length() - length)); + } +} + +int main(int argc, char **argv) { + vector<const char*> args; + argv_to_vec(argc, (const char **)argv, args); + + global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); + common_init_finish(g_ceph_context); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +// Local Variables: +// compile-command: "cd ../.. ; make -j4 && make unittest_erasure_code_jerasure && valgrind --tool=memcheck ./unittest_erasure_code_jerasure --gtest_filter=*.* --log-to-stderr=true --debug-osd=20" +// End: diff --git a/src/test/osd/TestErasureCodePluginExample.cc b/src/test/osd/TestErasureCodePluginExample.cc new file mode 100644 index 00000000000..67b41f2011a --- /dev/null +++ b/src/test/osd/TestErasureCodePluginExample.cc @@ -0,0 +1,51 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com> + * + * Author: Loic Dachary <loic@dachary.org> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include <errno.h> +#include "common/Thread.h" +#include "global/global_init.h" +#include "osd/ErasureCodePlugin.h" +#include "common/ceph_argparse.h" +#include "global/global_context.h" +#include "gtest/gtest.h" + +TEST(ErasureCodePluginRegistry, factory) +{ + map<std::string,std::string> parameters; + parameters["erasure-code-directory"] = ".libs"; + ErasureCodeInterfaceRef erasure_code; + ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); + EXPECT_FALSE(erasure_code); + EXPECT_EQ(0, instance.factory("example", parameters, &erasure_code)); + EXPECT_TRUE(erasure_code); + ErasureCodePlugin *plugin = 0; + EXPECT_EQ(-EEXIST, instance.load("example", parameters, &plugin)); +} + +int main(int argc, char **argv) { + vector<const char*> args; + argv_to_vec(argc, (const char **)argv, args); + + global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); + common_init_finish(g_ceph_context); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +// Local Variables: +// compile-command: "cd ../.. ; make -j4 && make unittest_erasure_code_plugin && ./unittest_erasure_code_plugin --gtest_filter=*.* --log-to-stderr=true --debug-osd=20" +// End: diff --git a/src/test/osd/TestErasureCodePluginJerasure.cc b/src/test/osd/TestErasureCodePluginJerasure.cc new file mode 100644 index 00000000000..fe819c71a39 --- /dev/null +++ b/src/test/osd/TestErasureCodePluginJerasure.cc @@ -0,0 +1,67 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com> + * + * Author: Loic Dachary <loic@dachary.org> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include <errno.h> +#include "global/global_init.h" +#include "osd/ErasureCodePlugin.h" +#include "common/ceph_argparse.h" +#include "global/global_context.h" +#include "gtest/gtest.h" + +TEST(ErasureCodePlugin, factory) +{ + ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); + map<std::string,std::string> parameters; + parameters["erasure-code-directory"] = ".libs"; + { + ErasureCodeInterfaceRef erasure_code; + EXPECT_FALSE(erasure_code); + EXPECT_EQ(-ENOENT, instance.factory("jerasure", parameters, &erasure_code)); + EXPECT_FALSE(erasure_code); + } + const char *techniques[] = { + "reed_sol_van", + "reed_sol_r6_op", + "cauchy_orig", + "cauchy_good", + "liberation", + "blaum_roth", + "liber8tion", + 0 + }; + for(const char **technique = techniques; *technique; technique++) { + ErasureCodeInterfaceRef erasure_code; + parameters["erasure-code-technique"] = *technique; + EXPECT_FALSE(erasure_code); + EXPECT_EQ(0, instance.factory("jerasure", parameters, &erasure_code)); + EXPECT_TRUE(erasure_code); + } +} + +int main(int argc, char **argv) { + vector<const char*> args; + argv_to_vec(argc, (const char **)argv, args); + + global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); + common_init_finish(g_ceph_context); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +// Local Variables: +// compile-command: "cd ../.. ; make -j4 && make unittest_erasure_code_jerasure_plugin && valgrind --tool=memcheck ./unittest_erasure_code_jerasure_plugin --gtest_filter=*.* --log-to-stderr=true --debug-osd=20" +// End: diff --git a/src/test/osd/TestRados.cc b/src/test/osd/TestRados.cc index 1deee23aa2c..1b6bd073a12 100644 --- a/src/test/osd/TestRados.cc +++ b/src/test/osd/TestRados.cc @@ -48,8 +48,8 @@ public: if (m_op <= m_objects) { stringstream oid; oid << m_op; - cout << m_op << ": Writing initial " << oid.str() << std::endl; - return new WriteOp(&context, oid.str()); + cout << m_op << ": write initial oid " << oid.str() << std::endl; + return new WriteOp(m_op, &context, oid.str()); } else if (m_op >= m_ops) { return NULL; } @@ -71,7 +71,6 @@ public: it != m_weight_sums.end(); ++it) { if (rand_val < it->second) { - cout << m_op << ": "; retval = gen_op(context, it->first); break; } @@ -85,37 +84,39 @@ private: TestOp *gen_op(RadosTestContext &context, TestOpType type) { string oid, oid2; - cout << "oids not in use " << context.oid_not_in_use.size() << std::endl; + //cout << "oids not in use " << context.oid_not_in_use.size() << std::endl; assert(context.oid_not_in_use.size()); + + cout << m_op << ": "; switch (type) { case TEST_OP_READ: oid = *(rand_choose(context.oid_not_in_use)); - cout << "Reading " << oid << std::endl; - return new ReadOp(&context, oid, m_stats); + cout << "read oid " << oid << std::endl; + return new ReadOp(m_op, &context, oid, m_stats); case TEST_OP_WRITE: oid = *(rand_choose(context.oid_not_in_use)); - cout << "Writing " << oid << " current snap is " + cout << "write oid " << oid << " current snap is " << context.current_snap << std::endl; - return new WriteOp(&context, oid, m_stats); + return new WriteOp(m_op, &context, oid, m_stats); case TEST_OP_DELETE: oid = *(rand_choose(context.oid_not_in_use)); - cout << "Deleting " << oid << " current snap is " + cout << "delete oid " << oid << " current snap is " << context.current_snap << std::endl; - return new DeleteOp(&context, oid, m_stats); + return new DeleteOp(m_op, &context, oid, m_stats); case TEST_OP_SNAP_CREATE: - cout << "Snapping" << std::endl; - return new SnapCreateOp(&context, m_stats); + cout << "snap_create" << std::endl; + return new SnapCreateOp(m_op, &context, m_stats); case TEST_OP_SNAP_REMOVE: if (context.snaps.empty()) { return NULL; } else { int snap = rand_choose(context.snaps)->first; - cout << "RemovingSnap " << snap << std::endl; - return new SnapRemoveOp(&context, snap, m_stats); + cout << "snap_remove snap " << snap << std::endl; + return new SnapRemoveOp(m_op, &context, snap, m_stats); } case TEST_OP_ROLLBACK: @@ -124,40 +125,40 @@ private: } else { int snap = rand_choose(context.snaps)->first; string oid = *(rand_choose(context.oid_not_in_use)); - cout << "RollingBack " << oid << " to " << snap << std::endl; - return new RollbackOp(&context, oid, snap); + cout << "rollback oid " << oid << " to " << snap << std::endl; + return new RollbackOp(m_op, &context, oid, snap); } case TEST_OP_SETATTR: oid = *(rand_choose(context.oid_not_in_use)); - cout << "Setting attrs on " << oid + cout << "setattr oid " << oid << " current snap is " << context.current_snap << std::endl; - return new SetAttrsOp(&context, oid, m_stats); + return new SetAttrsOp(m_op, &context, oid, m_stats); case TEST_OP_RMATTR: oid = *(rand_choose(context.oid_not_in_use)); - cout << "Removing attrs on " << oid + cout << "rmattr oid " << oid << " current snap is " << context.current_snap << std::endl; - return new RemoveAttrsOp(&context, oid, m_stats); + return new RemoveAttrsOp(m_op, &context, oid, m_stats); case TEST_OP_TMAPPUT: oid = *(rand_choose(context.oid_not_in_use)); - cout << "Setting tmap on " << oid + cout << "tmapput oid " << oid << " current snap is " << context.current_snap << std::endl; - return new TmapPutOp(&context, oid, m_stats); + return new TmapPutOp(m_op, &context, oid, m_stats); case TEST_OP_WATCH: oid = *(rand_choose(context.oid_not_in_use)); - cout << "Watching " << oid + cout << "watch oid " << oid << " current snap is " << context.current_snap << std::endl; - return new WatchOp(&context, oid, m_stats); + return new WatchOp(m_op, &context, oid, m_stats); case TEST_OP_COPY_FROM: oid = *(rand_choose(context.oid_not_in_use)); oid2 = *(rand_choose(context.oid_not_in_use)); - cout << "copy_from " << oid << " from " << oid2 + cout << "copy_from oid " << oid << " from oid " << oid2 << " current snap is " << context.current_snap << std::endl; - return new CopyFromOp(&context, oid, oid2, m_stats); + return new CopyFromOp(m_op, &context, oid, oid2, m_stats); default: cerr << "Invalid op type " << type << std::endl; diff --git a/src/test/perf_counters.cc b/src/test/perf_counters.cc index d0b05f9f049..c44a15ef856 100644 --- a/src/test/perf_counters.cc +++ b/src/test/perf_counters.cc @@ -11,6 +11,10 @@ * Foundation. See file COPYING. * */ +#include "include/int_types.h" +#include "include/types.h" // FIXME: ordering shouldn't be important, but right + // now, this include has to come before the others. + #include "common/perf_counters.h" #include "common/admin_socket_client.h" @@ -19,9 +23,6 @@ #include "common/errno.h" #include "common/safe_io.h" -#include "include/types.h" // FIXME: ordering shouldn't be important, but right - // now, this include has to come before the others. - #include "common/code_environment.h" #include "global/global_context.h" #include "global/global_init.h" @@ -30,7 +31,6 @@ #include <errno.h> #include <fcntl.h> -#include <inttypes.h> #include <map> #include <poll.h> #include <sstream> diff --git a/src/streamtest.cc b/src/test/streamtest.cc index 21693ac8713..21693ac8713 100644 --- a/src/streamtest.cc +++ b/src/test/streamtest.cc diff --git a/src/test/test_arch.c b/src/test/test_arch.c new file mode 100644 index 00000000000..549221e60f9 --- /dev/null +++ b/src/test/test_arch.c @@ -0,0 +1,19 @@ + +#include <stdio.h> +#include <stdlib.h> +#include <assert.h> + +#include "arch/probe.h" +#include "arch/intel.h" +#include "arch/neon.h" + +int main(int argc, char **argv) +{ + ceph_arch_probe(); + assert(ceph_arch_probed); + + printf("ceph_arch_intel_sse42 = %d\n", ceph_arch_intel_sse42); + printf("ceph_arch_neon = %d\n", ceph_arch_neon); + + return 0; +} diff --git a/src/test_trans.cc b/src/test/test_trans.cc index 43821c13aec..43821c13aec 100644 --- a/src/test_trans.cc +++ b/src/test/test_trans.cc diff --git a/src/testclass.cc b/src/test/testclass.cc index 22a97be6dcb..22a97be6dcb 100644 --- a/src/testclass.cc +++ b/src/test/testclass.cc diff --git a/src/testcrypto.cc b/src/test/testcrypto.cc index 0b7a9d54742..0b7a9d54742 100644 --- a/src/testcrypto.cc +++ b/src/test/testcrypto.cc diff --git a/src/testkeys.cc b/src/test/testkeys.cc index 27c38124ade..27c38124ade 100644 --- a/src/testkeys.cc +++ b/src/test/testkeys.cc diff --git a/src/testmsgr.cc b/src/test/testmsgr.cc index 4de779b5d7f..4de779b5d7f 100644 --- a/src/testmsgr.cc +++ b/src/test/testmsgr.cc diff --git a/src/tools/Makefile.am b/src/tools/Makefile.am new file mode 100644 index 00000000000..4b8da77951a --- /dev/null +++ b/src/tools/Makefile.am @@ -0,0 +1,87 @@ +ceph_osdomap_tool_SOURCES = tools/ceph-osdomap-tool.cc +ceph_osdomap_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) -lboost_program_options +bin_DEBUGPROGRAMS += ceph-osdomap-tool + +ceph_monstore_tool_SOURCES = tools/ceph-monstore-tool.cc +ceph_monstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) -lboost_program_options +bin_DEBUGPROGRAMS += ceph-monstore-tool + +ceph_filestore_dump_SOURCES = tools/ceph-filestore-dump.cc +ceph_filestore_dump_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) -lboost_program_options +if LINUX +ceph_filestore_dump_LDADD += -ldl +endif # LINUX +bin_PROGRAMS += ceph_filestore_dump + +monmaptool_SOURCES = tools/monmaptool.cc +monmaptool_LDADD = $(CEPH_GLOBAL) +bin_PROGRAMS += monmaptool + +crushtool_SOURCES = tools/crushtool.cc +crushtool_LDADD = $(CEPH_GLOBAL) +bin_PROGRAMS += crushtool + +osdmaptool_SOURCES = tools/osdmaptool.cc +osdmaptool_LDADD = $(CEPH_GLOBAL) +bin_PROGRAMS += osdmaptool + +ceph_scratchtool_SOURCES = tools/scratchtool.c +ceph_scratchtool_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_scratchtool + +ceph_scratchtoolpp_SOURCES = tools/scratchtoolpp.cc +ceph_scratchtoolpp_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_scratchtoolpp + +ceph_psim_SOURCES = tools/psim.cc +ceph_psim_LDADD = $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_psim + +ceph_dupstore_SOURCES = tools/dupstore.cc +ceph_dupstore_LDADD = $(LIBOS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_dupstore + +ceph_radosacl_SOURCES = tools/radosacl.cc +ceph_radosacl_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) +bin_DEBUGPROGRAMS += ceph_radosacl + +rados_SOURCES = \ + tools/rados/rados.cc \ + tools/rados/rados_import.cc \ + tools/rados/rados_export.cc \ + tools/rados/rados_sync.cc +rados_SOURCES += common/obj_bencher.cc # needs cleanup so it can go in libcommon.la +rados_LDADD = libcls_lock_client.la $(LIBRADOS) $(CEPH_GLOBAL) +bin_PROGRAMS += rados + +if WITH_REST_BENCH +rest_bench_SOURCES = tools/rest_bench.cc +rest_bench_SOURCES += common/obj_bencher.cc # needs cleanup so it can go in libcommon.la +rest_bench_LDADD = $(CEPH_GLOBAL) +bin_PROGRAMS += rest-bench + +if WITH_SYSTEM_LIBS3 +rest_bench_LDADD += -ls3 +else +rest_bench_LDADD += libs3/build/lib/libs3.a -lcurl -lxml2 +rest_bench_CXXFLAGS = ${AM_CXXFLAGS} -I$(top_srcdir)/src/libs3/inc +SUBDIRS += libs3 +endif # WITH_SYSTEM_LIBS3 +endif # WITH_REST_BENCH + +ceph_conf_SOURCES = tools/ceph_conf.cc +ceph_conf_LDADD = $(CEPH_GLOBAL) +bin_PROGRAMS += ceph-conf + +ceph_authtool_SOURCES = tools/ceph_authtool.cc +ceph_authtool_LDADD = $(CEPH_GLOBAL) +bin_PROGRAMS += ceph-authtool + +ceph_mon_store_converter_SOURCES = tools/mon_store_converter.cc +ceph_mon_store_converter_LDADD = $(LIBMON) $(LIBOS) $(CEPH_GLOBAL) +bin_PROGRAMS += ceph_mon_store_converter + +noinst_HEADERS += \ + tools/rados/rados_sync.h \ + tools/common.h + diff --git a/src/tools/ceph-filestore-dump.cc b/src/tools/ceph-filestore-dump.cc index 3badc2160b0..49b8d10bdba 100644 --- a/src/tools/ceph-filestore-dump.cc +++ b/src/tools/ceph-filestore-dump.cc @@ -1017,7 +1017,7 @@ int main(int argc, char **argv) ("pgid", po::value<string>(&pgidstr), "PG id, mandatory") ("type", po::value<string>(&type), - "Type which is 'info' or 'log', mandatory") + "Type one of info, log, export, or import, mandatory") ("file", po::value<string>(&file), "path of file to export or import") ("debug", "Enable diagnostic output to stderr") diff --git a/src/ceph_authtool.cc b/src/tools/ceph_authtool.cc index f66a3c66eee..f66a3c66eee 100644 --- a/src/ceph_authtool.cc +++ b/src/tools/ceph_authtool.cc diff --git a/src/ceph_conf.cc b/src/tools/ceph_conf.cc index b2286f4e094..b2286f4e094 100644 --- a/src/ceph_conf.cc +++ b/src/tools/ceph_conf.cc diff --git a/src/crushtool.cc b/src/tools/crushtool.cc index 75c26c098b6..03c83f24156 100644 --- a/src/crushtool.cc +++ b/src/tools/crushtool.cc @@ -569,7 +569,11 @@ int main(int argc, const char **argv) crush_bucket *b = crush_make_bucket(buckettype, CRUSH_HASH_DEFAULT, type, j, items, weights); assert(b); - int id = crush_add_bucket(crush.crush, 0, b); + int id; + int r = crush_add_bucket(crush.crush, 0, b, &id); + if (r < 0) { + dout(0) << "Couldn't add root bucket: " << strerror(-r) << dendl; + } rootid = id; char format[20]; diff --git a/src/dupstore.cc b/src/tools/dupstore.cc index e17eb2201a7..e17eb2201a7 100644 --- a/src/dupstore.cc +++ b/src/tools/dupstore.cc diff --git a/src/mon_store_converter.cc b/src/tools/mon_store_converter.cc index 1c0d3af98e2..1c0d3af98e2 100644 --- a/src/mon_store_converter.cc +++ b/src/tools/mon_store_converter.cc diff --git a/src/monmaptool.cc b/src/tools/monmaptool.cc index 57843aa350d..57843aa350d 100644 --- a/src/monmaptool.cc +++ b/src/tools/monmaptool.cc diff --git a/src/osdmaptool.cc b/src/tools/osdmaptool.cc index 2e55026076c..2e55026076c 100644 --- a/src/osdmaptool.cc +++ b/src/tools/osdmaptool.cc diff --git a/src/psim.cc b/src/tools/psim.cc index c1adc7580fb..c1adc7580fb 100644 --- a/src/psim.cc +++ b/src/tools/psim.cc diff --git a/src/rados.cc b/src/tools/rados/rados.cc index dd9b4de1794..e8386959349 100644 --- a/src/rados.cc +++ b/src/tools/rados/rados.cc @@ -879,7 +879,8 @@ protected: } public: - RadosBencher(librados::Rados& _r, librados::IoCtx& _i) : completions(NULL), rados(_r), io_ctx(_i), iterator_valid(false) {} + RadosBencher(CephContext *cct_, librados::Rados& _r, librados::IoCtx& _i) + : ObjBencher(cct_), completions(NULL), rados(_r), io_ctx(_i), iterator_valid(false) {} ~RadosBencher() { } }; @@ -1987,7 +1988,7 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts, operation = OP_RAND_READ; else usage_exit(); - RadosBencher bencher(rados, io_ctx); + RadosBencher bencher(g_ceph_context, rados, io_ctx); bencher.set_show_time(show_time); ret = bencher.aio_bench(operation, seconds, num_objs, concurrent_ios, op_size, cleanup); @@ -1998,7 +1999,7 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts, if (!pool_name || nargs.size() < 2) usage_exit(); const char *prefix = nargs[1]; - RadosBencher bencher(rados, io_ctx); + RadosBencher bencher(g_ceph_context, rados, io_ctx); ret = bencher.clean_up(prefix, concurrent_ios); if (ret != 0) cerr << "error during cleanup: " << ret << std::endl; diff --git a/src/rados_export.cc b/src/tools/rados/rados_export.cc index 5b8a899d3b8..bf6654114c5 100644 --- a/src/rados_export.cc +++ b/src/tools/rados/rados_export.cc @@ -11,6 +11,7 @@ * Foundation. See file COPYING. * */ +#include "include/int_types.h" #include "rados_sync.h" #include "common/errno.h" @@ -20,7 +21,6 @@ #include <dirent.h> #include <errno.h> #include <fstream> -#include <inttypes.h> #include <iostream> #include <sstream> #include <stdlib.h> diff --git a/src/rados_import.cc b/src/tools/rados/rados_import.cc index da968a7cc50..a6a398d767b 100644 --- a/src/rados_import.cc +++ b/src/tools/rados/rados_import.cc @@ -11,11 +11,11 @@ * Foundation. See file COPYING. * */ +#include "include/int_types.h" #include <dirent.h> #include <errno.h> #include <fstream> -#include <inttypes.h> #include <iostream> #include <sstream> #include <stdlib.h> diff --git a/src/rados_sync.cc b/src/tools/rados/rados_sync.cc index d16894c6ea9..03293d3402a 100644 --- a/src/rados_sync.cc +++ b/src/tools/rados/rados_sync.cc @@ -11,6 +11,7 @@ * Foundation. See file COPYING. * */ +#include "include/int_types.h" #include "common/ceph_argparse.h" #include "common/config.h" @@ -27,7 +28,6 @@ #include <dirent.h> #include <errno.h> #include <fstream> -#include <inttypes.h> #include <iostream> #include <memory> #include <sstream> diff --git a/src/rados_sync.h b/src/tools/rados/rados_sync.h index 0f7226e0239..0f7226e0239 100644 --- a/src/rados_sync.h +++ b/src/tools/rados/rados_sync.h diff --git a/src/radosacl.cc b/src/tools/radosacl.cc index d2f7ca5c488..d2f7ca5c488 100644 --- a/src/radosacl.cc +++ b/src/tools/radosacl.cc diff --git a/src/tools/rest_bench.cc b/src/tools/rest_bench.cc index 99fd16b5a34..feea4de4932 100644 --- a/src/tools/rest_bench.cc +++ b/src/tools/rest_bench.cc @@ -261,10 +261,12 @@ class RESTDispatcher { } req_wq; public: - RESTDispatcher(CephContext *cct, int num_threads) - : m_tp(cct, "RESTDispatcher::m_tp", num_threads), - req_wq(this, g_conf->rgw_op_thread_timeout, - g_conf->rgw_op_thread_suicide_timeout, &m_tp) { + CephContext *cct; + RESTDispatcher(CephContext *cct_, int num_threads) + : m_tp(cct_, "RESTDispatcher::m_tp", num_threads), + req_wq(this, cct_->_conf->rgw_op_thread_timeout, + cct_->_conf->rgw_op_thread_suicide_timeout, &m_tp), + cct(cct_) { response_handler.propertiesCallback = properties_callback; @@ -588,6 +590,7 @@ protected: public: RESTBencher(RESTDispatcher *_dispatcher) : + ObjBencher(_dispatcher->cct), dispatcher(_dispatcher), completions(NULL), list_start(NULL), diff --git a/src/scratchtool.c b/src/tools/scratchtool.c index 22cf2bdf531..22cf2bdf531 100644 --- a/src/scratchtool.c +++ b/src/tools/scratchtool.c diff --git a/src/scratchtoolpp.cc b/src/tools/scratchtoolpp.cc index 62096920300..62096920300 100644 --- a/src/scratchtoolpp.cc +++ b/src/tools/scratchtoolpp.cc |