Repository navigation
METADATA-split-recovery-7 test failure #52
Description
Activity
Saw this failure again:
METADATA-split-recovery-7 ................***Exception: Other 10.76 sec
This was a debug build. The RangeServer seg faulted here (in AccessGroup.cc:range_dir_initialize()):
} catch (Exception &e) { => HT_FATAL_OUT << e << HT_END; }Here's the print of exception e:
$2 = (Hypertable::Exception &) @0x337e500: { = { = {_vptr.exception = 0x190e8d0 }, _M_msg = {s\ tatic npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0x337e558 "Erro\ r checking existence of FS path: /hypertable/tables/2/default/1n9e8xfm_krkXFRC"}}}, m_error = 65537, m_line = 941, m_func = 0x14a1c80 "virtual bool Hypertable::FsBroker::Lib::Client::exists(const String&)", m_file = 0x149f898 "/home/doug/src/hyperta\ ble/src/cc/FsBroker/Lib/Client.cc", prev = 0x337e5c0}And here's the stack trace:
#0 0x00000036cb80f5db in raise () from /lib64/libpthread.so.0 #1 0x0000000000f701e6 in Hypertable::AccessGroup::range_dir_initialize (this=0x337ddd0) at /home/doug/src/hypertable/src/cc/Hypertable/RangeServer/AccessGroup.cc:1094 #2 0x0000000000f676f4 in Hypertable::AccessGroup::AccessGroup (this=0x337ddd0, identifier=0x337d3c0, schema=..., ag_spec=0x3321360, range=0x7fff6a7c41d0, hints=0x0) at /home/doug/src/hypertable/src/cc/Hypertable/RangeServer/AccessGroup.cc:71 #3 0x0000000001077010 in __gnu_cxx::new_allocator::construct&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*, Hypertable::AccessGroup::Hints const*&> > (this=0x7fff6a7c3daf, __p=0x337ddd0) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/ext/new_allocator.h:120 #4 0x0000000001076ce9 in std::allocator_traits >::_S_construct&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*, Hypertable::AccessGroup::Hints const*&> >(std::allocator&, std::allocator_traits >::__construct_helper*, (Hypertable::AccessGroup&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*, Hypertable::AccessGroup::Hints const*&>&&)...) (__a=..., __p=0x337ddd0) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/alloc_traits.h:253 #5 0x00000000010769d7 in std::allocator_traits >::construct&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*, Hypertable::AccessGroup::Hints const*&> >(std::allocator&, Hypertable::AccessGroup&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*, Hypertable::AccessGroup::Hints const*&>*, (Hypertable::AccessGroup&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*, Hypertable::AccessGroup::Hints const*&>&&)...) (__a=..., __p=0x337ddd0) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/alloc_traits.h:399 #6 0x00000000010764b5 in std::_Sp_counted_ptr_inplace, (__gnu_cxx::_Lock_policy)2>::_Sp_counted_ptr_inplace&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*, Hypertable::AccessGroup::Hints const*&>(std::allocator, Hypertable::TableIdentifier*&&, std::shared_ptr&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*&&, Hypertable::AccessGroup::Hints const*&) (this=0x337ddc0, __a=...) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/shared_ptr_base.h:515 #7 0x0000000001075e7b in __gnu_cxx::new_allocator, (__gnu_cxx::_Lock_policy)2> >::construct, (__gnu_cxx::_Lock_policy)2> const, Hypertable::TableIdentifier*, std::shared_ptr&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*, Hypertable::AccessGroup::Hints const*&> > (this=0x7fff6a7c3f37, __p=0x337ddc0) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/ext/new_allocator.h:120 #8 0x000000000107584d in std::allocator_traits, (__gnu_cxx::_Lock_policy)2> > >::_S_construct, (__gnu_cxx::_Lock_policy)2> const, Hypertable::TableIdentifier*, std::shared_ptr&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*, Hypertable::AccessGroup::Hints const*&> >(std::allocator, (__gnu_cxx::_Lock_policy)2> >&, std::allocator_traits, (__gnu_cxx::_Lock_policy)2> > >::__construct_helper*, (std::_Sp_counted_ptr_inplace, (__gnu_cxx::_Lock_policy)2> const, Hypertable::TableIdentifier*, std::shared_ptr&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*, Hypertable::AccessGroup::Hints const*&>&&)...) (__a=..., __p=0x337ddc0) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/alloc_traits.h:253 #9 0x0000000001074f44 in std::allocator_traits, (__gnu_cxx::_Lock_policy)2> > >::construct, (__gnu_cxx::_Lock_policy)2> const, Hypertable::TableIdentifier*, std::shared_ptr&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*, Hypertable::AccessGroup::Hints const*&> >(std::allocator, (__gnu_cxx::_Lock_policy)2> >&, std::_Sp_counted_ptr_inplace, (__gnu_cxx::_Lock_policy)2> const, Hypertable::TableIdentifier*, std::shared_ptr&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*, Hypertable::AccessGroup::Hints const*&>*, (std::_Sp_counted_ptr_inplace, (__gnu_cxx::_Lock_policy)2> const, Hypertable::TableIdentifier*, std::shared_ptr&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*, Hypertable::AccessGroup::Hints const*&>&&)...) (__a=..., __p=0x337ddc0) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/alloc_traits.h:399 #10 0x000000000107407c in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count, Hypertable::TableIdentifier*, std::shared_ptr&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*, Hypertable::AccessGroup::Hints const*&>(std::_Sp_make_shared_tag, Hypertable::AccessGroup*, std::allocator const&, Hypertable::TableIdentifier*&&, std::shared_ptr&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*&&, Hypertable::AccessGroup::Hints const*&) (this=0x7fff6a7c4388, __a=...) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/shared_ptr_base.h:619 #11 0x0000000001072b18 in std::__shared_ptr::__shared_ptr, Hypertable::TableIdentifier*, std::shared_ptr&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*, Hypertable::AccessGroup::Hints const*&>(std::_Sp_make_shared_tag, std::allocator const&, Hypertable::TableIdentifier*&&, std::shared_ptr&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*&&, Hypertable::AccessGroup::Hints const*&) (this=0x7fff6a7c4380, __tag=..., __a=...) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/shared_ptr_base.h:1090 #12 0x0000000001070e5a in std::shared_ptr::shared_ptr, Hypertable::TableIdentifier*, std::shared_ptr&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*, Hypertable::AccessGroup::Hints const*&>(std::_Sp_make_shared_tag, std::allocator const&, Hypertable::TableIdentifier*&&, std::shared_ptr&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*&&, Hypertable::AccessGroup::Hints const*&) (this=0x7fff6a7c4380, __tag=..., __a=...) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/shared_ptr.h:316 #13 0x000000000106ec31 in std::allocate_shared, Hypertable::TableIdentifier*, std::shared_ptr&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*, Hypertable::AccessGroup::Hints const*&>(std::allocator const&, Hypertable::TableIdentifier*&&, std::shared_ptr&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*&&, Hypertable::AccessGroup::Hints const*&) (__a=...) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/shared_ptr.h:588 #14 0x000000000106d056 in std::make_shared&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*, Hypertable::AccessGroup::Hints const*&>(Hypertable::TableIdentifier*&&, std::shared_ptr&, Hypertable::AccessGroupSpec*&, Hypertable::RangeSpecManaged*&&, Hypertable::AccessGroup::Hints const*&) () at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/shared_ptr.h:604 #15 0x000000000105baaa in Hypertable::Range::initialize (this=0x337d2b0) at /home/doug/src/hypertable/src/cc/Hypertable/RangeServer/Range.cc:168 #16 0x000000000105af7a in Hypertable::Range::Range (this=0x337d2b0, master_client=..., schema=..., range_entity=..., range_set=0x3348e70) at /home/doug/src/hypertable/src/cc/Hypertable/RangeServer/Range.cc:90 #17 0x0000000000f2fb70 in __gnu_cxx::new_allocator::construct&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*> > (this=0x7fff6a7c469f, __p=0x337d2b0) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/ext/new_allocator.h:120 #18 0x0000000000f2e922 in std::allocator_traits >::_S_construct&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*> >(std::allocator&, std::allocator_traits >::__construct_helper*, (Hypertable::Range&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*>&&)...) (__a=..., __p=0x337d2b0) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/alloc_traits.h:253 #19 0x0000000000f2d93a in std::allocator_traits >::construct&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*> >(std::allocator&, Hypertable::Range&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*>*, (Hypertable::Range&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*>&&)...) (__a=..., __p=0x337d2b0) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/alloc_traits.h:399 #20 0x0000000000f2c214 in std::_Sp_counted_ptr_inplace, (__gnu_cxx::_Lock_policy)2>::_Sp_counted_ptr_inplace&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*>(std::allocator, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*&&) (this=0x337d2a0, __a=...) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/shared_ptr_base.h:515 #21 0x0000000000f29574 in __gnu_cxx::new_allocator, (__gnu_cxx::_Lock_policy)2> >::construct, (__gnu_cxx::_Lock_policy)2> const, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*> > (this=0x7fff6a7c4807, __p=0x337d2a0) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/ext/new_allocator.h:120 #22 0x0000000000f26e90 in std::allocator_traits, (__gnu_cxx::_Lock_policy)2> > >::_S_construct, (__gnu_cxx::_Lock_policy)2> const, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*> >(std::allocator, (__gnu_cxx::_Lock_policy)2> >&, std::allocator_traits, (__gnu_cxx::_Lock_policy)2> > >::__construct_helper*, (std::_Sp_counted_ptr_inplace, (__gnu_cxx::_Lock_policy)2> const, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*>&&)...) (__a=..., __p=0x337d2a0) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/alloc_traits.h:253 #23 0x0000000000f2398d in std::allocator_traits, (__gnu_cxx::_Lock_policy)2> > >::construct, (__gnu_cxx::_Lock_policy)2> const, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*> >(std::allocator, (__gnu_cxx::_Lock_policy)2> >&, std::_Sp_counted_ptr_inplace, (__gnu_cxx::_Lock_policy)2> const, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*>*, (std::_Sp_counted_ptr_inplace, (__gnu_cxx::_Lock_policy)2> const, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*>&&)...) (__a=..., __p=0x337d2a0) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/alloc_traits.h:399 #24 0x0000000000f1eae9 in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*>(std::_Sp_make_shared_tag, Hypertable::Range*, std::allocator const&, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*&&) (this=0x7fff6a7c5cf8, __a=...) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/shared_ptr_base.h:619 #25 0x0000000000f17905 in std::__shared_ptr::__shared_ptr, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*>(std::_Sp_make_shared_tag, std::allocator const&, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*&&) (this=0x7fff6a7c5cf0, __tag=..., __a=...) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/shared_ptr_base.h:1090 #26 0x0000000000f0e4a4 in std::shared_ptr::shared_ptr, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*>(std::_Sp_make_shared_tag, std::allocator const&, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*&&) (this=0x7fff6a7c5cf0, __tag=..., __a=...) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/shared_ptr.h:316 #27 0x0000000000f04908 in std::allocate_shared, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*>(std::allocator const&, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*&&) (__a=...) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/shared_ptr.h:588 #28 0x0000000000efb9c6 in std::make_shared&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*>(std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, Hypertable::TableInfo*&&) () at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/shared_ptr.h:604 #29 0x0000000000ec8924 in Hypertable::Apps::RangeServer::replay_load_range (this=0x3306e90, replay_map=..., range_entity=...) at /home/doug/src/hypertable/src/cc/Hypertable/RangeServer/RangeServer.cc:1044 #30 0x0000000000ec61b2 in Hypertable::Apps::RangeServer::local_recover (this=0x3306e90) at /home/doug/src/hypertable/src/cc/Hypertable/RangeServer/RangeServer.cc:865 #31 0x0000000000ec1268 in Hypertable::Apps::RangeServer::RangeServer (this=0x3306e90, props=..., conn_mgr=..., app_queue=..., hyperspace=...) at /home/doug/src/hypertable/src/cc/Hypertable/RangeServer/RangeServer.cc:364 #32 0x0000000000eac0f4 in __gnu_cxx::new_allocator::construct&, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&> > (this=0x7fff6a7c945f, __p=0x3306e90) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/ext/new_allocator.h:120 #33 0x0000000000eabbf6 in std::allocator_traits >::_S_construct&, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&> >(std::allocator&, std::allocator_traits >::__construct_helper*, (Hypertable::Apps::RangeServer&, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&>&&)...) (__a=..., __p=0x3306e90) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/alloc_traits.h:253 #34 0x0000000000eab8d0 in std::allocator_traits >::construct&, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&> >(std::allocator&, Hypertable::Apps::RangeServer&, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&>*, (Hypertable::Apps::RangeServer&, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&>&&)...) (__a=..., __p=0x3306e90) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/alloc_traits.h:399 #35 0x0000000000eab43a in std::_Sp_counted_ptr_inplace, (__gnu_cxx::_Lock_policy)2>::_Sp_counted_ptr_inplace&, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&> (this=0x3306e80, __a=...) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/shared_ptr_base.h:515 #36 0x0000000000eaacbc in __gnu_cxx::new_allocator, (__gnu_cxx::_Lock_policy)2> >::construct, (__gnu_cxx::_Lock_policy)2> const, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&> > (this=0x7fff6a7c95c7, __p=0x3306e80) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/ext/new_allocator.h:120 #37 0x0000000000eaa404 in std::allocator_traits, (__gnu_cxx::_Lock_policy)2> > >::_S_construct, (__gnu_cxx::_Lock_policy)2> const, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&> >(std::allocator, (__gnu_cxx::_Lock_policy)2> >&, std::allocator_traits, (__gnu_cxx::_Lock_policy)2> > >::__construct_helper*, (std::_Sp_counted_ptr_inplace, (__gnu_cxx::_Lock_policy)2> const, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&>&&)...) (__a=..., __p=0x3306e80) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/alloc_traits.h:253 #38 0x0000000000ea9233 in std::allocator_traits, (__gnu_cxx::_Lock_policy)2> > >::construct, (__gnu_cxx::_Lock_policy)2> const, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&> >(std::allocator, (__gnu_cxx::_Lock_policy)2> >&, std::_Sp_counted_ptr_inplace, (__gnu_cxx::_Lock_policy)2> const, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&>*, (std::_Sp_counted_ptr_inplace, (__gnu_cxx::_Lock_policy)2> const, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&>&&)...) (__a=..., __p=0x3306e80) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/alloc_traits.h:399 #39 0x0000000000ea8115 in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&, std::shared_ptr&> (this=0x7fff6a7ca948, __a=...) at /usr/lib/gcc/x86_64-redhat-linux/4.9.2/include/c++/bits/shared_ptr_base.h:619 #40 0x0000000000ea643f in std::__shared_ptr::__shared_ptr, std::shared_ptr&, std::shared_ptr&, std::shared_ptr
Here's the test output:
135/283 Testing: METADATA-split-recovery-7 135/283 Test: METADATA-split-recovery-7 Command: "/bin/env" "TEST_7=1" "SKIP_START_SERVERS=1" "INSTALL_DIR=/opt/hypertable/doug/0.9.8.8" "bash" "-x" "/home/doug/src/hypertable/tests/integration/metadata-split /run.sh" Directory: /home/doug/build/hypertable/debug/tests/integration/metadata-split "METADATA-split-recovery-7" start time: Jun 11 10:44 PDT Output: ---------------------------------------------------------- + HT_HOME=/opt/hypertable/doug/0.9.8.8 + HYPERTABLE_HOME=/opt/hypertable/doug/0.9.8.8 + PIDFILE=/opt/hypertable/doug/0.9.8.8/run/RangeServer.pid ++ dirname /home/doug/src/hypertable/tests/integration/metadata-split/run.sh + SCRIPT_DIR=/home/doug/src/hypertable/tests/integration/metadata-split + startlog=/tmp/start-metadata-split32189.log + . /opt/hypertable/doug/0.9.8.8/bin/ht-env.sh ++ ulimit -c unlimited ++ ulimit -v unlimited ++ '[' e == e ']' ++ RUNTIME_ROOT=/opt/hypertable/doug/0.9.8.8 ++ '[' /opt/hypertable/doug/0.9.8.8 ']' ++ '[' -d /opt/hypertable/doug/0.9.8.8/run ']' ++ '[' -d /opt/hypertable/doug/0.9.8.8/log ']' ++ export LD_LIBRARY_PATH=/opt/hypertable/doug/0.9.8.8/lib:/opt/hypertable/doug/0.9.8.8/lib: ++ LD_LIBRARY_PATH=/opt/hypertable/doug/0.9.8.8/lib:/opt/hypertable/doug/0.9.8.8/lib: ++ export DYLD_LIBRARY_PATH=/opt/hypertable/doug/0.9.8.8/lib:/opt/hypertable/doug/0.9.8.8/lib::/opt/hypertable/doug/0.9.8.8/lib:: ++ DYLD_LIBRARY_PATH=/opt/hypertable/doug/0.9.8.8/lib:/opt/hypertable/doug/0.9.8.8/lib::/opt/hypertable/doug/0.9.8.8/lib:: + env + grep '^TEST_[0-9]' TEST_7=1 + '[' '' ']' + '[' '' ']' + '[' '' ']' + '[' '' ']' + '[' '' ']' + '[' '' ']' + '[' '' ']' + '[' 1 ']' + run_test 7 --induce-failure=metadata-load-range-3:exit:2 + local TEST_ID=7 + shift + touch running.7 + stop_range_server + '[' -f /opt/hypertable/doug/0.9.8.8/run/RangeServer.pid ']' ++ cat /opt/hypertable/doug/0.9.8.8/run/Master.pid + kill -9 31425 + rm -f /opt/hypertable/doug/0.9.8.8/run/Master.pid + '[' 7 == 9 ']' + '[' 7 == 10 ']' + '[' 7 == 11 ']' + '[' 7 == 12 ']' + /opt/hypertable/doug/0.9.8.8/bin/ht-start-test-servers.sh --clear --no-rangeserver --no-thriftbroker --Hypertable.Master.Gc.Interval=30000 --Hypertable.RangeServer.Ra nge.SplitSize=18K --Hypertable.RangeServer.Range.MetadataSplitSize=10K FsBroker.local appears to be running (31256): doug 31256 31254 1 10:43 pts/0 00:00:01 /opt/hypertable/doug/0.9.8.8/bin/htFsBrokerLocal --pidfile /opt/hypertable/doug/0.9.8.8/run/FsBroker.local.pid --verbose --Hyper table.Master.Gc.Interval=30000 --Hypertable.RangeServer.Range.SplitSize=18K --Hypertable.RangeServer.Range.MetadataSplitSize=10K Killing Hyperspace.pid 31313 Shutdown ThriftBroker complete Shutdown RangeServer complete Shutdown Hyperspace complete Shutdown Master complete Removed fs://hypertable/servers Removed fs://hypertable/tables Cleared /opt/hypertable/doug/0.9.8.8/hyperspace Cleared /opt/hypertable/doug/0.9.8.8/run Cleared /opt/hypertable/doug/0.9.8.8/fs Shutdown FsBroker complete FsBroker (local) Started Test time = 10.76 sec ---------------------------------------------------------- Test Failed. "METADATA-split-recovery-7" end time: Jun 11 10:44 PDT "METADATA-split-recovery-7" time elapsed: 00:00:10
This happened on a centos64 build machine with a release build.
+ echo 'USE '\''/'\''; LOAD DATA INFILE '\''dump.tsv'\'' INTO TABLE LoadTest;' + /opt/hypertable/0.9.8.7.f0d7b70/bin/ht shell --batch Loading 693,292 bytes of input data... 0% 10 20 30 40 50 60 70 80 90 100% |----|----|----|----|----|----|----|----|----|----| *************************************************** Load complete. Elapsed time: 0.03 s Avg value size: 150.00 bytes Avg key size: 30.00 bytes Throughput: 21734594.07 bytes/s (25116545.30 bytes/s) Total cells: 3333 Throughput: 120747.74 cells/s Resends: 0 + echo 'USE '\''/'\''; SELECT * FROM LoadTest INTO FILE '\''select-b.7'\'';' + /opt/hypertable/0.9.8.7.f0d7b70/bin/ht shell --batch 1428054233 ERROR ht_hypertable : dump_error_history (/root/src/hypertable/src/cc/Hypertable/Lib/RangeLocator.h:157): Hypertable::Exception: Problem creating s canner for start row '0/0:3:000000000004580155557679878442' on METADATA[..??] - HYPERTABLE request timeout at int Hypertable::RangeLocator::find(const Hypertable::TableIdentifier*, const char*, Hypertable::RangeLocationInfo*, Hypertable::Timer&, bool) (/roo t/src/hypertable/src/cc/Hypertable/Lib/RangeLocator.cc:372) at void Hypertable::Lib::RangeServer::Client::do_create_scanner(const Hypertable::CommAddress&, const Hypertable::TableIdentifier&, const Hypertable:: RangeSpec&, const Hypertable::Lib::ScanSpec&, Hypertable::ScanBlock&, int32_t) (/root/src/hypertable/src/cc/Hypertable/Lib/RangeServer/Client.cc:282): RangeSe rver create_scanner() failure : Event: type=ERROR "HYPERTABLE request timeout" from=10.31.152.249:15865 1428054233 ERROR ht_hypertable : retry_or_abort (/root/src/hypertable/src/cc/Hypertable/Lib/IntervalScannerAsync.cc:397): Hypertable::Exception: Locating rang e for table 3 row='000000000004580155557679878442' - HYPERTABLE request timeout at void Hypertable::IntervalScannerAsync::find_range_and_start_scan(const char*, bool) (/root/src/hypertable/src/cc/Hypertable/Lib/IntervalScannerAsyn c.cc:248) at void Hypertable::RangeLocator::find_loop(const Hypertable::TableIdentifier*, const char*, Hypertable::RangeLocationInfo*, Hypertable::Timer&, bool) (/root/src/hypertable/src/cc/Hypertable/Lib/RangeLocator.cc:249): Locating range for table 3 row='000000000004580155557679878442' 1428054233 ERROR ht_hypertable : handle_error (/root/src/hypertable/src/cc/Hypertable/Lib/TableScannerAsync.cc:493): Hypertable::Exception: Event: type=MESSAG E version=1 total_len=86 header_len=38 header_checksum=-1925115986 flags=8 id=34734 gid=0 timeout_ms=0 payload_checksum=0 command=2 from=10.31.152.249:15865 - RANGE SERVER range not found Hypertable CRITICAL - RANGE SERVER range not found (Event: type=MESSAGE version=1 total_len=86 header_len=38 header_checksum=-1925115986 flags=8 id=34734 gid= 0 timeout_ms=0 payload_checksum=0 command=2 from=10.31.152.249:15865)