MB-19251: Fix race in updating Vbucket.file{SpaceUsed,Size} 71/62971/6
authorDave Rigby <daver@couchbase.com>
Wed, 12 Nov 2014 17:52:41 +0000 (17:52 +0000)
committerChiyoung Seo <chiyoung@couchbase.com>
Sat, 23 Apr 2016 01:04:47 +0000 (01:04 +0000)
These variables are used in the calculation of ep_db_data_size and
ep_db_file_size stats, and crucially those stats are used by ns_server
when determining if a bucket should be compacted.

It is possible that due to this issue, compaction may not be triggered
when expected, or triggered when it shouldn't.

As identified by ThreadSantizer:

    WARNING: ThreadSanitizer: data race (pid=4009)
      Write of size 8 at 0x7d440000c5b0 by thread T6 (mutexes: write M55):
#0 KVStatsCallback::callback(KVStatsCtx&) /repos/couchbase/server/source/ep-engine/src/ep.cc:933 (ep.so+0x0000000f0a22)
#1 CouchKVStore::commit2couchstore(Callback<KVStatsCtx>*, unsigned long, unsigned long) /repos/couchbase/server/source/ep-engine/src/couch-kvstore/couch-kvstore.cc:1697 (ep.so+0x0000001aa8c6)
#2 CouchKVStore::commit(Callback<KVStatsCtx>*, unsigned long, unsigned long) /repos/couchbase/server/source/ep-engine/src/couch-kvstore/couch-kvstore.cc:1040 (ep.so+0x0000001a6483)
#3 EventuallyPersistentStore::flushVBucket(unsigned short) /repos/couchbase/server/source/ep-engine/src/ep.cc:2909 (ep.so+0x0000000e780b)
#4 Flusher::flushVB() /repos/couchbase/server/source/ep-engine/src/flusher.cc:283 (ep.so+0x00000013f363)
#5 Flusher::step(GlobalTask*) /repos/couchbase/server/source/ep-engine/src/flusher.cc:174 (ep.so+0x00000013e9c8)
#6 FlusherTask::run() /repos/couchbase/server/source/ep-engine/src/tasks.cc:44 (ep.so+0x000000174a85)
#7 ExecutorThread::run() /repos/couchbase/server/source/ep-engine/src/executorthread.cc:110 (ep.so+0x00000014a0c1)
#8 launch_executor_thread /repos/couchbase/server/source/ep-engine/src/executorthread.cc:34 (ep.so+0x00000014990a)
#9 platform_thread_wrap /repos/couchbase/server/source/platform/src/cb_pthreads.c:19 (libplatform.so.0.1.0+0x000000002d8b)
#10 __tsan_write_range ??:0 (libtsan.so.0+0x00000001b1c9)

      Previous read of size 8 at 0x7d440000c5b0 by main thread (mutexes: write M193510842443017784):
#0 VBucketCountVisitor::visitBucket(RCPtr<VBucket>&) /repos/couchbase/server/source/ep-engine/src/ep_engine.cc:2889 (ep.so+0x00000010c631)
#1 VBucketCountAggregator::visitBucket(RCPtr<VBucket>&) /repos/couchbase/server/source/ep-engine/src/ep_engine.cc:2926 (ep.so+0x000000121392)
#2 EventuallyPersistentStore::visit(VBucketVisitor&) /repos/couchbase/server/source/ep-engine/src/ep.cc:3278 (ep.so+0x0000000e99d9)
#3 EventuallyPersistentEngine::doEngineStats(void const*, void (*)(char const*, unsigned short, char const*, unsigned int, void const*)) /repos/couchbase/server/source/ep-engine/src/ep_engine.cc:2955 (ep.so+0x00000010cb46)
#4 EventuallyPersistentEngine::getStats(void const*, char const*, int, void (*)(char const*, unsigned short, char const*, unsigned int, void const*)) /repos/couchbase/server/source/ep-engine/src/ep_engine.cc:4344 (ep.so+0x000000113c0f)
#5 EvpGetStats /repos/couchbase/server/source/ep-engine/src/ep_engine.cc:217 (ep.so+0x000000102b14)
#6 mock_get_stats /repos/couchbase/server/source/memcached/programs/engine_testapp/engine_testapp.c:195 (exe+0x0000000026de)
#7 get_int_stat(engine_interface*, engine_interface_v1*, char const*, char const*) /repos/couchbase/server/source/ep-engine/tests/ep_test_apis.cc:799 (ep_testsuite.so+0x0000000832d8)
#8 wait_for_flusher_to_settle(engine_interface*, engine_interface_v1*) /repos/couchbase/server/source/ep-engine/tests/ep_test_apis.cc:900 (ep_testsuite.so+0x000000083cd4)
#9 wait_for_persisted_value(engine_interface*, engine_interface_v1*, char const*, char const*, unsigned short) /repos/couchbase/server/source/ep-engine/tests/ep_test_apis.cc:917 (ep_testsuite.so+0x000000083de5)
#10 test_io_stats /repos/couchbase/server/source/ep-engine/tests/ep_testsuite.cc:6279 (ep_testsuite.so+0x0000000468b3)
#11 execute_test /repos/couchbase/server/source/memcached/programs/engine_testapp/engine_testapp.c:1055 (exe+0x0000000059fc)
#12 main /repos/couchbase/server/source/memcached/programs/engine_testapp/engine_testapp.c:1313 (exe+0x000000006606)

Change to AtomicValues to fix this.

Change-Id: Ie7f9a403f809e751ff1802cceb5d2a77a483a586
Reviewed-on: http://review.couchbase.org/62971
Well-Formed: buildbot <build@couchbase.com>
Reviewed-by: Jim Walker <jim@couchbase.com>
Reviewed-by: Will Gardner <will.gardner@couchbase.com>
Tested-by: buildbot <build@couchbase.com>
src/vbucket.h

index f41ecfe..864ef71 100644 (file)
@@ -426,8 +426,8 @@ public:
     AtomicValue<size_t>  metaDataDisk;
 
     AtomicValue<size_t>  numExpiredItems;
-    volatile size_t  fileSpaceUsed;
-    volatile size_t  fileSize;
+    AtomicValue<size_t>  fileSpaceUsed;
+    AtomicValue<size_t>  fileSize;
 
 private:
     template <typename T>