MB-20725: Update drift accounting stats

The bucket now reports a total of drift for all VBs which will
allow better tracking of drift (e.g. UI graphing).

Change-Id: I5600df323b32f3f1d11feb5be2cf8ddac5988be9
Reviewed-on: http://review.couchbase.org/69328
Reviewed-by: Dave Rigby <daver@couchbase.com>
Tested-by: buildbot <build@couchbase.com>

diff --git a/docs/stats.org b/docs/stats.org
index f2503a0..df5a0e4 100644 (file)
--- a/docs/stats.org
+++ b/docs/stats.org
@@ -335,7 +335,24 @@ For introductory information on stats within Couchbase, start with the
 |                                    | remover will start cursor dropping     |
 | ep_cursors_dropped                 | Number of cursors dropped by the       |
 |                                    | checkpoint remover                     |
-
+| ep_active_hlc_drift                | The total absolute drift for all active|
+|                                    | vbuckets.                              |
+| ep_active_hlc_drift_count          | The number of updates applied to       |
+|                                    | ep_active_hlc_drift.                   |
+| ep_replica_hlc_drift               | The total absolute drift for all       |
+|                                    | replica vbucket.                       |
+| ep_replica_hlc_drift_count         | The number of updates applied to       |
+|                                    | ep_replica_hlc_drift.                  |
+| ep_active_ahead_exceptions         | The total number of ahead exceptions   |
+|                                    | for all active vbuckets.               |
+| ep_active_behind_exceptions        | The total number of behind exceptions  |
+|                                    | for all active vbuckets.               |
+| ep_replica_ahead_exceptions        | The total number of ahead exceptions   |
+|                                    | for all replica vbuckets.              |
+| ep_replica_behind_exceptions       | The total number of behind exceptions  |
+|                                    | for all replica vbuckets.              |
+| ep_clock_cas_drift_threshold_excee-| ep_active_ahead_exceptions +           |
+| ded                                | ep_replica_ahead_exceptions            |
 
 ** vBucket total stats
 
@@ -471,12 +488,23 @@ The stats below are listed for each vbucket.
 | bloom_filter_key_count        | Number of keys inserted into the bloom     |
 |                               | filter, considers overlapped items as one, |
 |                               | so this may not be accurate at times.      |
-| max_cas                       | Maximum CAS of all items in the vbucket    |
-| drift_counter                 | Drift counter value for vbucket used for   |
-|                               | time synchronization                       |
-| time_sync                     | Indicates if time synchronization is ON/OFF|
 | uuid                          | The current vbucket uuid                   |
 | rollback_item_count           | Num of items rolled back                   |
+| max_cas                       | Maximum CAS of all items in the vbucket.   |
+| total_abs_drift               | The accumulated absolute drift for this    |
+|                               | vbucket's hybrid logical clock.            |
+| total_abs_drift_count         | The number of updates applied to           |
+|                               | total_abs_drift.                           |
+| drift_ahead_threshold_exceeded| The number of HLC updates that had a value |
+|                               | ahead of the local HLC and were over the   |
+|                               | drift_ahead_threshold.                     |
+| drift_ahead_threshold         | The ahead threshold in us.                 |
+|drift_behind_threshold_exceeded| The number of HLC updates that had a value |
+|                               | behind the local HLC and were over the     |
+|                               | drift_behind_threshold.                    |
+| drift_behind_threshold        | The behind threshold in us.                |
+| logical_clock_ticks           | How many times this vbucket's HLC has      |
+|                               | returned logical clock ticks.              |
 
 ** vBucket seqno stats
 

diff --git a/src/ep_engine.cc b/src/ep_engine.cc
index 366344d..54972a5 100644 (file)
--- a/src/ep_engine.cc
+++ b/src/ep_engine.cc
@@ -3118,13 +3118,11 @@ bool VBucketCountVisitor::visitBucket(RCPtr<VBucket> &vb) {
         rollbackItemCount += vb->getRollbackItemCount();
 
         /*
-         * The bucket stat only reports the largest drift of the vbuckets.
+         * The bucket stat reports the total drift of the vbuckets.
          */
-        auto driftStats = vb->getHLCDriftStats();
-        // If this vbucket's max is bigger than ours
-        if (driftStats.total > maxAbsHLCDrift.total) {
-            maxAbsHLCDrift = driftStats;
-        }
+        auto absHLCDrift = vb->getHLCDriftStats();
+        totalAbsHLCDrift.total += absHLCDrift.total;
+        totalAbsHLCDrift.updates += absHLCDrift.updates;
 
         /*
          * Total up the exceptions
@@ -3713,13 +3711,13 @@ ENGINE_ERROR_CODE EventuallyPersistentEngine::doEngineStats(const void *cookie,
 
     // Add stats for tracking HLC drift
     add_casted_stat("ep_active_hlc_drift",
-        activeCountVisitor.getMaxAbsHLCDrift().total, add_stat, cookie);
+        activeCountVisitor.getTotalAbsHLCDrift().total, add_stat, cookie);
     add_casted_stat("ep_active_hlc_drift_count",
-        activeCountVisitor.getMaxAbsHLCDrift().updates, add_stat, cookie);
+        activeCountVisitor.getTotalAbsHLCDrift().updates, add_stat, cookie);
     add_casted_stat("ep_replica_hlc_drift",
-        replicaCountVisitor.getMaxAbsHLCDrift().total, add_stat, cookie);
+        replicaCountVisitor.getTotalAbsHLCDrift().total, add_stat, cookie);
     add_casted_stat("ep_replica_hlc_drift_count",
-        replicaCountVisitor.getMaxAbsHLCDrift().updates, add_stat, cookie);
+        replicaCountVisitor.getTotalAbsHLCDrift().updates, add_stat, cookie);
 
     add_casted_stat("ep_active_ahead_exceptions",
         activeCountVisitor.getTotalHLCDriftExceptionCounters().ahead,

diff --git a/src/ep_engine.h b/src/ep_engine.h
index 56d9a00..629ef0b 100644 (file)
--- a/src/ep_engine.h
+++ b/src/ep_engine.h
@@ -107,7 +107,7 @@ public:
         queueFill(0), queueDrain(0),
         pendingWrites(0), chkPersistRemaining(0),
         fileSpaceUsed(0), fileSize(0),
-        rollbackItemCount(0), maxAbsHLCDrift(),
+        rollbackItemCount(0), totalAbsHLCDrift(),
         totalHLCDriftExceptionCounters()
     { }
 
@@ -164,7 +164,7 @@ public:
 
     uint64_t getRollbackItemCount() { return rollbackItemCount; }
 
-    HLC::DriftStats getMaxAbsHLCDrift() {return maxAbsHLCDrift;}
+    HLC::DriftStats getTotalAbsHLCDrift() {return totalAbsHLCDrift;}
     HLC::DriftExceptions getTotalHLCDriftExceptionCounters() {return totalHLCDriftExceptionCounters;}
 
 private:
@@ -200,7 +200,7 @@ private:
     size_t fileSize;
 
     uint64_t rollbackItemCount;
-    HLC::DriftStats maxAbsHLCDrift;
+    HLC::DriftStats totalAbsHLCDrift;
     HLC::DriftExceptions totalHLCDriftExceptionCounters;
 };
 

diff --git a/tests/ep_testsuite_xdcr.cc b/tests/ep_testsuite_xdcr.cc
index ef64673..5500492 100644 (file)
--- a/tests/ep_testsuite_xdcr.cc
+++ b/tests/ep_testsuite_xdcr.cc
@@ -1606,7 +1606,7 @@ static enum test_result test_set_with_meta_and_check_drift_stats(ENGINE_HANDLE *
     // Bucket stats should report drift
     checkge(get_ull_stat(h, h1, "ep_active_hlc_drift"), uint64_t(0),
             "Expected drift above zero");
-    checkeq(uint64_t(n_keys), get_ull_stat(h, h1, "ep_active_hlc_drift_count"),
+    checkeq(uint64_t(n_keys*n_vbuckets), get_ull_stat(h, h1, "ep_active_hlc_drift_count"),
             "Expected ahead counter to match mutations");
 
     // Victim VBs should have exceptions
@@ -1713,7 +1713,7 @@ static enum test_result test_del_with_meta_and_check_drift_stats(ENGINE_HANDLE *
     // Bucket stats should report drift
     checkge(get_ull_stat(h, h1, "ep_active_hlc_drift"), uint64_t(0),
             "Expected drift above zero");
-    checkeq(uint64_t(n_keys*2), get_ull_stat(h, h1, "ep_active_hlc_drift_count"),
+    checkeq(2*uint64_t(n_keys*n_vbuckets), get_ull_stat(h, h1, "ep_active_hlc_drift_count"),
             "Expected ahead counter to match mutations");
 
     // and should report total exception of all VBs