MB-22695: Add stats for datatypes

This commit adds counts for datatypes of documents whose Storedvalue is
resident in memory.

Adds the datatype stats to the 'all' engine stat call. This means that
they will be added to the ETS tables in ns_server.

Introduces a regression test for MB-23892.

This patch was originally e4606e8 and was reverted by 6501c72 due to
MB-23892.

Change-Id: Ic0961af1a3c18362004369db0ff0fa7a9eeba22c
Reviewed-on: http://review.couchbase.org/76617
Tested-by: Build Bot <build@couchbase.com>
Reviewed-by: Dave Rigby <daver@couchbase.com>

diff --git a/src/hash_table.cc b/src/hash_table.cc
index 456c0ad..5c2edd9 100644 (file)
--- a/src/hash_table.cc
+++ b/src/hash_table.cc
@@ -48,6 +48,7 @@ HashTable::HashTable(EPStats& st,
       numTotalItems(0),
       numNonResidentItems(0),
       numDeletedItems(0),
+      datatypeCounts(),
       numEjects(0),
       memSize(0),
       cacheSize(0),
@@ -105,6 +106,7 @@ void HashTable::clear(bool deactivate) {
 
     stats.currentSize.fetch_sub(clearedMemSize - clearedValSize);
 
+    datatypeCounts.fill(0);
     numTotalItems.store(0);
     numItems.store(0);
     numTempItems.store(0);
@@ -290,6 +292,13 @@ MutationStatus HashTable::unlocked_updateStoredValue(
         --numDeletedItems;
     }
 
+    // If the item we are replacing is resident then we need to make sure we
+    // appropriately alter the datatype stats.
+    if (v.getDatatype() != itm.getDataType()) {
+        --datatypeCounts[v.getDatatype()];
+        ++datatypeCounts[itm.getDataType()];
+    }
+
     /* setValue() will mark v as undeleted if required */
     v.setValue(itm, *this);
     return status;
@@ -316,6 +325,7 @@ StoredValue* HashTable::unlocked_addNewStoredValue(const HashBucketLock& hbl,
     } else {
         ++numItems;
         ++numTotalItems;
+        ++datatypeCounts[v->getDatatype()];
     }
     values[hbl.getBucketNum()] = std::move(v);
 
@@ -361,6 +371,8 @@ void HashTable::unlocked_softDelete(const std::unique_lock<std::mutex>& htLock,
         decrNumNonResidentItems();
     }
 
+    --datatypeCounts[v.getDatatype()];
+
     if (onlyMarkDeleted) {
         v.markDeleted();
     } else {
@@ -432,6 +444,7 @@ StoredValue::UniquePtr HashTable::unlocked_release(
     } else {
         decrNumItems();
         decrNumTotalItems();
+        --datatypeCounts[released->getDatatype()];
         if (released->isDeleted()) {
             --numDeletedItems;
         }
@@ -624,7 +637,6 @@ bool HashTable::unlocked_ejectItem(StoredValue*& vptr,
         throw std::invalid_argument("HashTable::unlocked_ejectItem: "
                 "Unable to delete NULL StoredValue");
     }
-
     if (policy == VALUE_ONLY) {
         bool rv = vptr->ejectValue(*this, policy);
         if (rv) {
@@ -656,6 +668,7 @@ bool HashTable::unlocked_ejectItem(StoredValue*& vptr,
                                            // fully evicted.
             }
             decrNumItems(); // Decrement because the item is fully evicted.
+            --datatypeCounts[vptr->getDatatype()];
             ++numEjects;
             updateMaxDeletedRevSeqno(vptr->getRevSeqno());
 
@@ -692,6 +705,7 @@ bool HashTable::unlocked_restoreValue(
         /* set it back to false as we created a temp item by setting it to true
            when bg fetch is scheduled (full eviction mode). */
         v.setNewCacheItem(false);
+        ++datatypeCounts[itm.getDataType()];
     } else {
         decrNumNonResidentItems();
     }
@@ -722,5 +736,6 @@ void HashTable::unlocked_restoreMeta(const std::unique_lock<std::mutex>& htLock,
         --numTempItems;
         ++numItems;
         ++numNonResidentItems;
+        ++datatypeCounts[v.getDatatype()];
     }
 }

diff --git a/src/hash_table.h b/src/hash_table.h
index 55b9f58..11a6214 100644 (file)
--- a/src/hash_table.h
+++ b/src/hash_table.h
@@ -20,6 +20,7 @@
 #include "config.h"
 #include "storeddockey.h"
 #include "stored-value.h"
+#include <platform/non_negative_counter.h>
 
 class AbstractStoredValueFactory;
 class HashTableStatVisitor;
@@ -584,6 +585,8 @@ public:
     std::atomic<size_t>       numTotalItems;
     cb::NonNegativeCounter<size_t> numNonResidentItems;
     cb::NonNegativeCounter<size_t> numDeletedItems;
+    std::array<cb::NonNegativeCounter<size_t>, mcbp::datatype::highest + 1>
+            datatypeCounts;
     std::atomic<size_t>       numEjects;
     //! Memory consumed by items in this hashtable.
     std::atomic<size_t>       memSize;

diff --git a/src/kv_bucket.cc b/src/kv_bucket.cc
index 1d5b34a..5e29ccc 100644 (file)
--- a/src/kv_bucket.cc
+++ b/src/kv_bucket.cc
@@ -1334,6 +1334,18 @@ void KVBucket::appendAggregatedVBucketStats(VBucketCountVisitor& active,
             active.getTotalHLCDriftExceptionCounters().ahead +
                     replica.getTotalHLCDriftExceptionCounters().ahead);
 
+    for (uint8_t ii = 0; ii < active.getNumDatatypes(); ++ii) {
+        std::string name = "ep_active_datatype_";
+        name += mcbp::datatype::to_string(ii);
+        DO_STAT(name.c_str(), active.getDatatypeCount(ii));
+    }
+
+    for (uint8_t ii = 0; ii < replica.getNumDatatypes(); ++ii) {
+        std::string name = "ep_replica_datatype_";
+        name += mcbp::datatype::to_string(ii);
+        DO_STAT(name.c_str(), replica.getDatatypeCount(ii));
+    }
+
 #undef DO_STAT
 }
 

diff --git a/src/vb_count_visitor.cc b/src/vb_count_visitor.cc
index 29eedaa..7b2a1a8 100644 (file)
--- a/src/vb_count_visitor.cc
+++ b/src/vb_count_visitor.cc
@@ -63,5 +63,10 @@ void VBucketCountVisitor::visitBucket(VBucketPtr& vb) {
         auto driftExceptionCounters = vb->getHLCDriftExceptionCounters();
         totalHLCDriftExceptionCounters.ahead += driftExceptionCounters.ahead;
         totalHLCDriftExceptionCounters.behind += driftExceptionCounters.behind;
+
+        // Iterate over each datatype combination
+        for (uint8_t ii = 0; ii < datatypeCounts.size(); ++ii) {
+            datatypeCounts[ii] += vb->ht.datatypeCounts[ii];
+        }
     }
 }

diff --git a/src/vb_count_visitor.h b/src/vb_count_visitor.h
index 8508ea7..e3a4d75 100644 (file)
--- a/src/vb_count_visitor.h
+++ b/src/vb_count_visitor.h
@@ -54,6 +54,7 @@ public:
           backfillQueueSize(0),
           pendingWrites(0),
           chkPersistRemaining(0),
+          datatypeCounts{{0}},
           queueAge(0),
           rollbackItemCount(0),
           numHpVBReqs(0),
@@ -154,6 +155,13 @@ public:
         return chkPersistRemaining;
     }
 
+    size_t getDatatypeCount(protocol_binary_datatype_t datatype) const {
+        return datatypeCounts[datatype];
+    }
+    const size_t getNumDatatypes() const {
+        return datatypeCounts.size();
+    }
+
     uint64_t getRollbackItemCount() {
         return rollbackItemCount;
     }
@@ -197,6 +205,7 @@ private:
     size_t backfillQueueSize;
     size_t pendingWrites;
     size_t chkPersistRemaining;
+    std::array<size_t, mcbp::datatype::highest + 1> datatypeCounts;
     uint64_t queueAge;
     uint64_t rollbackItemCount;
     size_t numHpVBReqs;

diff --git a/tests/ep_testsuite.cc b/tests/ep_testsuite.cc
index f610538..1f2aef8 100644 (file)
--- a/tests/ep_testsuite.cc
+++ b/tests/ep_testsuite.cc
@@ -6500,6 +6500,14 @@ static enum test_result test_mb19687_fixed(ENGINE_HANDLE* h,
                 "ep_access_scanner_task_time",
                 "ep_active_ahead_exceptions",
                 "ep_active_behind_exceptions",
+                "ep_active_datatype_json",
+                "ep_active_datatype_json,xattr",
+                "ep_active_datatype_raw",
+                "ep_active_datatype_snappy",
+                "ep_active_datatype_snappy,json",
+                "ep_active_datatype_snappy,json,xattr",
+                "ep_active_datatype_snappy,xattr",
+                "ep_active_datatype_xattr",
                 "ep_active_hlc_drift",
                 "ep_active_hlc_drift_count",
                 "ep_backend",
@@ -6654,6 +6662,14 @@ static enum test_result test_mb19687_fixed(ENGINE_HANDLE* h,
                 "ep_queue_size",
                 "ep_replica_ahead_exceptions",
                 "ep_replica_behind_exceptions",
+                "ep_replica_datatype_json",
+                "ep_replica_datatype_json,xattr",
+                "ep_replica_datatype_raw",
+                "ep_replica_datatype_snappy",
+                "ep_replica_datatype_snappy,json",
+                "ep_replica_datatype_snappy,json,xattr",
+                "ep_replica_datatype_snappy,xattr",
+                "ep_replica_datatype_xattr",
                 "ep_replica_hlc_drift",
                 "ep_replica_hlc_drift_count",
                 "ep_replication_throttle_cap_pcnt",

diff --git a/tests/module_tests/stats_test.cc b/tests/module_tests/stats_test.cc
index c4717bd..af6030a 100644 (file)
--- a/tests/module_tests/stats_test.cc
+++ b/tests/module_tests/stats_test.cc
@@ -20,6 +20,8 @@
  */
 
 #include "stats_test.h"
+#include "evp_store_single_threaded_test.h"
+#include "test_helpers.h"
 
 #include <gmock/gmock.h>
 
@@ -57,6 +59,15 @@ std::map<std::string, std::string> StatTest::get_stat(const char* statkey) {
     return stats;
 }
 
+class DatatypeStatTest : public StatTest,
+                         public ::testing::WithParamInterface<std::string> {
+protected:
+    void SetUp() override {
+        config_string += std::string{"item_eviction_policy="} + GetParam();
+        StatTest::SetUp();
+    }
+};
+
 TEST_F(StatTest, vbucket_seqno_stats_test) {
     using namespace testing;
     const std::string vbucket = "vb_" + std::to_string(vbid);
@@ -71,3 +82,205 @@ TEST_F(StatTest, vbucket_seqno_stats_test) {
             Pair(vbucket + ":last_persisted_snap_start", "0"),
             Pair(vbucket + ":last_persisted_snap_end", "0")));
 }
+
+TEST_P(DatatypeStatTest, datatypesInitiallyZero) {
+    // Check that the datatype stats initialise to 0
+    auto vals = get_stat(nullptr);
+    EXPECT_EQ(0, std::stoi(vals["ep_active_datatype_snappy"]));
+    EXPECT_EQ(0, std::stoi(vals["ep_active_datatype_snappy,json"]));
+    EXPECT_EQ(0, std::stoi(vals["ep_active_datatype_snappy,xattr"]));
+    EXPECT_EQ(0, std::stoi(vals["ep_active_datatype_json"]));
+    EXPECT_EQ(0, std::stoi(vals["ep_active_datatype_json,xattr"]));
+    EXPECT_EQ(0, std::stoi(vals["ep_active_datatype_raw"]));
+    EXPECT_EQ(0, std::stoi(vals["ep_active_datatype_xattr"]));
+    EXPECT_EQ(0, std::stoi(vals["ep_active_datatype_snappy,json,xattr"]));
+
+    EXPECT_EQ(0, std::stoi(vals["ep_replica_datatype_snappy"]));
+    EXPECT_EQ(0, std::stoi(vals["ep_replica_datatype_snappy,json"]));
+    EXPECT_EQ(0, std::stoi(vals["ep_replica_datatype_snappy,xattr"]));
+    EXPECT_EQ(0, std::stoi(vals["ep_replica_datatype_json"]));
+    EXPECT_EQ(0, std::stoi(vals["ep_replica_datatype_json,xattr"]));
+    EXPECT_EQ(0, std::stoi(vals["ep_replica_datatype_raw"]));
+    EXPECT_EQ(0, std::stoi(vals["ep_replica_datatype_xattr"]));
+    EXPECT_EQ(0, std::stoi(vals["ep_replica_datatype_snappy,json,xattr"]));
+}
+
+void setDatatypeItem(KVBucket* store,
+                     const void* cookie,
+                     protocol_binary_datatype_t datatype,
+                     std::string name, std::string val = "[0]") {
+    Item item(make_item(
+            0, {name, DocNamespace::DefaultCollection}, val, 0, datatype));
+    store->set(item, cookie);
+}
+
+TEST_P(DatatypeStatTest, datatypeJsonToXattr) {
+    setDatatypeItem(store, cookie, PROTOCOL_BINARY_DATATYPE_JSON, "jsonDoc");
+    auto vals = get_stat(nullptr);
+    EXPECT_EQ(1, std::stoi(vals["ep_active_datatype_json"]));
+
+    // Check that updating an items datatype works
+    setDatatypeItem(store, cookie, PROTOCOL_BINARY_DATATYPE_XATTR, "jsonDoc");
+    vals = get_stat(nullptr);
+
+    EXPECT_EQ(1, std::stoi(vals["ep_active_datatype_xattr"]));
+    EXPECT_EQ(0, std::stoi(vals["ep_active_datatype_json"]));
+}
+
+TEST_P(DatatypeStatTest, datatypeRawStatTest) {
+    setDatatypeItem(store, cookie, 0, "rawDoc");
+    auto vals = get_stat(nullptr);
+    EXPECT_EQ(1, std::stoi(vals["ep_active_datatype_raw"]));
+}
+
+TEST_P(DatatypeStatTest, datatypeXattrStatTest) {
+    setDatatypeItem(store, cookie, PROTOCOL_BINARY_DATATYPE_XATTR, "xattrDoc");
+    auto vals = get_stat(nullptr);
+    EXPECT_EQ(1, std::stoi(vals["ep_active_datatype_xattr"]));
+    // Update the same key with a different value. The datatype stat should
+    // stay the same
+    setDatatypeItem(store, cookie, PROTOCOL_BINARY_DATATYPE_XATTR,
+                    "xattrDoc", "[2]");
+    vals = get_stat(nullptr);
+    EXPECT_EQ(1, std::stoi(vals["ep_active_datatype_xattr"]));
+}
+
+TEST_P(DatatypeStatTest, datatypeCompressedStatTest) {
+    setDatatypeItem(store,
+                    cookie,
+                    PROTOCOL_BINARY_DATATYPE_SNAPPY,
+                    "compressedDoc");
+    auto vals = get_stat(nullptr);
+    EXPECT_EQ(1, std::stoi(vals["ep_active_datatype_snappy"]));
+}
+
+TEST_P(DatatypeStatTest, datatypeCompressedJson) {
+    setDatatypeItem(
+            store,
+            cookie,
+            PROTOCOL_BINARY_DATATYPE_JSON | PROTOCOL_BINARY_DATATYPE_SNAPPY,
+            "jsonCompressedDoc");
+    auto vals = get_stat(nullptr);
+    EXPECT_EQ(1, std::stoi(vals["ep_active_datatype_snappy,json"]));
+}
+
+TEST_P(DatatypeStatTest, datatypeCompressedXattr) {
+    setDatatypeItem(store,
+                    cookie,
+                    PROTOCOL_BINARY_DATATYPE_XATTR |
+                            PROTOCOL_BINARY_DATATYPE_SNAPPY,
+                    "xattrCompressedDoc");
+    auto vals = get_stat(nullptr);
+    EXPECT_EQ(1, std::stoi(vals["ep_active_datatype_snappy,xattr"]));
+}
+
+TEST_P(DatatypeStatTest, datatypeJsonXattr) {
+    setDatatypeItem(
+            store,
+            cookie,
+            PROTOCOL_BINARY_DATATYPE_JSON | PROTOCOL_BINARY_DATATYPE_XATTR,
+            "jsonXattrDoc");
+    auto vals = get_stat(nullptr);
+    EXPECT_EQ(1, std::stoi(vals["ep_active_datatype_json,xattr"]));
+}
+
+TEST_P(DatatypeStatTest, datatypeDeletion) {
+    setDatatypeItem(
+            store,
+            cookie,
+            PROTOCOL_BINARY_DATATYPE_JSON | PROTOCOL_BINARY_DATATYPE_XATTR,
+            "jsonXattrDoc");
+    auto vals = get_stat(nullptr);
+    EXPECT_EQ(1, std::stoi(vals["ep_active_datatype_json,xattr"]));
+    uint64_t cas = 0;
+    store->deleteItem({"jsonXattrDoc", DocNamespace::DefaultCollection},
+                      cas,
+                      0,
+                      cookie,
+                      nullptr,
+                      nullptr,
+                      nullptr);
+    vals = get_stat(nullptr);
+    EXPECT_EQ(0, std::stoi(vals["ep_active_datatype_json,xattr"]));
+}
+
+TEST_P(DatatypeStatTest, datatypeCompressedJsonXattr) {
+    setDatatypeItem(store,
+                    cookie,
+                    PROTOCOL_BINARY_DATATYPE_JSON |
+                            PROTOCOL_BINARY_DATATYPE_SNAPPY |
+                            PROTOCOL_BINARY_DATATYPE_XATTR,
+                    "jsonCompressedXattrDoc");
+    auto vals = get_stat(nullptr);
+    EXPECT_EQ(1, std::stoi(vals["ep_active_datatype_snappy,json,xattr"]));
+}
+
+TEST_P(DatatypeStatTest, datatypeExpireItem) {
+    Item item(make_item(
+            0, {"expiryDoc", DocNamespace::DefaultCollection}, "[0]", 1,
+            PROTOCOL_BINARY_DATATYPE_JSON));
+    store->set(item, cookie);
+    store->get({"expiryDoc", DocNamespace::DefaultCollection}, 0, cookie, NONE);
+    auto vals = get_stat(nullptr);
+
+    //Should be 0, becuase the doc should have expired
+    EXPECT_EQ(0, std::stoi(vals["ep_active_datatype_json"]));
+}
+
+
+TEST_P(DatatypeStatTest, datatypeEviction) {
+    const DocKey key = {"jsonXattrDoc", DocNamespace::DefaultCollection};
+    setDatatypeItem(
+            store,
+            cookie,
+            PROTOCOL_BINARY_DATATYPE_JSON | PROTOCOL_BINARY_DATATYPE_XATTR,
+            "jsonXattrDoc");
+    auto vals = get_stat(nullptr);
+    EXPECT_EQ(1, std::stoi(vals["ep_active_datatype_json,xattr"]));
+    store->flushVBucket(0);
+    const char* msg;
+    store->evictKey(key, 0, &msg);
+    vals = get_stat(nullptr);
+    if (GetParam() == "value_only"){
+        // Should still be 1 as only value is evicted
+        EXPECT_EQ(1, std::stoi(vals["ep_active_datatype_json,xattr"]));
+    } else {
+        // Should be 0 as everything is evicted
+        EXPECT_EQ(0, std::stoi(vals["ep_active_datatype_json,xattr"]));
+    }
+
+    store->get(key, 0, cookie, QUEUE_BG_FETCH);
+    if (GetParam() == "full_eviction") {
+        // Run the bgfetch to restore the item from disk
+        ExTask task = new SingleBGFetcherTask(
+                engine.get(), key, 0, cookie, false, 0, false);
+        task_executor->schedule(task);
+        runNextTask(*task_executor->getLpTaskQ()[READER_TASK_IDX]);
+    }
+    vals = get_stat(nullptr);
+    // The item should be restored to memory, hence added back to the stats
+    EXPECT_EQ(1, std::stoi(vals["ep_active_datatype_json,xattr"]));
+}
+
+TEST_P(DatatypeStatTest, MB23892) {
+    // This test checks that updating a document with a different datatype is
+    // safe to do after an eviction (where the blob is now null)
+    const DocKey key = {"jsonXattrDoc", DocNamespace::DefaultCollection};
+    setDatatypeItem(
+            store,
+            cookie,
+            PROTOCOL_BINARY_DATATYPE_JSON | PROTOCOL_BINARY_DATATYPE_XATTR,
+            "jsonXattrDoc");
+    auto vals = get_stat(nullptr);
+    EXPECT_EQ(1, std::stoi(vals["ep_active_datatype_json,xattr"]));
+    store->flushVBucket(0);
+    const char* msg;
+    store->evictKey(key, 0, &msg);
+    store->flushVBucket(0);
+    setDatatypeItem(store, cookie, PROTOCOL_BINARY_DATATYPE_JSON, "jsonXattrDoc", "[1]");
+}
+
+INSTANTIATE_TEST_CASE_P(FullAndValueEviction, DatatypeStatTest,
+                        ::testing::Values("value_only", "full_eviction"), []
+                                (const ::testing::TestParamInfo<std::string>&
+                                info) {return info.param;});